fix(queue): 自动救援失活 Worker 遗留任务

新增基于任务状态、执行租约和 Worker 心跳的孤儿 River Job 精确识别，每 15 秒将确认失活的 Job 恢复为可重试，避免等待 River 一小时通用救援窗口。运行时恢复同步清除中断及终态任务的 admission、执行令牌和并发租约，防止状态残留污染队列指标。正常运行中的长任务仍由 running 状态和活跃心跳保护。验证通过完整 Go 测试、go vet、竞态检查、迁移安全检查和 PostgreSQL 18 集成测试。
2026-07-29 23:44:29 +08:00
parent 98820378b7
commit f3dd7cd262
6 changed files with 369 additions and 3 deletions
@@ -171,3 +171,105 @@ WHERE instance_id = $1`, strings.TrimSpace(instanceID))
 	}
 	return nil
 }
+
+// RecoverOrphanedAsyncRiverJobs retries River jobs whose owning worker is no
+// longer alive after the gateway execution lease has already returned the task
+// to queued. Normal long-running jobs remain protected by their running task
+// state and the active worker heartbeat.
+func (s *Store) RecoverOrphanedAsyncRiverJobs(
+	ctx context.Context,
+	workerStaleAfter time.Duration,
+	limit int,
+) (int64, error) {
+	if workerStaleAfter < workerHeartbeatStaleAfter {
+		workerStaleAfter = workerHeartbeatStaleAfter
+	}
+	if limit <= 0 || limit > 1000 {
+		limit = 100
+	}
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer tx.Rollback(ctx)
+	if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('gateway-river-orphan-recovery', 0))`); err != nil {
+		return 0, err
+	}
+	var recovered int64
+	if err := tx.QueryRow(ctx, `
+WITH orphaned AS MATERIALIZED (
+  SELECT job.id AS job_id,
+         task.id AS task_id,
+         job.attempt
+  FROM river_job job
+  JOIN gateway_tasks task ON task.river_job_id = job.id
+  WHERE job.queue = 'gateway_tasks'
+    AND job.kind = 'gateway_task_run'
+    AND job.state = 'running'
+    AND job.attempted_at <= now() - $1::interval
+    AND task.async_mode = true
+    AND task.status = 'queued'
+    AND (task.execution_lease_expires_at IS NULL OR task.execution_lease_expires_at <= now())
+    AND NOT EXISTS (
+      SELECT 1
+      FROM gateway_worker_instances worker
+      WHERE worker.status = 'active'
+        AND worker.heartbeat_at > now() - $1::interval
+        AND COALESCE(job.attempted_by[array_length(job.attempted_by, 1)], '')
+              LIKE worker.instance_id || '-exec-%'
+    )
+  ORDER BY job.attempted_at ASC, job.id ASC
+  LIMIT $2
+  FOR UPDATE OF job SKIP LOCKED
+),
+released_leases AS (
+  UPDATE gateway_concurrency_leases lease
+  SET released_at = now()
+  FROM orphaned
+  WHERE lease.task_id = orphaned.task_id
+    AND lease.released_at IS NULL
+),
+cleared_admissions AS (
+  DELETE FROM gateway_task_admissions admission
+  USING orphaned
+  WHERE admission.task_id = orphaned.task_id
+),
+reset_tasks AS (
+  UPDATE gateway_tasks task
+  SET locked_by = NULL,
+      locked_at = NULL,
+      heartbeat_at = NULL,
+      execution_token = NULL,
+      execution_lease_expires_at = NULL,
+      next_run_at = now(),
+      updated_at = now()
+  FROM orphaned
+  WHERE task.id = orphaned.task_id
+),
+recovered_jobs AS (
+  UPDATE river_job job
+  SET errors = array_append(
+        job.errors,
+        jsonb_build_object(
+          'at', now(),
+          'attempt', orphaned.attempt,
+          'error', 'Orphaned gateway job recovered after worker heartbeat expired',
+          'trace', ''
+        )
+      ),
+      finalized_at = NULL,
+      scheduled_at = now(),
+      state = 'retryable'
+  FROM orphaned
+  WHERE job.id = orphaned.job_id
+  RETURNING job.id
+)
+SELECT count(*)::bigint
+FROM recovered_jobs`, workerStaleAfter.String(), limit).Scan(&recovered); err != nil {
+		return 0, err
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return 0, err
+	}
+	return recovered, nil
+}