fix(queue): 自动救援失活 Worker 遗留任务

新增基于任务状态、执行租约和 Worker 心跳的孤儿 River Job 精确识别，每 15 秒将确认失活的 Job 恢复为可重试，避免等待 River 一小时通用救援窗口。运行时恢复同步清除中断及终态任务的 admission、执行令牌和并发租约，防止状态残留污染队列指标。正常运行中的长任务仍由 running 状态和活跃心跳保护。验证通过完整 Go 测试、go vet、竞态检查、迁移安全检查和 PostgreSQL 18 集成测试。
2026-07-29 23:44:29 +08:00
parent 98820378b7
commit f3dd7cd262
6 changed files with 369 additions and 3 deletions
@@ -446,6 +446,83 @@ WHERE id = $1::uuid`, atomicTask.ID).Scan(&riverJobID); err != nil {
 	if err := first.DeleteTaskAdmission(ctx, atomicTask.ID); err != nil {
 		t.Fatalf("release atomic admitted hook task: %v", err)
 	}
+
+	recoveryTask := createTask(true)
+	result, err = first.TryTaskAdmission(ctx, inputFor(recoveryTask, 100, ""))
+	if err != nil || !result.Admitted {
+		t.Fatalf("admit interrupted recovery task: result=%+v err=%v", result, err)
+	}
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_tasks
+SET status = 'running',
+    execution_token = $2::uuid,
+    execution_lease_expires_at = now() - interval '1 second'
+WHERE id = $1::uuid`, recoveryTask.ID, uuid.NewString()); err != nil {
+		t.Fatalf("expire interrupted recovery task lease: %v", err)
+	}
+	recovery, err := first.RecoverInterruptedRuntimeState(ctx)
+	if err != nil {
+		t.Fatalf("recover interrupted admitted task: %v", err)
+	}
+	if recovery.RequeuedAsyncTasks < 1 || recovery.CleanedTaskAdmissions < 1 {
+		t.Fatalf("interrupted task recovery = %+v, want requeue and admission cleanup", recovery)
+	}
+	var recoveredStatus string
+	var recoveredExecutionToken *string
+	if err := first.pool.QueryRow(ctx, `
+SELECT status, execution_token::text
+FROM gateway_tasks
+WHERE id = $1::uuid`, recoveryTask.ID).Scan(&recoveredStatus, &recoveredExecutionToken); err != nil {
+		t.Fatalf("read recovered task state: %v", err)
+	}
+	if recoveredStatus != "queued" || recoveredExecutionToken != nil {
+		t.Fatalf("recovered task status=%s execution_token=%v, want queued without token", recoveredStatus, recoveredExecutionToken)
+	}
+	var recoveredAdmissions, recoveredLeases int
+	if err := first.pool.QueryRow(ctx, `
+SELECT
+  (SELECT count(*) FROM gateway_task_admissions WHERE task_id = $1::uuid),
+  (SELECT count(*) FROM gateway_concurrency_leases WHERE task_id = $1::uuid AND released_at IS NULL)`,
+		recoveryTask.ID,
+	).Scan(&recoveredAdmissions, &recoveredLeases); err != nil {
+		t.Fatalf("read recovered task cleanup: %v", err)
+	}
+	if recoveredAdmissions != 0 || recoveredLeases != 0 {
+		t.Fatalf("recovered task left admissions=%d leases=%d", recoveredAdmissions, recoveredLeases)
+	}
+
+	terminalResidue := createTask(true)
+	result, err = first.TryTaskAdmission(ctx, inputFor(terminalResidue, 100, ""))
+	if err != nil || !result.Admitted {
+		t.Fatalf("admit terminal residue task: result=%+v err=%v", result, err)
+	}
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_tasks
+SET status = 'failed',
+    error_code = 'integration_test',
+    finished_at = now(),
+    updated_at = now()
+WHERE id = $1::uuid`, terminalResidue.ID); err != nil {
+		t.Fatalf("mark terminal residue task failed: %v", err)
+	}
+	recovery, err = first.RecoverInterruptedRuntimeState(ctx)
+	if err != nil {
+		t.Fatalf("clean terminal admission residue: %v", err)
+	}
+	if recovery.CleanedTaskAdmissions < 1 {
+		t.Fatalf("terminal recovery = %+v, want admission cleanup", recovery)
+	}
+	if err := first.pool.QueryRow(ctx, `
+SELECT
+  (SELECT count(*) FROM gateway_task_admissions WHERE task_id = $1::uuid),
+  (SELECT count(*) FROM gateway_concurrency_leases WHERE task_id = $1::uuid AND released_at IS NULL)`,
+		terminalResidue.ID,
+	).Scan(&recoveredAdmissions, &recoveredLeases); err != nil {
+		t.Fatalf("count terminal admission residue: %v", err)
+	}
+	if recoveredAdmissions != 0 || recoveredLeases != 0 {
+		t.Fatalf("terminal task left admissions=%d leases=%d", recoveredAdmissions, recoveredLeases)
+	}
 }

 func TestWorkerCapacityAllocationAndFailover(t *testing.T) {
@@ -16,6 +16,7 @@ type RuntimeRecoveryResult struct {
 	FailedAttempts            int64 `json:"failedAttempts"`
 	FailedTasks               int64 `json:"failedTasks"`
 	RequeuedAsyncTasks        int64 `json:"requeuedAsyncTasks"`
+	CleanedTaskAdmissions     int64 `json:"cleanedTaskAdmissions"`
 }

 var ErrConcurrencyLeaseLost = errors.New("concurrency lease lost")
@@ -549,6 +550,8 @@ SET status = 'queued',
    locked_by = NULL,
    locked_at = NULL,
    heartbeat_at = NULL,
+    execution_token = NULL,
+    execution_lease_expires_at = NULL,
    next_run_at = now(),
    finished_at = NULL,
    updated_at = now()
@@ -612,6 +615,24 @@ WHERE task.id = $1::uuid
 		}
 	}
 	result.RequeuedAsyncTasks = int64(len(asyncTaskIDs))
+	if len(asyncTaskIDs) > 0 {
+		tag, err = tx.Exec(ctx, `
+UPDATE gateway_concurrency_leases
+SET released_at = now()
+WHERE task_id = ANY($1::uuid[])
+  AND released_at IS NULL`, asyncTaskIDs)
+		if err != nil {
+			return RuntimeRecoveryResult{}, err
+		}
+		result.ReleasedConcurrencyLeases += tag.RowsAffected()
+		tag, err = tx.Exec(ctx, `
+DELETE FROM gateway_task_admissions
+WHERE task_id = ANY($1::uuid[])`, asyncTaskIDs)
+		if err != nil {
+			return RuntimeRecoveryResult{}, err
+		}
+		result.CleanedTaskAdmissions += tag.RowsAffected()
+	}

 	taskRows, err := tx.Query(ctx, `
 UPDATE gateway_tasks
@@ -620,6 +641,8 @@ SET status = 'failed',
    error_code = 'server_restarted',
    error_message = 'task interrupted by service restart',
    remote_task_payload = '{}'::jsonb,
+    execution_token = NULL,
+    execution_lease_expires_at = NULL,
    finished_at = now(),
    updated_at = now()
 WHERE async_mode = false
@@ -663,6 +686,26 @@ VALUES (
 		}
 	}
 	result.FailedTasks = int64(len(taskIDs))
+	tag, err = tx.Exec(ctx, `
+UPDATE gateway_concurrency_leases lease
+SET released_at = now()
+FROM gateway_tasks task
+WHERE lease.task_id = task.id
+  AND lease.released_at IS NULL
+  AND task.status IN ('succeeded', 'failed', 'cancelled')`)
+	if err != nil {
+		return RuntimeRecoveryResult{}, err
+	}
+	result.ReleasedConcurrencyLeases += tag.RowsAffected()
+	tag, err = tx.Exec(ctx, `
+DELETE FROM gateway_task_admissions admission
+USING gateway_tasks task
+WHERE admission.task_id = task.id
+  AND task.status IN ('succeeded', 'failed', 'cancelled')`)
+	if err != nil {
+		return RuntimeRecoveryResult{}, err
+	}
+	result.CleanedTaskAdmissions += tag.RowsAffected()

 	return result, tx.Commit(ctx)
 }
@@ -171,3 +171,105 @@ WHERE instance_id = $1`, strings.TrimSpace(instanceID))
 	}
 	return nil
 }
+
+// RecoverOrphanedAsyncRiverJobs retries River jobs whose owning worker is no
+// longer alive after the gateway execution lease has already returned the task
+// to queued. Normal long-running jobs remain protected by their running task
+// state and the active worker heartbeat.
+func (s *Store) RecoverOrphanedAsyncRiverJobs(
+	ctx context.Context,
+	workerStaleAfter time.Duration,
+	limit int,
+) (int64, error) {
+	if workerStaleAfter < workerHeartbeatStaleAfter {
+		workerStaleAfter = workerHeartbeatStaleAfter
+	}
+	if limit <= 0 || limit > 1000 {
+		limit = 100
+	}
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer tx.Rollback(ctx)
+	if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('gateway-river-orphan-recovery', 0))`); err != nil {
+		return 0, err
+	}
+	var recovered int64
+	if err := tx.QueryRow(ctx, `
+WITH orphaned AS MATERIALIZED (
+  SELECT job.id AS job_id,
+         task.id AS task_id,
+         job.attempt
+  FROM river_job job
+  JOIN gateway_tasks task ON task.river_job_id = job.id
+  WHERE job.queue = 'gateway_tasks'
+    AND job.kind = 'gateway_task_run'
+    AND job.state = 'running'
+    AND job.attempted_at <= now() - $1::interval
+    AND task.async_mode = true
+    AND task.status = 'queued'
+    AND (task.execution_lease_expires_at IS NULL OR task.execution_lease_expires_at <= now())
+    AND NOT EXISTS (
+      SELECT 1
+      FROM gateway_worker_instances worker
+      WHERE worker.status = 'active'
+        AND worker.heartbeat_at > now() - $1::interval
+        AND COALESCE(job.attempted_by[array_length(job.attempted_by, 1)], '')
+              LIKE worker.instance_id || '-exec-%'
+    )
+  ORDER BY job.attempted_at ASC, job.id ASC
+  LIMIT $2
+  FOR UPDATE OF job SKIP LOCKED
+),
+released_leases AS (
+  UPDATE gateway_concurrency_leases lease
+  SET released_at = now()
+  FROM orphaned
+  WHERE lease.task_id = orphaned.task_id
+    AND lease.released_at IS NULL
+),
+cleared_admissions AS (
+  DELETE FROM gateway_task_admissions admission
+  USING orphaned
+  WHERE admission.task_id = orphaned.task_id
+),
+reset_tasks AS (
+  UPDATE gateway_tasks task
+  SET locked_by = NULL,
+      locked_at = NULL,
+      heartbeat_at = NULL,
+      execution_token = NULL,
+      execution_lease_expires_at = NULL,
+      next_run_at = now(),
+      updated_at = now()
+  FROM orphaned
+  WHERE task.id = orphaned.task_id
+),
+recovered_jobs AS (
+  UPDATE river_job job
+  SET errors = array_append(
+        job.errors,
+        jsonb_build_object(
+          'at', now(),
+          'attempt', orphaned.attempt,
+          'error', 'Orphaned gateway job recovered after worker heartbeat expired',
+          'trace', ''
+        )
+      ),
+      finalized_at = NULL,
+      scheduled_at = now(),
+      state = 'retryable'
+  FROM orphaned
+  WHERE job.id = orphaned.job_id
+  RETURNING job.id
+)
+SELECT count(*)::bigint
+FROM recovered_jobs`, workerStaleAfter.String(), limit).Scan(&recovered); err != nil {
+		return 0, err
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return 0, err
+	}
+	return recovered, nil
+}