fix(worker): 防止事务泄漏并恢复滞留队列

为 PostgreSQL 连接增加可配置的事务空闲与锁等待超时，并在请求取消或 Worker 退出后使用独立有界上下文回滚事务。\n\n恢复过期任务时按批次使用 SKIP LOCKED，周期重建缺失或终态 River Job；锁等待超时只在确认尚未提交上游时安全重排，避免重复执行和重复结算。\n\n验证：完整 Go 测试、go vet、生产 Kustomize 渲染、gofmt 与 git diff --check 均通过。
2026-07-30 17:27:11 +08:00
parent bc44af751e
commit 6bab0f0749
32 changed files with 545 additions and 106 deletions
@@ -543,6 +543,160 @@ SELECT
 		t.Fatalf("recovered task left admissions=%d leases=%d", recoveredAdmissions, recoveredLeases)
 	}

+	lockedRecoveryTask := createTask(true)
+	unlockedRecoveryTask := createTask(true)
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_tasks
+SET status = 'running',
+    execution_token = gen_random_uuid(),
+    execution_lease_expires_at = now() - interval '1 second'
+WHERE id = ANY($1::uuid[])`, []string{lockedRecoveryTask.ID, unlockedRecoveryTask.ID}); err != nil {
+		t.Fatalf("prepare locked recovery tasks: %v", err)
+	}
+	lockTx, err := second.pool.Begin(ctx)
+	if err != nil {
+		t.Fatalf("begin task row lock: %v", err)
+	}
+	var lockedTaskID string
+	if err := lockTx.QueryRow(ctx, `
+SELECT id::text
+FROM gateway_tasks
+WHERE id = $1::uuid
+FOR UPDATE`, lockedRecoveryTask.ID).Scan(&lockedTaskID); err != nil {
+		rollbackTransaction(lockTx)
+		t.Fatalf("lock interrupted task row: %v", err)
+	}
+	recoveryCtx, recoveryCancel := context.WithTimeout(ctx, 3*time.Second)
+	recovery, err = first.RecoverInterruptedRuntimeState(recoveryCtx)
+	recoveryCancel()
+	if err != nil {
+		rollbackTransaction(lockTx)
+		t.Fatalf("recover unlocked task while another row is locked: %v", err)
+	}
+	if recovery.RequeuedAsyncTasks < 1 {
+		rollbackTransaction(lockTx)
+		t.Fatalf("SKIP LOCKED recovery = %+v, want at least one requeued task", recovery)
+	}
+	var lockedStatus, unlockedStatus string
+	if err := first.pool.QueryRow(ctx, `
+SELECT
+  (SELECT status FROM gateway_tasks WHERE id = $1::uuid),
+  (SELECT status FROM gateway_tasks WHERE id = $2::uuid)`,
+		lockedRecoveryTask.ID,
+		unlockedRecoveryTask.ID,
+	).Scan(&lockedStatus, &unlockedStatus); err != nil {
+		rollbackTransaction(lockTx)
+		t.Fatalf("read SKIP LOCKED recovery states: %v", err)
+	}
+	if lockedStatus != "running" || unlockedStatus != "queued" {
+		rollbackTransaction(lockTx)
+		t.Fatalf("SKIP LOCKED recovery statuses = %s/%s, want running/queued", lockedStatus, unlockedStatus)
+	}
+	rollbackTransaction(lockTx)
+	recovery, err = first.RecoverInterruptedRuntimeState(ctx)
+	if err != nil {
+		t.Fatalf("recover previously locked task: %v", err)
+	}
+	if recovery.RequeuedAsyncTasks < 1 {
+		t.Fatalf("unlocked recovery = %+v, want requeued task", recovery)
+	}
+	if err := first.pool.QueryRow(ctx, `
+SELECT status
+FROM gateway_tasks
+WHERE id = $1::uuid`, lockedRecoveryTask.ID).Scan(&lockedStatus); err != nil {
+		t.Fatalf("read unlocked recovery status: %v", err)
+	}
+	if lockedStatus != "queued" {
+		t.Fatalf("unlocked recovery status = %s, want queued", lockedStatus)
+	}
+
+	preSubmissionTask := createTask(true)
+	preSubmissionToken := uuid.NewString()
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_tasks
+SET status = 'running',
+    execution_token = $2::uuid,
+    execution_lease_expires_at = now() + interval '5 minutes'
+WHERE id = $1::uuid`,
+		preSubmissionTask.ID,
+		preSubmissionToken,
+	); err != nil {
+		t.Fatalf("prepare pre-submission lock timeout task: %v", err)
+	}
+	if _, err := first.pool.Exec(ctx, `
+INSERT INTO gateway_task_attempts (
+  task_id, attempt_no, queue_key, status, upstream_submission_status
+)
+VALUES ($1::uuid, 1, 'integration-test', 'running', 'not_submitted')`,
+		preSubmissionTask.ID,
+	); err != nil {
+		t.Fatalf("create pre-submission attempt: %v", err)
+	}
+	requeuedTask, changed, err := first.RequeueTaskBeforeUpstreamSubmission(
+		ctx,
+		preSubmissionTask.ID,
+		preSubmissionToken,
+		time.Second,
+	)
+	if err != nil || !changed || requeuedTask.Status != "queued" {
+		t.Fatalf("pre-submission lock timeout requeue task=%+v changed=%v err=%v", requeuedTask, changed, err)
+	}
+	var preSubmissionAttempts int
+	if err := first.pool.QueryRow(ctx, `
+SELECT count(*)
+FROM gateway_task_attempts
+WHERE task_id = $1::uuid`, preSubmissionTask.ID).Scan(&preSubmissionAttempts); err != nil {
+		t.Fatalf("count cleaned pre-submission attempts: %v", err)
+	}
+	if preSubmissionAttempts != 0 {
+		t.Fatalf("pre-submission lock timeout left %d attempts, want 0", preSubmissionAttempts)
+	}
+
+	ambiguousTask := createTask(true)
+	ambiguousToken := uuid.NewString()
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_tasks
+SET status = 'running',
+    execution_token = $2::uuid,
+    execution_lease_expires_at = now() + interval '5 minutes'
+WHERE id = $1::uuid`,
+		ambiguousTask.ID,
+		ambiguousToken,
+	); err != nil {
+		t.Fatalf("prepare ambiguous submission task: %v", err)
+	}
+	if _, err := first.pool.Exec(ctx, `
+INSERT INTO gateway_task_attempts (
+  task_id, attempt_no, queue_key, status, upstream_submission_status
+)
+VALUES ($1::uuid, 1, 'integration-test', 'running', 'submitting')`,
+		ambiguousTask.ID,
+	); err != nil {
+		t.Fatalf("create ambiguous submission attempt: %v", err)
+	}
+	_, changed, err = first.RequeueTaskBeforeUpstreamSubmission(
+		ctx,
+		ambiguousTask.ID,
+		ambiguousToken,
+		time.Second,
+	)
+	if err != nil {
+		t.Fatalf("guard ambiguous submission task requeue: %v", err)
+	}
+	if changed {
+		t.Fatal("ambiguous submission task was requeued")
+	}
+	var ambiguousStatus string
+	if err := first.pool.QueryRow(ctx, `
+SELECT status
+FROM gateway_tasks
+WHERE id = $1::uuid`, ambiguousTask.ID).Scan(&ambiguousStatus); err != nil {
+		t.Fatalf("read ambiguous submission task: %v", err)
+	}
+	if ambiguousStatus != "running" {
+		t.Fatalf("ambiguous submission task status = %s, want running", ambiguousStatus)
+	}
+
 	terminalResidue := createTask(true)
 	result, err = first.TryTaskAdmission(ctx, inputFor(terminalResidue, 100, ""))
 	if err != nil || !result.Admitted {