fix(queue): 自动救援失活 Worker 遗留任务

新增基于任务状态、执行租约和 Worker 心跳的孤儿 River Job 精确识别,每 15 秒将确认失活的 Job 恢复为可重试,避免等待 River 一小时通用救援窗口。运行时恢复同步清除中断及终态任务的 admission、执行令牌和并发租约,防止状态残留污染队列指标。正常运行中的长任务仍由 running 状态和活跃心跳保护。验证通过完整 Go 测试、go vet、竞态检查、迁移安全检查和 PostgreSQL 18 集成测试。
This commit is contained in:
2026-07-29 23:44:29 +08:00
parent 98820378b7
commit f3dd7cd262
6 changed files with 369 additions and 3 deletions
@@ -446,6 +446,83 @@ WHERE id = $1::uuid`, atomicTask.ID).Scan(&riverJobID); err != nil {
if err := first.DeleteTaskAdmission(ctx, atomicTask.ID); err != nil {
t.Fatalf("release atomic admitted hook task: %v", err)
}
recoveryTask := createTask(true)
result, err = first.TryTaskAdmission(ctx, inputFor(recoveryTask, 100, ""))
if err != nil || !result.Admitted {
t.Fatalf("admit interrupted recovery task: result=%+v err=%v", result, err)
}
if _, err := first.pool.Exec(ctx, `
UPDATE gateway_tasks
SET status = 'running',
execution_token = $2::uuid,
execution_lease_expires_at = now() - interval '1 second'
WHERE id = $1::uuid`, recoveryTask.ID, uuid.NewString()); err != nil {
t.Fatalf("expire interrupted recovery task lease: %v", err)
}
recovery, err := first.RecoverInterruptedRuntimeState(ctx)
if err != nil {
t.Fatalf("recover interrupted admitted task: %v", err)
}
if recovery.RequeuedAsyncTasks < 1 || recovery.CleanedTaskAdmissions < 1 {
t.Fatalf("interrupted task recovery = %+v, want requeue and admission cleanup", recovery)
}
var recoveredStatus string
var recoveredExecutionToken *string
if err := first.pool.QueryRow(ctx, `
SELECT status, execution_token::text
FROM gateway_tasks
WHERE id = $1::uuid`, recoveryTask.ID).Scan(&recoveredStatus, &recoveredExecutionToken); err != nil {
t.Fatalf("read recovered task state: %v", err)
}
if recoveredStatus != "queued" || recoveredExecutionToken != nil {
t.Fatalf("recovered task status=%s execution_token=%v, want queued without token", recoveredStatus, recoveredExecutionToken)
}
var recoveredAdmissions, recoveredLeases int
if err := first.pool.QueryRow(ctx, `
SELECT
(SELECT count(*) FROM gateway_task_admissions WHERE task_id = $1::uuid),
(SELECT count(*) FROM gateway_concurrency_leases WHERE task_id = $1::uuid AND released_at IS NULL)`,
recoveryTask.ID,
).Scan(&recoveredAdmissions, &recoveredLeases); err != nil {
t.Fatalf("read recovered task cleanup: %v", err)
}
if recoveredAdmissions != 0 || recoveredLeases != 0 {
t.Fatalf("recovered task left admissions=%d leases=%d", recoveredAdmissions, recoveredLeases)
}
terminalResidue := createTask(true)
result, err = first.TryTaskAdmission(ctx, inputFor(terminalResidue, 100, ""))
if err != nil || !result.Admitted {
t.Fatalf("admit terminal residue task: result=%+v err=%v", result, err)
}
if _, err := first.pool.Exec(ctx, `
UPDATE gateway_tasks
SET status = 'failed',
error_code = 'integration_test',
finished_at = now(),
updated_at = now()
WHERE id = $1::uuid`, terminalResidue.ID); err != nil {
t.Fatalf("mark terminal residue task failed: %v", err)
}
recovery, err = first.RecoverInterruptedRuntimeState(ctx)
if err != nil {
t.Fatalf("clean terminal admission residue: %v", err)
}
if recovery.CleanedTaskAdmissions < 1 {
t.Fatalf("terminal recovery = %+v, want admission cleanup", recovery)
}
if err := first.pool.QueryRow(ctx, `
SELECT
(SELECT count(*) FROM gateway_task_admissions WHERE task_id = $1::uuid),
(SELECT count(*) FROM gateway_concurrency_leases WHERE task_id = $1::uuid AND released_at IS NULL)`,
terminalResidue.ID,
).Scan(&recoveredAdmissions, &recoveredLeases); err != nil {
t.Fatalf("count terminal admission residue: %v", err)
}
if recoveredAdmissions != 0 || recoveredLeases != 0 {
t.Fatalf("terminal task left admissions=%d leases=%d", recoveredAdmissions, recoveredLeases)
}
}
func TestWorkerCapacityAllocationAndFailover(t *testing.T) {