fix(queue): 自动救援失活 Worker 遗留任务
新增基于任务状态、执行租约和 Worker 心跳的孤儿 River Job 精确识别,每 15 秒将确认失活的 Job 恢复为可重试,避免等待 River 一小时通用救援窗口。运行时恢复同步清除中断及终态任务的 admission、执行令牌和并发租约,防止状态残留污染队列指标。正常运行中的长任务仍由 running 状态和活跃心跳保护。验证通过完整 Go 测试、go vet、竞态检查、迁移安全检查和 PostgreSQL 18 集成测试。
This commit is contained in:
@@ -446,6 +446,83 @@ WHERE id = $1::uuid`, atomicTask.ID).Scan(&riverJobID); err != nil {
|
||||
if err := first.DeleteTaskAdmission(ctx, atomicTask.ID); err != nil {
|
||||
t.Fatalf("release atomic admitted hook task: %v", err)
|
||||
}
|
||||
|
||||
recoveryTask := createTask(true)
|
||||
result, err = first.TryTaskAdmission(ctx, inputFor(recoveryTask, 100, ""))
|
||||
if err != nil || !result.Admitted {
|
||||
t.Fatalf("admit interrupted recovery task: result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err := first.pool.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'running',
|
||||
execution_token = $2::uuid,
|
||||
execution_lease_expires_at = now() - interval '1 second'
|
||||
WHERE id = $1::uuid`, recoveryTask.ID, uuid.NewString()); err != nil {
|
||||
t.Fatalf("expire interrupted recovery task lease: %v", err)
|
||||
}
|
||||
recovery, err := first.RecoverInterruptedRuntimeState(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("recover interrupted admitted task: %v", err)
|
||||
}
|
||||
if recovery.RequeuedAsyncTasks < 1 || recovery.CleanedTaskAdmissions < 1 {
|
||||
t.Fatalf("interrupted task recovery = %+v, want requeue and admission cleanup", recovery)
|
||||
}
|
||||
var recoveredStatus string
|
||||
var recoveredExecutionToken *string
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT status, execution_token::text
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid`, recoveryTask.ID).Scan(&recoveredStatus, &recoveredExecutionToken); err != nil {
|
||||
t.Fatalf("read recovered task state: %v", err)
|
||||
}
|
||||
if recoveredStatus != "queued" || recoveredExecutionToken != nil {
|
||||
t.Fatalf("recovered task status=%s execution_token=%v, want queued without token", recoveredStatus, recoveredExecutionToken)
|
||||
}
|
||||
var recoveredAdmissions, recoveredLeases int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT count(*) FROM gateway_task_admissions WHERE task_id = $1::uuid),
|
||||
(SELECT count(*) FROM gateway_concurrency_leases WHERE task_id = $1::uuid AND released_at IS NULL)`,
|
||||
recoveryTask.ID,
|
||||
).Scan(&recoveredAdmissions, &recoveredLeases); err != nil {
|
||||
t.Fatalf("read recovered task cleanup: %v", err)
|
||||
}
|
||||
if recoveredAdmissions != 0 || recoveredLeases != 0 {
|
||||
t.Fatalf("recovered task left admissions=%d leases=%d", recoveredAdmissions, recoveredLeases)
|
||||
}
|
||||
|
||||
terminalResidue := createTask(true)
|
||||
result, err = first.TryTaskAdmission(ctx, inputFor(terminalResidue, 100, ""))
|
||||
if err != nil || !result.Admitted {
|
||||
t.Fatalf("admit terminal residue task: result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err := first.pool.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'failed',
|
||||
error_code = 'integration_test',
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid`, terminalResidue.ID); err != nil {
|
||||
t.Fatalf("mark terminal residue task failed: %v", err)
|
||||
}
|
||||
recovery, err = first.RecoverInterruptedRuntimeState(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("clean terminal admission residue: %v", err)
|
||||
}
|
||||
if recovery.CleanedTaskAdmissions < 1 {
|
||||
t.Fatalf("terminal recovery = %+v, want admission cleanup", recovery)
|
||||
}
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT count(*) FROM gateway_task_admissions WHERE task_id = $1::uuid),
|
||||
(SELECT count(*) FROM gateway_concurrency_leases WHERE task_id = $1::uuid AND released_at IS NULL)`,
|
||||
terminalResidue.ID,
|
||||
).Scan(&recoveredAdmissions, &recoveredLeases); err != nil {
|
||||
t.Fatalf("count terminal admission residue: %v", err)
|
||||
}
|
||||
if recoveredAdmissions != 0 || recoveredLeases != 0 {
|
||||
t.Fatalf("terminal task left admissions=%d leases=%d", recoveredAdmissions, recoveredLeases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerCapacityAllocationAndFailover(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user