fix(queue): 自动救援失活 Worker 遗留任务

新增基于任务状态、执行租约和 Worker 心跳的孤儿 River Job 精确识别,每 15 秒将确认失活的 Job 恢复为可重试,避免等待 River 一小时通用救援窗口。运行时恢复同步清除中断及终态任务的 admission、执行令牌和并发租约,防止状态残留污染队列指标。正常运行中的长任务仍由 running 状态和活跃心跳保护。验证通过完整 Go 测试、go vet、竞态检查、迁移安全检查和 PostgreSQL 18 集成测试。
This commit is contained in:
2026-07-29 23:44:29 +08:00
parent 98820378b7
commit f3dd7cd262
6 changed files with 369 additions and 3 deletions
+43
View File
@@ -16,6 +16,7 @@ type RuntimeRecoveryResult struct {
FailedAttempts int64 `json:"failedAttempts"`
FailedTasks int64 `json:"failedTasks"`
RequeuedAsyncTasks int64 `json:"requeuedAsyncTasks"`
CleanedTaskAdmissions int64 `json:"cleanedTaskAdmissions"`
}
var ErrConcurrencyLeaseLost = errors.New("concurrency lease lost")
@@ -549,6 +550,8 @@ SET status = 'queued',
locked_by = NULL,
locked_at = NULL,
heartbeat_at = NULL,
execution_token = NULL,
execution_lease_expires_at = NULL,
next_run_at = now(),
finished_at = NULL,
updated_at = now()
@@ -612,6 +615,24 @@ WHERE task.id = $1::uuid
}
}
result.RequeuedAsyncTasks = int64(len(asyncTaskIDs))
if len(asyncTaskIDs) > 0 {
tag, err = tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
WHERE task_id = ANY($1::uuid[])
AND released_at IS NULL`, asyncTaskIDs)
if err != nil {
return RuntimeRecoveryResult{}, err
}
result.ReleasedConcurrencyLeases += tag.RowsAffected()
tag, err = tx.Exec(ctx, `
DELETE FROM gateway_task_admissions
WHERE task_id = ANY($1::uuid[])`, asyncTaskIDs)
if err != nil {
return RuntimeRecoveryResult{}, err
}
result.CleanedTaskAdmissions += tag.RowsAffected()
}
taskRows, err := tx.Query(ctx, `
UPDATE gateway_tasks
@@ -620,6 +641,8 @@ SET status = 'failed',
error_code = 'server_restarted',
error_message = 'task interrupted by service restart',
remote_task_payload = '{}'::jsonb,
execution_token = NULL,
execution_lease_expires_at = NULL,
finished_at = now(),
updated_at = now()
WHERE async_mode = false
@@ -663,6 +686,26 @@ VALUES (
}
}
result.FailedTasks = int64(len(taskIDs))
tag, err = tx.Exec(ctx, `
UPDATE gateway_concurrency_leases lease
SET released_at = now()
FROM gateway_tasks task
WHERE lease.task_id = task.id
AND lease.released_at IS NULL
AND task.status IN ('succeeded', 'failed', 'cancelled')`)
if err != nil {
return RuntimeRecoveryResult{}, err
}
result.ReleasedConcurrencyLeases += tag.RowsAffected()
tag, err = tx.Exec(ctx, `
DELETE FROM gateway_task_admissions admission
USING gateway_tasks task
WHERE admission.task_id = task.id
AND task.status IN ('succeeded', 'failed', 'cancelled')`)
if err != nil {
return RuntimeRecoveryResult{}, err
}
result.CleanedTaskAdmissions += tag.RowsAffected()
return result, tx.Commit(ctx)
}