fix(queue): 自动救援失活 Worker 遗留任务
新增基于任务状态、执行租约和 Worker 心跳的孤儿 River Job 精确识别,每 15 秒将确认失活的 Job 恢复为可重试,避免等待 River 一小时通用救援窗口。运行时恢复同步清除中断及终态任务的 admission、执行令牌和并发租约,防止状态残留污染队列指标。正常运行中的长任务仍由 running 状态和活跃心跳保护。验证通过完整 Go 测试、go vet、竞态检查、迁移安全检查和 PostgreSQL 18 集成测试。
This commit is contained in:
@@ -16,6 +16,7 @@ type RuntimeRecoveryResult struct {
|
||||
FailedAttempts int64 `json:"failedAttempts"`
|
||||
FailedTasks int64 `json:"failedTasks"`
|
||||
RequeuedAsyncTasks int64 `json:"requeuedAsyncTasks"`
|
||||
CleanedTaskAdmissions int64 `json:"cleanedTaskAdmissions"`
|
||||
}
|
||||
|
||||
var ErrConcurrencyLeaseLost = errors.New("concurrency lease lost")
|
||||
@@ -549,6 +550,8 @@ SET status = 'queued',
|
||||
locked_by = NULL,
|
||||
locked_at = NULL,
|
||||
heartbeat_at = NULL,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
next_run_at = now(),
|
||||
finished_at = NULL,
|
||||
updated_at = now()
|
||||
@@ -612,6 +615,24 @@ WHERE task.id = $1::uuid
|
||||
}
|
||||
}
|
||||
result.RequeuedAsyncTasks = int64(len(asyncTaskIDs))
|
||||
if len(asyncTaskIDs) > 0 {
|
||||
tag, err = tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
WHERE task_id = ANY($1::uuid[])
|
||||
AND released_at IS NULL`, asyncTaskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.ReleasedConcurrencyLeases += tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
DELETE FROM gateway_task_admissions
|
||||
WHERE task_id = ANY($1::uuid[])`, asyncTaskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.CleanedTaskAdmissions += tag.RowsAffected()
|
||||
}
|
||||
|
||||
taskRows, err := tx.Query(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
@@ -620,6 +641,8 @@ SET status = 'failed',
|
||||
error_code = 'server_restarted',
|
||||
error_message = 'task interrupted by service restart',
|
||||
remote_task_payload = '{}'::jsonb,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE async_mode = false
|
||||
@@ -663,6 +686,26 @@ VALUES (
|
||||
}
|
||||
}
|
||||
result.FailedTasks = int64(len(taskIDs))
|
||||
tag, err = tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases lease
|
||||
SET released_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE lease.task_id = task.id
|
||||
AND lease.released_at IS NULL
|
||||
AND task.status IN ('succeeded', 'failed', 'cancelled')`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.ReleasedConcurrencyLeases += tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
DELETE FROM gateway_task_admissions admission
|
||||
USING gateway_tasks task
|
||||
WHERE admission.task_id = task.id
|
||||
AND task.status IN ('succeeded', 'failed', 'cancelled')`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.CleanedTaskAdmissions += tag.RowsAffected()
|
||||
|
||||
return result, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user