fix(queue): 回收终态任务的孤儿 River Job

原因:Worker 滚动切换后,执行租约恢复会终态化提交结果不确定的任务,但失活 Worker 持有的 River Job 仍需等待一小时通用救援窗口。\n\n影响:仅在任务已终态、执行租约清空、River owner 无活跃 Worker 且超过 45 秒时,将孤儿 Job 幂等标记为 completed;活跃 owner 不受影响。\n\n验证:PostgreSQL 集成测试覆盖失活 owner、活跃 owner 与重复回收;API 全量 go test 和 go vet 通过。
This commit is contained in:
2026-08-04 18:51:47 +08:00
parent 9bcfec890b
commit 484b05d005
3 changed files with 133 additions and 3 deletions
+40 -3
View File
@@ -254,14 +254,16 @@ func TestRecoverOrphanedAsyncRiverJobAndYieldStaleAdmission(t *testing.T) {
orphaned := createQueuedTask("orphaned")
protected := createQueuedTask("protected")
yielded := createQueuedTask("yielded")
terminalOrphaned := createQueuedTask("terminal-orphaned")
terminalProtected := createQueuedTask("terminal-protected")
staleWorkerID := "orphan-recovery-stale-" + suffix
activeWorkerID := "orphan-recovery-active-" + suffix
t.Cleanup(func() {
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cleanupCancel()
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM gateway_worker_instances WHERE instance_id = ANY($1::text[])`, []string{staleWorkerID, activeWorkerID})
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM river_job WHERE id = ANY($1::bigint[])`, []int64{orphaned.RiverJobID, protected.RiverJobID, yielded.RiverJobID})
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id = ANY($1::uuid[])`, []string{orphaned.ID, protected.ID, yielded.ID})
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM river_job WHERE id = ANY($1::bigint[])`, []int64{orphaned.RiverJobID, protected.RiverJobID, yielded.RiverJobID, terminalOrphaned.RiverJobID, terminalProtected.RiverJobID})
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id = ANY($1::uuid[])`, []string{orphaned.ID, protected.ID, yielded.ID, terminalOrphaned.ID, terminalProtected.ID})
})
if _, err := db.Pool().Exec(ctx, `
@@ -286,6 +288,8 @@ SET state = 'running',
WHEN $1::bigint THEN $3
WHEN $2::bigint THEN $4
WHEN $5::bigint THEN $4
WHEN $7::bigint THEN $3
WHEN $8::bigint THEN $4
END
]::text[]
WHERE id = ANY($6::bigint[])`,
@@ -294,10 +298,43 @@ WHERE id = ANY($6::bigint[])`,
staleWorkerID+"-exec-1-test",
activeWorkerID+"-exec-1-test",
yielded.RiverJobID,
[]int64{orphaned.RiverJobID, protected.RiverJobID, yielded.RiverJobID},
[]int64{orphaned.RiverJobID, protected.RiverJobID, yielded.RiverJobID, terminalOrphaned.RiverJobID, terminalProtected.RiverJobID},
terminalOrphaned.RiverJobID,
terminalProtected.RiverJobID,
); err != nil {
t.Fatalf("mark River jobs running: %v", err)
}
if _, err := db.Pool().Exec(ctx, `
UPDATE gateway_tasks
SET status = 'failed',
error_code = 'test_terminal',
finished_at = now(),
execution_token = NULL,
execution_lease_expires_at = NULL
WHERE id = ANY($1::uuid[])`, []string{terminalOrphaned.ID, terminalProtected.ID}); err != nil {
t.Fatalf("mark terminal River tasks failed: %v", err)
}
finalized, err := db.FinalizeOrphanedTerminalAsyncRiverJobs(ctx, 30*time.Second, 10)
if err != nil {
t.Fatalf("finalize orphaned terminal River jobs: %v", err)
}
if finalized != 1 {
t.Fatalf("finalized terminal jobs=%d, want 1", finalized)
}
var terminalOrphanedState, terminalProtectedState string
if err := db.Pool().QueryRow(ctx, `SELECT state::text FROM river_job WHERE id = $1`, terminalOrphaned.RiverJobID).Scan(&terminalOrphanedState); err != nil {
t.Fatalf("read terminal orphaned River state: %v", err)
}
if err := db.Pool().QueryRow(ctx, `SELECT state::text FROM river_job WHERE id = $1`, terminalProtected.RiverJobID).Scan(&terminalProtectedState); err != nil {
t.Fatalf("read terminal protected River state: %v", err)
}
if terminalOrphanedState != "completed" || terminalProtectedState != "running" {
t.Fatalf("terminal River states orphaned=%s protected=%s, want completed/running", terminalOrphanedState, terminalProtectedState)
}
finalized, err = db.FinalizeOrphanedTerminalAsyncRiverJobs(ctx, 30*time.Second, 10)
if err != nil || finalized != 0 {
t.Fatalf("second terminal River finalization count=%d err=%v, want idempotent zero", finalized, err)
}
var platformID, platformModelID string
if err := db.Pool().QueryRow(ctx, `
SELECT platform.id::text, model.id::text
+14
View File
@@ -694,6 +694,20 @@ func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
"cleanedTaskAdmissions", runtimeRecovery.CleanedTaskAdmissions,
)
}
finalizedTerminalJobs, err := s.coordinationStore.FinalizeOrphanedTerminalAsyncRiverJobs(
ctx,
orphanedRiverJobWorkerStaleAfter,
orphanedRiverJobRecoveryBatchSize,
)
if err != nil {
if ctx.Err() == nil {
s.logger.Warn("finalize orphaned terminal river jobs failed", "error", err)
}
return
}
if finalizedTerminalJobs > 0 {
s.logger.Warn("orphaned terminal river jobs finalized", "count", finalizedTerminalJobs)
}
yieldedAdmissions, err := s.coordinationStore.YieldStaleAsyncTaskAdmissions(
ctx,
orphanedRiverJobWorkerStaleAfter,