fix(queue): 回收终态任务的孤儿 River Job
原因:Worker 滚动切换后,执行租约恢复会终态化提交结果不确定的任务,但失活 Worker 持有的 River Job 仍需等待一小时通用救援窗口。\n\n影响:仅在任务已终态、执行租约清空、River owner 无活跃 Worker 且超过 45 秒时,将孤儿 Job 幂等标记为 completed;活跃 owner 不受影响。\n\n验证:PostgreSQL 集成测试覆盖失活 owner、活跃 owner 与重复回收;API 全量 go test 和 go vet 通过。
This commit is contained in:
@@ -254,14 +254,16 @@ func TestRecoverOrphanedAsyncRiverJobAndYieldStaleAdmission(t *testing.T) {
|
||||
orphaned := createQueuedTask("orphaned")
|
||||
protected := createQueuedTask("protected")
|
||||
yielded := createQueuedTask("yielded")
|
||||
terminalOrphaned := createQueuedTask("terminal-orphaned")
|
||||
terminalProtected := createQueuedTask("terminal-protected")
|
||||
staleWorkerID := "orphan-recovery-stale-" + suffix
|
||||
activeWorkerID := "orphan-recovery-active-" + suffix
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cleanupCancel()
|
||||
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM gateway_worker_instances WHERE instance_id = ANY($1::text[])`, []string{staleWorkerID, activeWorkerID})
|
||||
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM river_job WHERE id = ANY($1::bigint[])`, []int64{orphaned.RiverJobID, protected.RiverJobID, yielded.RiverJobID})
|
||||
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id = ANY($1::uuid[])`, []string{orphaned.ID, protected.ID, yielded.ID})
|
||||
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM river_job WHERE id = ANY($1::bigint[])`, []int64{orphaned.RiverJobID, protected.RiverJobID, yielded.RiverJobID, terminalOrphaned.RiverJobID, terminalProtected.RiverJobID})
|
||||
_, _ = db.Pool().Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id = ANY($1::uuid[])`, []string{orphaned.ID, protected.ID, yielded.ID, terminalOrphaned.ID, terminalProtected.ID})
|
||||
})
|
||||
|
||||
if _, err := db.Pool().Exec(ctx, `
|
||||
@@ -286,6 +288,8 @@ SET state = 'running',
|
||||
WHEN $1::bigint THEN $3
|
||||
WHEN $2::bigint THEN $4
|
||||
WHEN $5::bigint THEN $4
|
||||
WHEN $7::bigint THEN $3
|
||||
WHEN $8::bigint THEN $4
|
||||
END
|
||||
]::text[]
|
||||
WHERE id = ANY($6::bigint[])`,
|
||||
@@ -294,10 +298,43 @@ WHERE id = ANY($6::bigint[])`,
|
||||
staleWorkerID+"-exec-1-test",
|
||||
activeWorkerID+"-exec-1-test",
|
||||
yielded.RiverJobID,
|
||||
[]int64{orphaned.RiverJobID, protected.RiverJobID, yielded.RiverJobID},
|
||||
[]int64{orphaned.RiverJobID, protected.RiverJobID, yielded.RiverJobID, terminalOrphaned.RiverJobID, terminalProtected.RiverJobID},
|
||||
terminalOrphaned.RiverJobID,
|
||||
terminalProtected.RiverJobID,
|
||||
); err != nil {
|
||||
t.Fatalf("mark River jobs running: %v", err)
|
||||
}
|
||||
if _, err := db.Pool().Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'failed',
|
||||
error_code = 'test_terminal',
|
||||
finished_at = now(),
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL
|
||||
WHERE id = ANY($1::uuid[])`, []string{terminalOrphaned.ID, terminalProtected.ID}); err != nil {
|
||||
t.Fatalf("mark terminal River tasks failed: %v", err)
|
||||
}
|
||||
finalized, err := db.FinalizeOrphanedTerminalAsyncRiverJobs(ctx, 30*time.Second, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("finalize orphaned terminal River jobs: %v", err)
|
||||
}
|
||||
if finalized != 1 {
|
||||
t.Fatalf("finalized terminal jobs=%d, want 1", finalized)
|
||||
}
|
||||
var terminalOrphanedState, terminalProtectedState string
|
||||
if err := db.Pool().QueryRow(ctx, `SELECT state::text FROM river_job WHERE id = $1`, terminalOrphaned.RiverJobID).Scan(&terminalOrphanedState); err != nil {
|
||||
t.Fatalf("read terminal orphaned River state: %v", err)
|
||||
}
|
||||
if err := db.Pool().QueryRow(ctx, `SELECT state::text FROM river_job WHERE id = $1`, terminalProtected.RiverJobID).Scan(&terminalProtectedState); err != nil {
|
||||
t.Fatalf("read terminal protected River state: %v", err)
|
||||
}
|
||||
if terminalOrphanedState != "completed" || terminalProtectedState != "running" {
|
||||
t.Fatalf("terminal River states orphaned=%s protected=%s, want completed/running", terminalOrphanedState, terminalProtectedState)
|
||||
}
|
||||
finalized, err = db.FinalizeOrphanedTerminalAsyncRiverJobs(ctx, 30*time.Second, 10)
|
||||
if err != nil || finalized != 0 {
|
||||
t.Fatalf("second terminal River finalization count=%d err=%v, want idempotent zero", finalized, err)
|
||||
}
|
||||
var platformID, platformModelID string
|
||||
if err := db.Pool().QueryRow(ctx, `
|
||||
SELECT platform.id::text, model.id::text
|
||||
|
||||
@@ -694,6 +694,20 @@ func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
|
||||
"cleanedTaskAdmissions", runtimeRecovery.CleanedTaskAdmissions,
|
||||
)
|
||||
}
|
||||
finalizedTerminalJobs, err := s.coordinationStore.FinalizeOrphanedTerminalAsyncRiverJobs(
|
||||
ctx,
|
||||
orphanedRiverJobWorkerStaleAfter,
|
||||
orphanedRiverJobRecoveryBatchSize,
|
||||
)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
s.logger.Warn("finalize orphaned terminal river jobs failed", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if finalizedTerminalJobs > 0 {
|
||||
s.logger.Warn("orphaned terminal river jobs finalized", "count", finalizedTerminalJobs)
|
||||
}
|
||||
yieldedAdmissions, err := s.coordinationStore.YieldStaleAsyncTaskAdmissions(
|
||||
ctx,
|
||||
orphanedRiverJobWorkerStaleAfter,
|
||||
|
||||
Reference in New Issue
Block a user