fix(queue): 自动救援失活 Worker 遗留任务

新增基于任务状态、执行租约和 Worker 心跳的孤儿 River Job 精确识别,每 15 秒将确认失活的 Job 恢复为可重试,避免等待 River 一小时通用救援窗口。运行时恢复同步清除中断及终态任务的 admission、执行令牌和并发租约,防止状态残留污染队列指标。正常运行中的长任务仍由 running 状态和活跃心跳保护。验证通过完整 Go 测试、go vet、竞态检查、迁移安全检查和 PostgreSQL 18 集成测试。
This commit is contained in:
2026-07-29 23:44:29 +08:00
parent 98820378b7
commit f3dd7cd262
6 changed files with 369 additions and 3 deletions
+37 -1
View File
@@ -18,7 +18,12 @@ import (
"github.com/riverqueue/river/rivertype"
)
const asyncTaskQueueName = "gateway_tasks"
const (
asyncTaskQueueName = "gateway_tasks"
orphanedRiverJobScanInterval = 15 * time.Second
orphanedRiverJobWorkerStaleAfter = 45 * time.Second
orphanedRiverJobRecoveryBatchSize = 100
)
type asyncTaskArgs struct {
TaskID string `json:"task_id" river:"unique"`
@@ -173,6 +178,7 @@ func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error
go s.refreshAsyncWorkerCapacity(ctx)
go s.dispatchWaitingAsyncAdmissions(ctx)
go s.reapExpiredTaskAdmissions(ctx)
go s.recoverOrphanedAsyncRiverJobs(ctx)
go s.stopAsyncWorkersOnShutdown(ctx)
return nil
}
@@ -457,6 +463,36 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
return nil
}
func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
recoverJobs := func() {
recovered, err := s.store.RecoverOrphanedAsyncRiverJobs(
ctx,
orphanedRiverJobWorkerStaleAfter,
orphanedRiverJobRecoveryBatchSize,
)
if err != nil {
if ctx.Err() == nil {
s.logger.Warn("recover orphaned river jobs failed", "error", err)
}
return
}
if recovered > 0 {
s.logger.Warn("orphaned river jobs recovered", "count", recovered)
}
}
recoverJobs()
ticker := time.NewTicker(orphanedRiverJobScanInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
recoverJobs()
}
}
}
func asyncTaskInsertOpts(task store.GatewayTask) *river.InsertOpts {
priority := 2
if task.ID == "" {