fix(queue): 自动救援失活 Worker 遗留任务
新增基于任务状态、执行租约和 Worker 心跳的孤儿 River Job 精确识别,每 15 秒将确认失活的 Job 恢复为可重试,避免等待 River 一小时通用救援窗口。运行时恢复同步清除中断及终态任务的 admission、执行令牌和并发租约,防止状态残留污染队列指标。正常运行中的长任务仍由 running 状态和活跃心跳保护。验证通过完整 Go 测试、go vet、竞态检查、迁移安全检查和 PostgreSQL 18 集成测试。
This commit is contained in:
@@ -18,7 +18,12 @@ import (
|
||||
"github.com/riverqueue/river/rivertype"
|
||||
)
|
||||
|
||||
const asyncTaskQueueName = "gateway_tasks"
|
||||
const (
|
||||
asyncTaskQueueName = "gateway_tasks"
|
||||
orphanedRiverJobScanInterval = 15 * time.Second
|
||||
orphanedRiverJobWorkerStaleAfter = 45 * time.Second
|
||||
orphanedRiverJobRecoveryBatchSize = 100
|
||||
)
|
||||
|
||||
type asyncTaskArgs struct {
|
||||
TaskID string `json:"task_id" river:"unique"`
|
||||
@@ -173,6 +178,7 @@ func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error
|
||||
go s.refreshAsyncWorkerCapacity(ctx)
|
||||
go s.dispatchWaitingAsyncAdmissions(ctx)
|
||||
go s.reapExpiredTaskAdmissions(ctx)
|
||||
go s.recoverOrphanedAsyncRiverJobs(ctx)
|
||||
go s.stopAsyncWorkersOnShutdown(ctx)
|
||||
return nil
|
||||
}
|
||||
@@ -457,6 +463,36 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
|
||||
recoverJobs := func() {
|
||||
recovered, err := s.store.RecoverOrphanedAsyncRiverJobs(
|
||||
ctx,
|
||||
orphanedRiverJobWorkerStaleAfter,
|
||||
orphanedRiverJobRecoveryBatchSize,
|
||||
)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
s.logger.Warn("recover orphaned river jobs failed", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if recovered > 0 {
|
||||
s.logger.Warn("orphaned river jobs recovered", "count", recovered)
|
||||
}
|
||||
}
|
||||
recoverJobs()
|
||||
ticker := time.NewTicker(orphanedRiverJobScanInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
recoverJobs()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func asyncTaskInsertOpts(task store.GatewayTask) *river.InsertOpts {
|
||||
priority := 2
|
||||
if task.ID == "" {
|
||||
|
||||
Reference in New Issue
Block a user