fix(worker): 防止事务泄漏并恢复滞留队列
为 PostgreSQL 连接增加可配置的事务空闲与锁等待超时,并在请求取消或 Worker 退出后使用独立有界上下文回滚事务。\n\n恢复过期任务时按批次使用 SKIP LOCKED,周期重建缺失或终态 River Job;锁等待超时只在确认尚未提交上游时安全重排,避免重复执行和重复结算。\n\n验证:完整 Go 测试、go vet、生产 Kustomize 渲染、gofmt 与 git diff --check 均通过。
This commit is contained in:
@@ -80,6 +80,38 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
|
||||
w.service.logger.Debug("river async task interrupted and requeued", "taskID", task.ID, "status", queued.Status, "riverJobID", job.ID)
|
||||
return river.JobSnooze(0)
|
||||
}
|
||||
if store.IsPostgresLockTimeout(runErr) {
|
||||
queued, changed, queueErr := w.service.store.RequeueTaskBeforeUpstreamSubmission(
|
||||
context.WithoutCancel(ctx),
|
||||
task.ID,
|
||||
executionToken,
|
||||
time.Second,
|
||||
)
|
||||
if queueErr != nil {
|
||||
return queueErr
|
||||
}
|
||||
if changed {
|
||||
if eventErr := w.service.emit(
|
||||
context.WithoutCancel(ctx),
|
||||
task.ID,
|
||||
"task.queued",
|
||||
"queued",
|
||||
"database_lock_timeout",
|
||||
0.2,
|
||||
"async task queued after database lock timeout",
|
||||
map[string]any{"code": "database_lock_timeout"},
|
||||
task.RunMode == "simulation",
|
||||
); eventErr != nil {
|
||||
w.service.logger.Warn("record database lock timeout requeue event failed", "taskID", task.ID, "error", eventErr)
|
||||
}
|
||||
w.service.logger.Warn("river async task requeued after database lock timeout",
|
||||
"taskID", task.ID,
|
||||
"status", queued.Status,
|
||||
"riverJobID", job.ID,
|
||||
)
|
||||
return river.JobSnooze(time.Second)
|
||||
}
|
||||
}
|
||||
w.service.logger.Warn("river async task completed with failure", "taskID", task.ID, "error", runErr, "riverJobID", job.ID)
|
||||
return nil
|
||||
}
|
||||
@@ -570,6 +602,12 @@ func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
|
||||
if recovered > 0 {
|
||||
s.logger.Warn("orphaned river jobs recovered", "count", recovered)
|
||||
}
|
||||
if err := s.recoverAsyncRiverJobs(ctx); err != nil {
|
||||
if ctx.Err() == nil {
|
||||
s.logger.Warn("recover queued tasks without active river jobs failed", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
recoverJobs()
|
||||
ticker := time.NewTicker(orphanedRiverJobScanInterval)
|
||||
|
||||
Reference in New Issue
Block a user