fix(worker): 防止事务泄漏并恢复滞留队列

为 PostgreSQL 连接增加可配置的事务空闲与锁等待超时,并在请求取消或 Worker 退出后使用独立有界上下文回滚事务。\n\n恢复过期任务时按批次使用 SKIP LOCKED,周期重建缺失或终态 River Job;锁等待超时只在确认尚未提交上游时安全重排,避免重复执行和重复结算。\n\n验证:完整 Go 测试、go vet、生产 Kustomize 渲染、gofmt 与 git diff --check 均通过。
This commit is contained in:
2026-07-30 17:27:11 +08:00
parent bc44af751e
commit 6bab0f0749
32 changed files with 545 additions and 106 deletions
+38
View File
@@ -80,6 +80,38 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
w.service.logger.Debug("river async task interrupted and requeued", "taskID", task.ID, "status", queued.Status, "riverJobID", job.ID)
return river.JobSnooze(0)
}
if store.IsPostgresLockTimeout(runErr) {
queued, changed, queueErr := w.service.store.RequeueTaskBeforeUpstreamSubmission(
context.WithoutCancel(ctx),
task.ID,
executionToken,
time.Second,
)
if queueErr != nil {
return queueErr
}
if changed {
if eventErr := w.service.emit(
context.WithoutCancel(ctx),
task.ID,
"task.queued",
"queued",
"database_lock_timeout",
0.2,
"async task queued after database lock timeout",
map[string]any{"code": "database_lock_timeout"},
task.RunMode == "simulation",
); eventErr != nil {
w.service.logger.Warn("record database lock timeout requeue event failed", "taskID", task.ID, "error", eventErr)
}
w.service.logger.Warn("river async task requeued after database lock timeout",
"taskID", task.ID,
"status", queued.Status,
"riverJobID", job.ID,
)
return river.JobSnooze(time.Second)
}
}
w.service.logger.Warn("river async task completed with failure", "taskID", task.ID, "error", runErr, "riverJobID", job.ID)
return nil
}
@@ -570,6 +602,12 @@ func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
if recovered > 0 {
s.logger.Warn("orphaned river jobs recovered", "count", recovered)
}
if err := s.recoverAsyncRiverJobs(ctx); err != nil {
if ctx.Err() == nil {
s.logger.Warn("recover queued tasks without active river jobs failed", "error", err)
}
return
}
}
recoverJobs()
ticker := time.NewTicker(orphanedRiverJobScanInterval)