fix(worker): 增强队列启动与租约到期恢复
River 初始化遇到瞬时数据库超时时最多重试三次,并确保启动异常时先取消后台 LISTEN 和 Worker 上下文再关闭连接池,避免不就绪 Pod 卡在退出路径。Worker 维护循环每 15 秒执行幂等运行时恢复,使旧实例中断任务在执行租约到期后自动重排并继续触发孤儿 Job 救援。验证通过完整 Go 测试、go vet、竞态检查和迁移安全检查。
This commit is contained in:
@@ -23,6 +23,8 @@ const (
|
||||
orphanedRiverJobScanInterval = 15 * time.Second
|
||||
orphanedRiverJobWorkerStaleAfter = 45 * time.Second
|
||||
orphanedRiverJobRecoveryBatchSize = 100
|
||||
asyncQueueStartupMaxAttempts = 3
|
||||
asyncQueueStartupRetryDelay = 2 * time.Second
|
||||
)
|
||||
|
||||
type asyncTaskArgs struct {
|
||||
@@ -83,19 +85,79 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
|
||||
}
|
||||
|
||||
func (s *Service) StartAsyncQueueWorker(ctx context.Context) {
|
||||
if err := s.startRiverQueue(ctx, true); err != nil {
|
||||
if err := retryAsyncQueueStartup(
|
||||
ctx,
|
||||
asyncQueueStartupMaxAttempts,
|
||||
asyncQueueStartupRetryDelay,
|
||||
func() error { return s.startRiverQueue(ctx, true) },
|
||||
func(attempt int, err error) {
|
||||
s.logger.Warn("start river async queue failed; retrying",
|
||||
"attempt", attempt,
|
||||
"maxAttempts", asyncQueueStartupMaxAttempts,
|
||||
"error", err,
|
||||
)
|
||||
},
|
||||
); err != nil {
|
||||
s.logger.Error("start river async queue failed", "error", err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) StartAsyncQueueClient(ctx context.Context) {
|
||||
if err := s.startRiverQueue(ctx, false); err != nil {
|
||||
if err := retryAsyncQueueStartup(
|
||||
ctx,
|
||||
asyncQueueStartupMaxAttempts,
|
||||
asyncQueueStartupRetryDelay,
|
||||
func() error { return s.startRiverQueue(ctx, false) },
|
||||
func(attempt int, err error) {
|
||||
s.logger.Warn("start river async queue client failed; retrying",
|
||||
"attempt", attempt,
|
||||
"maxAttempts", asyncQueueStartupMaxAttempts,
|
||||
"error", err,
|
||||
)
|
||||
},
|
||||
); err != nil {
|
||||
s.logger.Error("start river async queue client failed", "error", err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func retryAsyncQueueStartup(
|
||||
ctx context.Context,
|
||||
maxAttempts int,
|
||||
retryDelay time.Duration,
|
||||
start func() error,
|
||||
onRetry func(attempt int, err error),
|
||||
) error {
|
||||
if maxAttempts < 1 {
|
||||
maxAttempts = 1
|
||||
}
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if err := start(); err != nil {
|
||||
lastErr = err
|
||||
if attempt == maxAttempts {
|
||||
break
|
||||
}
|
||||
if onRetry != nil {
|
||||
onRetry(attempt, err)
|
||||
}
|
||||
timer := time.NewTimer(retryDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error {
|
||||
driver := riverpgxv5.New(s.store.Pool())
|
||||
migrator, err := rivermigrate.New(driver, nil)
|
||||
@@ -465,6 +527,28 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
|
||||
|
||||
func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
|
||||
recoverJobs := func() {
|
||||
runtimeRecovery, err := s.store.RecoverInterruptedRuntimeState(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
s.logger.Warn("recover interrupted runtime state from worker failed", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if runtimeRecovery.ReleasedConcurrencyLeases > 0 ||
|
||||
runtimeRecovery.ReleasedRateReservations > 0 ||
|
||||
runtimeRecovery.FailedAttempts > 0 ||
|
||||
runtimeRecovery.FailedTasks > 0 ||
|
||||
runtimeRecovery.RequeuedAsyncTasks > 0 ||
|
||||
runtimeRecovery.CleanedTaskAdmissions > 0 {
|
||||
s.logger.Warn("interrupted runtime state recovered by worker",
|
||||
"releasedConcurrencyLeases", runtimeRecovery.ReleasedConcurrencyLeases,
|
||||
"releasedRateReservations", runtimeRecovery.ReleasedRateReservations,
|
||||
"failedAttempts", runtimeRecovery.FailedAttempts,
|
||||
"failedTasks", runtimeRecovery.FailedTasks,
|
||||
"requeuedAsyncTasks", runtimeRecovery.RequeuedAsyncTasks,
|
||||
"cleanedTaskAdmissions", runtimeRecovery.CleanedTaskAdmissions,
|
||||
)
|
||||
}
|
||||
recovered, err := s.store.RecoverOrphanedAsyncRiverJobs(
|
||||
ctx,
|
||||
orphanedRiverJobWorkerStaleAfter,
|
||||
|
||||
Reference in New Issue
Block a user