fix(worker): 增强队列启动与租约到期恢复

River 初始化遇到瞬时数据库超时时最多重试三次,并确保启动异常时先取消后台 LISTEN 和 Worker 上下文再关闭连接池,避免不就绪 Pod 卡在退出路径。Worker 维护循环每 15 秒执行幂等运行时恢复,使旧实例中断任务在执行租约到期后自动重排并继续触发孤儿 Job 救援。验证通过完整 Go 测试、go vet、竞态检查和迁移安全检查。
This commit is contained in:
2026-07-29 23:55:49 +08:00
parent f3dd7cd262
commit fd3b6bf042
3 changed files with 141 additions and 3 deletions
+86 -2
View File
@@ -23,6 +23,8 @@ const (
orphanedRiverJobScanInterval = 15 * time.Second
orphanedRiverJobWorkerStaleAfter = 45 * time.Second
orphanedRiverJobRecoveryBatchSize = 100
asyncQueueStartupMaxAttempts = 3
asyncQueueStartupRetryDelay = 2 * time.Second
)
type asyncTaskArgs struct {
@@ -83,19 +85,79 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
}
func (s *Service) StartAsyncQueueWorker(ctx context.Context) {
if err := s.startRiverQueue(ctx, true); err != nil {
if err := retryAsyncQueueStartup(
ctx,
asyncQueueStartupMaxAttempts,
asyncQueueStartupRetryDelay,
func() error { return s.startRiverQueue(ctx, true) },
func(attempt int, err error) {
s.logger.Warn("start river async queue failed; retrying",
"attempt", attempt,
"maxAttempts", asyncQueueStartupMaxAttempts,
"error", err,
)
},
); err != nil {
s.logger.Error("start river async queue failed", "error", err)
panic(err)
}
}
func (s *Service) StartAsyncQueueClient(ctx context.Context) {
if err := s.startRiverQueue(ctx, false); err != nil {
if err := retryAsyncQueueStartup(
ctx,
asyncQueueStartupMaxAttempts,
asyncQueueStartupRetryDelay,
func() error { return s.startRiverQueue(ctx, false) },
func(attempt int, err error) {
s.logger.Warn("start river async queue client failed; retrying",
"attempt", attempt,
"maxAttempts", asyncQueueStartupMaxAttempts,
"error", err,
)
},
); err != nil {
s.logger.Error("start river async queue client failed", "error", err)
panic(err)
}
}
func retryAsyncQueueStartup(
ctx context.Context,
maxAttempts int,
retryDelay time.Duration,
start func() error,
onRetry func(attempt int, err error),
) error {
if maxAttempts < 1 {
maxAttempts = 1
}
var lastErr error
for attempt := 1; attempt <= maxAttempts; attempt++ {
if err := start(); err != nil {
lastErr = err
if attempt == maxAttempts {
break
}
if onRetry != nil {
onRetry(attempt, err)
}
timer := time.NewTimer(retryDelay)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return ctx.Err()
case <-timer.C:
}
continue
}
return nil
}
return lastErr
}
func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error {
driver := riverpgxv5.New(s.store.Pool())
migrator, err := rivermigrate.New(driver, nil)
@@ -465,6 +527,28 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
recoverJobs := func() {
runtimeRecovery, err := s.store.RecoverInterruptedRuntimeState(ctx)
if err != nil {
if ctx.Err() == nil {
s.logger.Warn("recover interrupted runtime state from worker failed", "error", err)
}
return
}
if runtimeRecovery.ReleasedConcurrencyLeases > 0 ||
runtimeRecovery.ReleasedRateReservations > 0 ||
runtimeRecovery.FailedAttempts > 0 ||
runtimeRecovery.FailedTasks > 0 ||
runtimeRecovery.RequeuedAsyncTasks > 0 ||
runtimeRecovery.CleanedTaskAdmissions > 0 {
s.logger.Warn("interrupted runtime state recovered by worker",
"releasedConcurrencyLeases", runtimeRecovery.ReleasedConcurrencyLeases,
"releasedRateReservations", runtimeRecovery.ReleasedRateReservations,
"failedAttempts", runtimeRecovery.FailedAttempts,
"failedTasks", runtimeRecovery.FailedTasks,
"requeuedAsyncTasks", runtimeRecovery.RequeuedAsyncTasks,
"cleanedTaskAdmissions", runtimeRecovery.CleanedTaskAdmissions,
)
}
recovered, err := s.store.RecoverOrphanedAsyncRiverJobs(
ctx,
orphanedRiverJobWorkerStaleAfter,
@@ -21,6 +21,57 @@ type fakeAsyncExecutionClient struct {
stopCancelled atomic.Int64
}
func TestRetryAsyncQueueStartupRecoversTransientFailures(t *testing.T) {
transient := errors.New("transient River startup failure")
var attempts atomic.Int32
var retries atomic.Int32
err := retryAsyncQueueStartup(
context.Background(),
3,
time.Millisecond,
func() error {
if attempts.Add(1) < 3 {
return transient
}
return nil
},
func(attempt int, err error) {
if attempt < 1 || !errors.Is(err, transient) {
t.Fatalf("unexpected retry callback attempt=%d err=%v", attempt, err)
}
retries.Add(1)
},
)
if err != nil {
t.Fatalf("retry startup: %v", err)
}
if attempts.Load() != 3 || retries.Load() != 2 {
t.Fatalf("startup attempts=%d retries=%d, want 3/2", attempts.Load(), retries.Load())
}
}
func TestRetryAsyncQueueStartupStopsWhenContextIsCancelled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
var attempts atomic.Int32
err := retryAsyncQueueStartup(
ctx,
3,
time.Hour,
func() error {
attempts.Add(1)
cancel()
return errors.New("startup failed")
},
nil,
)
if !errors.Is(err, context.Canceled) {
t.Fatalf("retry startup error=%v, want context cancelled", err)
}
if attempts.Load() != 1 {
t.Fatalf("startup attempts=%d, want 1", attempts.Load())
}
}
func (c *fakeAsyncExecutionClient) Start(context.Context) error {
c.started.Add(1)
return c.startErr