fix(worker): 增强队列启动与租约到期恢复
River 初始化遇到瞬时数据库超时时最多重试三次,并确保启动异常时先取消后台 LISTEN 和 Worker 上下文再关闭连接池,避免不就绪 Pod 卡在退出路径。Worker 维护循环每 15 秒执行幂等运行时恢复,使旧实例中断任务在执行租约到期后自动重排并继续触发孤儿 Job 救援。验证通过完整 Go 测试、go vet、竞态检查和迁移安全检查。
This commit is contained in:
@@ -21,6 +21,57 @@ type fakeAsyncExecutionClient struct {
|
||||
stopCancelled atomic.Int64
|
||||
}
|
||||
|
||||
func TestRetryAsyncQueueStartupRecoversTransientFailures(t *testing.T) {
|
||||
transient := errors.New("transient River startup failure")
|
||||
var attempts atomic.Int32
|
||||
var retries atomic.Int32
|
||||
err := retryAsyncQueueStartup(
|
||||
context.Background(),
|
||||
3,
|
||||
time.Millisecond,
|
||||
func() error {
|
||||
if attempts.Add(1) < 3 {
|
||||
return transient
|
||||
}
|
||||
return nil
|
||||
},
|
||||
func(attempt int, err error) {
|
||||
if attempt < 1 || !errors.Is(err, transient) {
|
||||
t.Fatalf("unexpected retry callback attempt=%d err=%v", attempt, err)
|
||||
}
|
||||
retries.Add(1)
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("retry startup: %v", err)
|
||||
}
|
||||
if attempts.Load() != 3 || retries.Load() != 2 {
|
||||
t.Fatalf("startup attempts=%d retries=%d, want 3/2", attempts.Load(), retries.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryAsyncQueueStartupStopsWhenContextIsCancelled(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var attempts atomic.Int32
|
||||
err := retryAsyncQueueStartup(
|
||||
ctx,
|
||||
3,
|
||||
time.Hour,
|
||||
func() error {
|
||||
attempts.Add(1)
|
||||
cancel()
|
||||
return errors.New("startup failed")
|
||||
},
|
||||
nil,
|
||||
)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("retry startup error=%v, want context cancelled", err)
|
||||
}
|
||||
if attempts.Load() != 1 {
|
||||
t.Fatalf("startup attempts=%d, want 1", attempts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func (c *fakeAsyncExecutionClient) Start(context.Context) error {
|
||||
c.started.Add(1)
|
||||
return c.startErr
|
||||
|
||||
Reference in New Issue
Block a user