fix(worker): 防止事务泄漏并恢复滞留队列

为 PostgreSQL 连接增加可配置的事务空闲与锁等待超时,并在请求取消或 Worker 退出后使用独立有界上下文回滚事务。\n\n恢复过期任务时按批次使用 SKIP LOCKED,周期重建缺失或终态 River Job;锁等待超时只在确认尚未提交上游时安全重排,避免重复执行和重复结算。\n\n验证:完整 Go 测试、go vet、生产 Kustomize 渲染、gofmt 与 git diff --check 均通过。
This commit is contained in:
2026-07-30 17:27:11 +08:00
parent bc44af751e
commit 6bab0f0749
32 changed files with 545 additions and 106 deletions
+21 -10
View File
@@ -19,6 +19,8 @@ type RuntimeRecoveryResult struct {
CleanedTaskAdmissions int64 `json:"cleanedTaskAdmissions"`
}
const runtimeRecoveryBatchSize = 100
var ErrConcurrencyLeaseLost = errors.New("concurrency lease lost")
// CheckRateLimits performs a non-consuming admission preflight for fixed-window
@@ -104,7 +106,7 @@ func (s *Store) ReserveRateLimits(ctx context.Context, taskID string, attemptID
if err != nil {
return RateLimitResult{}, err
}
defer tx.Rollback(ctx)
defer rollbackTransaction(tx)
lockKeys := make([]string, 0)
lockKeySet := make(map[string]struct{})
@@ -414,7 +416,7 @@ func (s *Store) AttachRateLimitResultToAttempt(ctx context.Context, attemptID st
if err != nil {
return err
}
defer tx.Rollback(ctx)
defer rollbackTransaction(tx)
for _, reservation := range result.Reservations {
if reservation.ReservationID == "" {
@@ -458,7 +460,7 @@ func (s *Store) RecoverInterruptedRuntimeState(ctx context.Context) (RuntimeReco
if err != nil {
return RuntimeRecoveryResult{}, err
}
defer tx.Rollback(ctx)
defer rollbackTransaction(tx)
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('gateway-runtime-recovery', 0))`); err != nil {
return RuntimeRecoveryResult{}, err
}
@@ -542,7 +544,18 @@ WHERE status = 'running'
result.FailedAttempts = tag.RowsAffected()
asyncTaskRows, err := tx.Query(ctx, `
UPDATE gateway_tasks
WITH recoverable_async_tasks AS MATERIALIZED (
SELECT id
FROM gateway_tasks
WHERE async_mode = true
AND status = 'running'
AND execution_lease_expires_at IS NOT NULL
AND execution_lease_expires_at <= now()
ORDER BY execution_lease_expires_at ASC, id ASC
LIMIT $1
FOR UPDATE SKIP LOCKED
)
UPDATE gateway_tasks task
SET status = 'queued',
error = NULL,
error_code = NULL,
@@ -555,11 +568,9 @@ SET status = 'queued',
next_run_at = now(),
finished_at = NULL,
updated_at = now()
WHERE async_mode = true
AND status = 'running'
AND execution_lease_expires_at IS NOT NULL
AND execution_lease_expires_at <= now()
RETURNING id::text`)
FROM recoverable_async_tasks recoverable
WHERE task.id = recoverable.id
RETURNING task.id::text`, runtimeRecoveryBatchSize)
if err != nil {
return RuntimeRecoveryResult{}, err
}
@@ -718,7 +729,7 @@ func (s *Store) finishRateLimitReservations(ctx context.Context, reservations []
if err != nil {
return err
}
defer tx.Rollback(ctx)
defer rollbackTransaction(tx)
for _, reservation := range reservations {
if reservation.ReservationID == "" {