fix(worker): 防止事务泄漏并恢复滞留队列
为 PostgreSQL 连接增加可配置的事务空闲与锁等待超时,并在请求取消或 Worker 退出后使用独立有界上下文回滚事务。\n\n恢复过期任务时按批次使用 SKIP LOCKED,周期重建缺失或终态 River Job;锁等待超时只在确认尚未提交上游时安全重排,避免重复执行和重复结算。\n\n验证:完整 Go 测试、go vet、生产 Kustomize 渲染、gofmt 与 git diff --check 均通过。
This commit is contained in:
@@ -19,6 +19,8 @@ type RuntimeRecoveryResult struct {
|
||||
CleanedTaskAdmissions int64 `json:"cleanedTaskAdmissions"`
|
||||
}
|
||||
|
||||
const runtimeRecoveryBatchSize = 100
|
||||
|
||||
var ErrConcurrencyLeaseLost = errors.New("concurrency lease lost")
|
||||
|
||||
// CheckRateLimits performs a non-consuming admission preflight for fixed-window
|
||||
@@ -104,7 +106,7 @@ func (s *Store) ReserveRateLimits(ctx context.Context, taskID string, attemptID
|
||||
if err != nil {
|
||||
return RateLimitResult{}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
defer rollbackTransaction(tx)
|
||||
|
||||
lockKeys := make([]string, 0)
|
||||
lockKeySet := make(map[string]struct{})
|
||||
@@ -414,7 +416,7 @@ func (s *Store) AttachRateLimitResultToAttempt(ctx context.Context, attemptID st
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
defer rollbackTransaction(tx)
|
||||
|
||||
for _, reservation := range result.Reservations {
|
||||
if reservation.ReservationID == "" {
|
||||
@@ -458,7 +460,7 @@ func (s *Store) RecoverInterruptedRuntimeState(ctx context.Context) (RuntimeReco
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
defer rollbackTransaction(tx)
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('gateway-runtime-recovery', 0))`); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
@@ -542,7 +544,18 @@ WHERE status = 'running'
|
||||
result.FailedAttempts = tag.RowsAffected()
|
||||
|
||||
asyncTaskRows, err := tx.Query(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
WITH recoverable_async_tasks AS MATERIALIZED (
|
||||
SELECT id
|
||||
FROM gateway_tasks
|
||||
WHERE async_mode = true
|
||||
AND status = 'running'
|
||||
AND execution_lease_expires_at IS NOT NULL
|
||||
AND execution_lease_expires_at <= now()
|
||||
ORDER BY execution_lease_expires_at ASC, id ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_tasks task
|
||||
SET status = 'queued',
|
||||
error = NULL,
|
||||
error_code = NULL,
|
||||
@@ -555,11 +568,9 @@ SET status = 'queued',
|
||||
next_run_at = now(),
|
||||
finished_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE async_mode = true
|
||||
AND status = 'running'
|
||||
AND execution_lease_expires_at IS NOT NULL
|
||||
AND execution_lease_expires_at <= now()
|
||||
RETURNING id::text`)
|
||||
FROM recoverable_async_tasks recoverable
|
||||
WHERE task.id = recoverable.id
|
||||
RETURNING task.id::text`, runtimeRecoveryBatchSize)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
@@ -718,7 +729,7 @@ func (s *Store) finishRateLimitReservations(ctx context.Context, reservations []
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
defer rollbackTransaction(tx)
|
||||
|
||||
for _, reservation := range reservations {
|
||||
if reservation.ReservationID == "" {
|
||||
|
||||
Reference in New Issue
Block a user