fix(queue): 隔离异步准入故障并持久化退避

将 X-Async 调度改为逐任务阻塞协议，区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障，避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈，并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证：API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过；pnpm audit 命中未改动的 Nx 工具链既有漏洞。
2026-08-04 18:23:35 +08:00
parent 44d5cf2b9d
commit f4214dd489
12 changed files with 1654 additions and 133 deletions
@@ -545,17 +545,40 @@ WHERE id = $1::uuid
 }

 func (s *Store) FailQueuedTask(ctx context.Context, taskID string, code string, message string) (GatewayTask, error) {
+	return s.FailQueuedTaskWithCallback(ctx, taskID, code, message, "", false)
+}
+
+// FailQueuedTaskWithCallback terminally fails a task before upstream
+// submission and atomically releases every durable resource owned by its
+// admission. The task advisory lock makes competing dispatchers idempotent.
+func (s *Store) FailQueuedTaskWithCallback(
+	ctx context.Context,
+	taskID string,
+	code string,
+	message string,
+	callbackURL string,
+	simulated bool,
+) (GatewayTask, error) {
+	code = strings.TrimSpace(code)
+	message = truncateUTF8Bytes(message, 2048)
 	publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID)
-	tag, err := s.pool.Exec(ctx, `
+	var task GatewayTask
+	err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
+		if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, "task-admission:"+taskID); err != nil {
+			return err
+		}
+		var err error
+		task, err = scanGatewayTask(tx.QueryRow(ctx, `
 UPDATE gateway_tasks
 SET status = 'failed',
    error = NULL,
    error_code = NULLIF($2, ''),
    error_message = NULLIF($3, ''),
-	public_error = $4::jsonb,
+    public_error = $4::jsonb,
    billing_status = CASE
      WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
-      ELSE 'released'
+	  WHEN reservation_amount > 0 THEN 'pending'
+	  ELSE 'released'
    END,
    billing_updated_at = now(),
    locked_by = NULL,
@@ -566,14 +589,76 @@ SET status = 'failed',
    finished_at = now(),
    updated_at = now()
 WHERE id = $1::uuid
-  AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048), string(publicErrorJSON))
+  AND status = 'queued'
+  AND COALESCE(remote_task_id, '') = ''
+RETURNING `+gatewayTaskColumns, taskID, code, message, string(publicErrorJSON)))
+		if errors.Is(err, pgx.ErrNoRows) {
+			return ErrTaskExecutionFinished
+		}
+		if err != nil {
+			return err
+		}
+		if _, err := tx.Exec(ctx, `
+UPDATE gateway_concurrency_leases
+SET released_at = statement_timestamp()
+WHERE task_id = $1::uuid
+  AND released_at IS NULL`, taskID); err != nil {
+			return err
+		}
+		if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
+			return err
+		}
+		if _, err := releaseRuntimeRateReservationsTx(ctx, tx, []string{taskID}); err != nil {
+			return err
+		}
+		payloadJSON, _ := json.Marshal(map[string]any{"taskId": taskID, "reason": code})
+		if _, err := tx.Exec(ctx, `
+INSERT INTO settlement_outbox (
+  task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
+)
+SELECT id, 'task.billing.release', 'release', reservation_amount, billing_currency,
+       pricing_snapshot, $2::jsonb, 'pending', now()
+FROM gateway_tasks
+WHERE id = $1::uuid
+  AND run_mode IN ('production', 'acceptance', 'acceptance_canary')
+  AND gateway_user_id IS NOT NULL
+  AND reservation_amount > 0
+ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
+			return err
+		}
+		var eventID string
+		var eventSeq int64
+		if err := tx.QueryRow(ctx, `
+WITH next_seq AS (
+  SELECT COALESCE(MAX(seq), 0) + 1 AS seq
+  FROM gateway_task_events
+  WHERE task_id = $1::uuid
+)
+INSERT INTO gateway_task_events (
+  task_id, seq, event_type, status, phase, progress, message, payload, simulated
+)
+SELECT $1::uuid, next_seq.seq, 'task.failed', 'failed', 'admission', 0,
+       NULLIF($2, ''), '{}'::jsonb, $3
+FROM next_seq
+RETURNING id::text, seq`, taskID, message, simulated).Scan(&eventID, &eventSeq); err != nil {
+			return err
+		}
+		callbackURL = strings.TrimSpace(callbackURL)
+		if callbackURL != "" {
+			if _, err := tx.Exec(ctx, `
+INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
+VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
+ON CONFLICT (task_id, seq, callback_url) DO NOTHING`, taskID, eventID, eventSeq, callbackURL); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 	if err != nil {
 		return GatewayTask{}, err
 	}
-	if tag.RowsAffected() != 1 {
-		return GatewayTask{}, ErrTaskExecutionFinished
-	}
-	return s.GetTask(ctx, taskID)
+	s.notifyTaskAdmissionBestEffort(ctx, "*")
+	return task, nil
 }

 func (s *Store) RenewTaskExecutionLease(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) error {
@@ -888,6 +973,25 @@ WHERE attempt.id = $1::uuid
 }

 func (s *Store) CancelQueuedTask(ctx context.Context, taskID string, message string) (GatewayTask, bool, error) {
+	return s.cancelQueuedTask(ctx, taskID, message, "", false)
+}
+
+func (s *Store) CancelQueuedTaskWithCallback(
+	ctx context.Context,
+	taskID string,
+	message string,
+	callbackURL string,
+) (GatewayTask, bool, error) {
+	return s.cancelQueuedTask(ctx, taskID, message, callbackURL, true)
+}
+
+func (s *Store) cancelQueuedTask(
+	ctx context.Context,
+	taskID string,
+	message string,
+	callbackURL string,
+	emitTerminalEvent bool,
+) (GatewayTask, bool, error) {
 	message = strings.TrimSpace(message)
 	if message == "" {
 		message = "任务已取消"
@@ -941,8 +1045,11 @@ WHERE task_id = $1::uuid
 		if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
 			return err
 		}
+		if _, err := releaseRuntimeRateReservationsTx(ctx, tx, []string{taskID}); err != nil {
+			return err
+		}
 		payloadJSON, _ := json.Marshal(map[string]any{"taskId": taskID, "reason": "queued_cancelled"})
-		_, err = tx.Exec(ctx, `
+		if _, err = tx.Exec(ctx, `
 INSERT INTO settlement_outbox (
  task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
 )
@@ -953,7 +1060,37 @@ WHERE id = $1::uuid
  AND run_mode IN ('production', 'acceptance', 'acceptance_canary')
  AND gateway_user_id IS NOT NULL
  AND reservation_amount > 0
-ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON))
+ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
+			return err
+		}
+		if !emitTerminalEvent {
+			return nil
+		}
+		var eventID string
+		var eventSeq int64
+		if err := tx.QueryRow(ctx, `
+WITH next_seq AS (
+  SELECT COALESCE(MAX(seq), 0) + 1 AS seq
+  FROM gateway_task_events
+  WHERE task_id = $1::uuid
+)
+INSERT INTO gateway_task_events (
+  task_id, seq, event_type, status, phase, progress, message, payload, simulated
+)
+SELECT $1::uuid, next_seq.seq, 'task.cancelled', 'cancelled', 'admission', 0,
+       NULLIF($2, ''), '{}'::jsonb, $3
+FROM next_seq
+RETURNING id::text, seq`, taskID, message, task.RunMode == "simulation").Scan(&eventID, &eventSeq); err != nil {
+			return err
+		}
+		callbackURL = strings.TrimSpace(callbackURL)
+		if callbackURL == "" {
+			return nil
+		}
+		_, err = tx.Exec(ctx, `
+INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
+VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
+ON CONFLICT (task_id, seq, callback_url) DO NOTHING`, taskID, eventID, eventSeq, callbackURL)
 		return err
 	})
 	if err != nil {