fix(queue): 隔离异步准入故障并持久化退避

将 X-Async 调度改为逐任务阻塞协议，区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障，避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈，并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证：API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过；pnpm audit 命中未改动的 Nx 工具链既有漏洞。

apps/api/internal/runner/admission.go

@@ -745,17 +745,19 @@ func admissionInputFromSnapshot(admission store.TaskAdmission, workerCapacity in
 	}
 }
 
-func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []store.TaskAdmission) (bool, error) {
+func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []store.TaskAdmission) (bool, bool, error) {
 	if len(admissions) == 0 {
-		return false, nil
+		return false, false, nil
 	}
 	workerCapacity, err := s.coordinationStore.ActiveWorkerCapacity(ctx)
 	if err != nil {
-		return false, err
+		return false, false, err
 	}
 	if workerCapacity <= 0 {
-		return true, nil
+		s.observeAsyncAdmissionDispatch("global_wait")
+		return true, false, nil
 	}
+	progressed := false
 	microbatchSize := s.cfg.AsyncAdmissionMicrobatchSize
 	if microbatchSize <= 0 {
 		microbatchSize = 1
@@ -765,7 +767,7 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
 		inputs := make([]store.TaskAdmissionInput, 0, end-start)
 		tasksByID := make(map[string]store.GatewayTask, end-start)
 		for _, admission := range admissions[start:end] {
-			if admission.ReselectRequestedAt.IsZero() && len(admission.Scopes) > 0 {
+			if admission.ReselectRequestedAt.IsZero() && len(admission.Scopes) > 0 && !admission.SnapshotInvalid {
 				inputs = append(inputs, admissionInputFromSnapshot(admission, workerCapacity))
 				tasksByID[admission.TaskID] = store.GatewayTask{ID: admission.TaskID}
 				continue
@@ -773,78 +775,411 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
 			task, loadErr := s.store.GetTask(ctx, admission.TaskID)
 			if loadErr != nil {
 				if errors.Is(loadErr, pgx.ErrNoRows) {
+					progressed = true
 					continue
 				}
-				return false, loadErr
+				return false, progressed, loadErr
 			}
 			plan, planErr := s.buildTaskAdmissionPlanForCurrentBinding(ctx, task, authUserFromTask(task), &admission)
 			if planErr != nil {
-				return false, planErr
+				handled, handleErr := s.handleAsyncAdmissionTaskError(ctx, task, admission, planErr)
+				if handleErr != nil {
+					return false, progressed, handleErr
+				}
+				progressed = progressed || handled
+				continue
 			}
 			if !plan.Eligible {
-				if enqueueErr := s.EnqueueAsyncTask(ctx, task); enqueueErr != nil {
-					return false, enqueueErr
+				// This task already owns a durable admission row from its previous
+				// binding. Promote that row and insert the River job in the same
+				// transaction even when the replacement candidate has no business
+				// concurrency scope; direct enqueue would strand a waiting row and
+				// make every dispatcher enqueue it forever.
+				if plan.Candidate.PlatformID == "" || plan.Candidate.PlatformModelID == "" {
+					handled, handleErr := s.handleAsyncAdmissionTaskError(
+						ctx,
+						task,
+						admission,
+						errors.New("waiting admission rebuilt without a platform model candidate"),
+					)
+					if handleErr != nil {
+						return false, progressed, handleErr
+					}
+					progressed = progressed || handled
+					continue
 				}
-				continue
 			}
 			input := taskAdmissionInput(task, plan, "")
 			if _, rebindErr := s.store.RebindWaitingTaskAdmission(ctx, input); rebindErr != nil && !errors.Is(rebindErr, pgx.ErrNoRows) {
-				return false, rebindErr
+				handled, handleErr := s.handleAsyncAdmissionTaskError(ctx, task, admission, rebindErr)
+				if handleErr != nil {
+					return false, progressed, handleErr
+				}
+				progressed = progressed || handled
+				continue
 			}
+			progressed = true
 			inputs = append(inputs, input)
 			tasksByID[task.ID] = task
 		}
 		if len(inputs) == 0 {
 			continue
 		}
-		outcomes, err := s.store.TryTaskAdmissionAtomicBatchWithAdmittedHook(
+		enqueueHook := func(tx pgx.Tx, input store.TaskAdmissionInput) error {
+			task, ok := tasksByID[input.TaskID]
+			if !ok {
+				return fmt.Errorf("async admission batch task %s was not prepared", input.TaskID)
+			}
+			return s.enqueueAsyncTaskTx(ctx, tx, task.ID, asyncTaskInsertOpts(task))
+		}
+		outcomes, batchErr := s.store.TryTaskAdmissionAtomicBatchWithAdmittedHook(
 			ctx,
 			inputs,
-			func(tx pgx.Tx, input store.TaskAdmissionInput) error {
-				task, ok := tasksByID[input.TaskID]
-				if !ok {
-					return fmt.Errorf("async admission batch task %s was not prepared", input.TaskID)
-				}
-				return s.enqueueAsyncTaskTx(ctx, tx, task.ID, asyncTaskInsertOpts(task))
-			},
+			enqueueHook,
 		)
-		if err != nil {
-			return false, err
-		}
-		for _, outcome := range outcomes {
-			s.observeTaskAdmissionAttempt(outcome.Result, outcome.Err)
-			if outcome.Err != nil {
-				if errors.Is(outcome.Err, store.ErrTaskExecutionFinished) ||
-					errors.Is(outcome.Err, store.ErrQueueTimeout) {
+		if batchErr != nil {
+			if isAsyncAdmissionSystemError(batchErr) {
+				return false, progressed, batchErr
+			}
+			failedTaskID := asyncAdmissionFailedTaskID(outcomes)
+			if failedTaskID == "" {
+				return false, progressed, batchErr
+			}
+			failedTask, loadErr := s.asyncAdmissionDispatchTask(ctx, tasksByID[failedTaskID])
+			if loadErr != nil {
+				return false, progressed, loadErr
+			}
+			failedAdmission := admissionByTaskID(admissions[start:end], failedTaskID)
+			handled, handleErr := s.handleAsyncAdmissionTaskError(ctx, failedTask, failedAdmission, batchErr)
+			if handleErr != nil {
+				return false, progressed, handleErr
+			}
+			progressed = progressed || handled
+
+			for _, input := range inputs {
+				if input.TaskID == failedTaskID {
 					continue
 				}
-				return false, outcome.Err
+				result, oneErr := s.store.TryTaskAdmissionWithAdmittedHook(ctx, input, func(tx pgx.Tx) error {
+					return enqueueHook(tx, input)
+				})
+				globalSaturated, outcomeProgress, outcomeErr := s.handleAsyncAdmissionOutcome(
+					ctx,
+					store.TaskAdmissionBatchOutcome{TaskID: input.TaskID, Result: result, Err: oneErr},
+					input,
+					tasksByID,
+					admissionByTaskID(admissions[start:end], input.TaskID),
+				)
+				if outcomeErr != nil {
+					return false, progressed, outcomeErr
+				}
+				progressed = progressed || outcomeProgress
+				if globalSaturated {
+					return true, progressed, nil
+				}
 			}
-			if !outcome.Result.Admitted {
-				s.observeCandidateRouting("quota_race_rotated")
-				platformModelID := outcome.Result.Admission.PlatformModelID
-				if platformModelID == "" {
-					for _, input := range inputs {
-						if input.TaskID == outcome.TaskID {
-							platformModelID = input.PlatformModelID
-							break
-						}
-					}
-				}
-				if platformModelID != "" {
-					marked, markErr := s.store.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
-					if markErr != nil {
-						return false, markErr
-					}
-					if marked > 0 {
-						s.observeTaskAdmission("candidate_reselect_requested")
-					}
-				}
-				return true, nil
+			continue
+		}
+		for _, outcome := range outcomes {
+			input := inputByTaskID(inputs, outcome.TaskID)
+			globalSaturated, outcomeProgress, outcomeErr := s.handleAsyncAdmissionOutcome(
+				ctx,
+				outcome,
+				input,
+				tasksByID,
+				admissionByTaskID(admissions[start:end], outcome.TaskID),
+			)
+			if outcomeErr != nil {
+				return false, progressed, outcomeErr
+			}
+			progressed = progressed || outcomeProgress
+			if globalSaturated {
+				return true, progressed, nil
 			}
 		}
 	}
-	return false, nil
+	return false, progressed, nil
+}
+
+func (s *Service) handleAsyncAdmissionOutcome(
+	ctx context.Context,
+	outcome store.TaskAdmissionBatchOutcome,
+	input store.TaskAdmissionInput,
+	tasksByID map[string]store.GatewayTask,
+	admission store.TaskAdmission,
+) (bool, bool, error) {
+	s.observeTaskAdmissionAttempt(outcome.Result, outcome.Err)
+	if outcome.Err != nil {
+		if errors.Is(outcome.Err, store.ErrTaskExecutionFinished) || errors.Is(outcome.Err, store.ErrQueueTimeout) {
+			return false, true, nil
+		}
+		if isAsyncAdmissionSystemError(outcome.Err) {
+			return false, false, outcome.Err
+		}
+		task, err := s.asyncAdmissionDispatchTask(ctx, tasksByID[outcome.TaskID])
+		if err != nil {
+			return false, false, err
+		}
+		handled, handleErr := s.handleAsyncAdmissionTaskError(ctx, task, admission, outcome.Err)
+		return false, handled, handleErr
+	}
+	if outcome.Result.Admitted {
+		s.observeAsyncAdmissionDispatch("admitted")
+		return false, true, nil
+	}
+
+	globalBlocked := false
+	groupBlocked := false
+	platformSaturated := false
+	for _, blocker := range outcome.Result.Blockers {
+		switch blocker.ScopeType {
+		case "worker_capacity":
+			if blocker.Reason == "saturated" {
+				globalBlocked = true
+			}
+		case "user_group":
+			groupBlocked = true
+		case "platform_model":
+			if blocker.Reason == "saturated" {
+				platformSaturated = true
+			}
+		}
+	}
+	if globalBlocked {
+		s.observeAsyncAdmissionDispatch("global_wait")
+		return true, false, nil
+	}
+	if groupBlocked {
+		s.observeAsyncAdmissionDispatch("group_wait")
+		return false, false, nil
+	}
+	if platformSaturated {
+		platformModelID := outcome.Result.Admission.PlatformModelID
+		if platformModelID == "" {
+			platformModelID = input.PlatformModelID
+		}
+		_, marked, err := s.store.RequestTaskAdmissionReselect(ctx, outcome.TaskID, platformModelID)
+		if err != nil {
+			return false, false, err
+		}
+		s.observeAsyncAdmissionDispatch("platform_wait")
+		if marked {
+			s.observeTaskAdmission("candidate_reselect_requested")
+			s.observeAsyncAdmissionDispatch("reselect")
+			return false, true, nil
+		}
+		return false, false, nil
+	}
+	if len(outcome.Result.Blockers) > 0 {
+		s.observeAsyncAdmissionDispatch("platform_wait")
+		return false, false, nil
+	}
+
+	task, err := s.asyncAdmissionDispatchTask(ctx, tasksByID[outcome.TaskID])
+	if err != nil {
+		return false, false, err
+	}
+	handled, handleErr := s.handleAsyncAdmissionTaskError(
+		ctx,
+		task,
+		admission,
+		errors.New("task admission returned neither admitted nor a blocker"),
+	)
+	return false, handled, handleErr
+}
+
+func (s *Service) handleAsyncAdmissionTaskError(
+	ctx context.Context,
+	task store.GatewayTask,
+	admission store.TaskAdmission,
+	cause error,
+) (bool, error) {
+	if isAsyncAdmissionSystemError(cause) {
+		return false, cause
+	}
+	now := time.Now()
+	terminal := asyncAdmissionTaskErrorTerminal(cause)
+	code := asyncAdmissionTaskErrorCode(cause)
+	if !admission.WaitDeadlineAt.IsZero() && !admission.WaitDeadlineAt.After(now) {
+		terminal = true
+		if !errors.Is(cause, store.ErrNoModelCandidate) {
+			code = "admission_dispatch_failed"
+		}
+	}
+	if terminal {
+		callbackURL, callbackErr := s.taskCallbackURL(ctx, task.ID)
+		if callbackErr != nil {
+			return false, callbackErr
+		}
+		_, failErr := s.store.FailQueuedTaskWithCallback(
+			context.WithoutCancel(ctx),
+			task.ID,
+			code,
+			cause.Error(),
+			callbackURL,
+			task.RunMode == "simulation",
+		)
+		if errors.Is(failErr, store.ErrTaskExecutionFinished) {
+			return true, nil
+		}
+		if failErr != nil {
+			return false, failErr
+		}
+		s.observeAsyncAdmissionDispatch("terminal")
+		if s.logger != nil {
+			s.logger.Info("terminally failed waiting async admission",
+				"taskID", task.ID,
+				"error_category", code,
+			)
+		}
+		return true, nil
+	}
+
+	retryAt := asyncAdmissionRetryAt(now, task.ID, admission.DispatchFailureCount, cause)
+	updated, changed, deferErr := s.store.DeferWaitingTaskAdmission(
+		ctx,
+		task.ID,
+		admission.PlatformModelID,
+		retryAt,
+		"deferred",
+		code,
+	)
+	if deferErr != nil {
+		return false, deferErr
+	}
+	if !changed {
+		return true, nil
+	}
+	s.observeAsyncAdmissionDispatch("deferred")
+	s.observeAsyncAdmissionDispatch("task_error")
+	if s.logger != nil && (updated.DispatchFailureCount == 1 || updated.DispatchFailureCount == 5 || updated.DispatchFailureCount == 7 || updated.DispatchFailureCount%10 == 0) {
+		s.logger.Warn("deferred task-specific async admission failure",
+			"taskID", task.ID,
+			"failureCount", updated.DispatchFailureCount,
+			"nextDispatchAt", updated.DispatchNextAt.UTC().Format(time.RFC3339),
+			"error_category", code,
+		)
+	}
+	return true, nil
+}
+
+func asyncAdmissionTaskErrorTerminal(err error) bool {
+	if errors.Is(err, store.ErrNoModelCandidate) {
+		return store.ModelCandidateRetryAfter(err) <= 0 && store.ModelCandidateRecoveryAt(err).IsZero()
+	}
+	if errors.Is(err, store.ErrInvalidPlatformModelConfiguration) {
+		return true
+	}
+	if errors.Is(err, store.ErrRateLimited) {
+		return !store.RateLimitRetryable(err)
+	}
+	var clientErr *clients.ClientError
+	return errors.As(err, &clientErr) && !clientErr.Retryable
+}
+
+func asyncAdmissionTaskErrorCode(err error) string {
+	if errors.Is(err, store.ErrNoModelCandidate) {
+		return store.ModelCandidateErrorCode(err)
+	}
+	return clients.ErrorCode(err)
+}
+
+func asyncAdmissionRetryAt(now time.Time, taskID string, failureCount int, err error) time.Time {
+	if recoveryAt := store.ModelCandidateRecoveryAt(err); recoveryAt.After(now) {
+		return recoveryAt
+	}
+	if delay := store.ModelCandidateRetryAfter(err); delay > 0 {
+		return now.Add(delay)
+	}
+	if delay := store.RateLimitRetryAfter(err); delay > 0 {
+		return now.Add(delay)
+	}
+	delay := time.Second
+	for index := 0; index < failureCount && delay < time.Minute; index++ {
+		delay *= 2
+	}
+	if delay > time.Minute {
+		delay = time.Minute
+	}
+	hasher := fnv.New32a()
+	_, _ = hasher.Write([]byte(taskID))
+	jitterPermille := int64(hasher.Sum32()%401) - 200
+	delay += time.Duration(int64(delay) * jitterPermille / 1000)
+	if delay < time.Second {
+		delay = time.Second
+	}
+	if delay > time.Minute {
+		delay = time.Minute
+	}
+	return now.Add(delay)
+}
+
+func isAsyncAdmissionSystemError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || store.IsPostgresUnavailable(err) {
+		return true
+	}
+	var sqlState interface{ SQLState() string }
+	if !errors.As(err, &sqlState) {
+		return false
+	}
+	code := strings.TrimSpace(sqlState.SQLState())
+	return strings.HasPrefix(code, "08") || // connection exception
+		strings.HasPrefix(code, "25") || // invalid transaction state
+		strings.HasPrefix(code, "40") || // transaction rollback/deadlock
+		strings.HasPrefix(code, "53") || // insufficient resources
+		strings.HasPrefix(code, "58") || // system error
+		code == "55P03" || // lock not available
+		code == "57014" || // query cancelled/statement timeout
+		code == "57P01" || code == "57P02" || code == "57P03"
+}
+
+func asyncAdmissionFailedTaskID(outcomes []store.TaskAdmissionBatchOutcome) string {
+	for index := len(outcomes) - 1; index >= 0; index-- {
+		if outcomes[index].Err != nil {
+			return outcomes[index].TaskID
+		}
+	}
+	return ""
+}
+
+func inputByTaskID(inputs []store.TaskAdmissionInput, taskID string) store.TaskAdmissionInput {
+	for _, input := range inputs {
+		if input.TaskID == taskID {
+			return input
+		}
+	}
+	return store.TaskAdmissionInput{TaskID: taskID}
+}
+
+func admissionByTaskID(admissions []store.TaskAdmission, taskID string) store.TaskAdmission {
+	for _, admission := range admissions {
+		if admission.TaskID == taskID {
+			return admission
+		}
+	}
+	return store.TaskAdmission{TaskID: taskID}
+}
+
+func (s *Service) asyncAdmissionDispatchTask(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) {
+	if task.ID == "" {
+		return store.GatewayTask{}, pgx.ErrNoRows
+	}
+	if task.Kind != "" {
+		return task, nil
+	}
+	return s.store.GetTask(ctx, task.ID)
+}
+
+func (s *Service) observeAsyncAdmissionDispatch(outcome string) {
+	observer, ok := s.billingMetrics.(interface {
+		ObserveAsyncAdmissionDispatch(string)
+	})
+	if ok {
+		observer.ObserveAsyncAdmissionDispatch(outcome)
+	}
 }
 
 func (s *Service) cancelAsyncSubmissionIfDisconnected(ctx context.Context, taskID string) bool {
@@ -853,7 +1188,16 @@ func (s *Service) cancelAsyncSubmissionIfDisconnected(ctx context.Context, taskI
 	}
 	cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 	defer cancel()
-	if _, changed, err := s.store.CancelQueuedTask(cleanupCtx, taskID, "client disconnected before upstream submission"); err != nil {
+	callbackURL, callbackErr := s.taskCallbackURL(cleanupCtx, taskID)
+	if callbackErr != nil && s.logger != nil {
+		s.logger.Warn("resolve disconnected task callback failed", "taskID", taskID, "error", callbackErr)
+	}
+	if _, changed, err := s.store.CancelQueuedTaskWithCallback(
+		cleanupCtx,
+		taskID,
+		"client disconnected before upstream submission",
+		callbackURL,
+	); err != nil {
 		if s.logger != nil {
 			s.logger.Warn("cancel disconnected asynchronous task failed", "taskID", taskID, "error", err)
 		}
@@ -866,6 +1210,8 @@ func (s *Service) cancelAsyncSubmissionIfDisconnected(ctx context.Context, taskI
 func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
+	systemBackoff := time.Second
+	var systemRetryAt time.Time
 	for {
 		select {
 		case <-ctx.Done():
@@ -873,21 +1219,34 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
 		case <-s.asyncAdmissionWake:
 		case <-ticker.C:
 		}
+		if time.Now().Before(systemRetryAt) {
+			continue
+		}
 		for {
 			batchLimit := asyncAdmissionBatchLimit(s.cfg.AsyncWorkerHardLimit)
 			admissions, err := s.store.ListWaitingAsyncAdmissions(ctx, batchLimit)
 			if err != nil {
 				s.logger.Warn("list waiting async admissions failed", "error", err)
+				s.observeAsyncAdmissionDispatch("system_error")
+				systemRetryAt = time.Now().Add(systemBackoff)
+				systemBackoff = min(systemBackoff*2, 30*time.Second)
 				break
 			}
 			if len(admissions) == 0 {
+				systemBackoff = time.Second
+				systemRetryAt = time.Time{}
 				break
 			}
-			saturated, err := s.dispatchWaitingAsyncTasks(ctx, admissions)
+			saturated, progressed, err := s.dispatchWaitingAsyncTasks(ctx, admissions)
 			if err != nil {
 				s.logger.Warn("dispatch waiting async admission batch failed", "tasks", len(admissions), "error", err)
+				s.observeAsyncAdmissionDispatch("system_error")
+				systemRetryAt = time.Now().Add(systemBackoff)
+				systemBackoff = min(systemBackoff*2, 30*time.Second)
 				break
 			}
+			systemBackoff = time.Second
+			systemRetryAt = time.Time{}
 			if saturated {
 				// Every asynchronous task shares the global worker-capacity FIFO
 				// scope. Once its current head cannot be admitted, later tasks
@@ -895,6 +1254,9 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
 				s.observeTaskAdmission("dispatch_saturated")
 				break
 			}
+			if !progressed {
+				break
+			}
 			// PostgreSQL notifications are wake-up hints and collapse while a
 			// batch is being admitted. Keep filling consecutive batches until
 			// the global execution scope is actually saturated so a large burst
@@ -919,7 +1281,11 @@ func (s *Service) reapExpiredTaskAdmissions(ctx context.Context) {
 			return
 		case <-ticker.C:
 		}
-		result, err := s.store.ReapExpiredTaskAdmissions(ctx, 500)
+		defaultCallbackURL := ""
+		if s.cfg.TaskProgressCallbackEnabled {
+			defaultCallbackURL = s.cfg.TaskProgressCallbackURL
+		}
+		result, err := s.store.ReapExpiredTaskAdmissions(ctx, 500, defaultCallbackURL)
 		if err != nil {
 			if s.logger != nil {
 				s.logger.Warn("reap expired task admissions failed", "error", err)