refactor(runner): 收敛上游失败决策与轮转策略

将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。

apps/api/internal/runner/service.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
@@ -269,6 +270,20 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 		if task.Kind == "responses" && responseExecution.PreviousChain != nil {
 			err = responseChainUnavailableError()
 		}
+		if task.AsyncMode && store.ModelCandidateRetryAfter(err) > 0 {
+			queued, delay, queueErr := s.requeueModelCoolingTask(ctx, task, err)
+			if queueErr != nil {
+				return Result{}, queueErr
+			}
+			return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
+		}
+		if store.ModelCandidateRetryAfter(err) > 0 {
+			failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err)
+			if finishErr != nil {
+				return Result{}, finishErr
+			}
+			return Result{Task: failed, Output: failed.Result}, err
+		}
 		s.recordFailedAttempt(ctx, failedAttemptRecord{
 			Task:      task,
 			Body:      body,
@@ -539,18 +554,38 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 
 	maxPlatforms := maxPlatformsForCandidates(candidates, runnerPolicy)
 	maxFailoverDuration := maxFailoverDurationForCandidates(candidates, runnerPolicy)
-	singleSourceProtected := singleSourceProtectionActive(candidates, maxPlatforms, runnerPolicy)
+	singleSourceProtection := runnerSingleSourceProtectionEnabled(runnerPolicy)
+	var downstreamStarted atomic.Bool
+	candidateDelta := onDelta
+	if onDelta != nil {
+		candidateDelta = func(event clients.StreamDeltaEvent) error {
+			downstreamStarted.Store(true)
+			return onDelta(event)
+		}
+	}
 	var lastErr error
 	var lastCandidate store.RuntimeModelCandidate
 	var lastPreprocessing *parameterPreprocessingLog
+	platformsVisited := 0
 candidatesLoop:
 	for index, candidate := range candidates {
-		if index >= maxPlatforms {
+		if platformsVisited >= maxPlatforms {
 			break
 		}
+		available, availabilityErr := s.store.RuntimeCandidateAvailable(ctx, candidate.PlatformID, candidate.PlatformModelID)
+		if availabilityErr != nil {
+			return Result{}, availabilityErr
+		}
+		if !available {
+			continue
+		}
+		platformsVisited++
 		lastCandidate = candidate
 		clientAttempts := clientAttemptsForCandidate(candidate)
 		var candidateErr error
+		var candidateDecision failureDecision
+		var candidateDecisionAttempt int
+		var candidateClientAttempt int
 		for clientAttempt := 1; clientAttempt <= clientAttempts; clientAttempt++ {
 			nextAttemptNo := attemptNo + 1
 			preprocessing, ok := preprocessingByCandidate[pricingCandidateKey(candidate)]
@@ -579,7 +614,7 @@ candidatesLoop:
 			}
 			candidateBody := preprocessing.Body
 			candidatePricing := pricingByCandidate[pricingCandidateKey(candidate)]
-			response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, candidatePricing, nextAttemptNo, onDelta, responseExecution, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record)
+			response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, candidatePricing, nextAttemptNo, candidateDelta, responseExecution, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record)
 			if err != nil && isVolcesRemoteTaskCancellation(candidate, err) {
 				cancelled, changed, cancelErr := s.store.CancelSubmittedTask(ctx, task.ID, task.ExecutionToken, "任务已由火山引擎取消")
 				if cancelErr != nil {
@@ -748,7 +783,18 @@ candidatesLoop:
 			if isLocalRateLimitError(err) {
 				lastErr = err
 				candidateErr = err
-				if task.AsyncMode && store.RateLimitRetryable(err) {
+				candidateDecision = resolveCandidateFailure(resolveCandidateFailureInput{
+					RunnerPolicy:      runnerPolicy,
+					Candidate:         candidate,
+					Err:               err,
+					ClientAttempt:     clientAttempt,
+					MaxClientAttempts: clientAttempts,
+					HasNextCandidate:  platformsVisited < maxPlatforms && index+1 < len(candidates),
+					FailoverExpired:   failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration),
+					Async:             task.AsyncMode,
+					DownstreamStarted: downstreamStarted.Load(),
+				})
+				if candidateDecision.Route == "requeue" {
 					queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidate)
 					if queueErr != nil {
 						return Result{}, queueErr
@@ -768,104 +814,88 @@ candidatesLoop:
 					ExtraMetrics: []map[string]any{parameterPreprocessingMetrics(preprocessing.Log)},
 					ModelType:    candidate.ModelType,
 				})
+				candidateDecisionAttempt = attemptNo
+				candidateClientAttempt = clientAttempt
+				s.recordAttemptTrace(ctx, task.ID, attemptNo, failureDecisionTraceEntry(candidateDecision, candidate, clientAttempt, clientAttempts, false, ""))
 				break candidatesLoop
 			}
 			attemptNo = nextAttemptNo
 			lastErr = err
 			candidateErr = err
-			retryDecision := retryDecisionForCandidate(candidate, err)
-			retryAction := "retry"
-			if !retryDecision.Retry {
-				retryAction = "stop"
-			}
-			if clientAttempt >= clientAttempts {
-				retryDecision.Retry = false
-				retryDecision.Reason = "same_client_max_attempts"
-				retryDecision.Match = policyRuleMatch{
-					Source: "model_runtime_policy_sets.retry_policy",
-					Policy: "retryPolicy",
-					Rule:   "maxAttempts",
-					Value:  strconv.Itoa(clientAttempts),
-				}
-				retryDecision.Info = failureInfoFromError(err)
-				retryAction = "stop"
-			}
-			if failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration) {
-				retryDecision.Retry = false
-				retryDecision.Reason = "failover_time_budget_exceeded"
-				retryDecision.Match = policyRuleMatch{
-					Source: "gateway_runner_policies.failover_policy",
-					Policy: "failoverPolicy",
-					Rule:   "maxDurationSeconds",
-					Value:  strconv.Itoa(int(maxFailoverDuration.Seconds())),
-				}
-				retryDecision.Info = failureInfoFromError(err)
-				retryAction = "stop"
-			}
-			s.recordAttemptTrace(ctx, task.ID, attemptNo, retryTraceEntry(candidate, retryDecision, retryAction, clientAttempt, clientAttempts))
-			if !retryDecision.Retry {
+			candidateDecision = resolveCandidateFailure(resolveCandidateFailureInput{
+				RunnerPolicy:      runnerPolicy,
+				Candidate:         candidate,
+				Err:               err,
+				ClientAttempt:     clientAttempt,
+				MaxClientAttempts: clientAttempts,
+				HasNextCandidate:  platformsVisited < maxPlatforms && index+1 < len(candidates),
+				FailoverExpired:   failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration),
+				Async:             task.AsyncMode,
+				DownstreamStarted: downstreamStarted.Load(),
+			})
+			candidateDecisionAttempt = attemptNo
+			candidateClientAttempt = clientAttempt
+			if candidateDecision.Route != "retry_same" {
 				break
 			}
+			s.recordAttemptTrace(ctx, task.ID, attemptNo, failureDecisionTraceEntry(candidateDecision, candidate, clientAttempt, clientAttempts, false, ""))
 			if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.45, "retrying same client", addPolicyTracePayload(map[string]any{
 				"attempt":       attemptNo,
 				"clientAttempt": clientAttempt,
 				"clientId":      candidate.ClientID,
 				"error":         err.Error(),
-				"reason":        retryDecision.Reason,
+				"reason":        candidateDecision.Reason,
 				"scope":         "same_client",
-			}, retryDecision.Match, retryDecision.Info), isSimulation(task, candidate)); err != nil {
+			}, candidateDecision.Match, candidateDecision.Info), isSimulation(task, candidate)); err != nil {
 				return Result{}, err
 			}
 		}
-		if candidateErr == nil || index+1 >= len(candidates) || index+1 >= maxPlatforms {
-			if candidateErr != nil {
-				s.applyPriorityDemotePolicy(ctx, task.ID, attemptNo, runnerPolicy, candidate, task.Model, candidateErr, isSimulation(task, candidate))
-				decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
-				if decision.Retry {
-					decision.Retry = false
-					decision.Action = "stop"
-					decision.Reason = "no_next_platform"
-					decision.Match = policyRuleMatch{Source: "runner_candidates", Policy: "candidateSelection", Rule: "candidateCount", Value: strconv.Itoa(len(candidates))}
-					if index+1 >= maxPlatforms {
-						decision.Reason = "max_platforms_reached"
-						decision.Match = policyRuleMatch{Source: "gateway_runner_policies.failover_policy", Policy: "failoverPolicy", Rule: "maxPlatforms", Value: strconv.Itoa(maxPlatforms)}
-					}
+		if candidateErr == nil {
+			break
+		}
+		effectResult, effectErr := s.applyFailureDecision(
+			context.WithoutCancel(ctx),
+			task.ID,
+			task.Model,
+			candidate,
+			candidateDecision,
+			singleSourceProtection,
+			isSimulation(task, candidate),
+		)
+		if effectErr != nil {
+			s.logger.Warn("apply candidate failure effect failed", "taskID", task.ID, "effect", candidateDecision.Effect, "error", effectErr)
+			effectResult.GuardReason = "effect_error"
+		}
+		s.recordAttemptTrace(ctx, task.ID, candidateDecisionAttempt, failureDecisionTraceEntry(
+			candidateDecision,
+			candidate,
+			candidateClientAttempt,
+			clientAttempts,
+			effectResult.Applied,
+			effectResult.GuardReason,
+		))
+		if candidateDecision.Route != "next" {
+			if candidateDecision.Reason == "failover_time_budget_exceeded" {
+				elapsedSeconds := int(time.Since(executeStartedAt).Seconds())
+				if err := s.emit(ctx, task.ID, "task.failover.stopped", "running", "retry", 0.55, "failover time budget exceeded", map[string]any{
+					"elapsedSeconds":     elapsedSeconds,
+					"maxDurationSeconds": int(maxFailoverDuration.Seconds()),
+					"scope":              "next_platform",
+					"statusCode":         clients.ErrorResponseMetadata(candidateErr).StatusCode,
+				}, isSimulation(task, candidate)); err != nil {
+					return Result{}, err
 				}
-				s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision, candidate))
 			}
 			break
 		}
-		s.applyPriorityDemotePolicy(ctx, task.ID, attemptNo, runnerPolicy, candidate, task.Model, candidateErr, isSimulation(task, candidate))
-		if failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration) {
-			elapsedSeconds := int(time.Since(executeStartedAt).Seconds())
-			maxDurationSeconds := int(maxFailoverDuration.Seconds())
-			s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTimeBudgetTraceEntry(elapsedSeconds, maxDurationSeconds, failureInfoFromError(candidateErr), candidate))
-			if err := s.emit(ctx, task.ID, "task.failover.stopped", "running", "retry", 0.55, "failover time budget exceeded", map[string]any{
-				"elapsedSeconds":     elapsedSeconds,
-				"maxDurationSeconds": maxDurationSeconds,
-				"scope":              "next_platform",
-				"statusCode":         clients.ErrorResponseMetadata(candidateErr).StatusCode,
-			}, isSimulation(task, candidate)); err != nil {
-				return Result{}, err
-			}
-			break
-		}
-		decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
-		if !decision.Retry && !isResultPersistenceFailure(candidateErr) && hasLoadAvoidanceFallback(candidates, index, maxPlatforms) {
-			decision = loadAvoidanceFallbackDecision(candidateErr)
-		}
-		s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision, candidate))
-		if !decision.Retry {
-			break
-		}
-		s.applyFailoverAction(ctx, task.ID, candidate, task.Model, decision, isSimulation(task, candidate), singleSourceProtected)
 		if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.55, "retrying next client", addPolicyTracePayload(map[string]any{
 			"attempt": attemptNo,
-			"action":  decision.Action,
+			"route":   candidateDecision.Route,
+			"effect":  candidateDecision.Effect,
 			"error":   candidateErr.Error(),
-			"reason":  decision.Reason,
+			"reason":  candidateDecision.Reason,
 			"scope":   "next_platform",
-		}, decision.Match, decision.Info), isSimulation(task, candidate)); err != nil {
+		}, candidateDecision.Match, candidateDecision.Info), isSimulation(task, candidate)); err != nil {
 			return Result{}, err
 		}
 	}
@@ -951,7 +981,7 @@ func billingItemsFixedTotal(items []any) fixedAmount {
 	return total
 }
 
-func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, pricing resolvedPricing, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) {
+func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, pricing resolvedPricing, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) {
 	var err error
 	candidate, err = candidateWithEnvironmentCredentials(candidate)
 	if err != nil {
@@ -1230,7 +1260,6 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 		if !simulated && submissionStatus == "submitting" {
 			return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: err}
 		}
-		s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated, singleSourceProtected)
 		return clients.Response{}, err
 	}
 	uploadedResult, err := s.uploadGeneratedAssets(ctx, task.ID, task.Kind, response.Result)
@@ -1669,6 +1698,34 @@ func (s *Service) requeueRateLimitedTask(ctx context.Context, task store.Gateway
 	return queued, delay, nil
 }
 
+func (s *Service) requeueModelCoolingTask(ctx context.Context, task store.GatewayTask, cause error) (store.GatewayTask, time.Duration, error) {
+	delay := store.ModelCandidateRetryAfter(cause)
+	recoveryAt := store.ModelCandidateRecoveryAt(cause)
+	if recoveryAt.IsZero() {
+		recoveryAt = time.Now().Add(delay)
+	}
+	if delay <= 0 {
+		delay = time.Until(recoveryAt)
+	}
+	if delay < time.Second {
+		delay = time.Second
+		recoveryAt = time.Now().Add(delay)
+	}
+	queued, err := s.store.RequeueTaskUntil(ctx, task.ID, task.ExecutionToken, recoveryAt, "")
+	if err != nil {
+		return store.GatewayTask{}, 0, err
+	}
+	payload := map[string]any{
+		"code":              store.ModelCandidateErrorCode(cause),
+		"retryAfterSeconds": int((delay + time.Second - 1) / time.Second),
+		"recoveryAt":        recoveryAt.UTC().Format(time.RFC3339),
+	}
+	if eventErr := s.emit(ctx, task.ID, "task.queued", "queued", "model_cooling_down", 0.2, "task queued until a model candidate recovers", payload, task.RunMode == "simulation"); eventErr != nil {
+		return store.GatewayTask{}, 0, eventErr
+	}
+	return queued, delay, nil
+}
+
 func taskRetryJitter(taskID string) time.Duration {
 	var sum uint32
 	for _, value := range []byte(taskID) {