refactor(runner): 收敛上游失败决策与轮转策略

将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。

apps/api/internal/runner/retry_decision.go

@@ -42,6 +42,29 @@ type failoverDecision struct {
 	Info            failureInfo
 }
 
+type failureDecision struct {
+	Route           string
+	Effect          string
+	Target          string
+	CooldownSeconds int
+	DemoteSteps     int
+	Reason          string
+	Match           policyRuleMatch
+	Info            failureInfo
+}
+
+type resolveCandidateFailureInput struct {
+	RunnerPolicy      store.RunnerPolicy
+	Candidate         store.RuntimeModelCandidate
+	Err               error
+	ClientAttempt     int
+	MaxClientAttempts int
+	HasNextCandidate  bool
+	FailoverExpired   bool
+	Async             bool
+	DownstreamStarted bool
+}
+
 type priorityDemoteDecision struct {
 	Demote      bool
 	Reason      string
@@ -104,7 +127,7 @@ func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate sto
 	if errors.Is(err, store.ErrRateLimited) {
 		return failoverDecision{Retry: false, Action: "queue", Reason: "local_rate_limit_wait_queue", Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"}, Info: info}
 	}
-	if ruleDecision, ok := failoverActionRuleDecisionWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
+	if ruleDecision, ok := candidateFailoverActionRuleDecision(runnerPolicy.FailoverPolicy, overridePolicy, candidate, info); ok {
 		return ruleDecision
 	}
 	if match, ok := failoverDenyMatchWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
@@ -125,6 +148,122 @@ func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate sto
 	return failoverDecision{Retry: false, Action: "stop", Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
 }
 
+func resolveCandidateFailure(input resolveCandidateFailureInput) failureDecision {
+	info := failureInfoFromError(input.Err)
+	if isResultPersistenceFailure(input.Err) {
+		return failureDecision{
+			Route:  "stop",
+			Effect: "none",
+			Reason: "result_persistence_failed",
+			Match:  policyRuleMatch{Source: "gateway_result_storage", Policy: "persistence", Rule: "providerFailover", Value: "disabled"},
+			Info:   info,
+		}
+	}
+	if errors.Is(input.Err, store.ErrRateLimited) {
+		route := "stop"
+		if input.Async && store.RateLimitRetryable(input.Err) {
+			route = "requeue"
+		}
+		return failureDecision{
+			Route:  route,
+			Effect: "none",
+			Reason: "local_rate_limit_wait_queue",
+			Match:  policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"},
+			Info:   info,
+		}
+	}
+
+	failover := failoverDecisionForCandidate(input.RunnerPolicy, input.Candidate, input.Err)
+	effect := failoverEffect(failover.Action)
+	if input.DownstreamStarted {
+		return failureDecision{
+			Route:           "stop",
+			Effect:          effect,
+			Target:          failover.Target,
+			CooldownSeconds: failover.CooldownSeconds,
+			DemoteSteps:     failover.DemoteSteps,
+			Reason:          "downstream_response_started",
+			Match:           policyRuleMatch{Source: "gateway_stream", Policy: "streamSafety", Rule: "downstreamStarted", Value: "true"},
+			Info:            info,
+		}
+	}
+	explicitStop := failover.Action == "stop" && (failover.Reason == "failover_action_rule" ||
+		failover.Reason == "hard_stop_policy" ||
+		failover.Reason == "failover_deny_policy" ||
+		failover.Reason == "failover_disabled" ||
+		failover.Reason == "runner_policy_disabled")
+	if explicitStop {
+		return failureDecision{
+			Route:           "stop",
+			Effect:          effect,
+			Target:          failover.Target,
+			CooldownSeconds: failover.CooldownSeconds,
+			DemoteSteps:     failover.DemoteSteps,
+			Reason:          failover.Reason,
+			Match:           failover.Match,
+			Info:            failover.Info,
+		}
+	}
+
+	immediateCandidateInvalidation := failover.Action == "cooldown_and_next" || failover.Action == "disable_and_next"
+	retry := retryDecisionForCandidate(input.Candidate, input.Err)
+	if !immediateCandidateInvalidation && retry.Retry && input.ClientAttempt < input.MaxClientAttempts {
+		return failureDecision{
+			Route:  "retry_same",
+			Effect: "none",
+			Reason: retry.Reason,
+			Match:  retry.Match,
+			Info:   retry.Info,
+		}
+	}
+
+	route := "next"
+	reason := failover.Reason
+	match := failover.Match
+	if input.FailoverExpired {
+		route = "stop"
+		reason = "failover_time_budget_exceeded"
+		match = policyRuleMatch{
+			Source: "gateway_runner_policies.failover_policy",
+			Policy: "failoverPolicy",
+			Rule:   "maxDurationSeconds",
+			Value:  "exceeded",
+		}
+	} else if !input.HasNextCandidate {
+		route = "stop"
+		reason = "no_next_platform"
+		match = policyRuleMatch{
+			Source: "runner_candidates",
+			Policy: "candidateSelection",
+			Rule:   "candidateCount",
+			Value:  "exhausted",
+		}
+	}
+	return failureDecision{
+		Route:           route,
+		Effect:          effect,
+		Target:          failover.Target,
+		CooldownSeconds: failover.CooldownSeconds,
+		DemoteSteps:     failover.DemoteSteps,
+		Reason:          reason,
+		Match:           match,
+		Info:            failover.Info,
+	}
+}
+
+func failoverEffect(action string) string {
+	switch action {
+	case "cooldown_and_next":
+		return "cooldown"
+	case "demote_and_next":
+		return "demote"
+	case "disable_and_next":
+		return "disable"
+	default:
+		return "none"
+	}
+}
+
 func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) bool {
 	decision := priorityDemoteDecisionForCandidate(runnerPolicy, err)
 	return decision.Demote
@@ -346,6 +485,46 @@ func failoverActionRuleDecisionWithSources(base map[string]any, override map[str
 	return failoverActionRuleDecision(actionRulesFromPolicy(base), "gateway_runner_policies.failover_policy", info)
 }
 
+func candidateFailoverActionRuleDecision(base map[string]any, override map[string]any, candidate store.RuntimeModelCandidate, info failureInfo) (failoverDecision, bool) {
+	if rules := actionRulesFromPolicy(override); len(rules) > 0 {
+		if decision, ok := failoverActionRuleDecision(rules, "runtime_policy_override.failoverPolicy", info); ok {
+			return decision, true
+		}
+	}
+	autoDisablePolicy := effectiveRuntimePolicy(candidate.AutoDisablePolicy, candidate.RuntimePolicyOverride, "autoDisablePolicy")
+	if boolFromPolicy(autoDisablePolicy, "enabled", false) && intFromPolicy(autoDisablePolicy, "threshold") <= 1 {
+		if value, ok := matchingKeywordValue(autoDisablePolicy, "keywords", info.Target); ok {
+			return failoverDecision{
+				Retry:  true,
+				Action: "disable_and_next",
+				Target: "platform",
+				Reason: "legacy_auto_disable_policy",
+				Match:  policyRuleMatch{Source: "model_runtime_policy_sets.auto_disable_policy", Policy: "autoDisablePolicy", Rule: "keywords", Value: value},
+				Info:   info,
+			}, true
+		}
+	}
+	degradePolicy := effectiveRuntimePolicy(candidate.DegradePolicy, candidate.RuntimePolicyOverride, "degradePolicy")
+	if boolFromPolicy(degradePolicy, "enabled", false) {
+		if value, ok := matchingKeywordValue(degradePolicy, "keywords", info.Target); ok {
+			cooldownSeconds := intFromPolicy(degradePolicy, "cooldownSeconds")
+			if cooldownSeconds <= 0 {
+				cooldownSeconds = 300
+			}
+			return failoverDecision{
+				Retry:           true,
+				Action:          "cooldown_and_next",
+				Target:          "model",
+				Reason:          "legacy_degrade_policy",
+				CooldownSeconds: cooldownSeconds,
+				Match:           policyRuleMatch{Source: "model_runtime_policy_sets.degrade_policy", Policy: "degradePolicy", Rule: "keywords", Value: value},
+				Info:            info,
+			}, true
+		}
+	}
+	return failoverActionRuleDecision(actionRulesFromPolicy(base), "gateway_runner_policies.failover_policy", info)
+}
+
 func failoverActionRuleDecision(rules []map[string]any, source string, info failureInfo) (failoverDecision, bool) {
 	for index, rule := range rules {
 		if !boolFromPolicy(rule, "enabled", true) {

apps/api/internal/runner/retry_decision_test.go

@@ -334,3 +334,151 @@ func TestPriorityDemotePolicyIsSkippedWhenFailoverActionRulesExist(t *testing.T)
 		t.Fatalf("legacy priority demotion should be skipped when actionRules are active, got %+v", decision)
 	}
 }
+
+func TestResolveCandidateFailureCooldownSkipsSameClientRetry(t *testing.T) {
+	decision := resolveCandidateFailure(resolveCandidateFailureInput{
+		RunnerPolicy: store.RunnerPolicy{
+			Status: "active",
+			FailoverPolicy: map[string]any{
+				"enabled": true,
+				"actionRules": []any{
+					map[string]any{
+						"statusCodes":     []any{429},
+						"action":          "cooldown_and_next",
+						"target":          "model",
+						"cooldownSeconds": 60,
+					},
+				},
+			},
+		},
+		Candidate: store.RuntimeModelCandidate{
+			ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3},
+		},
+		Err:               &clients.ClientError{Code: "resource_exhausted", StatusCode: 429, Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 3,
+		HasNextCandidate:  true,
+	})
+
+	if decision.Route != "next" || decision.Effect != "cooldown" || decision.Target != "model" || decision.CooldownSeconds != 60 {
+		t.Fatalf("429 cooldown rule should invalidate the candidate immediately, got %+v", decision)
+	}
+}
+
+func TestResolveCandidateFailureDemoteRetriesBeforeNext(t *testing.T) {
+	input := resolveCandidateFailureInput{
+		RunnerPolicy: store.RunnerPolicy{
+			Status: "active",
+			FailoverPolicy: map[string]any{
+				"enabled": true,
+				"actionRules": []any{
+					map[string]any{
+						"statusCodes": []any{500},
+						"action":      "demote_and_next",
+						"target":      "platform",
+						"demoteSteps": 2,
+					},
+				},
+			},
+		},
+		Candidate: store.RuntimeModelCandidate{
+			ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2},
+		},
+		Err:               &clients.ClientError{Code: "server_error", StatusCode: 500, Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 2,
+		HasNextCandidate:  true,
+	}
+
+	first := resolveCandidateFailure(input)
+	if first.Route != "retry_same" || first.Effect != "none" {
+		t.Fatalf("demotion must wait until same-client retries are exhausted, got %+v", first)
+	}
+
+	input.ClientAttempt = 2
+	second := resolveCandidateFailure(input)
+	if second.Route != "next" || second.Effect != "demote" || second.DemoteSteps != 2 {
+		t.Fatalf("exhausted retry should demote once and rotate, got %+v", second)
+	}
+}
+
+func TestResolveCandidateFailureStopsAfterDownstreamStarts(t *testing.T) {
+	decision := resolveCandidateFailure(resolveCandidateFailureInput{
+		RunnerPolicy: store.RunnerPolicy{
+			Status: "active",
+			FailoverPolicy: map[string]any{
+				"enabled": true,
+				"actionRules": []any{
+					map[string]any{
+						"categories": []any{"stream_error"},
+						"action":     "next",
+					},
+				},
+			},
+		},
+		Candidate:         store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3}},
+		Err:               &clients.ClientError{Code: "stream_read_error", Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 3,
+		HasNextCandidate:  true,
+		DownstreamStarted: true,
+	})
+
+	if decision.Route != "stop" || decision.Reason != "downstream_response_started" {
+		t.Fatalf("stream output must prevent retries and failover, got %+v", decision)
+	}
+}
+
+func TestResolveCandidateFailureLegacyPolicyPrecedence(t *testing.T) {
+	candidate := store.RuntimeModelCandidate{
+		DegradePolicy: map[string]any{
+			"enabled":         true,
+			"keywords":        []any{"resource_exhausted"},
+			"cooldownSeconds": 45,
+		},
+	}
+	decision := resolveCandidateFailure(resolveCandidateFailureInput{
+		RunnerPolicy: store.RunnerPolicy{
+			Status: "active",
+			FailoverPolicy: map[string]any{
+				"enabled": true,
+				"actionRules": []any{
+					map[string]any{
+						"statusCodes":     []any{429},
+						"action":          "cooldown_and_next",
+						"target":          "platform",
+						"cooldownSeconds": 90,
+					},
+				},
+			},
+		},
+		Candidate:         candidate,
+		Err:               &clients.ClientError{Code: "resource_exhausted", Message: "resource_exhausted", StatusCode: 429, Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 3,
+		HasNextCandidate:  true,
+	})
+
+	if decision.Match.Policy != "degradePolicy" || decision.Target != "model" || decision.CooldownSeconds != 45 {
+		t.Fatalf("legacy compatibility rule should precede global action rules, got %+v", decision)
+	}
+}
+
+func TestResolveCandidateFailureUnmatchedRetryableUsesSameThenNext(t *testing.T) {
+	input := resolveCandidateFailureInput{
+		RunnerPolicy:      store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true, "actionRules": []any{}}},
+		Candidate:         store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2}},
+		Err:               &clients.ClientError{Code: "unknown_transient", Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 2,
+		HasNextCandidate:  true,
+	}
+
+	if decision := resolveCandidateFailure(input); decision.Route != "retry_same" {
+		t.Fatalf("unmatched retryable error should retry the same client first, got %+v", decision)
+	}
+	input.ClientAttempt = 2
+	if decision := resolveCandidateFailure(input); decision.Route != "next" || decision.Effect != "none" {
+		t.Fatalf("unmatched exhausted error should rotate without side effects, got %+v", decision)
+	}
+}

apps/api/internal/runner/runtime_policy.go

@@ -2,153 +2,59 @@ package runner
 
 import (
 	"context"
-	"errors"
 	"strings"
 
-	"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
 )
 
-func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, cause error, simulated bool, singleSourceProtected bool) {
-	code := clients.ErrorCode(cause)
-	message := ""
-	if cause != nil {
-		message = cause.Error()
+func (s *Service) applyFailureDecision(ctx context.Context, taskID string, requestedModel string, candidate store.RuntimeModelCandidate, decision failureDecision, singleSourceProtection bool, simulated bool) (store.CandidateFailureEffectResult, error) {
+	result, err := s.store.ApplyCandidateFailureEffect(ctx, store.CandidateFailureEffectInput{
+		Effect:                 decision.Effect,
+		Target:                 decision.Target,
+		PlatformID:             candidate.PlatformID,
+		PlatformModelID:        candidate.PlatformModelID,
+		RequestedModel:         requestedModel,
+		ModelType:              candidate.ModelType,
+		CooldownSeconds:        decision.CooldownSeconds,
+		DemoteSteps:            decision.DemoteSteps,
+		SingleSourceProtection: singleSourceProtection,
+	})
+	if err != nil {
+		return result, err
 	}
-
-	autoDisablePolicy := effectiveRuntimePolicy(candidate.AutoDisablePolicy, candidate.RuntimePolicyOverride, "autoDisablePolicy")
-	if failurePolicyMatches(autoDisablePolicy, code, message) && intFromPolicy(autoDisablePolicy, "threshold") <= 1 {
-		if singleSourceProtected {
-			s.emitSingleSourceProtected(ctx, taskID, candidate, "auto_disable", code, simulated)
-		} else if err := s.store.DisableCandidatePlatform(ctx, candidate.PlatformID); err == nil {
-			_ = s.emit(ctx, taskID, "task.policy.auto_disabled", "running", "auto_disable", 0.48, "candidate platform disabled by failure policy", map[string]any{
-				"platformId":      candidate.PlatformID,
-				"platformModelId": candidate.PlatformModelID,
-				"code":            code,
-			}, simulated)
-		}
-	}
-
-	degradePolicy := effectiveRuntimePolicy(candidate.DegradePolicy, candidate.RuntimePolicyOverride, "degradePolicy")
-	if failurePolicyMatches(degradePolicy, code, message) {
-		cooldownSeconds := intFromPolicy(degradePolicy, "cooldownSeconds")
-		if singleSourceProtected {
-			s.emitSingleSourceProtected(ctx, taskID, candidate, "degrade_cooldown", code, simulated)
-		} else if err := s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, cooldownSeconds); err == nil {
-			_ = s.emit(ctx, taskID, "task.policy.degraded", "running", "degrade", 0.5, "candidate model cooled down by failure policy", map[string]any{
-				"platformId":      candidate.PlatformID,
-				"platformModelId": candidate.PlatformModelID,
-				"cooldownSeconds": cooldownSeconds,
-				"code":            code,
-			}, simulated)
-		}
-	}
-}
-
-func (s *Service) applyFailoverAction(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, requestedModel string, decision failoverDecision, simulated bool, singleSourceProtected bool) {
-	switch decision.Action {
-	case "disable_and_next":
-		if singleSourceProtected {
-			s.emitSingleSourceProtected(ctx, taskID, candidate, decision.Action, decision.Info.Code, simulated)
-			return
-		}
-		target := decision.Target
-		if target == "" {
-			target = "platform"
-		}
-		var err error
-		if target == "model" {
-			err = s.store.DisableCandidatePlatformModel(ctx, candidate.PlatformModelID)
-		} else {
-			err = s.store.DisableCandidatePlatform(ctx, candidate.PlatformID)
-		}
-		if err == nil {
-			_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate platform disabled by runner failover policy", addPolicyTracePayload(map[string]any{
-				"platformId":      candidate.PlatformID,
-				"platformModelId": candidate.PlatformModelID,
-				"action":          decision.Action,
-				"target":          target,
-				"reason":          decision.Reason,
-			}, decision.Match, decision.Info), simulated)
-		}
-	case "cooldown_and_next":
-		if singleSourceProtected {
-			s.emitSingleSourceProtected(ctx, taskID, candidate, decision.Action, decision.Info.Code, simulated)
-			return
-		}
-		target := decision.Target
-		if target == "" {
-			target = "model"
-		}
-		var err error
-		if target == "platform" {
-			err = s.store.CooldownCandidatePlatform(ctx, candidate.PlatformID, decision.CooldownSeconds)
-		} else {
-			err = s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, decision.CooldownSeconds)
-		}
-		if err == nil {
-			_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate model cooled down by runner failover policy", addPolicyTracePayload(map[string]any{
-				"platformId":      candidate.PlatformID,
-				"platformModelId": candidate.PlatformModelID,
-				"cooldownSeconds": decision.CooldownSeconds,
-				"action":          decision.Action,
-				"target":          target,
-				"reason":          decision.Reason,
-			}, decision.Match, decision.Info), simulated)
-		}
-	case "demote_and_next":
-		demoteSteps := decision.DemoteSteps
-		if demoteSteps <= 0 {
-			demoteSteps = 1
-		}
-		if dynamicPriority, err := s.store.DemoteCandidatePlatformPriorityBySteps(ctx, candidate.PlatformID, candidate.PlatformModelID, requestedModel, candidate.ModelType, demoteSteps); err == nil {
-			_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner failover policy", addPolicyTracePayload(map[string]any{
-				"platformId":      candidate.PlatformID,
-				"platformModelId": candidate.PlatformModelID,
-				"dynamicPriority": dynamicPriority,
-				"demoteSteps":     demoteSteps,
-				"action":          decision.Action,
-				"target":          "platform",
-				"reason":          decision.Reason,
-				"errorMessage":    decision.Info.Message,
-			}, decision.Match, decision.Info), simulated)
-		}
-	}
-}
-
-func (s *Service) emitSingleSourceProtected(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, action string, code string, simulated bool) {
-	_ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", map[string]any{
-		"platformId":      candidate.PlatformID,
-		"platformModelId": candidate.PlatformModelID,
-		"action":          action,
-		"code":            code,
-	}, simulated)
-}
-
-func (s *Service) applyPriorityDemotePolicy(ctx context.Context, taskID string, attemptNo int, runnerPolicy store.RunnerPolicy, candidate store.RuntimeModelCandidate, requestedModel string, cause error, simulated bool) {
-	if errors.Is(cause, store.ErrRateLimited) {
-		return
-	}
-	decision := priorityDemoteDecisionForCandidate(runnerPolicy, cause)
-	if !decision.Demote {
-		return
-	}
-	demoteSteps := decision.DemoteSteps
-	if demoteSteps <= 0 {
-		demoteSteps = 99
-	}
-	if dynamicPriority, err := s.store.DemoteCandidatePlatformPriorityBySteps(ctx, candidate.PlatformID, candidate.PlatformModelID, requestedModel, candidate.ModelType, demoteSteps); err == nil {
-		s.recordAttemptTrace(ctx, taskID, attemptNo, priorityDemoteTraceEntry(decision, candidate.PlatformID, candidate.PlatformModelID, dynamicPriority))
-		_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner policy", addPolicyTracePayload(map[string]any{
+	if result.GuardReason == "single_source" {
+		_ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", addPolicyTracePayload(map[string]any{
 			"platformId":      candidate.PlatformID,
 			"platformModelId": candidate.PlatformModelID,
-			"dynamicPriority": dynamicPriority,
-			"demoteSteps":     demoteSteps,
-			"code":            clients.ErrorCode(cause),
-			"reason":          decision.Reason,
-			"errorMessage":    messageFromError(cause),
+			"effect":          decision.Effect,
+			"target":          decision.Target,
+			"guardReason":     result.GuardReason,
 		}, decision.Match, decision.Info), simulated)
+		return result, nil
 	}
+	if !result.Applied {
+		return result, nil
+	}
+
+	payload := addPolicyTracePayload(map[string]any{
+		"platformId":      candidate.PlatformID,
+		"platformModelId": candidate.PlatformModelID,
+		"effect":          decision.Effect,
+		"target":          decision.Target,
+		"reason":          decision.Reason,
+	}, decision.Match, decision.Info)
+	switch decision.Effect {
+	case "disable":
+		_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate disabled by runner failover policy", payload, simulated)
+	case "cooldown":
+		payload["cooldownSeconds"] = decision.CooldownSeconds
+		_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate cooled down by runner failover policy", payload, simulated)
+	case "demote":
+		payload["demoteSteps"] = decision.DemoteSteps
+		payload["dynamicPriority"] = result.DynamicPriority
+		_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner failover policy", payload, simulated)
+	}
+	return result, nil
 }
 
 func messageFromError(err error) string {

apps/api/internal/runner/service.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
@@ -269,6 +270,20 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 		if task.Kind == "responses" && responseExecution.PreviousChain != nil {
 			err = responseChainUnavailableError()
 		}
+		if task.AsyncMode && store.ModelCandidateRetryAfter(err) > 0 {
+			queued, delay, queueErr := s.requeueModelCoolingTask(ctx, task, err)
+			if queueErr != nil {
+				return Result{}, queueErr
+			}
+			return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
+		}
+		if store.ModelCandidateRetryAfter(err) > 0 {
+			failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err)
+			if finishErr != nil {
+				return Result{}, finishErr
+			}
+			return Result{Task: failed, Output: failed.Result}, err
+		}
 		s.recordFailedAttempt(ctx, failedAttemptRecord{
 			Task:      task,
 			Body:      body,
@@ -539,18 +554,38 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 
 	maxPlatforms := maxPlatformsForCandidates(candidates, runnerPolicy)
 	maxFailoverDuration := maxFailoverDurationForCandidates(candidates, runnerPolicy)
-	singleSourceProtected := singleSourceProtectionActive(candidates, maxPlatforms, runnerPolicy)
+	singleSourceProtection := runnerSingleSourceProtectionEnabled(runnerPolicy)
+	var downstreamStarted atomic.Bool
+	candidateDelta := onDelta
+	if onDelta != nil {
+		candidateDelta = func(event clients.StreamDeltaEvent) error {
+			downstreamStarted.Store(true)
+			return onDelta(event)
+		}
+	}
 	var lastErr error
 	var lastCandidate store.RuntimeModelCandidate
 	var lastPreprocessing *parameterPreprocessingLog
+	platformsVisited := 0
 candidatesLoop:
 	for index, candidate := range candidates {
-		if index >= maxPlatforms {
+		if platformsVisited >= maxPlatforms {
 			break
 		}
+		available, availabilityErr := s.store.RuntimeCandidateAvailable(ctx, candidate.PlatformID, candidate.PlatformModelID)
+		if availabilityErr != nil {
+			return Result{}, availabilityErr
+		}
+		if !available {
+			continue
+		}
+		platformsVisited++
 		lastCandidate = candidate
 		clientAttempts := clientAttemptsForCandidate(candidate)
 		var candidateErr error
+		var candidateDecision failureDecision
+		var candidateDecisionAttempt int
+		var candidateClientAttempt int
 		for clientAttempt := 1; clientAttempt <= clientAttempts; clientAttempt++ {
 			nextAttemptNo := attemptNo + 1
 			preprocessing, ok := preprocessingByCandidate[pricingCandidateKey(candidate)]
@@ -579,7 +614,7 @@ candidatesLoop:
 			}
 			candidateBody := preprocessing.Body
 			candidatePricing := pricingByCandidate[pricingCandidateKey(candidate)]
-			response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, candidatePricing, nextAttemptNo, onDelta, responseExecution, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record)
+			response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, candidatePricing, nextAttemptNo, candidateDelta, responseExecution, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record)
 			if err != nil && isVolcesRemoteTaskCancellation(candidate, err) {
 				cancelled, changed, cancelErr := s.store.CancelSubmittedTask(ctx, task.ID, task.ExecutionToken, "任务已由火山引擎取消")
 				if cancelErr != nil {
@@ -748,7 +783,18 @@ candidatesLoop:
 			if isLocalRateLimitError(err) {
 				lastErr = err
 				candidateErr = err
-				if task.AsyncMode && store.RateLimitRetryable(err) {
+				candidateDecision = resolveCandidateFailure(resolveCandidateFailureInput{
+					RunnerPolicy:      runnerPolicy,
+					Candidate:         candidate,
+					Err:               err,
+					ClientAttempt:     clientAttempt,
+					MaxClientAttempts: clientAttempts,
+					HasNextCandidate:  platformsVisited < maxPlatforms && index+1 < len(candidates),
+					FailoverExpired:   failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration),
+					Async:             task.AsyncMode,
+					DownstreamStarted: downstreamStarted.Load(),
+				})
+				if candidateDecision.Route == "requeue" {
 					queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidate)
 					if queueErr != nil {
 						return Result{}, queueErr
@@ -768,104 +814,88 @@ candidatesLoop:
 					ExtraMetrics: []map[string]any{parameterPreprocessingMetrics(preprocessing.Log)},
 					ModelType:    candidate.ModelType,
 				})
+				candidateDecisionAttempt = attemptNo
+				candidateClientAttempt = clientAttempt
+				s.recordAttemptTrace(ctx, task.ID, attemptNo, failureDecisionTraceEntry(candidateDecision, candidate, clientAttempt, clientAttempts, false, ""))
 				break candidatesLoop
 			}
 			attemptNo = nextAttemptNo
 			lastErr = err
 			candidateErr = err
-			retryDecision := retryDecisionForCandidate(candidate, err)
-			retryAction := "retry"
-			if !retryDecision.Retry {
-				retryAction = "stop"
-			}
-			if clientAttempt >= clientAttempts {
-				retryDecision.Retry = false
-				retryDecision.Reason = "same_client_max_attempts"
-				retryDecision.Match = policyRuleMatch{
-					Source: "model_runtime_policy_sets.retry_policy",
-					Policy: "retryPolicy",
-					Rule:   "maxAttempts",
-					Value:  strconv.Itoa(clientAttempts),
-				}
-				retryDecision.Info = failureInfoFromError(err)
-				retryAction = "stop"
-			}
-			if failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration) {
-				retryDecision.Retry = false
-				retryDecision.Reason = "failover_time_budget_exceeded"
-				retryDecision.Match = policyRuleMatch{
-					Source: "gateway_runner_policies.failover_policy",
-					Policy: "failoverPolicy",
-					Rule:   "maxDurationSeconds",
-					Value:  strconv.Itoa(int(maxFailoverDuration.Seconds())),
-				}
-				retryDecision.Info = failureInfoFromError(err)
-				retryAction = "stop"
-			}
-			s.recordAttemptTrace(ctx, task.ID, attemptNo, retryTraceEntry(candidate, retryDecision, retryAction, clientAttempt, clientAttempts))
-			if !retryDecision.Retry {
+			candidateDecision = resolveCandidateFailure(resolveCandidateFailureInput{
+				RunnerPolicy:      runnerPolicy,
+				Candidate:         candidate,
+				Err:               err,
+				ClientAttempt:     clientAttempt,
+				MaxClientAttempts: clientAttempts,
+				HasNextCandidate:  platformsVisited < maxPlatforms && index+1 < len(candidates),
+				FailoverExpired:   failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration),
+				Async:             task.AsyncMode,
+				DownstreamStarted: downstreamStarted.Load(),
+			})
+			candidateDecisionAttempt = attemptNo
+			candidateClientAttempt = clientAttempt
+			if candidateDecision.Route != "retry_same" {
 				break
 			}
+			s.recordAttemptTrace(ctx, task.ID, attemptNo, failureDecisionTraceEntry(candidateDecision, candidate, clientAttempt, clientAttempts, false, ""))
 			if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.45, "retrying same client", addPolicyTracePayload(map[string]any{
 				"attempt":       attemptNo,
 				"clientAttempt": clientAttempt,
 				"clientId":      candidate.ClientID,
 				"error":         err.Error(),
-				"reason":        retryDecision.Reason,
+				"reason":        candidateDecision.Reason,
 				"scope":         "same_client",
-			}, retryDecision.Match, retryDecision.Info), isSimulation(task, candidate)); err != nil {
+			}, candidateDecision.Match, candidateDecision.Info), isSimulation(task, candidate)); err != nil {
 				return Result{}, err
 			}
 		}
-		if candidateErr == nil || index+1 >= len(candidates) || index+1 >= maxPlatforms {
-			if candidateErr != nil {
-				s.applyPriorityDemotePolicy(ctx, task.ID, attemptNo, runnerPolicy, candidate, task.Model, candidateErr, isSimulation(task, candidate))
-				decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
-				if decision.Retry {
-					decision.Retry = false
-					decision.Action = "stop"
-					decision.Reason = "no_next_platform"
-					decision.Match = policyRuleMatch{Source: "runner_candidates", Policy: "candidateSelection", Rule: "candidateCount", Value: strconv.Itoa(len(candidates))}
-					if index+1 >= maxPlatforms {
-						decision.Reason = "max_platforms_reached"
-						decision.Match = policyRuleMatch{Source: "gateway_runner_policies.failover_policy", Policy: "failoverPolicy", Rule: "maxPlatforms", Value: strconv.Itoa(maxPlatforms)}
-					}
+		if candidateErr == nil {
+			break
+		}
+		effectResult, effectErr := s.applyFailureDecision(
+			context.WithoutCancel(ctx),
+			task.ID,
+			task.Model,
+			candidate,
+			candidateDecision,
+			singleSourceProtection,
+			isSimulation(task, candidate),
+		)
+		if effectErr != nil {
+			s.logger.Warn("apply candidate failure effect failed", "taskID", task.ID, "effect", candidateDecision.Effect, "error", effectErr)
+			effectResult.GuardReason = "effect_error"
+		}
+		s.recordAttemptTrace(ctx, task.ID, candidateDecisionAttempt, failureDecisionTraceEntry(
+			candidateDecision,
+			candidate,
+			candidateClientAttempt,
+			clientAttempts,
+			effectResult.Applied,
+			effectResult.GuardReason,
+		))
+		if candidateDecision.Route != "next" {
+			if candidateDecision.Reason == "failover_time_budget_exceeded" {
+				elapsedSeconds := int(time.Since(executeStartedAt).Seconds())
+				if err := s.emit(ctx, task.ID, "task.failover.stopped", "running", "retry", 0.55, "failover time budget exceeded", map[string]any{
+					"elapsedSeconds":     elapsedSeconds,
+					"maxDurationSeconds": int(maxFailoverDuration.Seconds()),
+					"scope":              "next_platform",
+					"statusCode":         clients.ErrorResponseMetadata(candidateErr).StatusCode,
+				}, isSimulation(task, candidate)); err != nil {
+					return Result{}, err
 				}
-				s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision, candidate))
 			}
 			break
 		}
-		s.applyPriorityDemotePolicy(ctx, task.ID, attemptNo, runnerPolicy, candidate, task.Model, candidateErr, isSimulation(task, candidate))
-		if failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration) {
-			elapsedSeconds := int(time.Since(executeStartedAt).Seconds())
-			maxDurationSeconds := int(maxFailoverDuration.Seconds())
-			s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTimeBudgetTraceEntry(elapsedSeconds, maxDurationSeconds, failureInfoFromError(candidateErr), candidate))
-			if err := s.emit(ctx, task.ID, "task.failover.stopped", "running", "retry", 0.55, "failover time budget exceeded", map[string]any{
-				"elapsedSeconds":     elapsedSeconds,
-				"maxDurationSeconds": maxDurationSeconds,
-				"scope":              "next_platform",
-				"statusCode":         clients.ErrorResponseMetadata(candidateErr).StatusCode,
-			}, isSimulation(task, candidate)); err != nil {
-				return Result{}, err
-			}
-			break
-		}
-		decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
-		if !decision.Retry && !isResultPersistenceFailure(candidateErr) && hasLoadAvoidanceFallback(candidates, index, maxPlatforms) {
-			decision = loadAvoidanceFallbackDecision(candidateErr)
-		}
-		s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision, candidate))
-		if !decision.Retry {
-			break
-		}
-		s.applyFailoverAction(ctx, task.ID, candidate, task.Model, decision, isSimulation(task, candidate), singleSourceProtected)
 		if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.55, "retrying next client", addPolicyTracePayload(map[string]any{
 			"attempt": attemptNo,
-			"action":  decision.Action,
+			"route":   candidateDecision.Route,
+			"effect":  candidateDecision.Effect,
 			"error":   candidateErr.Error(),
-			"reason":  decision.Reason,
+			"reason":  candidateDecision.Reason,
 			"scope":   "next_platform",
-		}, decision.Match, decision.Info), isSimulation(task, candidate)); err != nil {
+		}, candidateDecision.Match, candidateDecision.Info), isSimulation(task, candidate)); err != nil {
 			return Result{}, err
 		}
 	}
@@ -951,7 +981,7 @@ func billingItemsFixedTotal(items []any) fixedAmount {
 	return total
 }
 
-func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, pricing resolvedPricing, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) {
+func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, pricing resolvedPricing, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) {
 	var err error
 	candidate, err = candidateWithEnvironmentCredentials(candidate)
 	if err != nil {
@@ -1230,7 +1260,6 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 		if !simulated && submissionStatus == "submitting" {
 			return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: err}
 		}
-		s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated, singleSourceProtected)
 		return clients.Response{}, err
 	}
 	uploadedResult, err := s.uploadGeneratedAssets(ctx, task.ID, task.Kind, response.Result)
@@ -1669,6 +1698,34 @@ func (s *Service) requeueRateLimitedTask(ctx context.Context, task store.Gateway
 	return queued, delay, nil
 }
 
+func (s *Service) requeueModelCoolingTask(ctx context.Context, task store.GatewayTask, cause error) (store.GatewayTask, time.Duration, error) {
+	delay := store.ModelCandidateRetryAfter(cause)
+	recoveryAt := store.ModelCandidateRecoveryAt(cause)
+	if recoveryAt.IsZero() {
+		recoveryAt = time.Now().Add(delay)
+	}
+	if delay <= 0 {
+		delay = time.Until(recoveryAt)
+	}
+	if delay < time.Second {
+		delay = time.Second
+		recoveryAt = time.Now().Add(delay)
+	}
+	queued, err := s.store.RequeueTaskUntil(ctx, task.ID, task.ExecutionToken, recoveryAt, "")
+	if err != nil {
+		return store.GatewayTask{}, 0, err
+	}
+	payload := map[string]any{
+		"code":              store.ModelCandidateErrorCode(cause),
+		"retryAfterSeconds": int((delay + time.Second - 1) / time.Second),
+		"recoveryAt":        recoveryAt.UTC().Format(time.RFC3339),
+	}
+	if eventErr := s.emit(ctx, task.ID, "task.queued", "queued", "model_cooling_down", 0.2, "task queued until a model candidate recovers", payload, task.RunMode == "simulation"); eventErr != nil {
+		return store.GatewayTask{}, 0, eventErr
+	}
+	return queued, delay, nil
+}
+
 func taskRetryJitter(taskID string) time.Duration {
 	var sum uint32
 	for _, value := range []byte(taskID) {

apps/api/internal/runner/trace.go

@@ -48,6 +48,29 @@ func failoverTraceEntry(decision failoverDecision, candidate store.RuntimeModelC
 	return entry
 }
 
+func failureDecisionTraceEntry(decision failureDecision, candidate store.RuntimeModelCandidate, clientAttempt int, maxAttempts int, applied bool, guardReason string) map[string]any {
+	entry := policyTraceEntry("failure_decision", "candidate_failure", decision.Route, decision.Reason, decision.Match, decision.Info)
+	entry["route"] = decision.Route
+	entry["effect"] = decision.Effect
+	entry["applied"] = applied
+	entry["guardReason"] = guardReason
+	entry["clientAttempt"] = clientAttempt
+	entry["maxAttempts"] = maxAttempts
+	entry["rule"] = decision.Match.Rule
+	entry["value"] = decision.Match.Value
+	if decision.Target != "" {
+		entry["target"] = decision.Target
+	}
+	if decision.CooldownSeconds > 0 {
+		entry["cooldownSeconds"] = decision.CooldownSeconds
+	}
+	if decision.DemoteSteps > 0 {
+		entry["demoteSteps"] = decision.DemoteSteps
+	}
+	addCandidatePriorityTraceFields(entry, candidate)
+	return entry
+}
+
 func priorityDemoteTraceEntry(decision priorityDemoteDecision, platformID string, platformModelID string, dynamicPriority int) map[string]any {
 	entry := policyTraceEntry("priority_demoted", "priority_demote", "demote", decision.Reason, decision.Match, decision.Info)
 	entry["demote"] = decision.Demote