refactor(runner): 收敛上游失败决策与轮转策略

将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
2026-07-27 23:50:17 +08:00
parent 046c16fc69
commit 31565af07a
19 changed files with 1988 additions and 307 deletions
@@ -334,3 +334,151 @@ func TestPriorityDemotePolicyIsSkippedWhenFailoverActionRulesExist(t *testing.T)
 		t.Fatalf("legacy priority demotion should be skipped when actionRules are active, got %+v", decision)
 	}
 }
+
+func TestResolveCandidateFailureCooldownSkipsSameClientRetry(t *testing.T) {
+	decision := resolveCandidateFailure(resolveCandidateFailureInput{
+		RunnerPolicy: store.RunnerPolicy{
+			Status: "active",
+			FailoverPolicy: map[string]any{
+				"enabled": true,
+				"actionRules": []any{
+					map[string]any{
+						"statusCodes":     []any{429},
+						"action":          "cooldown_and_next",
+						"target":          "model",
+						"cooldownSeconds": 60,
+					},
+				},
+			},
+		},
+		Candidate: store.RuntimeModelCandidate{
+			ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3},
+		},
+		Err:               &clients.ClientError{Code: "resource_exhausted", StatusCode: 429, Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 3,
+		HasNextCandidate:  true,
+	})
+
+	if decision.Route != "next" || decision.Effect != "cooldown" || decision.Target != "model" || decision.CooldownSeconds != 60 {
+		t.Fatalf("429 cooldown rule should invalidate the candidate immediately, got %+v", decision)
+	}
+}
+
+func TestResolveCandidateFailureDemoteRetriesBeforeNext(t *testing.T) {
+	input := resolveCandidateFailureInput{
+		RunnerPolicy: store.RunnerPolicy{
+			Status: "active",
+			FailoverPolicy: map[string]any{
+				"enabled": true,
+				"actionRules": []any{
+					map[string]any{
+						"statusCodes": []any{500},
+						"action":      "demote_and_next",
+						"target":      "platform",
+						"demoteSteps": 2,
+					},
+				},
+			},
+		},
+		Candidate: store.RuntimeModelCandidate{
+			ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2},
+		},
+		Err:               &clients.ClientError{Code: "server_error", StatusCode: 500, Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 2,
+		HasNextCandidate:  true,
+	}
+
+	first := resolveCandidateFailure(input)
+	if first.Route != "retry_same" || first.Effect != "none" {
+		t.Fatalf("demotion must wait until same-client retries are exhausted, got %+v", first)
+	}
+
+	input.ClientAttempt = 2
+	second := resolveCandidateFailure(input)
+	if second.Route != "next" || second.Effect != "demote" || second.DemoteSteps != 2 {
+		t.Fatalf("exhausted retry should demote once and rotate, got %+v", second)
+	}
+}
+
+func TestResolveCandidateFailureStopsAfterDownstreamStarts(t *testing.T) {
+	decision := resolveCandidateFailure(resolveCandidateFailureInput{
+		RunnerPolicy: store.RunnerPolicy{
+			Status: "active",
+			FailoverPolicy: map[string]any{
+				"enabled": true,
+				"actionRules": []any{
+					map[string]any{
+						"categories": []any{"stream_error"},
+						"action":     "next",
+					},
+				},
+			},
+		},
+		Candidate:         store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3}},
+		Err:               &clients.ClientError{Code: "stream_read_error", Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 3,
+		HasNextCandidate:  true,
+		DownstreamStarted: true,
+	})
+
+	if decision.Route != "stop" || decision.Reason != "downstream_response_started" {
+		t.Fatalf("stream output must prevent retries and failover, got %+v", decision)
+	}
+}
+
+func TestResolveCandidateFailureLegacyPolicyPrecedence(t *testing.T) {
+	candidate := store.RuntimeModelCandidate{
+		DegradePolicy: map[string]any{
+			"enabled":         true,
+			"keywords":        []any{"resource_exhausted"},
+			"cooldownSeconds": 45,
+		},
+	}
+	decision := resolveCandidateFailure(resolveCandidateFailureInput{
+		RunnerPolicy: store.RunnerPolicy{
+			Status: "active",
+			FailoverPolicy: map[string]any{
+				"enabled": true,
+				"actionRules": []any{
+					map[string]any{
+						"statusCodes":     []any{429},
+						"action":          "cooldown_and_next",
+						"target":          "platform",
+						"cooldownSeconds": 90,
+					},
+				},
+			},
+		},
+		Candidate:         candidate,
+		Err:               &clients.ClientError{Code: "resource_exhausted", Message: "resource_exhausted", StatusCode: 429, Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 3,
+		HasNextCandidate:  true,
+	})
+
+	if decision.Match.Policy != "degradePolicy" || decision.Target != "model" || decision.CooldownSeconds != 45 {
+		t.Fatalf("legacy compatibility rule should precede global action rules, got %+v", decision)
+	}
+}
+
+func TestResolveCandidateFailureUnmatchedRetryableUsesSameThenNext(t *testing.T) {
+	input := resolveCandidateFailureInput{
+		RunnerPolicy:      store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true, "actionRules": []any{}}},
+		Candidate:         store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2}},
+		Err:               &clients.ClientError{Code: "unknown_transient", Retryable: true},
+		ClientAttempt:     1,
+		MaxClientAttempts: 2,
+		HasNextCandidate:  true,
+	}
+
+	if decision := resolveCandidateFailure(input); decision.Route != "retry_same" {
+		t.Fatalf("unmatched retryable error should retry the same client first, got %+v", decision)
+	}
+	input.ClientAttempt = 2
+	if decision := resolveCandidateFailure(input); decision.Route != "next" || decision.Effect != "none" {
+		t.Fatalf("unmatched exhausted error should rotate without side effects, got %+v", decision)
+	}
+}