feat: add runner failover rules and cache affinity

2026-06-28 20:50:23 +08:00
parent 41bb3a6525
commit 0e675e259c
31 changed files with 2939 additions and 470 deletions
@@ -197,6 +197,97 @@ func TestProviderAuthErrorsFailOverInsteadOfHardStop(t *testing.T) {
 	}
 }

+func TestFailoverActionRulesControlPerRuleCooldownAndDemotion(t *testing.T) {
+	runnerPolicy := store.RunnerPolicy{
+		Status: "active",
+		FailoverPolicy: map[string]any{
+			"enabled": true,
+			"actionRules": []any{
+				map[string]any{
+					"enabled":         true,
+					"categories":      []any{"rate_limit"},
+					"action":          "cooldown_and_next",
+					"target":          "platform",
+					"cooldownSeconds": 45,
+				},
+				map[string]any{
+					"enabled":     true,
+					"keywords":    []any{"temporary overload"},
+					"action":      "demote_and_next",
+					"demoteSteps": 3,
+				},
+			},
+		},
+	}
+
+	decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "rate_limit", StatusCode: 429, Retryable: false})
+	if !decision.Retry || decision.Action != "cooldown_and_next" || decision.Target != "platform" || decision.CooldownSeconds != 45 {
+		t.Fatalf("rate limit should use per-rule cooldown action, got %+v", decision)
+	}
+
+	decision = failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "server_error", Message: "temporary overload from upstream", StatusCode: 503, Retryable: false})
+	if !decision.Retry || decision.Action != "demote_and_next" || decision.DemoteSteps != 3 {
+		t.Fatalf("keyword match should use per-rule demote action, got %+v", decision)
+	}
+}
+
+func TestFailoverActionRulesCanStopAndMatchStringStatusCodes(t *testing.T) {
+	runnerPolicy := store.RunnerPolicy{
+		Status: "active",
+		FailoverPolicy: map[string]any{
+			"enabled": true,
+			"actionRules": []any{
+				map[string]any{
+					"enabled":     true,
+					"statusCodes": []any{"400"},
+					"keywords":    []any{"bad request"},
+					"action":      "stop",
+				},
+				map[string]any{
+					"enabled":    true,
+					"categories": []any{"request_error"},
+					"action":     "next",
+				},
+			},
+		},
+	}
+
+	decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "bad_request", Message: "bad request", StatusCode: 400, Retryable: true})
+	if decision.Retry || decision.Action != "stop" || decision.Reason != "failover_action_rule" || decision.Match.Rule != "statusCodes" {
+		t.Fatalf("first stop rule should win before later retry rules, got %+v", decision)
+	}
+}
+
+func TestFailoverActionRulesKeepOrderBeforeDerivedHardStopPolicy(t *testing.T) {
+	runnerPolicy := store.RunnerPolicy{
+		Status: "active",
+		FailoverPolicy: map[string]any{
+			"enabled": true,
+			"actionRules": []any{
+				map[string]any{
+					"enabled":  true,
+					"keywords": []any{"bad request retryable"},
+					"action":   "next",
+				},
+				map[string]any{
+					"enabled":     true,
+					"statusCodes": []any{400},
+					"action":      "stop",
+				},
+			},
+		},
+		HardStopPolicy: map[string]any{
+			"enabled":     true,
+			"statusCodes": []any{400},
+		},
+	}
+
+	decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "bad_request", Message: "bad request retryable", StatusCode: 400, Retryable: true})
+	if !decision.Retry || decision.Action != "next" || decision.Match.Rule != "keywords" {
+		t.Fatalf("actionRules should keep list order before derived hard stop compatibility fields, got %+v", decision)
+	}
+}
+
 func TestPriorityDemotePolicyIsKeywordGatedAndHardStopSafe(t *testing.T) {
 	runnerPolicy := store.RunnerPolicy{
 		Status: "active",
@@ -218,3 +309,28 @@ func TestPriorityDemotePolicyIsKeywordGatedAndHardStopSafe(t *testing.T) {
 		t.Fatal("priority demotion should not run for hard-stop request errors")
 	}
 }
+
+func TestPriorityDemotePolicyIsSkippedWhenFailoverActionRulesExist(t *testing.T) {
+	runnerPolicy := store.RunnerPolicy{
+		Status: "active",
+		FailoverPolicy: map[string]any{
+			"enabled": true,
+			"actionRules": []any{
+				map[string]any{
+					"enabled":    true,
+					"categories": []any{"rate_limit"},
+					"action":     "demote_and_next",
+				},
+			},
+		},
+		PriorityDemotePolicy: map[string]any{
+			"enabled":  true,
+			"keywords": []any{"rate_limit"},
+		},
+	}
+
+	decision := priorityDemoteDecisionForCandidate(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true})
+	if decision.Demote || decision.Reason != "failover_action_rules_enabled" {
+		t.Fatalf("legacy priority demotion should be skipped when actionRules are active, got %+v", decision)
+	}
+}