feat: add runner failover rules and cache affinity

2026-06-28 20:50:23 +08:00
parent 41bb3a6525
commit 0e675e259c
31 changed files with 2939 additions and 470 deletions
@@ -106,6 +106,156 @@ func TestRuntimeCandidateSortingAvoidsFullCandidatesButKeepsFallback(t *testing.
 	}
 }

+func TestRuntimeCandidateSortingPrefersCacheAffinityWithinAvailableCandidates(t *testing.T) {
+	candidates := []RuntimeModelCandidate{
+		{
+			PlatformID:       "base-priority",
+			PlatformPriority: 10,
+		},
+		{
+			PlatformID:       "cache-affinity",
+			PlatformPriority: 20,
+		},
+	}
+	applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
+		CacheAffinityKey:    "explicit:test",
+		CacheAffinityPolicy: map[string]any{"enabled": true, "minSamples": 3, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}},
+	}, runtimeCandidateCacheAffinityInput{
+		RequestCount: 1,
+		EMAHitRatio:  1,
+	})
+	applyRuntimeCandidateCacheAffinity(&candidates[1], ListModelCandidatesOptions{
+		CacheAffinityKey:    "explicit:test",
+		CacheAffinityPolicy: map[string]any{"enabled": true, "minSamples": 3, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}},
+	}, runtimeCandidateCacheAffinityInput{
+		RequestCount: 3,
+		EMAHitRatio:  0.9,
+	})
+
+	sortRuntimeModelCandidates(candidates)
+
+	if candidates[0].PlatformID != "cache-affinity" || !candidates[0].CacheAffinity.Applied {
+		t.Fatalf("expected cache affinity candidate first, got %+v", candidates)
+	}
+	if candidates[1].PlatformID != "base-priority" || candidates[1].CacheAffinity.Applied {
+		t.Fatalf("expected low sample candidate to keep base priority, got %+v", candidates)
+	}
+}
+
+func TestRuntimeCandidateSortingPromotesObservedCacheHitAbovePriority(t *testing.T) {
+	candidates := []RuntimeModelCandidate{
+		{
+			PlatformID:       "volces-priority",
+			PlatformPriority: 1,
+		},
+		{
+			PlatformID:       "deepseek-cache-hit",
+			PlatformPriority: 100,
+		},
+	}
+	policy := map[string]any{"enabled": true, "minSamples": 3, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}}
+	applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
+		CacheAffinityKey:    "conversation:test",
+		CacheAffinityPolicy: policy,
+	}, runtimeCandidateCacheAffinityInput{})
+	applyRuntimeCandidateCacheAffinity(&candidates[1], ListModelCandidatesOptions{
+		CacheAffinityKey:    "conversation:test",
+		CacheAffinityPolicy: policy,
+	}, runtimeCandidateCacheAffinityInput{
+		RequestCount:      1,
+		InputTokens:       7945,
+		CachedInputTokens: 7296,
+		EMAHitRatio:       0.918,
+		LastHitRatio:      0.918,
+	})
+
+	sortRuntimeModelCandidates(candidates)
+
+	if candidates[0].PlatformID != "deepseek-cache-hit" || !candidates[0].CacheAffinity.Applied {
+		t.Fatalf("expected observed cache hit to outrank platform priority, got %+v", candidates)
+	}
+	if candidates[0].CacheAffinity.Score <= 0 || candidates[0].CacheAffinity.Boost <= 0 {
+		t.Fatalf("expected observed cache hit to carry score and boost, got %+v", candidates[0].CacheAffinity)
+	}
+	if candidates[1].PlatformID != "volces-priority" || candidates[1].CacheAffinity.Applied {
+		t.Fatalf("expected priority-only candidate second, got %+v", candidates)
+	}
+}
+
+func TestRuntimeCandidateSortingOrdersMultipleCacheAffinityCandidatesByScore(t *testing.T) {
+	candidates := []RuntimeModelCandidate{
+		{
+			PlatformID:       "lower-cache-score",
+			PlatformPriority: 1,
+		},
+		{
+			PlatformID:       "higher-cache-score",
+			PlatformPriority: 50,
+		},
+	}
+	policy := map[string]any{"enabled": true, "minSamples": 1, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}}
+	applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
+		CacheAffinityKey:    "conversation:test",
+		CacheAffinityPolicy: policy,
+	}, runtimeCandidateCacheAffinityInput{
+		RequestCount:      2,
+		InputTokens:       8000,
+		CachedInputTokens: 1000,
+		EMAHitRatio:       0.5,
+		LastHitRatio:      0.5,
+	})
+	applyRuntimeCandidateCacheAffinity(&candidates[1], ListModelCandidatesOptions{
+		CacheAffinityKey:    "conversation:test",
+		CacheAffinityPolicy: policy,
+	}, runtimeCandidateCacheAffinityInput{
+		RequestCount:      1,
+		InputTokens:       8000,
+		CachedInputTokens: 7000,
+		EMAHitRatio:       0.75,
+		LastHitRatio:      0.75,
+	})
+
+	sortRuntimeModelCandidates(candidates)
+
+	if candidates[0].PlatformID != "higher-cache-score" {
+		t.Fatalf("expected higher cache score candidate first, got %+v", candidates)
+	}
+	if candidates[0].CacheAffinity.Score <= candidates[1].CacheAffinity.Score {
+		t.Fatalf("expected score to drive cache-affinity order, got %+v", candidates)
+	}
+}
+
+func TestRuntimeCandidateSortingKeepsFullCacheAffinityCandidateAvoided(t *testing.T) {
+	candidates := []RuntimeModelCandidate{
+		{
+			PlatformID:       "cache-affinity-full",
+			PlatformPriority: 50,
+			LoadLimited:      true,
+			LoadRatio:        1.2,
+		},
+		{
+			PlatformID:       "available",
+			PlatformPriority: 10,
+		},
+	}
+	applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
+		CacheAffinityKey:    "explicit:test",
+		CacheAffinityPolicy: map[string]any{"enabled": true, "minSamples": 1, "maxPriorityBoost": 100, "modelTypes": []any{"text_generate"}},
+	}, runtimeCandidateCacheAffinityInput{
+		RequestCount: 10,
+		EMAHitRatio:  1,
+	})
+
+	sortRuntimeModelCandidates(candidates)
+
+	if candidates[0].PlatformID != "available" {
+		t.Fatalf("expected available candidate before full cache-affinity candidate, got %+v", candidates)
+	}
+	if candidates[1].PlatformID != "cache-affinity-full" || !candidates[1].LoadAvoided {
+		t.Fatalf("expected full cache-affinity candidate to remain avoided fallback, got %+v", candidates)
+	}
+}
+
 func TestDefaultRunnerPriorityDemotePolicyUsesAutoMode(t *testing.T) {
 	policy := defaultRunnerPriorityDemotePolicy()

@@ -115,4 +265,190 @@ func TestDefaultRunnerPriorityDemotePolicyUsesAutoMode(t *testing.T) {
 	if policy["enabled"] != true {
 		t.Fatalf("expected default priority demotion to stay enabled, got %+v", policy["enabled"])
 	}
+	if policy["deprecated"] != true || policy["replacedBy"] != "failoverPolicy.actionRules" {
+		t.Fatalf("expected legacy priority demotion policy to be marked deprecated, got %+v", policy)
+	}
+}
+
+func TestDefaultRunnerFailoverPolicyCarriesComprehensiveDefaultsInActionRules(t *testing.T) {
+	policy := defaultRunnerFailoverPolicy()
+	if policy["legacyFieldsDeprecated"] != true || policy["replacedBy"] != "actionRules" {
+		t.Fatalf("expected legacy failover summary fields to be marked deprecated, got %+v", policy)
+	}
+	rules, ok := policy["actionRules"].([]any)
+	if !ok || len(rules) < 5 {
+		t.Fatalf("expected default action rules to be present, got %+v", policy["actionRules"])
+	}
+
+	cooldownRule := defaultActionRuleByAction(t, rules, "cooldown_and_next")
+	for _, code := range []string{"rate_limit", "too_many_requests", "resource_exhausted", "throttled", "quota_exceeded", "rpm_limit", "tpm_limit"} {
+		if !actionRuleHasString(cooldownRule, "codes", code) {
+			t.Fatalf("expected cooldown rule to include rate-limit code %q, got %+v", code, cooldownRule["codes"])
+		}
+	}
+	for _, keyword := range []string{"rate limit", "too many requests", "resource exhausted", "request limit", "requests per minute", "tokens per minute"} {
+		if !actionRuleHasString(cooldownRule, "keywords", keyword) {
+			t.Fatalf("expected cooldown rule to include rate-limit keyword %q, got %+v", keyword, cooldownRule["keywords"])
+		}
+	}
+	if !actionRuleHasInt(cooldownRule, "statusCodes", 429) {
+		t.Fatalf("expected cooldown rule to include 429, got %+v", cooldownRule["statusCodes"])
+	}
+
+	disableRule := defaultActionRuleByAction(t, rules, "disable_and_next")
+	for _, code := range []string{"auth_failed", "invalid_api_key", "missing_credentials", "account_disabled", "billing_not_active", "permission_denied"} {
+		if !actionRuleHasString(disableRule, "codes", code) {
+			t.Fatalf("expected disable rule to include auth/platform code %q, got %+v", code, disableRule["codes"])
+		}
+	}
+	for _, keyword := range []string{"invalid api key", "unauthorized", "credential", "permission denied", "account disabled", "billing_not_active", "余额不足"} {
+		if !actionRuleHasString(disableRule, "keywords", keyword) {
+			t.Fatalf("expected disable rule to include auth/platform keyword %q, got %+v", keyword, disableRule["keywords"])
+		}
+	}
+	for _, status := range []int{401, 403} {
+		if !actionRuleHasInt(disableRule, "statusCodes", status) {
+			t.Fatalf("expected disable rule to include status %d, got %+v", status, disableRule["statusCodes"])
+		}
+	}
+
+	demoteRule := defaultActionRuleByAction(t, rules, "demote_and_next")
+	for _, category := range []string{"timeout", "provider_5xx", "provider_overloaded"} {
+		if !actionRuleHasString(demoteRule, "categories", category) {
+			t.Fatalf("expected demote rule to include transient category %q, got %+v", category, demoteRule["categories"])
+		}
+	}
+	for _, code := range []string{"timeout", "server_error", "overloaded", "provider_failed", "invalid_response", "script_timeout", "upload_failed"} {
+		if !actionRuleHasString(demoteRule, "codes", code) {
+			t.Fatalf("expected demote rule to include transient code %q, got %+v", code, demoteRule["codes"])
+		}
+	}
+	for _, status := range []int{408, 500, 502, 503, 504, 520, 522, 524, 529, 530} {
+		if !actionRuleHasInt(demoteRule, "statusCodes", status) {
+			t.Fatalf("expected demote rule to include transient status %d, got %+v", status, demoteRule["statusCodes"])
+		}
+	}
+	for _, keyword := range []string{"deadline exceeded", "temporarily unavailable", "service unavailable", "bad gateway", "upstream timeout", "try again later"} {
+		if !actionRuleHasString(demoteRule, "keywords", keyword) {
+			t.Fatalf("expected demote rule to include transient keyword %q, got %+v", keyword, demoteRule["keywords"])
+		}
+	}
+
+	nextRule := defaultActionRuleByAction(t, rules, "next")
+	for _, category := range []string{"network", "stream_error"} {
+		if !actionRuleHasString(nextRule, "categories", category) {
+			t.Fatalf("expected next rule to include network category %q, got %+v", category, nextRule["categories"])
+		}
+	}
+	for _, code := range []string{"network", "stream_read_error", "upload_network", "upload_config_failed", "upload_source_fetch_failed"} {
+		if !actionRuleHasString(nextRule, "codes", code) {
+			t.Fatalf("expected next rule to include network code %q, got %+v", code, nextRule["codes"])
+		}
+	}
+	for _, keyword := range []string{"socket hang up", "connection reset", "econnreset", "enotfound", "unexpected eof", "stream error", "fetch failed"} {
+		if !actionRuleHasString(nextRule, "keywords", keyword) {
+			t.Fatalf("expected next rule to include network keyword %q, got %+v", keyword, nextRule["keywords"])
+		}
+	}
+
+	stopRule := defaultActionRuleByAction(t, rules, "stop")
+	for _, code := range []string{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "request_asset_input_format_unsupported", "upload_source_too_large", "invalid_multipart_image", "script_error", "insufficient_balance", "permission_denied"} {
+		if !actionRuleHasString(stopRule, "codes", code) {
+			t.Fatalf("expected stop rule to include legacy hard-stop code %q, got %+v", code, stopRule["codes"])
+		}
+	}
+	for _, status := range []int{400, 404, 405, 409, 413, 415, 422} {
+		if !actionRuleHasInt(stopRule, "statusCodes", status) {
+			t.Fatalf("expected stop rule to include request-error status %d, got %+v", status, stopRule["statusCodes"])
+		}
+	}
+	for _, keyword := range []string{"invalid parameter", "missing required", "not supported", "image is required", "too large", "not found"} {
+		if !actionRuleHasString(stopRule, "keywords", keyword) {
+			t.Fatalf("expected stop rule to include hard-stop keyword %q, got %+v", keyword, stopRule["keywords"])
+		}
+	}
+}
+
+func TestLegacyDefaultRunnerActionRulesAreDetectedForUpgrade(t *testing.T) {
+	legacy := []any{
+		map[string]any{
+			"enabled":    true,
+			"categories": []any{"network", "timeout", "stream_error", "provider_5xx", "provider_overloaded"},
+			"action":     "next",
+		},
+		map[string]any{
+			"enabled":         true,
+			"categories":      []any{"rate_limit"},
+			"codes":           []any{"rate_limit"},
+			"statusCodes":     []any{429},
+			"keywords":        []any{"rate_limit", "429", "too many requests"},
+			"action":          "cooldown_and_next",
+			"target":          "model",
+			"cooldownSeconds": 300,
+		},
+		map[string]any{
+			"enabled":     true,
+			"keywords":    []any{"rate limit", "too many requests", "temporarily_unavailable"},
+			"action":      "demote_and_next",
+			"target":      "platform",
+			"demoteSteps": 1,
+		},
+		map[string]any{
+			"enabled":     true,
+			"categories":  []any{"auth_error"},
+			"codes":       []any{"auth_failed", "invalid_api_key", "missing_credentials"},
+			"statusCodes": []any{401, 403},
+			"keywords":    []any{"invalid api key", "api key is invalid", "unauthorized", "forbidden", "permission denied"},
+			"action":      "disable_and_next",
+			"target":      "platform",
+		},
+		map[string]any{
+			"enabled":    true,
+			"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
+			"codes":      []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "insufficient_balance", "permission_denied"},
+			"keywords":   []any{"invalid_parameter", "missing required", "bad request", "insufficient balance"},
+			"action":     "stop",
+		},
+	}
+	if !runnerActionRulesLookLikeLegacyDefaults(legacy) {
+		t.Fatal("expected old compact default action rules to be detected")
+	}
+	if runnerActionRulesLookLikeLegacyDefaults(defaultRunnerFailoverPolicy()["actionRules"]) {
+		t.Fatal("enhanced defaults should not be treated as legacy")
+	}
+}
+
+func defaultActionRuleByAction(t *testing.T, rules []any, action string) map[string]any {
+	t.Helper()
+	for _, raw := range rules {
+		rule, ok := raw.(map[string]any)
+		if ok && rule["action"] == action {
+			return rule
+		}
+	}
+	t.Fatalf("expected action rule %q in %+v", action, rules)
+	return nil
+}
+
+func actionRuleHasString(rule map[string]any, key string, want string) bool {
+	for _, raw := range ruleSlice(rule, key) {
+		if raw == want {
+			return true
+		}
+	}
+	return false
+}
+
+func actionRuleHasInt(rule map[string]any, key string, want int) bool {
+	for _, raw := range ruleSlice(rule, key) {
+		if raw == want {
+			return true
+		}
+	}
+	return false
+}
+
+func ruleSlice(rule map[string]any, key string) []any {
+	values, _ := rule[key].([]any)
+	return values
 }