package runner import ( "testing" "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) func TestShouldRetrySameClientUsesRuntimeRetryPolicyKeywords(t *testing.T) { candidate := store.RuntimeModelCandidate{ ModelRetryPolicy: map[string]any{ "enabled": true, "allowKeywords": []any{"temporary vendor blip"}, "denyKeywords": []any{"bad request"}, }, } if shouldRetrySameClient(candidate, &clients.ClientError{Code: "bad_request", Message: "bad request timeout", Retryable: true}) { t.Fatal("deny keywords should block same-client retry even when the client marks the error retryable") } if !shouldRetrySameClient(candidate, &clients.ClientError{Code: "custom_error", Message: "temporary vendor blip", Retryable: false}) { t.Fatal("allow keywords should allow same-client retry when the client does not mark the error retryable") } } func TestFailoverBudgetDefaults(t *testing.T) { candidates := make([]store.RuntimeModelCandidate, 150) runnerPolicy := store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true}} if got := maxPlatformsForCandidates(candidates, runnerPolicy); got != 99 { t.Fatalf("default max platform budget should be 99, got %d", got) } if got := maxFailoverDurationForCandidates(candidates, runnerPolicy); got != 10*time.Minute { t.Fatalf("default max failover duration should be 10 minutes, got %s", got) } } func TestSingleSourceProtectionDefaultsOn(t *testing.T) { runnerPolicy := store.RunnerPolicy{Status: "active"} if !singleSourceProtectionActive([]store.RuntimeModelCandidate{{PlatformID: "only-source"}}, 99, runnerPolicy) { t.Fatal("single source protection should default to enabled") } } func TestSingleSourceProtectionCanBeDisabled(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", SingleSourcePolicy: map[string]any{"enabled": false}, } if singleSourceProtectionActive([]store.RuntimeModelCandidate{{PlatformID: "only-source"}}, 99, runnerPolicy) { t.Fatal("single source protection should respect the global toggle") } } func TestSingleSourceProtectionUsesEffectivePlatformBudget(t *testing.T) { candidates := []store.RuntimeModelCandidate{ {PlatformID: "first-source"}, {PlatformID: "second-source"}, } runnerPolicy := store.RunnerPolicy{Status: "active"} if !singleSourceProtectionActive(candidates, 1, runnerPolicy) { t.Fatal("maxPlatforms=1 should make this execution a single-source run") } if singleSourceProtectionActive(candidates, 2, runnerPolicy) { t.Fatal("multiple effective sources should allow disable and cooldown actions") } } func TestFailoverTimeBudgetExceeded(t *testing.T) { if !failoverTimeBudgetExceeded(time.Now().Add(-601*time.Second), 10*time.Minute) { t.Fatal("failover time budget should stop retries after the configured duration") } if failoverTimeBudgetExceeded(time.Now().Add(-590*time.Second), 10*time.Minute) { t.Fatal("failover time budget should allow retries before the configured duration") } } func TestLoadAvoidanceFallbackContinuesToAvoidedCandidate(t *testing.T) { candidates := []store.RuntimeModelCandidate{ {PlatformID: "available-candidate"}, {PlatformID: "avoided-full-candidate", LoadAvoided: true}, } if !hasLoadAvoidanceFallback(candidates, 0, 99) { t.Fatal("expected non-avoided candidate to fall back to later avoided candidate") } if hasLoadAvoidanceFallback(candidates, 1, 99) { t.Fatal("avoided candidate should not force another load-avoidance fallback") } decision := loadAvoidanceFallbackDecision(&clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: false}) if !decision.Retry || decision.Reason != "load_avoidance_fallback" || decision.Action != "next" { t.Fatalf("expected active load avoidance fallback to force next candidate, got %+v", decision) } } func TestFailoverHardStopBeatsModelOverride(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "allowCategories": []any{"request_error"}, }, HardStopPolicy: map[string]any{ "enabled": true, "categories": []any{"request_error"}, }, } candidate := store.RuntimeModelCandidate{ RuntimePolicyOverride: map[string]any{ "failoverPolicy": map[string]any{ "enabled": true, "allowCategories": []any{"request_error"}, }, }, } decision := failoverDecisionForCandidate(runnerPolicy, candidate, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true}) if decision.Retry || decision.Reason != "hard_stop_policy" { t.Fatalf("hard stop should block model-level failover override, got %+v", decision) } } func TestFailoverPolicyAllowsModelOverride(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{"enabled": true}, HardStopPolicy: map[string]any{"enabled": true, "categories": []any{"request_error"}}, } candidate := store.RuntimeModelCandidate{ RuntimePolicyOverride: map[string]any{ "failoverPolicy": map[string]any{ "allowKeywords": []any{"temporary upstream outage"}, }, }, } decision := failoverDecisionForCandidate(runnerPolicy, candidate, &clients.ClientError{Code: "custom_error", Message: "temporary upstream outage", Retryable: false}) if !decision.Retry || decision.Reason != "failover_allow_policy" { t.Fatalf("model failoverPolicy override should allow cross-platform failover, got %+v", decision) } } func TestLocalRateLimitWaitsInQueueWithoutRetryOrFailover(t *testing.T) { err := &localRateLimitError{ clientErr: &clients.ClientError{Code: "rate_limit", Message: "local capacity exceeded", Retryable: true}, cause: &store.RateLimitExceededError{Metric: "concurrent", Retryable: true}, } retryDecision := retryDecisionForCandidate(store.RuntimeModelCandidate{}, err) if retryDecision.Retry || retryDecision.Reason != "local_rate_limit_wait_queue" { t.Fatalf("local rate limit should not be same-client retry, got %+v", retryDecision) } failoverDecision := failoverDecisionForCandidate(store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{"enabled": true, "allowCategories": []any{"rate_limit"}}, }, store.RuntimeModelCandidate{}, err) if failoverDecision.Retry || failoverDecision.Action != "queue" || failoverDecision.Reason != "local_rate_limit_wait_queue" { t.Fatalf("local rate limit should wait in queue without failover, got %+v", failoverDecision) } } func TestProviderAuthErrorsFailOverInsteadOfHardStop(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "allowCategories": []any{"auth_error"}, "allowCodes": []any{"auth_failed", "invalid_api_key", "missing_credentials"}, "allowStatusCodes": []any{ 401, 403, }, "actions": map[string]any{"auth_error": "disable_and_next"}, }, HardStopPolicy: map[string]any{ "enabled": true, "categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"}, "codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "permission_denied"}, "statusCodes": []any{}, "keywords": []any{"invalid_parameter", "missing required", "bad request", "insufficient balance"}, }, } decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "auth_failed", StatusCode: 401, Retryable: false}) if !decision.Retry || decision.Action != "disable_and_next" || decision.Reason != "failover_allow_policy" { t.Fatalf("provider auth failures should switch platform, got %+v", decision) } decision = failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "http_400", Message: "invalid api key", StatusCode: 400, Retryable: false}) if !decision.Retry || decision.Info.Category != "auth_error" { t.Fatalf("provider auth-looking 400 should switch platform, got %+v", decision) } } func TestFailoverActionRulesControlPerRuleCooldownAndDemotion(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "actionRules": []any{ map[string]any{ "enabled": true, "categories": []any{"rate_limit"}, "action": "cooldown_and_next", "target": "platform", "cooldownSeconds": 45, }, map[string]any{ "enabled": true, "keywords": []any{"temporary overload"}, "action": "demote_and_next", "demoteSteps": 3, }, }, }, } decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "rate_limit", StatusCode: 429, Retryable: false}) if !decision.Retry || decision.Action != "cooldown_and_next" || decision.Target != "platform" || decision.CooldownSeconds != 45 { t.Fatalf("rate limit should use per-rule cooldown action, got %+v", decision) } decision = failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "server_error", Message: "temporary overload from upstream", StatusCode: 503, Retryable: false}) if !decision.Retry || decision.Action != "demote_and_next" || decision.DemoteSteps != 3 { t.Fatalf("keyword match should use per-rule demote action, got %+v", decision) } } func TestFailoverActionRulesCanStopAndMatchStringStatusCodes(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "actionRules": []any{ map[string]any{ "enabled": true, "statusCodes": []any{"400"}, "keywords": []any{"bad request"}, "action": "stop", }, map[string]any{ "enabled": true, "categories": []any{"request_error"}, "action": "next", }, }, }, } decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "bad_request", Message: "bad request", StatusCode: 400, Retryable: true}) if decision.Retry || decision.Action != "stop" || decision.Reason != "failover_action_rule" || decision.Match.Rule != "statusCodes" { t.Fatalf("first stop rule should win before later retry rules, got %+v", decision) } } func TestFailoverActionRulesKeepOrderBeforeDerivedHardStopPolicy(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "actionRules": []any{ map[string]any{ "enabled": true, "keywords": []any{"bad request retryable"}, "action": "next", }, map[string]any{ "enabled": true, "statusCodes": []any{400}, "action": "stop", }, }, }, HardStopPolicy: map[string]any{ "enabled": true, "statusCodes": []any{400}, }, } decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "bad_request", Message: "bad request retryable", StatusCode: 400, Retryable: true}) if !decision.Retry || decision.Action != "next" || decision.Match.Rule != "keywords" { t.Fatalf("actionRules should keep list order before derived hard stop compatibility fields, got %+v", decision) } } func TestPriorityDemotePolicyIsKeywordGatedAndHardStopSafe(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", HardStopPolicy: map[string]any{ "enabled": true, "categories": []any{"request_error"}, }, PriorityDemotePolicy: map[string]any{ "enabled": true, "keywords": []any{"rate_limit"}, }, } if !shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true}) { t.Fatal("priority demotion should be enabled only by matched policy") } if shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true}) { t.Fatal("priority demotion should not run for hard-stop request errors") } } func TestPriorityDemotePolicyIsSkippedWhenFailoverActionRulesExist(t *testing.T) { runnerPolicy := store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "actionRules": []any{ map[string]any{ "enabled": true, "categories": []any{"rate_limit"}, "action": "demote_and_next", }, }, }, PriorityDemotePolicy: map[string]any{ "enabled": true, "keywords": []any{"rate_limit"}, }, } decision := priorityDemoteDecisionForCandidate(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true}) if decision.Demote || decision.Reason != "failover_action_rules_enabled" { t.Fatalf("legacy priority demotion should be skipped when actionRules are active, got %+v", decision) } } func TestResolveCandidateFailureCooldownSkipsSameClientRetry(t *testing.T) { decision := resolveCandidateFailure(resolveCandidateFailureInput{ RunnerPolicy: store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "actionRules": []any{ map[string]any{ "statusCodes": []any{429}, "action": "cooldown_and_next", "target": "model", "cooldownSeconds": 60, }, }, }, }, Candidate: store.RuntimeModelCandidate{ ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3}, }, Err: &clients.ClientError{Code: "resource_exhausted", StatusCode: 429, Retryable: true}, ClientAttempt: 1, MaxClientAttempts: 3, HasNextCandidate: true, }) if decision.Route != "next" || decision.Effect != "cooldown" || decision.Target != "model" || decision.CooldownSeconds != 60 { t.Fatalf("429 cooldown rule should invalidate the candidate immediately, got %+v", decision) } } func TestResolveCandidateFailureDemoteRetriesBeforeNext(t *testing.T) { input := resolveCandidateFailureInput{ RunnerPolicy: store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "actionRules": []any{ map[string]any{ "statusCodes": []any{500}, "action": "demote_and_next", "target": "platform", "demoteSteps": 2, }, }, }, }, Candidate: store.RuntimeModelCandidate{ ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2}, }, Err: &clients.ClientError{Code: "server_error", StatusCode: 500, Retryable: true}, ClientAttempt: 1, MaxClientAttempts: 2, HasNextCandidate: true, } first := resolveCandidateFailure(input) if first.Route != "retry_same" || first.Effect != "none" { t.Fatalf("demotion must wait until same-client retries are exhausted, got %+v", first) } input.ClientAttempt = 2 second := resolveCandidateFailure(input) if second.Route != "next" || second.Effect != "demote" || second.DemoteSteps != 2 { t.Fatalf("exhausted retry should demote once and rotate, got %+v", second) } } func TestResolveCandidateFailureStopsAfterDownstreamStarts(t *testing.T) { decision := resolveCandidateFailure(resolveCandidateFailureInput{ RunnerPolicy: store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "actionRules": []any{ map[string]any{ "categories": []any{"stream_error"}, "action": "next", }, }, }, }, Candidate: store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3}}, Err: &clients.ClientError{Code: "stream_read_error", Retryable: true}, ClientAttempt: 1, MaxClientAttempts: 3, HasNextCandidate: true, DownstreamStarted: true, }) if decision.Route != "stop" || decision.Reason != "downstream_response_started" { t.Fatalf("stream output must prevent retries and failover, got %+v", decision) } } func TestResolveCandidateFailureLegacyPolicyPrecedence(t *testing.T) { candidate := store.RuntimeModelCandidate{ DegradePolicy: map[string]any{ "enabled": true, "keywords": []any{"resource_exhausted"}, "cooldownSeconds": 45, }, } decision := resolveCandidateFailure(resolveCandidateFailureInput{ RunnerPolicy: store.RunnerPolicy{ Status: "active", FailoverPolicy: map[string]any{ "enabled": true, "actionRules": []any{ map[string]any{ "statusCodes": []any{429}, "action": "cooldown_and_next", "target": "platform", "cooldownSeconds": 90, }, }, }, }, Candidate: candidate, Err: &clients.ClientError{Code: "resource_exhausted", Message: "resource_exhausted", StatusCode: 429, Retryable: true}, ClientAttempt: 1, MaxClientAttempts: 3, HasNextCandidate: true, }) if decision.Match.Policy != "degradePolicy" || decision.Target != "model" || decision.CooldownSeconds != 45 { t.Fatalf("legacy compatibility rule should precede global action rules, got %+v", decision) } } func TestResolveCandidateFailureUnmatchedRetryableUsesSameThenNext(t *testing.T) { input := resolveCandidateFailureInput{ RunnerPolicy: store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true, "actionRules": []any{}}}, Candidate: store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2}}, Err: &clients.ClientError{Code: "unknown_transient", Retryable: true}, ClientAttempt: 1, MaxClientAttempts: 2, HasNextCandidate: true, } if decision := resolveCandidateFailure(input); decision.Route != "retry_same" { t.Fatalf("unmatched retryable error should retry the same client first, got %+v", decision) } input.ClientAttempt = 2 if decision := resolveCandidateFailure(input); decision.Route != "next" || decision.Effect != "none" { t.Fatalf("unmatched exhausted error should rotate without side effects, got %+v", decision) } }