wangbo
31565af07a
将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
485 lines
18 KiB
Go
485 lines
18 KiB
Go
package runner
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
func TestShouldRetrySameClientUsesRuntimeRetryPolicyKeywords(t *testing.T) {
|
|
candidate := store.RuntimeModelCandidate{
|
|
ModelRetryPolicy: map[string]any{
|
|
"enabled": true,
|
|
"allowKeywords": []any{"temporary vendor blip"},
|
|
"denyKeywords": []any{"bad request"},
|
|
},
|
|
}
|
|
|
|
if shouldRetrySameClient(candidate, &clients.ClientError{Code: "bad_request", Message: "bad request timeout", Retryable: true}) {
|
|
t.Fatal("deny keywords should block same-client retry even when the client marks the error retryable")
|
|
}
|
|
if !shouldRetrySameClient(candidate, &clients.ClientError{Code: "custom_error", Message: "temporary vendor blip", Retryable: false}) {
|
|
t.Fatal("allow keywords should allow same-client retry when the client does not mark the error retryable")
|
|
}
|
|
}
|
|
|
|
func TestFailoverBudgetDefaults(t *testing.T) {
|
|
candidates := make([]store.RuntimeModelCandidate, 150)
|
|
runnerPolicy := store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true}}
|
|
|
|
if got := maxPlatformsForCandidates(candidates, runnerPolicy); got != 99 {
|
|
t.Fatalf("default max platform budget should be 99, got %d", got)
|
|
}
|
|
if got := maxFailoverDurationForCandidates(candidates, runnerPolicy); got != 10*time.Minute {
|
|
t.Fatalf("default max failover duration should be 10 minutes, got %s", got)
|
|
}
|
|
}
|
|
|
|
func TestSingleSourceProtectionDefaultsOn(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{Status: "active"}
|
|
|
|
if !singleSourceProtectionActive([]store.RuntimeModelCandidate{{PlatformID: "only-source"}}, 99, runnerPolicy) {
|
|
t.Fatal("single source protection should default to enabled")
|
|
}
|
|
}
|
|
|
|
func TestSingleSourceProtectionCanBeDisabled(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
SingleSourcePolicy: map[string]any{"enabled": false},
|
|
}
|
|
|
|
if singleSourceProtectionActive([]store.RuntimeModelCandidate{{PlatformID: "only-source"}}, 99, runnerPolicy) {
|
|
t.Fatal("single source protection should respect the global toggle")
|
|
}
|
|
}
|
|
|
|
func TestSingleSourceProtectionUsesEffectivePlatformBudget(t *testing.T) {
|
|
candidates := []store.RuntimeModelCandidate{
|
|
{PlatformID: "first-source"},
|
|
{PlatformID: "second-source"},
|
|
}
|
|
runnerPolicy := store.RunnerPolicy{Status: "active"}
|
|
|
|
if !singleSourceProtectionActive(candidates, 1, runnerPolicy) {
|
|
t.Fatal("maxPlatforms=1 should make this execution a single-source run")
|
|
}
|
|
if singleSourceProtectionActive(candidates, 2, runnerPolicy) {
|
|
t.Fatal("multiple effective sources should allow disable and cooldown actions")
|
|
}
|
|
}
|
|
|
|
func TestFailoverTimeBudgetExceeded(t *testing.T) {
|
|
if !failoverTimeBudgetExceeded(time.Now().Add(-601*time.Second), 10*time.Minute) {
|
|
t.Fatal("failover time budget should stop retries after the configured duration")
|
|
}
|
|
if failoverTimeBudgetExceeded(time.Now().Add(-590*time.Second), 10*time.Minute) {
|
|
t.Fatal("failover time budget should allow retries before the configured duration")
|
|
}
|
|
}
|
|
|
|
func TestLoadAvoidanceFallbackContinuesToAvoidedCandidate(t *testing.T) {
|
|
candidates := []store.RuntimeModelCandidate{
|
|
{PlatformID: "available-candidate"},
|
|
{PlatformID: "avoided-full-candidate", LoadAvoided: true},
|
|
}
|
|
|
|
if !hasLoadAvoidanceFallback(candidates, 0, 99) {
|
|
t.Fatal("expected non-avoided candidate to fall back to later avoided candidate")
|
|
}
|
|
if hasLoadAvoidanceFallback(candidates, 1, 99) {
|
|
t.Fatal("avoided candidate should not force another load-avoidance fallback")
|
|
}
|
|
decision := loadAvoidanceFallbackDecision(&clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: false})
|
|
if !decision.Retry || decision.Reason != "load_avoidance_fallback" || decision.Action != "next" {
|
|
t.Fatalf("expected active load avoidance fallback to force next candidate, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverHardStopBeatsModelOverride(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"allowCategories": []any{"request_error"},
|
|
},
|
|
HardStopPolicy: map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"request_error"},
|
|
},
|
|
}
|
|
candidate := store.RuntimeModelCandidate{
|
|
RuntimePolicyOverride: map[string]any{
|
|
"failoverPolicy": map[string]any{
|
|
"enabled": true,
|
|
"allowCategories": []any{"request_error"},
|
|
},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, candidate, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true})
|
|
if decision.Retry || decision.Reason != "hard_stop_policy" {
|
|
t.Fatalf("hard stop should block model-level failover override, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverPolicyAllowsModelOverride(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{"enabled": true},
|
|
HardStopPolicy: map[string]any{"enabled": true, "categories": []any{"request_error"}},
|
|
}
|
|
candidate := store.RuntimeModelCandidate{
|
|
RuntimePolicyOverride: map[string]any{
|
|
"failoverPolicy": map[string]any{
|
|
"allowKeywords": []any{"temporary upstream outage"},
|
|
},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, candidate, &clients.ClientError{Code: "custom_error", Message: "temporary upstream outage", Retryable: false})
|
|
if !decision.Retry || decision.Reason != "failover_allow_policy" {
|
|
t.Fatalf("model failoverPolicy override should allow cross-platform failover, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestLocalRateLimitWaitsInQueueWithoutRetryOrFailover(t *testing.T) {
|
|
err := &localRateLimitError{
|
|
clientErr: &clients.ClientError{Code: "rate_limit", Message: "local capacity exceeded", Retryable: true},
|
|
cause: &store.RateLimitExceededError{Metric: "concurrent", Retryable: true},
|
|
}
|
|
retryDecision := retryDecisionForCandidate(store.RuntimeModelCandidate{}, err)
|
|
if retryDecision.Retry || retryDecision.Reason != "local_rate_limit_wait_queue" {
|
|
t.Fatalf("local rate limit should not be same-client retry, got %+v", retryDecision)
|
|
}
|
|
|
|
failoverDecision := failoverDecisionForCandidate(store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{"enabled": true, "allowCategories": []any{"rate_limit"}},
|
|
}, store.RuntimeModelCandidate{}, err)
|
|
if failoverDecision.Retry || failoverDecision.Action != "queue" || failoverDecision.Reason != "local_rate_limit_wait_queue" {
|
|
t.Fatalf("local rate limit should wait in queue without failover, got %+v", failoverDecision)
|
|
}
|
|
}
|
|
|
|
func TestProviderAuthErrorsFailOverInsteadOfHardStop(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"allowCategories": []any{"auth_error"},
|
|
"allowCodes": []any{"auth_failed", "invalid_api_key", "missing_credentials"},
|
|
"allowStatusCodes": []any{
|
|
401,
|
|
403,
|
|
},
|
|
"actions": map[string]any{"auth_error": "disable_and_next"},
|
|
},
|
|
HardStopPolicy: map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
|
"codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "permission_denied"},
|
|
"statusCodes": []any{},
|
|
"keywords": []any{"invalid_parameter", "missing required", "bad request", "insufficient balance"},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "auth_failed", StatusCode: 401, Retryable: false})
|
|
if !decision.Retry || decision.Action != "disable_and_next" || decision.Reason != "failover_allow_policy" {
|
|
t.Fatalf("provider auth failures should switch platform, got %+v", decision)
|
|
}
|
|
|
|
decision = failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "http_400", Message: "invalid api key", StatusCode: 400, Retryable: false})
|
|
if !decision.Retry || decision.Info.Category != "auth_error" {
|
|
t.Fatalf("provider auth-looking 400 should switch platform, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverActionRulesControlPerRuleCooldownAndDemotion(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"rate_limit"},
|
|
"action": "cooldown_and_next",
|
|
"target": "platform",
|
|
"cooldownSeconds": 45,
|
|
},
|
|
map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"temporary overload"},
|
|
"action": "demote_and_next",
|
|
"demoteSteps": 3,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "rate_limit", StatusCode: 429, Retryable: false})
|
|
if !decision.Retry || decision.Action != "cooldown_and_next" || decision.Target != "platform" || decision.CooldownSeconds != 45 {
|
|
t.Fatalf("rate limit should use per-rule cooldown action, got %+v", decision)
|
|
}
|
|
|
|
decision = failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "server_error", Message: "temporary overload from upstream", StatusCode: 503, Retryable: false})
|
|
if !decision.Retry || decision.Action != "demote_and_next" || decision.DemoteSteps != 3 {
|
|
t.Fatalf("keyword match should use per-rule demote action, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverActionRulesCanStopAndMatchStringStatusCodes(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"enabled": true,
|
|
"statusCodes": []any{"400"},
|
|
"keywords": []any{"bad request"},
|
|
"action": "stop",
|
|
},
|
|
map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"request_error"},
|
|
"action": "next",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "bad_request", Message: "bad request", StatusCode: 400, Retryable: true})
|
|
if decision.Retry || decision.Action != "stop" || decision.Reason != "failover_action_rule" || decision.Match.Rule != "statusCodes" {
|
|
t.Fatalf("first stop rule should win before later retry rules, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverActionRulesKeepOrderBeforeDerivedHardStopPolicy(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"bad request retryable"},
|
|
"action": "next",
|
|
},
|
|
map[string]any{
|
|
"enabled": true,
|
|
"statusCodes": []any{400},
|
|
"action": "stop",
|
|
},
|
|
},
|
|
},
|
|
HardStopPolicy: map[string]any{
|
|
"enabled": true,
|
|
"statusCodes": []any{400},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "bad_request", Message: "bad request retryable", StatusCode: 400, Retryable: true})
|
|
if !decision.Retry || decision.Action != "next" || decision.Match.Rule != "keywords" {
|
|
t.Fatalf("actionRules should keep list order before derived hard stop compatibility fields, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestPriorityDemotePolicyIsKeywordGatedAndHardStopSafe(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
HardStopPolicy: map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"request_error"},
|
|
},
|
|
PriorityDemotePolicy: map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"rate_limit"},
|
|
},
|
|
}
|
|
|
|
if !shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true}) {
|
|
t.Fatal("priority demotion should be enabled only by matched policy")
|
|
}
|
|
|
|
if shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true}) {
|
|
t.Fatal("priority demotion should not run for hard-stop request errors")
|
|
}
|
|
}
|
|
|
|
func TestPriorityDemotePolicyIsSkippedWhenFailoverActionRulesExist(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"rate_limit"},
|
|
"action": "demote_and_next",
|
|
},
|
|
},
|
|
},
|
|
PriorityDemotePolicy: map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"rate_limit"},
|
|
},
|
|
}
|
|
|
|
decision := priorityDemoteDecisionForCandidate(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true})
|
|
if decision.Demote || decision.Reason != "failover_action_rules_enabled" {
|
|
t.Fatalf("legacy priority demotion should be skipped when actionRules are active, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureCooldownSkipsSameClientRetry(t *testing.T) {
|
|
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"statusCodes": []any{429},
|
|
"action": "cooldown_and_next",
|
|
"target": "model",
|
|
"cooldownSeconds": 60,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Candidate: store.RuntimeModelCandidate{
|
|
ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3},
|
|
},
|
|
Err: &clients.ClientError{Code: "resource_exhausted", StatusCode: 429, Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 3,
|
|
HasNextCandidate: true,
|
|
})
|
|
|
|
if decision.Route != "next" || decision.Effect != "cooldown" || decision.Target != "model" || decision.CooldownSeconds != 60 {
|
|
t.Fatalf("429 cooldown rule should invalidate the candidate immediately, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureDemoteRetriesBeforeNext(t *testing.T) {
|
|
input := resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"statusCodes": []any{500},
|
|
"action": "demote_and_next",
|
|
"target": "platform",
|
|
"demoteSteps": 2,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Candidate: store.RuntimeModelCandidate{
|
|
ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2},
|
|
},
|
|
Err: &clients.ClientError{Code: "server_error", StatusCode: 500, Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 2,
|
|
HasNextCandidate: true,
|
|
}
|
|
|
|
first := resolveCandidateFailure(input)
|
|
if first.Route != "retry_same" || first.Effect != "none" {
|
|
t.Fatalf("demotion must wait until same-client retries are exhausted, got %+v", first)
|
|
}
|
|
|
|
input.ClientAttempt = 2
|
|
second := resolveCandidateFailure(input)
|
|
if second.Route != "next" || second.Effect != "demote" || second.DemoteSteps != 2 {
|
|
t.Fatalf("exhausted retry should demote once and rotate, got %+v", second)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureStopsAfterDownstreamStarts(t *testing.T) {
|
|
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"categories": []any{"stream_error"},
|
|
"action": "next",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Candidate: store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3}},
|
|
Err: &clients.ClientError{Code: "stream_read_error", Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 3,
|
|
HasNextCandidate: true,
|
|
DownstreamStarted: true,
|
|
})
|
|
|
|
if decision.Route != "stop" || decision.Reason != "downstream_response_started" {
|
|
t.Fatalf("stream output must prevent retries and failover, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureLegacyPolicyPrecedence(t *testing.T) {
|
|
candidate := store.RuntimeModelCandidate{
|
|
DegradePolicy: map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"resource_exhausted"},
|
|
"cooldownSeconds": 45,
|
|
},
|
|
}
|
|
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"statusCodes": []any{429},
|
|
"action": "cooldown_and_next",
|
|
"target": "platform",
|
|
"cooldownSeconds": 90,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Candidate: candidate,
|
|
Err: &clients.ClientError{Code: "resource_exhausted", Message: "resource_exhausted", StatusCode: 429, Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 3,
|
|
HasNextCandidate: true,
|
|
})
|
|
|
|
if decision.Match.Policy != "degradePolicy" || decision.Target != "model" || decision.CooldownSeconds != 45 {
|
|
t.Fatalf("legacy compatibility rule should precede global action rules, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureUnmatchedRetryableUsesSameThenNext(t *testing.T) {
|
|
input := resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true, "actionRules": []any{}}},
|
|
Candidate: store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2}},
|
|
Err: &clients.ClientError{Code: "unknown_transient", Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 2,
|
|
HasNextCandidate: true,
|
|
}
|
|
|
|
if decision := resolveCandidateFailure(input); decision.Route != "retry_same" {
|
|
t.Fatalf("unmatched retryable error should retry the same client first, got %+v", decision)
|
|
}
|
|
input.ClientAttempt = 2
|
|
if decision := resolveCandidateFailure(input); decision.Route != "next" || decision.Effect != "none" {
|
|
t.Fatalf("unmatched exhausted error should rotate without side effects, got %+v", decision)
|
|
}
|
|
}
|