wangbo
394cc6288f
修正非语义字段长 Base64 元数据被误判为媒体的问题，并允许按文件签名识别真实媒体后转存 OSS；同时将网关本地存储失败与上游失败分离，避免重复调用和错误降权。 异步任务复用已分配并发名额前立即刷新租约，租约已丢失时重新排队获取，避免排队耗时侵蚀租约后错误调用上游。生产环境关闭 server-main 尚未实现的进度回调，保留 Gateway 任务详情与事件查询。 验证：API 全量 go test、go vet、gofmt、Kubernetes 渲染、迁移安全检查及真实阿里云 OSS 读写/生命周期验收通过。
532 lines
19 KiB
Go
532 lines
19 KiB
Go
package runner
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
func TestShouldRetrySameClientUsesRuntimeRetryPolicyKeywords(t *testing.T) {
|
|
candidate := store.RuntimeModelCandidate{
|
|
ModelRetryPolicy: map[string]any{
|
|
"enabled": true,
|
|
"allowKeywords": []any{"temporary vendor blip"},
|
|
"denyKeywords": []any{"bad request"},
|
|
},
|
|
}
|
|
|
|
if shouldRetrySameClient(candidate, &clients.ClientError{Code: "bad_request", Message: "bad request timeout", Retryable: true}) {
|
|
t.Fatal("deny keywords should block same-client retry even when the client marks the error retryable")
|
|
}
|
|
if !shouldRetrySameClient(candidate, &clients.ClientError{Code: "custom_error", Message: "temporary vendor blip", Retryable: false}) {
|
|
t.Fatal("allow keywords should allow same-client retry when the client does not mark the error retryable")
|
|
}
|
|
}
|
|
|
|
func TestFailoverBudgetDefaults(t *testing.T) {
|
|
candidates := make([]store.RuntimeModelCandidate, 150)
|
|
runnerPolicy := store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true}}
|
|
|
|
if got := maxPlatformsForCandidates(candidates, runnerPolicy); got != 99 {
|
|
t.Fatalf("default max platform budget should be 99, got %d", got)
|
|
}
|
|
if got := maxFailoverDurationForCandidates(candidates, runnerPolicy); got != 10*time.Minute {
|
|
t.Fatalf("default max failover duration should be 10 minutes, got %s", got)
|
|
}
|
|
}
|
|
|
|
func TestSingleSourceProtectionDefaultsOn(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{Status: "active"}
|
|
|
|
if !singleSourceProtectionActive([]store.RuntimeModelCandidate{{PlatformID: "only-source"}}, 99, runnerPolicy) {
|
|
t.Fatal("single source protection should default to enabled")
|
|
}
|
|
}
|
|
|
|
func TestSingleSourceProtectionCanBeDisabled(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
SingleSourcePolicy: map[string]any{"enabled": false},
|
|
}
|
|
|
|
if singleSourceProtectionActive([]store.RuntimeModelCandidate{{PlatformID: "only-source"}}, 99, runnerPolicy) {
|
|
t.Fatal("single source protection should respect the global toggle")
|
|
}
|
|
}
|
|
|
|
func TestSingleSourceProtectionUsesEffectivePlatformBudget(t *testing.T) {
|
|
candidates := []store.RuntimeModelCandidate{
|
|
{PlatformID: "first-source"},
|
|
{PlatformID: "second-source"},
|
|
}
|
|
runnerPolicy := store.RunnerPolicy{Status: "active"}
|
|
|
|
if !singleSourceProtectionActive(candidates, 1, runnerPolicy) {
|
|
t.Fatal("maxPlatforms=1 should make this execution a single-source run")
|
|
}
|
|
if singleSourceProtectionActive(candidates, 2, runnerPolicy) {
|
|
t.Fatal("multiple effective sources should allow disable and cooldown actions")
|
|
}
|
|
}
|
|
|
|
func TestFailoverTimeBudgetExceeded(t *testing.T) {
|
|
if !failoverTimeBudgetExceeded(time.Now().Add(-601*time.Second), 10*time.Minute) {
|
|
t.Fatal("failover time budget should stop retries after the configured duration")
|
|
}
|
|
if failoverTimeBudgetExceeded(time.Now().Add(-590*time.Second), 10*time.Minute) {
|
|
t.Fatal("failover time budget should allow retries before the configured duration")
|
|
}
|
|
}
|
|
|
|
func TestFailoverHardStopBeatsModelOverride(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"allowCategories": []any{"request_error"},
|
|
},
|
|
HardStopPolicy: map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"request_error"},
|
|
},
|
|
}
|
|
candidate := store.RuntimeModelCandidate{
|
|
RuntimePolicyOverride: map[string]any{
|
|
"failoverPolicy": map[string]any{
|
|
"enabled": true,
|
|
"allowCategories": []any{"request_error"},
|
|
},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, candidate, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true})
|
|
if decision.Retry || decision.Reason != "hard_stop_policy" {
|
|
t.Fatalf("hard stop should block model-level failover override, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverPolicyAllowsModelOverride(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{"enabled": true},
|
|
HardStopPolicy: map[string]any{"enabled": true, "categories": []any{"request_error"}},
|
|
}
|
|
candidate := store.RuntimeModelCandidate{
|
|
RuntimePolicyOverride: map[string]any{
|
|
"failoverPolicy": map[string]any{
|
|
"allowKeywords": []any{"temporary upstream outage"},
|
|
},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, candidate, &clients.ClientError{Code: "custom_error", Message: "temporary upstream outage", Retryable: false})
|
|
if !decision.Retry || decision.Reason != "failover_allow_policy" {
|
|
t.Fatalf("model failoverPolicy override should allow cross-platform failover, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestLocalRateLimitWaitsInQueueWithoutRetryOrFailover(t *testing.T) {
|
|
err := &localRateLimitError{
|
|
clientErr: &clients.ClientError{Code: "rate_limit", Message: "local capacity exceeded", Retryable: true},
|
|
cause: &store.RateLimitExceededError{ScopeType: "user_group", Metric: "concurrent", Retryable: true},
|
|
}
|
|
retryDecision := retryDecisionForCandidate(store.RuntimeModelCandidate{}, err)
|
|
if retryDecision.Retry || retryDecision.Reason != "local_rate_limit_wait_queue" {
|
|
t.Fatalf("local rate limit should not be same-client retry, got %+v", retryDecision)
|
|
}
|
|
|
|
failoverDecision := failoverDecisionForCandidate(store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{"enabled": true, "allowCategories": []any{"rate_limit"}},
|
|
}, store.RuntimeModelCandidate{}, err)
|
|
if failoverDecision.Retry || failoverDecision.Action != "queue" || failoverDecision.Reason != "local_rate_limit_wait_queue" {
|
|
t.Fatalf("local rate limit should wait in queue without failover, got %+v", failoverDecision)
|
|
}
|
|
}
|
|
|
|
func TestPlatformModelRateLimitRaceRotatesWithoutFailureEffect(t *testing.T) {
|
|
err := &localRateLimitError{
|
|
clientErr: &clients.ClientError{Code: "rate_limit", Message: "platform capacity raced", Retryable: true},
|
|
cause: &store.RateLimitExceededError{
|
|
ScopeType: "platform_model",
|
|
Metric: "concurrent",
|
|
Retryable: true,
|
|
RetryAfter: time.Second,
|
|
},
|
|
}
|
|
|
|
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{Status: "active"},
|
|
Err: err,
|
|
HasNextCandidate: true,
|
|
Async: true,
|
|
})
|
|
if decision.Route != "next" || decision.Effect != "none" || decision.Reason != "quota_race_rotated" {
|
|
t.Fatalf("platform quota race should rotate without mutation: %+v", decision)
|
|
}
|
|
|
|
last := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{Status: "active"},
|
|
Err: err,
|
|
Async: true,
|
|
})
|
|
if last.Route != "requeue" || last.Effect != "none" {
|
|
t.Fatalf("last saturated platform should requeue: %+v", last)
|
|
}
|
|
}
|
|
|
|
func TestProviderAuthErrorsFailOverInsteadOfHardStop(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"allowCategories": []any{"auth_error"},
|
|
"allowCodes": []any{"auth_failed", "invalid_api_key", "missing_credentials"},
|
|
"allowStatusCodes": []any{
|
|
401,
|
|
403,
|
|
},
|
|
"actions": map[string]any{"auth_error": "disable_and_next"},
|
|
},
|
|
HardStopPolicy: map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
|
"codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "permission_denied"},
|
|
"statusCodes": []any{},
|
|
"keywords": []any{"invalid_parameter", "missing required", "bad request", "insufficient balance"},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "auth_failed", StatusCode: 401, Retryable: false})
|
|
if !decision.Retry || decision.Action != "disable_and_next" || decision.Reason != "failover_allow_policy" {
|
|
t.Fatalf("provider auth failures should switch platform, got %+v", decision)
|
|
}
|
|
|
|
decision = failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "http_400", Message: "invalid api key", StatusCode: 400, Retryable: false})
|
|
if !decision.Retry || decision.Info.Category != "auth_error" {
|
|
t.Fatalf("provider auth-looking 400 should switch platform, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverActionRulesControlPerRuleCooldownAndDemotion(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"rate_limit"},
|
|
"action": "cooldown_and_next",
|
|
"target": "platform",
|
|
"cooldownSeconds": 45,
|
|
},
|
|
map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"temporary overload"},
|
|
"action": "demote_and_next",
|
|
"demoteSteps": 3,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "rate_limit", StatusCode: 429, Retryable: false})
|
|
if !decision.Retry || decision.Action != "cooldown_and_next" || decision.Target != "platform" || decision.CooldownSeconds != 45 {
|
|
t.Fatalf("rate limit should use per-rule cooldown action, got %+v", decision)
|
|
}
|
|
|
|
decision = failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "server_error", Message: "temporary overload from upstream", StatusCode: 503, Retryable: false})
|
|
if !decision.Retry || decision.Action != "demote_and_next" || decision.DemoteSteps != 3 {
|
|
t.Fatalf("keyword match should use per-rule demote action, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverActionRulesCanStopAndMatchStringStatusCodes(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"enabled": true,
|
|
"statusCodes": []any{"400"},
|
|
"keywords": []any{"bad request"},
|
|
"action": "stop",
|
|
},
|
|
map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"request_error"},
|
|
"action": "next",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "bad_request", Message: "bad request", StatusCode: 400, Retryable: true})
|
|
if decision.Retry || decision.Action != "stop" || decision.Reason != "failover_action_rule" || decision.Match.Rule != "statusCodes" {
|
|
t.Fatalf("first stop rule should win before later retry rules, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestFailoverActionRulesKeepOrderBeforeDerivedHardStopPolicy(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"bad request retryable"},
|
|
"action": "next",
|
|
},
|
|
map[string]any{
|
|
"enabled": true,
|
|
"statusCodes": []any{400},
|
|
"action": "stop",
|
|
},
|
|
},
|
|
},
|
|
HardStopPolicy: map[string]any{
|
|
"enabled": true,
|
|
"statusCodes": []any{400},
|
|
},
|
|
}
|
|
|
|
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "bad_request", Message: "bad request retryable", StatusCode: 400, Retryable: true})
|
|
if !decision.Retry || decision.Action != "next" || decision.Match.Rule != "keywords" {
|
|
t.Fatalf("actionRules should keep list order before derived hard stop compatibility fields, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestPriorityDemotePolicyIsKeywordGatedAndHardStopSafe(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
HardStopPolicy: map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"request_error"},
|
|
},
|
|
PriorityDemotePolicy: map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"rate_limit"},
|
|
},
|
|
}
|
|
|
|
if !shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true}) {
|
|
t.Fatal("priority demotion should be enabled only by matched policy")
|
|
}
|
|
|
|
if shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true}) {
|
|
t.Fatal("priority demotion should not run for hard-stop request errors")
|
|
}
|
|
}
|
|
|
|
func TestPriorityDemotePolicyIsSkippedWhenFailoverActionRulesExist(t *testing.T) {
|
|
runnerPolicy := store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"enabled": true,
|
|
"categories": []any{"rate_limit"},
|
|
"action": "demote_and_next",
|
|
},
|
|
},
|
|
},
|
|
PriorityDemotePolicy: map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"rate_limit"},
|
|
},
|
|
}
|
|
|
|
decision := priorityDemoteDecisionForCandidate(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true})
|
|
if decision.Demote || decision.Reason != "failover_action_rules_enabled" {
|
|
t.Fatalf("legacy priority demotion should be skipped when actionRules are active, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureCooldownSkipsSameClientRetry(t *testing.T) {
|
|
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"statusCodes": []any{429},
|
|
"action": "cooldown_and_next",
|
|
"target": "model",
|
|
"cooldownSeconds": 60,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Candidate: store.RuntimeModelCandidate{
|
|
ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3},
|
|
},
|
|
Err: &clients.ClientError{Code: "resource_exhausted", StatusCode: 429, Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 3,
|
|
HasNextCandidate: true,
|
|
})
|
|
|
|
if decision.Route != "next" || decision.Effect != "cooldown" || decision.Target != "model" || decision.CooldownSeconds != 60 {
|
|
t.Fatalf("429 cooldown rule should invalidate the candidate immediately, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureDemoteRetriesBeforeNext(t *testing.T) {
|
|
input := resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"statusCodes": []any{500},
|
|
"action": "demote_and_next",
|
|
"target": "platform",
|
|
"demoteSteps": 2,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Candidate: store.RuntimeModelCandidate{
|
|
ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2},
|
|
},
|
|
Err: &clients.ClientError{Code: "server_error", StatusCode: 500, Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 2,
|
|
HasNextCandidate: true,
|
|
}
|
|
|
|
first := resolveCandidateFailure(input)
|
|
if first.Route != "retry_same" || first.Effect != "none" {
|
|
t.Fatalf("demotion must wait until same-client retries are exhausted, got %+v", first)
|
|
}
|
|
|
|
input.ClientAttempt = 2
|
|
second := resolveCandidateFailure(input)
|
|
if second.Route != "next" || second.Effect != "demote" || second.DemoteSteps != 2 {
|
|
t.Fatalf("exhausted retry should demote once and rotate, got %+v", second)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureStopsAfterDownstreamStarts(t *testing.T) {
|
|
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"categories": []any{"stream_error"},
|
|
"action": "next",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Candidate: store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 3}},
|
|
Err: &clients.ClientError{Code: "stream_read_error", Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 3,
|
|
HasNextCandidate: true,
|
|
DownstreamStarted: true,
|
|
})
|
|
|
|
if decision.Route != "stop" || decision.Reason != "downstream_response_started" {
|
|
t.Fatalf("stream output must prevent retries and failover, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureLegacyPolicyPrecedence(t *testing.T) {
|
|
candidate := store.RuntimeModelCandidate{
|
|
DegradePolicy: map[string]any{
|
|
"enabled": true,
|
|
"keywords": []any{"resource_exhausted"},
|
|
"cooldownSeconds": 45,
|
|
},
|
|
}
|
|
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{
|
|
"enabled": true,
|
|
"actionRules": []any{
|
|
map[string]any{
|
|
"statusCodes": []any{429},
|
|
"action": "cooldown_and_next",
|
|
"target": "platform",
|
|
"cooldownSeconds": 90,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Candidate: candidate,
|
|
Err: &clients.ClientError{Code: "resource_exhausted", Message: "resource_exhausted", StatusCode: 429, Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 3,
|
|
HasNextCandidate: true,
|
|
})
|
|
|
|
if decision.Match.Policy != "degradePolicy" || decision.Target != "model" || decision.CooldownSeconds != 45 {
|
|
t.Fatalf("legacy compatibility rule should precede global action rules, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestResolveCandidateFailureUnmatchedRetryableUsesSameThenNext(t *testing.T) {
|
|
input := resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true, "actionRules": []any{}}},
|
|
Candidate: store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{"enabled": true, "maxAttempts": 2}},
|
|
Err: &clients.ClientError{Code: "unknown_transient", Retryable: true},
|
|
ClientAttempt: 1,
|
|
MaxClientAttempts: 2,
|
|
HasNextCandidate: true,
|
|
}
|
|
|
|
if decision := resolveCandidateFailure(input); decision.Route != "retry_same" {
|
|
t.Fatalf("unmatched retryable error should retry the same client first, got %+v", decision)
|
|
}
|
|
input.ClientAttempt = 2
|
|
if decision := resolveCandidateFailure(input); decision.Route != "next" || decision.Effect != "none" {
|
|
t.Fatalf("unmatched exhausted error should rotate without side effects, got %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestGatewayStorageFailureNeverRetriesProvider(t *testing.T) {
|
|
err := &clients.ClientError{
|
|
Code: "storage_write_failed",
|
|
Message: "generated binary result could not be written to object storage",
|
|
StatusCode: 503,
|
|
Retryable: true,
|
|
}
|
|
candidate := store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{
|
|
"enabled": true,
|
|
"allowKeywords": []any{"5xx"},
|
|
}}
|
|
|
|
retry := retryDecisionForCandidate(candidate, err)
|
|
if retry.Retry || retry.Reason != "result_persistence_failed" {
|
|
t.Fatalf("Gateway storage failure must not repeat an upstream request: %+v", retry)
|
|
}
|
|
failover := failoverDecisionForCandidate(store.RunnerPolicy{
|
|
Status: "active",
|
|
FailoverPolicy: map[string]any{"enabled": true, "allowCategories": []any{"provider_5xx"}},
|
|
}, candidate, err)
|
|
if failover.Retry || failover.Reason != "result_persistence_failed" {
|
|
t.Fatalf("Gateway storage failure must not fail over to another provider: %+v", failover)
|
|
}
|
|
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
|
RunnerPolicy: store.RunnerPolicy{Status: "active"},
|
|
Err: err,
|
|
HasNextCandidate: true,
|
|
Async: true,
|
|
})
|
|
if decision.Route != "stop" || decision.Reason != "result_persistence_failed" || decision.Info.Category != "gateway_storage" {
|
|
t.Fatalf("Gateway storage failure classification is incorrect: %+v", decision)
|
|
}
|
|
}
|