BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(runner): 收敛上游失败决策与轮转策略

Browse Source 
将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
This commit is contained in:
wangbo
2026-07-27 23:50:17 +08:00
parent 046c16fc69
commit 31565af07a
19 changed files with 1988 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/runner/retry_decision.go
+180 -1
View File
@@ -42,6 +42,29 @@ type failoverDecision struct {
Info failureInfo
}
type failureDecision struct {
Route string
Effect string
Target string
CooldownSeconds int
DemoteSteps int
Reason string
Match policyRuleMatch
Info failureInfo
}
type resolveCandidateFailureInput struct {
RunnerPolicy store.RunnerPolicy
Candidate store.RuntimeModelCandidate
Err error
ClientAttempt int
MaxClientAttempts int
HasNextCandidate bool
FailoverExpired bool
Async bool
DownstreamStarted bool
}
type priorityDemoteDecision struct {
Demote bool
Reason string
@@ -104,7 +127,7 @@ func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate sto
if errors.Is(err, store.ErrRateLimited) {
return failoverDecision{Retry: false, Action: "queue", Reason: "local_rate_limit_wait_queue", Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"}, Info: info}
}
if ruleDecision, ok := failoverActionRuleDecisionWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
if ruleDecision, ok := candidateFailoverActionRuleDecision(runnerPolicy.FailoverPolicy, overridePolicy, candidate, info); ok {
return ruleDecision
}
if match, ok := failoverDenyMatchWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
@@ -125,6 +148,122 @@ func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate sto
return failoverDecision{Retry: false, Action: "stop", Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
}
func resolveCandidateFailure(input resolveCandidateFailureInput) failureDecision {
info := failureInfoFromError(input.Err)
if isResultPersistenceFailure(input.Err) {
return failureDecision{
Route: "stop",
Effect: "none",
Reason: "result_persistence_failed",
Match: policyRuleMatch{Source: "gateway_result_storage", Policy: "persistence", Rule: "providerFailover", Value: "disabled"},
Info: info,
}
}
if errors.Is(input.Err, store.ErrRateLimited) {
route := "stop"
if input.Async && store.RateLimitRetryable(input.Err) {
route = "requeue"
}
return failureDecision{
Route: route,
Effect: "none",
Reason: "local_rate_limit_wait_queue",
Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"},
Info: info,
}
}
failover := failoverDecisionForCandidate(input.RunnerPolicy, input.Candidate, input.Err)
effect := failoverEffect(failover.Action)
if input.DownstreamStarted {
return failureDecision{
Route: "stop",
Effect: effect,
Target: failover.Target,
CooldownSeconds: failover.CooldownSeconds,
DemoteSteps: failover.DemoteSteps,
Reason: "downstream_response_started",
Match: policyRuleMatch{Source: "gateway_stream", Policy: "streamSafety", Rule: "downstreamStarted", Value: "true"},
Info: info,
}
}
explicitStop := failover.Action == "stop" && (failover.Reason == "failover_action_rule" ||
failover.Reason == "hard_stop_policy" ||
failover.Reason == "failover_deny_policy" ||
failover.Reason == "failover_disabled" ||
failover.Reason == "runner_policy_disabled")
if explicitStop {
return failureDecision{
Route: "stop",
Effect: effect,
Target: failover.Target,
CooldownSeconds: failover.CooldownSeconds,
DemoteSteps: failover.DemoteSteps,
Reason: failover.Reason,
Match: failover.Match,
Info: failover.Info,
}
}
immediateCandidateInvalidation := failover.Action == "cooldown_and_next" || failover.Action == "disable_and_next"
retry := retryDecisionForCandidate(input.Candidate, input.Err)
if !immediateCandidateInvalidation && retry.Retry && input.ClientAttempt < input.MaxClientAttempts {
return failureDecision{
Route: "retry_same",
Effect: "none",
Reason: retry.Reason,
Match: retry.Match,
Info: retry.Info,
}
}
route := "next"
reason := failover.Reason
match := failover.Match
if input.FailoverExpired {
route = "stop"
reason = "failover_time_budget_exceeded"
match = policyRuleMatch{
Source: "gateway_runner_policies.failover_policy",
Policy: "failoverPolicy",
Rule: "maxDurationSeconds",
Value: "exceeded",
}
} else if !input.HasNextCandidate {
route = "stop"
reason = "no_next_platform"
match = policyRuleMatch{
Source: "runner_candidates",
Policy: "candidateSelection",
Rule: "candidateCount",
Value: "exhausted",
}
}
return failureDecision{
Route: route,
Effect: effect,
Target: failover.Target,
CooldownSeconds: failover.CooldownSeconds,
DemoteSteps: failover.DemoteSteps,
Reason: reason,
Match: match,
Info: failover.Info,
}
}
func failoverEffect(action string) string {
switch action {
case "cooldown_and_next":
return "cooldown"
case "demote_and_next":
return "demote"
case "disable_and_next":
return "disable"
default:
return "none"
}
}
func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) bool {
decision := priorityDemoteDecisionForCandidate(runnerPolicy, err)
return decision.Demote
@@ -346,6 +485,46 @@ func failoverActionRuleDecisionWithSources(base map[string]any, override map[str
return failoverActionRuleDecision(actionRulesFromPolicy(base), "gateway_runner_policies.failover_policy", info)
}
func candidateFailoverActionRuleDecision(base map[string]any, override map[string]any, candidate store.RuntimeModelCandidate, info failureInfo) (failoverDecision, bool) {
if rules := actionRulesFromPolicy(override); len(rules) > 0 {
if decision, ok := failoverActionRuleDecision(rules, "runtime_policy_override.failoverPolicy", info); ok {
return decision, true
}
}
autoDisablePolicy := effectiveRuntimePolicy(candidate.AutoDisablePolicy, candidate.RuntimePolicyOverride, "autoDisablePolicy")
if boolFromPolicy(autoDisablePolicy, "enabled", false) && intFromPolicy(autoDisablePolicy, "threshold") <= 1 {
if value, ok := matchingKeywordValue(autoDisablePolicy, "keywords", info.Target); ok {
return failoverDecision{
Retry: true,
Action: "disable_and_next",
Target: "platform",
Reason: "legacy_auto_disable_policy",
Match: policyRuleMatch{Source: "model_runtime_policy_sets.auto_disable_policy", Policy: "autoDisablePolicy", Rule: "keywords", Value: value},
Info: info,
}, true
}
}
degradePolicy := effectiveRuntimePolicy(candidate.DegradePolicy, candidate.RuntimePolicyOverride, "degradePolicy")
if boolFromPolicy(degradePolicy, "enabled", false) {
if value, ok := matchingKeywordValue(degradePolicy, "keywords", info.Target); ok {
cooldownSeconds := intFromPolicy(degradePolicy, "cooldownSeconds")
if cooldownSeconds <= 0 {
cooldownSeconds = 300
}
return failoverDecision{
Retry: true,
Action: "cooldown_and_next",
Target: "model",
Reason: "legacy_degrade_policy",
CooldownSeconds: cooldownSeconds,
Match: policyRuleMatch{Source: "model_runtime_policy_sets.degrade_policy", Policy: "degradePolicy", Rule: "keywords", Value: value},
Info: info,
}, true
}
}
return failoverActionRuleDecision(actionRulesFromPolicy(base), "gateway_runner_policies.failover_policy", info)
}
func failoverActionRuleDecision(rules []map[string]any, source string, info failureInfo) (failoverDecision, bool) {
for index, rule := range rules {
if !boolFromPolicy(rule, "enabled", true) {