BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add runner failover rules and cache affinity

Browse Source
This commit is contained in:
wangbo
2026-06-28 20:50:23 +08:00
parent 41bb3a6525
commit 0e675e259c
31 changed files with 2939 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/runner/retry_decision.go
+141 -14
View File
@@ -34,17 +34,20 @@ type retryDecision struct {
type failoverDecision struct {
Retry bool
Action string
Target string
Reason string
CooldownSeconds int
DemoteSteps int
Match policyRuleMatch
Info failureInfo
}
type priorityDemoteDecision struct {
Demote bool
Reason string
Match policyRuleMatch
Info failureInfo
Demote bool
Reason string
DemoteSteps int
Match policyRuleMatch
Info failureInfo
}
func shouldRetrySameClient(candidate store.RuntimeModelCandidate, err error) bool {
@@ -77,10 +80,13 @@ func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate sto
if strings.TrimSpace(runnerPolicy.Status) != "" && runnerPolicy.Status != "active" {
return failoverDecision{Retry: false, Action: "stop", Reason: "runner_policy_disabled", Match: policyRuleMatch{Source: "gateway_runner_policies", Policy: "runnerPolicy", Rule: "status", Value: runnerPolicy.Status}, Info: info}
}
if match, ok := hardStopPolicyMatch(runnerPolicy.HardStopPolicy, info); ok {
return failoverDecision{Retry: false, Action: "stop", Reason: "hard_stop_policy", Match: match, Info: info}
}
overridePolicy := failoverOverridePolicy(candidate.RuntimePolicyOverride)
hasActionRules := len(actionRulesFromPolicy(runnerPolicy.FailoverPolicy)) > 0 || len(actionRulesFromPolicy(overridePolicy)) > 0
if !hasActionRules {
if match, ok := hardStopPolicyMatch(runnerPolicy.HardStopPolicy, info); ok {
return failoverDecision{Retry: false, Action: "stop", Reason: "hard_stop_policy", Match: match, Info: info}
}
}
policy := effectiveFailoverPolicy(runnerPolicy.FailoverPolicy, candidate.RuntimePolicyOverride)
if !boolFromPolicy(policy, "enabled", true) {
source := "gateway_runner_policies.failover_policy"
@@ -89,22 +95,26 @@ func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate sto
}
return failoverDecision{Retry: false, Action: "stop", Reason: "failover_disabled", Match: policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "enabled", Value: "false"}, Info: info}
}
if errors.Is(err, store.ErrRateLimited) {
return failoverDecision{Retry: false, Action: "queue", Reason: "local_rate_limit_wait_queue", Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"}, Info: info}
}
if ruleDecision, ok := failoverActionRuleDecisionWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
return ruleDecision
}
if match, ok := failoverDenyMatchWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
return failoverDecision{Retry: false, Action: "stop", Reason: "failover_deny_policy", Match: match, Info: info}
}
action := failoverAction(policy, info)
target := defaultFailoverActionTarget(action)
cooldownSeconds := intFromPolicy(policy, "cooldownSeconds")
if cooldownSeconds <= 0 {
cooldownSeconds = 300
}
if errors.Is(err, store.ErrRateLimited) {
return failoverDecision{Retry: false, Action: "queue", Reason: "local_rate_limit_wait_queue", Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"}, Info: info}
}
if match, ok := failoverAllowMatchWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
return failoverDecision{Retry: true, Action: action, Reason: "failover_allow_policy", CooldownSeconds: cooldownSeconds, Match: match, Info: info}
return failoverDecision{Retry: true, Action: action, Target: target, Reason: "failover_allow_policy", CooldownSeconds: cooldownSeconds, DemoteSteps: 1, Match: match, Info: info}
}
if clients.IsRetryable(err) {
return failoverDecision{Retry: true, Action: action, Reason: "client_retryable", CooldownSeconds: cooldownSeconds, Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "true"}, Info: info}
return failoverDecision{Retry: true, Action: action, Target: target, Reason: "client_retryable", CooldownSeconds: cooldownSeconds, DemoteSteps: 1, Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "true"}, Info: info}
}
return failoverDecision{Retry: false, Action: "stop", Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
}
@@ -122,12 +132,15 @@ func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err err
if hardStopPolicyMatches(runnerPolicy.HardStopPolicy, info) {
return priorityDemoteDecision{Demote: false, Reason: "hard_stop_policy", Info: info}
}
if len(actionRulesFromPolicy(runnerPolicy.FailoverPolicy)) > 0 {
return priorityDemoteDecision{Demote: false, Reason: "failover_action_rules_enabled", Info: info}
}
policy := runnerPolicy.PriorityDemotePolicy
if !boolFromPolicy(policy, "enabled", false) {
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_disabled", Info: info}
}
if match, ok := priorityDemotePolicyMatch(policy, info); ok {
return priorityDemoteDecision{Demote: true, Reason: "priority_demote_policy", Match: match, Info: info}
return priorityDemoteDecision{Demote: true, Reason: "priority_demote_policy", DemoteSteps: 99, Match: match, Info: info}
}
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_no_match", Info: info}
}
@@ -299,11 +312,118 @@ func priorityDemotePolicyMatch(policy map[string]any, info failureInfo) (policyR
func failoverAction(policy map[string]any, info failureInfo) string {
actions, _ := policy["actions"].(map[string]any)
if action := stringFromAny(actions[info.Category]); action != "" {
return action
return normalizeFailoverAction(action)
}
return "next"
}
func failoverActionRuleDecisionWithSources(base map[string]any, override map[string]any, info failureInfo) (failoverDecision, bool) {
if rules := actionRulesFromPolicy(override); len(rules) > 0 {
return failoverActionRuleDecision(rules, "runtime_policy_override.failoverPolicy", info)
}
return failoverActionRuleDecision(actionRulesFromPolicy(base), "gateway_runner_policies.failover_policy", info)
}
func failoverActionRuleDecision(rules []map[string]any, source string, info failureInfo) (failoverDecision, bool) {
for index, rule := range rules {
if !boolFromPolicy(rule, "enabled", true) {
continue
}
match, ok := failoverActionRuleMatch(rule, info, source, index)
if !ok {
continue
}
action := normalizeFailoverAction(stringFromAny(rule["action"]))
target := normalizeFailoverTarget(stringFromAny(rule["target"]), action)
cooldownSeconds := intFromPolicy(rule, "cooldownSeconds")
if cooldownSeconds <= 0 {
cooldownSeconds = 300
}
demoteSteps := intFromPolicy(rule, "demoteSteps")
if demoteSteps <= 0 {
demoteSteps = 1
}
if action == "stop" {
return failoverDecision{Retry: false, Action: "stop", Target: target, Reason: "failover_action_rule", CooldownSeconds: cooldownSeconds, DemoteSteps: demoteSteps, Match: match, Info: info}, true
}
return failoverDecision{Retry: true, Action: action, Target: target, Reason: "failover_action_rule", CooldownSeconds: cooldownSeconds, DemoteSteps: demoteSteps, Match: match, Info: info}, true
}
return failoverDecision{}, false
}
func actionRulesFromPolicy(policy map[string]any) []map[string]any {
raw, ok := policy["actionRules"].([]any)
if !ok {
return nil
}
rules := make([]map[string]any, 0, len(raw))
for _, item := range raw {
rule, ok := item.(map[string]any)
if !ok {
continue
}
rules = append(rules, rule)
}
return rules
}
func failoverActionRuleMatch(rule map[string]any, info failureInfo, source string, index int) (policyRuleMatch, bool) {
source = fmt.Sprintf("%s.actionRules[%d]", source, index)
if value, ok := matchingStringListValue(rule, "categories", info.Category); ok {
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "categories", Value: value}, true
}
if value, ok := matchingStringListValue(rule, "codes", info.Code); ok {
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "codes", Value: value}, true
}
if value, ok := matchingStringListValue(rule, "errorCodes", info.Code); ok {
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "errorCodes", Value: value}, true
}
if value, ok := matchingStringListValue(rule, "errorCodes", info.Category); ok {
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "errorCodes", Value: value}, true
}
if value, ok := matchingIntListValue(rule, "statusCodes", info.Status); ok {
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "statusCodes", Value: fmt.Sprintf("%d", value)}, true
}
if value, ok := matchingKeywordValue(rule, "keywords", info.Target); ok {
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "keywords", Value: value}, true
}
return policyRuleMatch{}, false
}
func normalizeFailoverAction(action string) string {
switch strings.TrimSpace(action) {
case "rotate":
return "next"
case "cooldown_and_rotate":
return "cooldown_and_next"
case "demote_and_rotate":
return "demote_and_next"
case "disable_and_rotate":
return "disable_and_next"
case "cooldown_and_next", "demote_and_next", "disable_and_next", "stop", "next":
return strings.TrimSpace(action)
default:
return "next"
}
}
func normalizeFailoverTarget(target string, action string) string {
target = strings.ToLower(strings.TrimSpace(target))
if target == "platform" || target == "model" {
return target
}
return defaultFailoverActionTarget(action)
}
func defaultFailoverActionTarget(action string) string {
switch action {
case "cooldown_and_next":
return "model"
default:
return "platform"
}
}
func boolFromPolicy(policy map[string]any, key string, fallback bool) bool {
value, ok := policy[key].(bool)
if !ok {
@@ -442,6 +562,13 @@ func intListFromPolicy(policy map[string]any, key string) []int {
out = append(out, typed)
case float64:
out = append(out, int(typed))
case string:
if parsed := strings.TrimSpace(typed); parsed != "" {
var value int
if _, err := fmt.Sscanf(parsed, "%d", &value); err == nil && value > 0 {
out = append(out, value)
}
}
}
}
return out