wangbo
31565af07a
将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
777 lines
29 KiB
Go
777 lines
29 KiB
Go
package runner
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
type failureInfo struct {
|
|
Code string
|
|
Message string
|
|
Status int
|
|
Category string
|
|
Target string
|
|
}
|
|
|
|
type policyRuleMatch struct {
|
|
Source string
|
|
Policy string
|
|
Rule string
|
|
Value string
|
|
}
|
|
|
|
type retryDecision struct {
|
|
Retry bool
|
|
Reason string
|
|
Match policyRuleMatch
|
|
Info failureInfo
|
|
}
|
|
|
|
type failoverDecision struct {
|
|
Retry bool
|
|
Action string
|
|
Target string
|
|
Reason string
|
|
CooldownSeconds int
|
|
DemoteSteps int
|
|
Match policyRuleMatch
|
|
Info failureInfo
|
|
}
|
|
|
|
type failureDecision struct {
|
|
Route string
|
|
Effect string
|
|
Target string
|
|
CooldownSeconds int
|
|
DemoteSteps int
|
|
Reason string
|
|
Match policyRuleMatch
|
|
Info failureInfo
|
|
}
|
|
|
|
type resolveCandidateFailureInput struct {
|
|
RunnerPolicy store.RunnerPolicy
|
|
Candidate store.RuntimeModelCandidate
|
|
Err error
|
|
ClientAttempt int
|
|
MaxClientAttempts int
|
|
HasNextCandidate bool
|
|
FailoverExpired bool
|
|
Async bool
|
|
DownstreamStarted bool
|
|
}
|
|
|
|
type priorityDemoteDecision struct {
|
|
Demote bool
|
|
Reason string
|
|
DemoteSteps int
|
|
Match policyRuleMatch
|
|
Info failureInfo
|
|
}
|
|
|
|
func shouldRetrySameClient(candidate store.RuntimeModelCandidate, err error) bool {
|
|
return retryDecisionForCandidate(candidate, err).Retry
|
|
}
|
|
|
|
func retryDecisionForCandidate(candidate store.RuntimeModelCandidate, err error) retryDecision {
|
|
policy := effectiveRetryPolicy(candidate)
|
|
info := failureInfoFromError(err)
|
|
if isResultPersistenceFailure(err) {
|
|
return retryDecision{Retry: false, Reason: "result_persistence_failed", Match: policyRuleMatch{Source: "gateway_result_storage", Policy: "persistence", Rule: "providerRetry", Value: "disabled"}, Info: info}
|
|
}
|
|
if errors.Is(err, store.ErrRateLimited) {
|
|
return retryDecision{Retry: false, Reason: "local_rate_limit_wait_queue", Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"}, Info: info}
|
|
}
|
|
if !boolFromPolicy(policy, "enabled", true) {
|
|
return retryDecision{Retry: false, Reason: "retry_disabled", Match: policyRuleMatch{Source: "model_runtime_policy_sets.retry_policy", Policy: "retryPolicy", Rule: "enabled", Value: "false"}, Info: info}
|
|
}
|
|
if match, ok := retryPolicyDenyMatch(policy, info, "model_runtime_policy_sets.retry_policy", "retryPolicy"); ok {
|
|
return retryDecision{Retry: false, Reason: "retry_deny_policy", Match: match, Info: info}
|
|
}
|
|
if match, ok := retryPolicyAllowMatch(policy, info, "model_runtime_policy_sets.retry_policy", "retryPolicy"); ok {
|
|
return retryDecision{Retry: true, Reason: "retry_allow_policy", Match: match, Info: info}
|
|
}
|
|
if clients.IsRetryable(err) {
|
|
return retryDecision{Retry: true, Reason: "client_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "true"}, Info: info}
|
|
}
|
|
return retryDecision{Retry: false, Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
|
|
}
|
|
|
|
func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate store.RuntimeModelCandidate, err error) failoverDecision {
|
|
info := failureInfoFromError(err)
|
|
if isResultPersistenceFailure(err) {
|
|
return failoverDecision{Retry: false, Action: "stop", Reason: "result_persistence_failed", Match: policyRuleMatch{Source: "gateway_result_storage", Policy: "persistence", Rule: "providerFailover", Value: "disabled"}, Info: info}
|
|
}
|
|
if strings.TrimSpace(runnerPolicy.Status) != "" && runnerPolicy.Status != "active" {
|
|
return failoverDecision{Retry: false, Action: "stop", Reason: "runner_policy_disabled", Match: policyRuleMatch{Source: "gateway_runner_policies", Policy: "runnerPolicy", Rule: "status", Value: runnerPolicy.Status}, Info: info}
|
|
}
|
|
overridePolicy := failoverOverridePolicy(candidate.RuntimePolicyOverride)
|
|
hasActionRules := len(actionRulesFromPolicy(runnerPolicy.FailoverPolicy)) > 0 || len(actionRulesFromPolicy(overridePolicy)) > 0
|
|
if !hasActionRules {
|
|
if match, ok := hardStopPolicyMatch(runnerPolicy.HardStopPolicy, info); ok {
|
|
return failoverDecision{Retry: false, Action: "stop", Reason: "hard_stop_policy", Match: match, Info: info}
|
|
}
|
|
}
|
|
policy := effectiveFailoverPolicy(runnerPolicy.FailoverPolicy, candidate.RuntimePolicyOverride)
|
|
if !boolFromPolicy(policy, "enabled", true) {
|
|
source := "gateway_runner_policies.failover_policy"
|
|
if _, ok := overridePolicy["enabled"]; ok {
|
|
source = "runtime_policy_override.failoverPolicy"
|
|
}
|
|
return failoverDecision{Retry: false, Action: "stop", Reason: "failover_disabled", Match: policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "enabled", Value: "false"}, Info: info}
|
|
}
|
|
if errors.Is(err, store.ErrRateLimited) {
|
|
return failoverDecision{Retry: false, Action: "queue", Reason: "local_rate_limit_wait_queue", Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"}, Info: info}
|
|
}
|
|
if ruleDecision, ok := candidateFailoverActionRuleDecision(runnerPolicy.FailoverPolicy, overridePolicy, candidate, info); ok {
|
|
return ruleDecision
|
|
}
|
|
if match, ok := failoverDenyMatchWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
|
|
return failoverDecision{Retry: false, Action: "stop", Reason: "failover_deny_policy", Match: match, Info: info}
|
|
}
|
|
action := failoverAction(policy, info)
|
|
target := defaultFailoverActionTarget(action)
|
|
cooldownSeconds := intFromPolicy(policy, "cooldownSeconds")
|
|
if cooldownSeconds <= 0 {
|
|
cooldownSeconds = 300
|
|
}
|
|
if match, ok := failoverAllowMatchWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
|
|
return failoverDecision{Retry: true, Action: action, Target: target, Reason: "failover_allow_policy", CooldownSeconds: cooldownSeconds, DemoteSteps: 1, Match: match, Info: info}
|
|
}
|
|
if clients.IsRetryable(err) {
|
|
return failoverDecision{Retry: true, Action: action, Target: target, Reason: "client_retryable", CooldownSeconds: cooldownSeconds, DemoteSteps: 1, Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "true"}, Info: info}
|
|
}
|
|
return failoverDecision{Retry: false, Action: "stop", Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
|
|
}
|
|
|
|
func resolveCandidateFailure(input resolveCandidateFailureInput) failureDecision {
|
|
info := failureInfoFromError(input.Err)
|
|
if isResultPersistenceFailure(input.Err) {
|
|
return failureDecision{
|
|
Route: "stop",
|
|
Effect: "none",
|
|
Reason: "result_persistence_failed",
|
|
Match: policyRuleMatch{Source: "gateway_result_storage", Policy: "persistence", Rule: "providerFailover", Value: "disabled"},
|
|
Info: info,
|
|
}
|
|
}
|
|
if errors.Is(input.Err, store.ErrRateLimited) {
|
|
route := "stop"
|
|
if input.Async && store.RateLimitRetryable(input.Err) {
|
|
route = "requeue"
|
|
}
|
|
return failureDecision{
|
|
Route: route,
|
|
Effect: "none",
|
|
Reason: "local_rate_limit_wait_queue",
|
|
Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"},
|
|
Info: info,
|
|
}
|
|
}
|
|
|
|
failover := failoverDecisionForCandidate(input.RunnerPolicy, input.Candidate, input.Err)
|
|
effect := failoverEffect(failover.Action)
|
|
if input.DownstreamStarted {
|
|
return failureDecision{
|
|
Route: "stop",
|
|
Effect: effect,
|
|
Target: failover.Target,
|
|
CooldownSeconds: failover.CooldownSeconds,
|
|
DemoteSteps: failover.DemoteSteps,
|
|
Reason: "downstream_response_started",
|
|
Match: policyRuleMatch{Source: "gateway_stream", Policy: "streamSafety", Rule: "downstreamStarted", Value: "true"},
|
|
Info: info,
|
|
}
|
|
}
|
|
explicitStop := failover.Action == "stop" && (failover.Reason == "failover_action_rule" ||
|
|
failover.Reason == "hard_stop_policy" ||
|
|
failover.Reason == "failover_deny_policy" ||
|
|
failover.Reason == "failover_disabled" ||
|
|
failover.Reason == "runner_policy_disabled")
|
|
if explicitStop {
|
|
return failureDecision{
|
|
Route: "stop",
|
|
Effect: effect,
|
|
Target: failover.Target,
|
|
CooldownSeconds: failover.CooldownSeconds,
|
|
DemoteSteps: failover.DemoteSteps,
|
|
Reason: failover.Reason,
|
|
Match: failover.Match,
|
|
Info: failover.Info,
|
|
}
|
|
}
|
|
|
|
immediateCandidateInvalidation := failover.Action == "cooldown_and_next" || failover.Action == "disable_and_next"
|
|
retry := retryDecisionForCandidate(input.Candidate, input.Err)
|
|
if !immediateCandidateInvalidation && retry.Retry && input.ClientAttempt < input.MaxClientAttempts {
|
|
return failureDecision{
|
|
Route: "retry_same",
|
|
Effect: "none",
|
|
Reason: retry.Reason,
|
|
Match: retry.Match,
|
|
Info: retry.Info,
|
|
}
|
|
}
|
|
|
|
route := "next"
|
|
reason := failover.Reason
|
|
match := failover.Match
|
|
if input.FailoverExpired {
|
|
route = "stop"
|
|
reason = "failover_time_budget_exceeded"
|
|
match = policyRuleMatch{
|
|
Source: "gateway_runner_policies.failover_policy",
|
|
Policy: "failoverPolicy",
|
|
Rule: "maxDurationSeconds",
|
|
Value: "exceeded",
|
|
}
|
|
} else if !input.HasNextCandidate {
|
|
route = "stop"
|
|
reason = "no_next_platform"
|
|
match = policyRuleMatch{
|
|
Source: "runner_candidates",
|
|
Policy: "candidateSelection",
|
|
Rule: "candidateCount",
|
|
Value: "exhausted",
|
|
}
|
|
}
|
|
return failureDecision{
|
|
Route: route,
|
|
Effect: effect,
|
|
Target: failover.Target,
|
|
CooldownSeconds: failover.CooldownSeconds,
|
|
DemoteSteps: failover.DemoteSteps,
|
|
Reason: reason,
|
|
Match: match,
|
|
Info: failover.Info,
|
|
}
|
|
}
|
|
|
|
func failoverEffect(action string) string {
|
|
switch action {
|
|
case "cooldown_and_next":
|
|
return "cooldown"
|
|
case "demote_and_next":
|
|
return "demote"
|
|
case "disable_and_next":
|
|
return "disable"
|
|
default:
|
|
return "none"
|
|
}
|
|
}
|
|
|
|
func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) bool {
|
|
decision := priorityDemoteDecisionForCandidate(runnerPolicy, err)
|
|
return decision.Demote
|
|
}
|
|
|
|
func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err error) priorityDemoteDecision {
|
|
info := failureInfoFromError(err)
|
|
if isResultPersistenceFailure(err) {
|
|
return priorityDemoteDecision{Demote: false, Reason: "result_persistence_failed", Info: info}
|
|
}
|
|
if strings.TrimSpace(runnerPolicy.Status) != "" && runnerPolicy.Status != "active" {
|
|
return priorityDemoteDecision{Demote: false, Reason: "runner_policy_disabled", Info: info}
|
|
}
|
|
if hardStopPolicyMatches(runnerPolicy.HardStopPolicy, info) {
|
|
return priorityDemoteDecision{Demote: false, Reason: "hard_stop_policy", Info: info}
|
|
}
|
|
if len(actionRulesFromPolicy(runnerPolicy.FailoverPolicy)) > 0 {
|
|
return priorityDemoteDecision{Demote: false, Reason: "failover_action_rules_enabled", Info: info}
|
|
}
|
|
policy := runnerPolicy.PriorityDemotePolicy
|
|
if !boolFromPolicy(policy, "enabled", false) {
|
|
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_disabled", Info: info}
|
|
}
|
|
if match, ok := priorityDemotePolicyMatch(policy, info); ok {
|
|
return priorityDemoteDecision{Demote: true, Reason: "priority_demote_policy", DemoteSteps: 99, Match: match, Info: info}
|
|
}
|
|
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_no_match", Info: info}
|
|
}
|
|
|
|
func isResultPersistenceFailure(err error) bool {
|
|
switch strings.ToLower(strings.TrimSpace(clients.ErrorCode(err))) {
|
|
case "local_result_storage_unavailable",
|
|
"binary_result_too_large",
|
|
"binary_result_corrupted",
|
|
"binary_result_expired",
|
|
"result_binary_not_materialized":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func effectiveFailoverPolicy(base map[string]any, override map[string]any) map[string]any {
|
|
policy := base
|
|
if nested := failoverOverridePolicy(override); len(nested) > 0 {
|
|
policy = mergeMap(policy, nested)
|
|
}
|
|
return policy
|
|
}
|
|
|
|
func failoverOverridePolicy(override map[string]any) map[string]any {
|
|
if nested, ok := override["failoverPolicy"].(map[string]any); ok {
|
|
return nested
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func failureInfoFromError(err error) failureInfo {
|
|
code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err)))
|
|
message := ""
|
|
if err != nil {
|
|
message = err.Error()
|
|
}
|
|
status := clients.ErrorResponseMetadata(err).StatusCode
|
|
category := failureCategory(code, status, message)
|
|
target := strings.ToLower(strings.TrimSpace(fmt.Sprintf("%s %s %d %s", code, category, status, message)))
|
|
return failureInfo{
|
|
Code: code,
|
|
Message: message,
|
|
Status: status,
|
|
Category: category,
|
|
Target: target,
|
|
}
|
|
}
|
|
|
|
func failureCategory(code string, status int, message string) string {
|
|
target := strings.ToLower(code + " " + message)
|
|
switch {
|
|
case code == "insufficient_balance":
|
|
return "insufficient_balance"
|
|
case code == "rate_limit" || status == 429:
|
|
return "rate_limit"
|
|
case code == "network":
|
|
return "network"
|
|
case code == "timeout" || status == 408 || strings.Contains(target, "timeout"):
|
|
return "timeout"
|
|
case code == "stream_read_error":
|
|
return "stream_error"
|
|
case code == "overloaded" || strings.Contains(target, "overloaded"):
|
|
return "provider_overloaded"
|
|
case status >= 500 || code == "server_error":
|
|
return "provider_5xx"
|
|
case code == "permission_denied":
|
|
return "user_permission"
|
|
case code == "auth_failed" || code == "invalid_api_key" || code == "missing_credentials" || status == 401 || status == 403 || providerAuthMessage(target):
|
|
return "auth_error"
|
|
case strings.Contains(code, "unsupported"):
|
|
return "unsupported_model"
|
|
case status == 400 || code == "bad_request" || code == "invalid_request" || code == "invalid_parameter" || code == "missing_required":
|
|
return "request_error"
|
|
case status > 400 && status < 500:
|
|
return "request_error"
|
|
default:
|
|
return "client_error"
|
|
}
|
|
}
|
|
|
|
func providerAuthMessage(target string) bool {
|
|
return strings.Contains(target, "api key") ||
|
|
strings.Contains(target, "apikey") ||
|
|
strings.Contains(target, "unauthorized") ||
|
|
strings.Contains(target, "authentication") ||
|
|
strings.Contains(target, "auth failed") ||
|
|
strings.Contains(target, "credential")
|
|
}
|
|
|
|
func hardStopPolicyMatches(policy map[string]any, info failureInfo) bool {
|
|
_, ok := hardStopPolicyMatch(policy, info)
|
|
return ok
|
|
}
|
|
|
|
func hardStopPolicyMatch(policy map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
|
if !boolFromPolicy(policy, "enabled", true) {
|
|
return policyRuleMatch{}, false
|
|
}
|
|
return firstPolicyMatch(policy, info, "gateway_runner_policies.hard_stop_policy", "hardStopPolicy", []policyMatchSpec{
|
|
{Key: "categories", Value: info.Category, Kind: "string"},
|
|
{Key: "codes", Value: info.Code, Kind: "string"},
|
|
{Key: "statusCodes", IntValue: info.Status, Kind: "int"},
|
|
{Key: "keywords", Value: info.Target, Kind: "keyword"},
|
|
})
|
|
}
|
|
|
|
func retryPolicyDenyMatches(policy map[string]any, info failureInfo) bool {
|
|
_, ok := retryPolicyDenyMatch(policy, info, "", "")
|
|
return ok
|
|
}
|
|
|
|
func retryPolicyDenyMatch(policy map[string]any, info failureInfo, source string, policyName string) (policyRuleMatch, bool) {
|
|
return firstPolicyMatch(policy, info, firstNonEmptyString(source, "effective_retry_policy"), firstNonEmptyString(policyName, "retryPolicy"), []policyMatchSpec{
|
|
{Key: "denyCategories", Value: info.Category, Kind: "string"},
|
|
{Key: "denyCodes", Value: info.Code, Kind: "string"},
|
|
{Key: "denyStatusCodes", IntValue: info.Status, Kind: "int"},
|
|
{Key: "denyKeywords", Value: info.Target, Kind: "keyword"},
|
|
})
|
|
}
|
|
|
|
func retryPolicyAllowMatches(policy map[string]any, info failureInfo) bool {
|
|
_, ok := retryPolicyAllowMatch(policy, info, "", "")
|
|
return ok
|
|
}
|
|
|
|
func retryPolicyAllowMatch(policy map[string]any, info failureInfo, source string, policyName string) (policyRuleMatch, bool) {
|
|
return firstPolicyMatch(policy, info, firstNonEmptyString(source, "effective_retry_policy"), firstNonEmptyString(policyName, "retryPolicy"), []policyMatchSpec{
|
|
{Key: "allowCategories", Value: info.Category, Kind: "string"},
|
|
{Key: "allowCodes", Value: info.Code, Kind: "string"},
|
|
{Key: "allowStatusCodes", IntValue: info.Status, Kind: "int"},
|
|
{Key: "allowKeywords", Value: info.Target, Kind: "keyword"},
|
|
})
|
|
}
|
|
|
|
func failoverDenyMatches(policy map[string]any, info failureInfo) bool {
|
|
_, ok := failoverDenyMatch(policy, info)
|
|
return ok
|
|
}
|
|
|
|
func failoverDenyMatch(policy map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
|
return retryPolicyDenyMatch(policy, info, "gateway_runner_policies.failover_policy", "failoverPolicy")
|
|
}
|
|
|
|
func failoverDenyMatchWithSources(base map[string]any, override map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
|
return retryPolicyMatchWithSources(base, override, "gateway_runner_policies.failover_policy", "runtime_policy_override.failoverPolicy", "failoverPolicy", []policyMatchSpec{
|
|
{Key: "denyCategories", Value: info.Category, Kind: "string"},
|
|
{Key: "denyCodes", Value: info.Code, Kind: "string"},
|
|
{Key: "denyStatusCodes", IntValue: info.Status, Kind: "int"},
|
|
{Key: "denyKeywords", Value: info.Target, Kind: "keyword"},
|
|
})
|
|
}
|
|
|
|
func failoverAllowMatches(policy map[string]any, info failureInfo) bool {
|
|
_, ok := failoverAllowMatch(policy, info)
|
|
return ok
|
|
}
|
|
|
|
func failoverAllowMatch(policy map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
|
return retryPolicyAllowMatch(policy, info, "gateway_runner_policies.failover_policy", "failoverPolicy")
|
|
}
|
|
|
|
func failoverAllowMatchWithSources(base map[string]any, override map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
|
return retryPolicyMatchWithSources(base, override, "gateway_runner_policies.failover_policy", "runtime_policy_override.failoverPolicy", "failoverPolicy", []policyMatchSpec{
|
|
{Key: "allowCategories", Value: info.Category, Kind: "string"},
|
|
{Key: "allowCodes", Value: info.Code, Kind: "string"},
|
|
{Key: "allowStatusCodes", IntValue: info.Status, Kind: "int"},
|
|
{Key: "allowKeywords", Value: info.Target, Kind: "keyword"},
|
|
})
|
|
}
|
|
|
|
func priorityDemotePolicyMatch(policy map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
|
return firstPolicyMatch(policy, info, "gateway_runner_policies.priority_demote_policy", "priorityDemotePolicy", []policyMatchSpec{
|
|
{Key: "categories", Value: info.Category, Kind: "string"},
|
|
{Key: "codes", Value: info.Code, Kind: "string"},
|
|
{Key: "statusCodes", IntValue: info.Status, Kind: "int"},
|
|
{Key: "keywords", Value: info.Target, Kind: "keyword"},
|
|
})
|
|
}
|
|
|
|
func failoverAction(policy map[string]any, info failureInfo) string {
|
|
actions, _ := policy["actions"].(map[string]any)
|
|
if action := stringFromAny(actions[info.Category]); action != "" {
|
|
return normalizeFailoverAction(action)
|
|
}
|
|
return "next"
|
|
}
|
|
|
|
func failoverActionRuleDecisionWithSources(base map[string]any, override map[string]any, info failureInfo) (failoverDecision, bool) {
|
|
if rules := actionRulesFromPolicy(override); len(rules) > 0 {
|
|
return failoverActionRuleDecision(rules, "runtime_policy_override.failoverPolicy", info)
|
|
}
|
|
return failoverActionRuleDecision(actionRulesFromPolicy(base), "gateway_runner_policies.failover_policy", info)
|
|
}
|
|
|
|
func candidateFailoverActionRuleDecision(base map[string]any, override map[string]any, candidate store.RuntimeModelCandidate, info failureInfo) (failoverDecision, bool) {
|
|
if rules := actionRulesFromPolicy(override); len(rules) > 0 {
|
|
if decision, ok := failoverActionRuleDecision(rules, "runtime_policy_override.failoverPolicy", info); ok {
|
|
return decision, true
|
|
}
|
|
}
|
|
autoDisablePolicy := effectiveRuntimePolicy(candidate.AutoDisablePolicy, candidate.RuntimePolicyOverride, "autoDisablePolicy")
|
|
if boolFromPolicy(autoDisablePolicy, "enabled", false) && intFromPolicy(autoDisablePolicy, "threshold") <= 1 {
|
|
if value, ok := matchingKeywordValue(autoDisablePolicy, "keywords", info.Target); ok {
|
|
return failoverDecision{
|
|
Retry: true,
|
|
Action: "disable_and_next",
|
|
Target: "platform",
|
|
Reason: "legacy_auto_disable_policy",
|
|
Match: policyRuleMatch{Source: "model_runtime_policy_sets.auto_disable_policy", Policy: "autoDisablePolicy", Rule: "keywords", Value: value},
|
|
Info: info,
|
|
}, true
|
|
}
|
|
}
|
|
degradePolicy := effectiveRuntimePolicy(candidate.DegradePolicy, candidate.RuntimePolicyOverride, "degradePolicy")
|
|
if boolFromPolicy(degradePolicy, "enabled", false) {
|
|
if value, ok := matchingKeywordValue(degradePolicy, "keywords", info.Target); ok {
|
|
cooldownSeconds := intFromPolicy(degradePolicy, "cooldownSeconds")
|
|
if cooldownSeconds <= 0 {
|
|
cooldownSeconds = 300
|
|
}
|
|
return failoverDecision{
|
|
Retry: true,
|
|
Action: "cooldown_and_next",
|
|
Target: "model",
|
|
Reason: "legacy_degrade_policy",
|
|
CooldownSeconds: cooldownSeconds,
|
|
Match: policyRuleMatch{Source: "model_runtime_policy_sets.degrade_policy", Policy: "degradePolicy", Rule: "keywords", Value: value},
|
|
Info: info,
|
|
}, true
|
|
}
|
|
}
|
|
return failoverActionRuleDecision(actionRulesFromPolicy(base), "gateway_runner_policies.failover_policy", info)
|
|
}
|
|
|
|
func failoverActionRuleDecision(rules []map[string]any, source string, info failureInfo) (failoverDecision, bool) {
|
|
for index, rule := range rules {
|
|
if !boolFromPolicy(rule, "enabled", true) {
|
|
continue
|
|
}
|
|
match, ok := failoverActionRuleMatch(rule, info, source, index)
|
|
if !ok {
|
|
continue
|
|
}
|
|
action := normalizeFailoverAction(stringFromAny(rule["action"]))
|
|
target := normalizeFailoverTarget(stringFromAny(rule["target"]), action)
|
|
cooldownSeconds := intFromPolicy(rule, "cooldownSeconds")
|
|
if cooldownSeconds <= 0 {
|
|
cooldownSeconds = 300
|
|
}
|
|
demoteSteps := intFromPolicy(rule, "demoteSteps")
|
|
if demoteSteps <= 0 {
|
|
demoteSteps = 1
|
|
}
|
|
if action == "stop" {
|
|
return failoverDecision{Retry: false, Action: "stop", Target: target, Reason: "failover_action_rule", CooldownSeconds: cooldownSeconds, DemoteSteps: demoteSteps, Match: match, Info: info}, true
|
|
}
|
|
return failoverDecision{Retry: true, Action: action, Target: target, Reason: "failover_action_rule", CooldownSeconds: cooldownSeconds, DemoteSteps: demoteSteps, Match: match, Info: info}, true
|
|
}
|
|
return failoverDecision{}, false
|
|
}
|
|
|
|
func actionRulesFromPolicy(policy map[string]any) []map[string]any {
|
|
raw, ok := policy["actionRules"].([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
rules := make([]map[string]any, 0, len(raw))
|
|
for _, item := range raw {
|
|
rule, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
rules = append(rules, rule)
|
|
}
|
|
return rules
|
|
}
|
|
|
|
func failoverActionRuleMatch(rule map[string]any, info failureInfo, source string, index int) (policyRuleMatch, bool) {
|
|
source = fmt.Sprintf("%s.actionRules[%d]", source, index)
|
|
if value, ok := matchingStringListValue(rule, "categories", info.Category); ok {
|
|
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "categories", Value: value}, true
|
|
}
|
|
if value, ok := matchingStringListValue(rule, "codes", info.Code); ok {
|
|
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "codes", Value: value}, true
|
|
}
|
|
if value, ok := matchingStringListValue(rule, "errorCodes", info.Code); ok {
|
|
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "errorCodes", Value: value}, true
|
|
}
|
|
if value, ok := matchingStringListValue(rule, "errorCodes", info.Category); ok {
|
|
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "errorCodes", Value: value}, true
|
|
}
|
|
if value, ok := matchingIntListValue(rule, "statusCodes", info.Status); ok {
|
|
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "statusCodes", Value: fmt.Sprintf("%d", value)}, true
|
|
}
|
|
if value, ok := matchingKeywordValue(rule, "keywords", info.Target); ok {
|
|
return policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "keywords", Value: value}, true
|
|
}
|
|
return policyRuleMatch{}, false
|
|
}
|
|
|
|
func normalizeFailoverAction(action string) string {
|
|
switch strings.TrimSpace(action) {
|
|
case "rotate":
|
|
return "next"
|
|
case "cooldown_and_rotate":
|
|
return "cooldown_and_next"
|
|
case "demote_and_rotate":
|
|
return "demote_and_next"
|
|
case "disable_and_rotate":
|
|
return "disable_and_next"
|
|
case "cooldown_and_next", "demote_and_next", "disable_and_next", "stop", "next":
|
|
return strings.TrimSpace(action)
|
|
default:
|
|
return "next"
|
|
}
|
|
}
|
|
|
|
func normalizeFailoverTarget(target string, action string) string {
|
|
target = strings.ToLower(strings.TrimSpace(target))
|
|
if target == "platform" || target == "model" {
|
|
return target
|
|
}
|
|
return defaultFailoverActionTarget(action)
|
|
}
|
|
|
|
func defaultFailoverActionTarget(action string) string {
|
|
switch action {
|
|
case "cooldown_and_next":
|
|
return "model"
|
|
default:
|
|
return "platform"
|
|
}
|
|
}
|
|
|
|
func boolFromPolicy(policy map[string]any, key string, fallback bool) bool {
|
|
value, ok := policy[key].(bool)
|
|
if !ok {
|
|
return fallback
|
|
}
|
|
return value
|
|
}
|
|
|
|
type policyMatchSpec struct {
|
|
Key string
|
|
Kind string
|
|
Value string
|
|
IntValue int
|
|
}
|
|
|
|
func firstPolicyMatch(policy map[string]any, info failureInfo, source string, policyName string, specs []policyMatchSpec) (policyRuleMatch, bool) {
|
|
for _, spec := range specs {
|
|
switch spec.Kind {
|
|
case "string":
|
|
if value, ok := matchingStringListValue(policy, spec.Key, spec.Value); ok {
|
|
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: value}, true
|
|
}
|
|
case "int":
|
|
if value, ok := matchingIntListValue(policy, spec.Key, spec.IntValue); ok {
|
|
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: fmt.Sprintf("%d", value)}, true
|
|
}
|
|
case "keyword":
|
|
if value, ok := matchingKeywordValue(policy, spec.Key, spec.Value); ok {
|
|
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: value}, true
|
|
}
|
|
}
|
|
}
|
|
return policyRuleMatch{}, false
|
|
}
|
|
|
|
func retryPolicyMatchWithSources(base map[string]any, override map[string]any, baseSource string, overrideSource string, policyName string, specs []policyMatchSpec) (policyRuleMatch, bool) {
|
|
for _, spec := range specs {
|
|
if _, ok := override[spec.Key]; ok {
|
|
if match, matched := policyMatchSpecValue(override, spec, overrideSource, policyName); matched {
|
|
return match, true
|
|
}
|
|
continue
|
|
}
|
|
if match, matched := policyMatchSpecValue(base, spec, baseSource, policyName); matched {
|
|
return match, true
|
|
}
|
|
}
|
|
return policyRuleMatch{}, false
|
|
}
|
|
|
|
func policyMatchSpecValue(policy map[string]any, spec policyMatchSpec, source string, policyName string) (policyRuleMatch, bool) {
|
|
switch spec.Kind {
|
|
case "string":
|
|
if value, ok := matchingStringListValue(policy, spec.Key, spec.Value); ok {
|
|
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: value}, true
|
|
}
|
|
case "int":
|
|
if value, ok := matchingIntListValue(policy, spec.Key, spec.IntValue); ok {
|
|
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: fmt.Sprintf("%d", value)}, true
|
|
}
|
|
case "keyword":
|
|
if value, ok := matchingKeywordValue(policy, spec.Key, spec.Value); ok {
|
|
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: value}, true
|
|
}
|
|
}
|
|
return policyRuleMatch{}, false
|
|
}
|
|
|
|
func stringListContains(policy map[string]any, key string, value string) bool {
|
|
_, ok := matchingStringListValue(policy, key, value)
|
|
return ok
|
|
}
|
|
|
|
func matchingStringListValue(policy map[string]any, key string, value string) (string, bool) {
|
|
value = strings.ToLower(strings.TrimSpace(value))
|
|
if value == "" {
|
|
return "", false
|
|
}
|
|
for _, item := range stringListFromPolicy(policy, key) {
|
|
item = strings.TrimSpace(item)
|
|
if strings.ToLower(item) == value {
|
|
return item, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func keywordListMatches(policy map[string]any, key string, target string) bool {
|
|
_, ok := matchingKeywordValue(policy, key, target)
|
|
return ok
|
|
}
|
|
|
|
func matchingKeywordValue(policy map[string]any, key string, target string) (string, bool) {
|
|
target = strings.ToLower(strings.TrimSpace(target))
|
|
if target == "" {
|
|
return "", false
|
|
}
|
|
for _, keyword := range stringListFromPolicy(policy, key) {
|
|
keyword = strings.TrimSpace(keyword)
|
|
if keyword != "" && strings.Contains(target, strings.ToLower(keyword)) {
|
|
return keyword, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func intListContains(policy map[string]any, key string, value int) bool {
|
|
_, ok := matchingIntListValue(policy, key, value)
|
|
return ok
|
|
}
|
|
|
|
func matchingIntListValue(policy map[string]any, key string, value int) (int, bool) {
|
|
if value == 0 {
|
|
return 0, false
|
|
}
|
|
for _, item := range intListFromPolicy(policy, key) {
|
|
if item == value {
|
|
return item, true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func intListFromPolicy(policy map[string]any, key string) []int {
|
|
raw, ok := policy[key].([]any)
|
|
if !ok {
|
|
if typed, ok := policy[key].([]int); ok {
|
|
return typed
|
|
}
|
|
return nil
|
|
}
|
|
out := make([]int, 0, len(raw))
|
|
for _, item := range raw {
|
|
switch typed := item.(type) {
|
|
case int:
|
|
out = append(out, typed)
|
|
case float64:
|
|
out = append(out, int(typed))
|
|
case string:
|
|
if parsed := strings.TrimSpace(typed); parsed != "" {
|
|
var value int
|
|
if _, err := fmt.Sscanf(parsed, "%d", &value); err == nil && value > 0 {
|
|
out = append(out, value)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|