feat: add runner failover policies and traces
This commit is contained in:
@@ -182,9 +182,12 @@ func failureMetrics(err error, simulated bool) (string, map[string]any, time.Tim
|
||||
metrics := map[string]any{
|
||||
"simulated": simulated,
|
||||
}
|
||||
retryable := clients.IsRetryable(err)
|
||||
if err != nil {
|
||||
info := failureInfoFromError(err)
|
||||
metrics["error"] = err.Error()
|
||||
metrics["retryable"] = clients.IsRetryable(err)
|
||||
metrics["errorCategory"] = info.Category
|
||||
metrics["retryable"] = retryable
|
||||
}
|
||||
if meta.StatusCode > 0 {
|
||||
metrics["statusCode"] = meta.StatusCode
|
||||
@@ -195,6 +198,9 @@ func failureMetrics(err error, simulated bool) (string, map[string]any, time.Tim
|
||||
if meta.ResponseDurationMS > 0 {
|
||||
metrics["responseDurationMs"] = meta.ResponseDurationMS
|
||||
}
|
||||
if err != nil {
|
||||
metrics["trace"] = []any{failureTraceEntry(err, retryable)}
|
||||
}
|
||||
return meta.RequestID, metrics, meta.ResponseStartedAt, meta.ResponseFinishedAt, meta.ResponseDurationMS
|
||||
}
|
||||
|
||||
@@ -227,12 +233,16 @@ func summarizeAttempts(attempts []store.TaskAttempt) []map[string]any {
|
||||
"requestId": attempt.RequestID,
|
||||
"retryable": attempt.Retryable,
|
||||
"simulated": attempt.Simulated,
|
||||
"statusCode": attempt.StatusCode,
|
||||
"errorCode": attempt.ErrorCode,
|
||||
"errorMessage": attempt.ErrorMessage,
|
||||
"responseDurationMs": attempt.ResponseDurationMS,
|
||||
"startedAt": attempt.StartedAt,
|
||||
"finishedAt": attempt.FinishedAt,
|
||||
}
|
||||
if trace, ok := attempt.Metrics["trace"]; ok {
|
||||
item["trace"] = trace
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
type failureInfo struct {
|
||||
Code string
|
||||
Message string
|
||||
Status int
|
||||
Category string
|
||||
Target string
|
||||
}
|
||||
|
||||
type policyRuleMatch struct {
|
||||
Source string
|
||||
Policy string
|
||||
Rule string
|
||||
Value string
|
||||
}
|
||||
|
||||
type retryDecision struct {
|
||||
Retry bool
|
||||
Reason string
|
||||
Match policyRuleMatch
|
||||
Info failureInfo
|
||||
}
|
||||
|
||||
type failoverDecision struct {
|
||||
Retry bool
|
||||
Action string
|
||||
Reason string
|
||||
CooldownSeconds int
|
||||
Match policyRuleMatch
|
||||
Info failureInfo
|
||||
}
|
||||
|
||||
type priorityDemoteDecision struct {
|
||||
Demote bool
|
||||
Reason string
|
||||
Step int
|
||||
Match policyRuleMatch
|
||||
Info failureInfo
|
||||
}
|
||||
|
||||
func shouldRetrySameClient(candidate store.RuntimeModelCandidate, err error) bool {
|
||||
return retryDecisionForCandidate(candidate, err).Retry
|
||||
}
|
||||
|
||||
func retryDecisionForCandidate(candidate store.RuntimeModelCandidate, err error) retryDecision {
|
||||
policy := effectiveRetryPolicy(candidate)
|
||||
info := failureInfoFromError(err)
|
||||
if !boolFromPolicy(policy, "enabled", true) {
|
||||
return retryDecision{Retry: false, Reason: "retry_disabled", Match: policyRuleMatch{Source: "model_runtime_policy_sets.retry_policy", Policy: "retryPolicy", Rule: "enabled", Value: "false"}, Info: info}
|
||||
}
|
||||
if match, ok := retryPolicyDenyMatch(policy, info, "model_runtime_policy_sets.retry_policy", "retryPolicy"); ok {
|
||||
return retryDecision{Retry: false, Reason: "retry_deny_policy", Match: match, Info: info}
|
||||
}
|
||||
if match, ok := retryPolicyAllowMatch(policy, info, "model_runtime_policy_sets.retry_policy", "retryPolicy"); ok {
|
||||
return retryDecision{Retry: true, Reason: "retry_allow_policy", Match: match, Info: info}
|
||||
}
|
||||
if clients.IsRetryable(err) {
|
||||
return retryDecision{Retry: true, Reason: "client_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "true"}, Info: info}
|
||||
}
|
||||
return retryDecision{Retry: false, Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
|
||||
}
|
||||
|
||||
func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate store.RuntimeModelCandidate, err error) failoverDecision {
|
||||
info := failureInfoFromError(err)
|
||||
if strings.TrimSpace(runnerPolicy.Status) != "" && runnerPolicy.Status != "active" {
|
||||
return failoverDecision{Retry: false, Action: "stop", Reason: "runner_policy_disabled", Match: policyRuleMatch{Source: "gateway_runner_policies", Policy: "runnerPolicy", Rule: "status", Value: runnerPolicy.Status}, Info: info}
|
||||
}
|
||||
if match, ok := hardStopPolicyMatch(runnerPolicy.HardStopPolicy, info); ok {
|
||||
return failoverDecision{Retry: false, Action: "stop", Reason: "hard_stop_policy", Match: match, Info: info}
|
||||
}
|
||||
overridePolicy := failoverOverridePolicy(candidate.RuntimePolicyOverride)
|
||||
policy := effectiveFailoverPolicy(runnerPolicy.FailoverPolicy, candidate.RuntimePolicyOverride)
|
||||
if !boolFromPolicy(policy, "enabled", true) {
|
||||
source := "gateway_runner_policies.failover_policy"
|
||||
if _, ok := overridePolicy["enabled"]; ok {
|
||||
source = "runtime_policy_override.failoverPolicy"
|
||||
}
|
||||
return failoverDecision{Retry: false, Action: "stop", Reason: "failover_disabled", Match: policyRuleMatch{Source: source, Policy: "failoverPolicy", Rule: "enabled", Value: "false"}, Info: info}
|
||||
}
|
||||
if match, ok := failoverDenyMatchWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
|
||||
return failoverDecision{Retry: false, Action: "stop", Reason: "failover_deny_policy", Match: match, Info: info}
|
||||
}
|
||||
action := failoverAction(policy, info)
|
||||
cooldownSeconds := intFromPolicy(policy, "cooldownSeconds")
|
||||
if cooldownSeconds <= 0 {
|
||||
cooldownSeconds = 300
|
||||
}
|
||||
if match, ok := failoverAllowMatchWithSources(runnerPolicy.FailoverPolicy, overridePolicy, info); ok {
|
||||
return failoverDecision{Retry: true, Action: action, Reason: "failover_allow_policy", CooldownSeconds: cooldownSeconds, Match: match, Info: info}
|
||||
}
|
||||
if clients.IsRetryable(err) {
|
||||
return failoverDecision{Retry: true, Action: action, Reason: "client_retryable", CooldownSeconds: cooldownSeconds, Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "true"}, Info: info}
|
||||
}
|
||||
return failoverDecision{Retry: false, Action: "stop", Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
|
||||
}
|
||||
|
||||
func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) (bool, int) {
|
||||
decision := priorityDemoteDecisionForCandidate(runnerPolicy, err)
|
||||
return decision.Demote, decision.Step
|
||||
}
|
||||
|
||||
func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err error) priorityDemoteDecision {
|
||||
info := failureInfoFromError(err)
|
||||
if strings.TrimSpace(runnerPolicy.Status) != "" && runnerPolicy.Status != "active" {
|
||||
return priorityDemoteDecision{Demote: false, Reason: "runner_policy_disabled", Info: info}
|
||||
}
|
||||
if hardStopPolicyMatches(runnerPolicy.HardStopPolicy, info) {
|
||||
return priorityDemoteDecision{Demote: false, Reason: "hard_stop_policy", Info: info}
|
||||
}
|
||||
policy := runnerPolicy.PriorityDemotePolicy
|
||||
if !boolFromPolicy(policy, "enabled", false) {
|
||||
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_disabled", Info: info}
|
||||
}
|
||||
if match, ok := priorityDemotePolicyMatch(policy, info); ok {
|
||||
step := intFromPolicy(policy, "demoteStep")
|
||||
if step <= 0 {
|
||||
step = 100
|
||||
}
|
||||
return priorityDemoteDecision{Demote: true, Reason: "priority_demote_policy", Step: step, Match: match, Info: info}
|
||||
}
|
||||
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_no_match", Info: info}
|
||||
}
|
||||
|
||||
func effectiveFailoverPolicy(base map[string]any, override map[string]any) map[string]any {
|
||||
policy := base
|
||||
if nested := failoverOverridePolicy(override); len(nested) > 0 {
|
||||
policy = mergeMap(policy, nested)
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
func failoverOverridePolicy(override map[string]any) map[string]any {
|
||||
if nested, ok := override["failoverPolicy"].(map[string]any); ok {
|
||||
return nested
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func failureInfoFromError(err error) failureInfo {
|
||||
code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err)))
|
||||
message := ""
|
||||
if err != nil {
|
||||
message = err.Error()
|
||||
}
|
||||
status := clients.ErrorResponseMetadata(err).StatusCode
|
||||
category := failureCategory(code, status, message)
|
||||
target := strings.ToLower(strings.TrimSpace(fmt.Sprintf("%s %s %d %s", code, category, status, message)))
|
||||
return failureInfo{
|
||||
Code: code,
|
||||
Message: message,
|
||||
Status: status,
|
||||
Category: category,
|
||||
Target: target,
|
||||
}
|
||||
}
|
||||
|
||||
func failureCategory(code string, status int, message string) string {
|
||||
target := strings.ToLower(code + " " + message)
|
||||
switch {
|
||||
case code == "insufficient_balance":
|
||||
return "insufficient_balance"
|
||||
case code == "rate_limit" || status == 429:
|
||||
return "rate_limit"
|
||||
case code == "network":
|
||||
return "network"
|
||||
case code == "timeout" || status == 408 || strings.Contains(target, "timeout"):
|
||||
return "timeout"
|
||||
case code == "stream_read_error":
|
||||
return "stream_error"
|
||||
case code == "overloaded" || strings.Contains(target, "overloaded"):
|
||||
return "provider_overloaded"
|
||||
case status >= 500 || code == "server_error":
|
||||
return "provider_5xx"
|
||||
case code == "permission_denied":
|
||||
return "user_permission"
|
||||
case code == "auth_failed" || code == "invalid_api_key" || code == "missing_credentials" || status == 401 || status == 403 || providerAuthMessage(target):
|
||||
return "auth_error"
|
||||
case strings.Contains(code, "unsupported"):
|
||||
return "unsupported_model"
|
||||
case status == 400 || code == "bad_request" || code == "invalid_request" || code == "invalid_parameter" || code == "missing_required":
|
||||
return "request_error"
|
||||
case status > 400 && status < 500:
|
||||
return "request_error"
|
||||
default:
|
||||
return "client_error"
|
||||
}
|
||||
}
|
||||
|
||||
func providerAuthMessage(target string) bool {
|
||||
return strings.Contains(target, "api key") ||
|
||||
strings.Contains(target, "apikey") ||
|
||||
strings.Contains(target, "unauthorized") ||
|
||||
strings.Contains(target, "authentication") ||
|
||||
strings.Contains(target, "auth failed") ||
|
||||
strings.Contains(target, "credential")
|
||||
}
|
||||
|
||||
func hardStopPolicyMatches(policy map[string]any, info failureInfo) bool {
|
||||
_, ok := hardStopPolicyMatch(policy, info)
|
||||
return ok
|
||||
}
|
||||
|
||||
func hardStopPolicyMatch(policy map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
||||
if !boolFromPolicy(policy, "enabled", true) {
|
||||
return policyRuleMatch{}, false
|
||||
}
|
||||
return firstPolicyMatch(policy, info, "gateway_runner_policies.hard_stop_policy", "hardStopPolicy", []policyMatchSpec{
|
||||
{Key: "categories", Value: info.Category, Kind: "string"},
|
||||
{Key: "codes", Value: info.Code, Kind: "string"},
|
||||
{Key: "statusCodes", IntValue: info.Status, Kind: "int"},
|
||||
{Key: "keywords", Value: info.Target, Kind: "keyword"},
|
||||
})
|
||||
}
|
||||
|
||||
func retryPolicyDenyMatches(policy map[string]any, info failureInfo) bool {
|
||||
_, ok := retryPolicyDenyMatch(policy, info, "", "")
|
||||
return ok
|
||||
}
|
||||
|
||||
func retryPolicyDenyMatch(policy map[string]any, info failureInfo, source string, policyName string) (policyRuleMatch, bool) {
|
||||
return firstPolicyMatch(policy, info, firstNonEmptyString(source, "effective_retry_policy"), firstNonEmptyString(policyName, "retryPolicy"), []policyMatchSpec{
|
||||
{Key: "denyCategories", Value: info.Category, Kind: "string"},
|
||||
{Key: "denyCodes", Value: info.Code, Kind: "string"},
|
||||
{Key: "denyStatusCodes", IntValue: info.Status, Kind: "int"},
|
||||
{Key: "denyKeywords", Value: info.Target, Kind: "keyword"},
|
||||
})
|
||||
}
|
||||
|
||||
func retryPolicyAllowMatches(policy map[string]any, info failureInfo) bool {
|
||||
_, ok := retryPolicyAllowMatch(policy, info, "", "")
|
||||
return ok
|
||||
}
|
||||
|
||||
func retryPolicyAllowMatch(policy map[string]any, info failureInfo, source string, policyName string) (policyRuleMatch, bool) {
|
||||
return firstPolicyMatch(policy, info, firstNonEmptyString(source, "effective_retry_policy"), firstNonEmptyString(policyName, "retryPolicy"), []policyMatchSpec{
|
||||
{Key: "allowCategories", Value: info.Category, Kind: "string"},
|
||||
{Key: "allowCodes", Value: info.Code, Kind: "string"},
|
||||
{Key: "allowStatusCodes", IntValue: info.Status, Kind: "int"},
|
||||
{Key: "allowKeywords", Value: info.Target, Kind: "keyword"},
|
||||
})
|
||||
}
|
||||
|
||||
func failoverDenyMatches(policy map[string]any, info failureInfo) bool {
|
||||
_, ok := failoverDenyMatch(policy, info)
|
||||
return ok
|
||||
}
|
||||
|
||||
func failoverDenyMatch(policy map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
||||
return retryPolicyDenyMatch(policy, info, "gateway_runner_policies.failover_policy", "failoverPolicy")
|
||||
}
|
||||
|
||||
func failoverDenyMatchWithSources(base map[string]any, override map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
||||
return retryPolicyMatchWithSources(base, override, "gateway_runner_policies.failover_policy", "runtime_policy_override.failoverPolicy", "failoverPolicy", []policyMatchSpec{
|
||||
{Key: "denyCategories", Value: info.Category, Kind: "string"},
|
||||
{Key: "denyCodes", Value: info.Code, Kind: "string"},
|
||||
{Key: "denyStatusCodes", IntValue: info.Status, Kind: "int"},
|
||||
{Key: "denyKeywords", Value: info.Target, Kind: "keyword"},
|
||||
})
|
||||
}
|
||||
|
||||
func failoverAllowMatches(policy map[string]any, info failureInfo) bool {
|
||||
_, ok := failoverAllowMatch(policy, info)
|
||||
return ok
|
||||
}
|
||||
|
||||
func failoverAllowMatch(policy map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
||||
return retryPolicyAllowMatch(policy, info, "gateway_runner_policies.failover_policy", "failoverPolicy")
|
||||
}
|
||||
|
||||
func failoverAllowMatchWithSources(base map[string]any, override map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
||||
return retryPolicyMatchWithSources(base, override, "gateway_runner_policies.failover_policy", "runtime_policy_override.failoverPolicy", "failoverPolicy", []policyMatchSpec{
|
||||
{Key: "allowCategories", Value: info.Category, Kind: "string"},
|
||||
{Key: "allowCodes", Value: info.Code, Kind: "string"},
|
||||
{Key: "allowStatusCodes", IntValue: info.Status, Kind: "int"},
|
||||
{Key: "allowKeywords", Value: info.Target, Kind: "keyword"},
|
||||
})
|
||||
}
|
||||
|
||||
func priorityDemotePolicyMatch(policy map[string]any, info failureInfo) (policyRuleMatch, bool) {
|
||||
return firstPolicyMatch(policy, info, "gateway_runner_policies.priority_demote_policy", "priorityDemotePolicy", []policyMatchSpec{
|
||||
{Key: "categories", Value: info.Category, Kind: "string"},
|
||||
{Key: "codes", Value: info.Code, Kind: "string"},
|
||||
{Key: "statusCodes", IntValue: info.Status, Kind: "int"},
|
||||
{Key: "keywords", Value: info.Target, Kind: "keyword"},
|
||||
})
|
||||
}
|
||||
|
||||
func failoverAction(policy map[string]any, info failureInfo) string {
|
||||
actions, _ := policy["actions"].(map[string]any)
|
||||
if action := stringFromAny(actions[info.Category]); action != "" {
|
||||
return action
|
||||
}
|
||||
return "next"
|
||||
}
|
||||
|
||||
func boolFromPolicy(policy map[string]any, key string, fallback bool) bool {
|
||||
value, ok := policy[key].(bool)
|
||||
if !ok {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type policyMatchSpec struct {
|
||||
Key string
|
||||
Kind string
|
||||
Value string
|
||||
IntValue int
|
||||
}
|
||||
|
||||
func firstPolicyMatch(policy map[string]any, info failureInfo, source string, policyName string, specs []policyMatchSpec) (policyRuleMatch, bool) {
|
||||
for _, spec := range specs {
|
||||
switch spec.Kind {
|
||||
case "string":
|
||||
if value, ok := matchingStringListValue(policy, spec.Key, spec.Value); ok {
|
||||
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: value}, true
|
||||
}
|
||||
case "int":
|
||||
if value, ok := matchingIntListValue(policy, spec.Key, spec.IntValue); ok {
|
||||
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: fmt.Sprintf("%d", value)}, true
|
||||
}
|
||||
case "keyword":
|
||||
if value, ok := matchingKeywordValue(policy, spec.Key, spec.Value); ok {
|
||||
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: value}, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return policyRuleMatch{}, false
|
||||
}
|
||||
|
||||
func retryPolicyMatchWithSources(base map[string]any, override map[string]any, baseSource string, overrideSource string, policyName string, specs []policyMatchSpec) (policyRuleMatch, bool) {
|
||||
for _, spec := range specs {
|
||||
if _, ok := override[spec.Key]; ok {
|
||||
if match, matched := policyMatchSpecValue(override, spec, overrideSource, policyName); matched {
|
||||
return match, true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if match, matched := policyMatchSpecValue(base, spec, baseSource, policyName); matched {
|
||||
return match, true
|
||||
}
|
||||
}
|
||||
return policyRuleMatch{}, false
|
||||
}
|
||||
|
||||
func policyMatchSpecValue(policy map[string]any, spec policyMatchSpec, source string, policyName string) (policyRuleMatch, bool) {
|
||||
switch spec.Kind {
|
||||
case "string":
|
||||
if value, ok := matchingStringListValue(policy, spec.Key, spec.Value); ok {
|
||||
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: value}, true
|
||||
}
|
||||
case "int":
|
||||
if value, ok := matchingIntListValue(policy, spec.Key, spec.IntValue); ok {
|
||||
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: fmt.Sprintf("%d", value)}, true
|
||||
}
|
||||
case "keyword":
|
||||
if value, ok := matchingKeywordValue(policy, spec.Key, spec.Value); ok {
|
||||
return policyRuleMatch{Source: source, Policy: policyName, Rule: spec.Key, Value: value}, true
|
||||
}
|
||||
}
|
||||
return policyRuleMatch{}, false
|
||||
}
|
||||
|
||||
func stringListContains(policy map[string]any, key string, value string) bool {
|
||||
_, ok := matchingStringListValue(policy, key, value)
|
||||
return ok
|
||||
}
|
||||
|
||||
func matchingStringListValue(policy map[string]any, key string, value string) (string, bool) {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value == "" {
|
||||
return "", false
|
||||
}
|
||||
for _, item := range stringListFromPolicy(policy, key) {
|
||||
item = strings.TrimSpace(item)
|
||||
if strings.ToLower(item) == value {
|
||||
return item, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func keywordListMatches(policy map[string]any, key string, target string) bool {
|
||||
_, ok := matchingKeywordValue(policy, key, target)
|
||||
return ok
|
||||
}
|
||||
|
||||
func matchingKeywordValue(policy map[string]any, key string, target string) (string, bool) {
|
||||
target = strings.ToLower(strings.TrimSpace(target))
|
||||
if target == "" {
|
||||
return "", false
|
||||
}
|
||||
for _, keyword := range stringListFromPolicy(policy, key) {
|
||||
keyword = strings.TrimSpace(keyword)
|
||||
if keyword != "" && strings.Contains(target, strings.ToLower(keyword)) {
|
||||
return keyword, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func intListContains(policy map[string]any, key string, value int) bool {
|
||||
_, ok := matchingIntListValue(policy, key, value)
|
||||
return ok
|
||||
}
|
||||
|
||||
func matchingIntListValue(policy map[string]any, key string, value int) (int, bool) {
|
||||
if value == 0 {
|
||||
return 0, false
|
||||
}
|
||||
for _, item := range intListFromPolicy(policy, key) {
|
||||
if item == value {
|
||||
return item, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func intListFromPolicy(policy map[string]any, key string) []int {
|
||||
raw, ok := policy[key].([]any)
|
||||
if !ok {
|
||||
if typed, ok := policy[key].([]int); ok {
|
||||
return typed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
out := make([]int, 0, len(raw))
|
||||
for _, item := range raw {
|
||||
switch typed := item.(type) {
|
||||
case int:
|
||||
out = append(out, typed)
|
||||
case float64:
|
||||
out = append(out, int(typed))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestShouldRetrySameClientUsesRuntimeRetryPolicyKeywords(t *testing.T) {
|
||||
candidate := store.RuntimeModelCandidate{
|
||||
ModelRetryPolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"allowKeywords": []any{"temporary vendor blip"},
|
||||
"denyKeywords": []any{"bad request"},
|
||||
},
|
||||
}
|
||||
|
||||
if shouldRetrySameClient(candidate, &clients.ClientError{Code: "bad_request", Message: "bad request timeout", Retryable: true}) {
|
||||
t.Fatal("deny keywords should block same-client retry even when the client marks the error retryable")
|
||||
}
|
||||
if !shouldRetrySameClient(candidate, &clients.ClientError{Code: "custom_error", Message: "temporary vendor blip", Retryable: false}) {
|
||||
t.Fatal("allow keywords should allow same-client retry when the client does not mark the error retryable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverBudgetDefaults(t *testing.T) {
|
||||
candidates := make([]store.RuntimeModelCandidate, 150)
|
||||
runnerPolicy := store.RunnerPolicy{Status: "active", FailoverPolicy: map[string]any{"enabled": true}}
|
||||
|
||||
if got := maxPlatformsForCandidates(candidates, runnerPolicy); got != 99 {
|
||||
t.Fatalf("default max platform budget should be 99, got %d", got)
|
||||
}
|
||||
if got := maxFailoverDurationForCandidates(candidates, runnerPolicy); got != 10*time.Minute {
|
||||
t.Fatalf("default max failover duration should be 10 minutes, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverTimeBudgetExceeded(t *testing.T) {
|
||||
if !failoverTimeBudgetExceeded(time.Now().Add(-601*time.Second), 10*time.Minute) {
|
||||
t.Fatal("failover time budget should stop retries after the configured duration")
|
||||
}
|
||||
if failoverTimeBudgetExceeded(time.Now().Add(-590*time.Second), 10*time.Minute) {
|
||||
t.Fatal("failover time budget should allow retries before the configured duration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverHardStopBeatsModelOverride(t *testing.T) {
|
||||
runnerPolicy := store.RunnerPolicy{
|
||||
Status: "active",
|
||||
FailoverPolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"allowCategories": []any{"request_error"},
|
||||
},
|
||||
HardStopPolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"request_error"},
|
||||
},
|
||||
}
|
||||
candidate := store.RuntimeModelCandidate{
|
||||
RuntimePolicyOverride: map[string]any{
|
||||
"failoverPolicy": map[string]any{
|
||||
"enabled": true,
|
||||
"allowCategories": []any{"request_error"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
decision := failoverDecisionForCandidate(runnerPolicy, candidate, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true})
|
||||
if decision.Retry || decision.Reason != "hard_stop_policy" {
|
||||
t.Fatalf("hard stop should block model-level failover override, got %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverPolicyAllowsModelOverride(t *testing.T) {
|
||||
runnerPolicy := store.RunnerPolicy{
|
||||
Status: "active",
|
||||
FailoverPolicy: map[string]any{"enabled": true},
|
||||
HardStopPolicy: map[string]any{"enabled": true, "categories": []any{"request_error"}},
|
||||
}
|
||||
candidate := store.RuntimeModelCandidate{
|
||||
RuntimePolicyOverride: map[string]any{
|
||||
"failoverPolicy": map[string]any{
|
||||
"allowKeywords": []any{"temporary upstream outage"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
decision := failoverDecisionForCandidate(runnerPolicy, candidate, &clients.ClientError{Code: "custom_error", Message: "temporary upstream outage", Retryable: false})
|
||||
if !decision.Retry || decision.Reason != "failover_allow_policy" {
|
||||
t.Fatalf("model failoverPolicy override should allow cross-platform failover, got %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderAuthErrorsFailOverInsteadOfHardStop(t *testing.T) {
|
||||
runnerPolicy := store.RunnerPolicy{
|
||||
Status: "active",
|
||||
FailoverPolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"allowCategories": []any{"auth_error"},
|
||||
"allowCodes": []any{"auth_failed", "invalid_api_key", "missing_credentials"},
|
||||
"allowStatusCodes": []any{
|
||||
401,
|
||||
403,
|
||||
},
|
||||
"actions": map[string]any{"auth_error": "disable_and_next"},
|
||||
},
|
||||
HardStopPolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "permission_denied"},
|
||||
"statusCodes": []any{},
|
||||
"keywords": []any{"invalid_parameter", "missing required", "bad request", "insufficient balance"},
|
||||
},
|
||||
}
|
||||
|
||||
decision := failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "auth_failed", StatusCode: 401, Retryable: false})
|
||||
if !decision.Retry || decision.Action != "disable_and_next" || decision.Reason != "failover_allow_policy" {
|
||||
t.Fatalf("provider auth failures should switch platform, got %+v", decision)
|
||||
}
|
||||
|
||||
decision = failoverDecisionForCandidate(runnerPolicy, store.RuntimeModelCandidate{}, &clients.ClientError{Code: "http_400", Message: "invalid api key", StatusCode: 400, Retryable: false})
|
||||
if !decision.Retry || decision.Info.Category != "auth_error" {
|
||||
t.Fatalf("provider auth-looking 400 should switch platform, got %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPriorityDemotePolicyIsKeywordGatedAndHardStopSafe(t *testing.T) {
|
||||
runnerPolicy := store.RunnerPolicy{
|
||||
Status: "active",
|
||||
HardStopPolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"request_error"},
|
||||
},
|
||||
PriorityDemotePolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"demoteStep": 25,
|
||||
"keywords": []any{"rate_limit"},
|
||||
},
|
||||
}
|
||||
|
||||
shouldDemote, step := shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true})
|
||||
if !shouldDemote || step != 25 {
|
||||
t.Fatalf("priority demotion should be enabled only by matched policy, got shouldDemote=%v step=%d", shouldDemote, step)
|
||||
}
|
||||
|
||||
shouldDemote, _ = shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true})
|
||||
if shouldDemote {
|
||||
t.Fatal("priority demotion should not run for hard-stop request errors")
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,46 @@ func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID stri
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) applyFailoverAction(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, decision failoverDecision, simulated bool) {
|
||||
switch decision.Action {
|
||||
case "disable_and_next":
|
||||
if err := s.store.DisableCandidatePlatform(ctx, candidate.PlatformID); err == nil {
|
||||
_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate platform disabled by runner failover policy", addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"action": decision.Action,
|
||||
"reason": decision.Reason,
|
||||
}, decision.Match, decision.Info), simulated)
|
||||
}
|
||||
case "cooldown_and_next":
|
||||
if err := s.store.CooldownCandidatePlatform(ctx, candidate.PlatformID, decision.CooldownSeconds); err == nil {
|
||||
_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate platform cooled down by runner failover policy", addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"cooldownSeconds": decision.CooldownSeconds,
|
||||
"action": decision.Action,
|
||||
"reason": decision.Reason,
|
||||
}, decision.Match, decision.Info), simulated)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) applyPriorityDemotePolicy(ctx context.Context, taskID string, attemptNo int, runnerPolicy store.RunnerPolicy, candidate store.RuntimeModelCandidate, cause error, simulated bool) {
|
||||
decision := priorityDemoteDecisionForCandidate(runnerPolicy, cause)
|
||||
if !decision.Demote {
|
||||
return
|
||||
}
|
||||
if err := s.store.DemoteCandidatePlatformPriority(ctx, candidate.PlatformID, decision.Step); err == nil {
|
||||
s.recordAttemptTrace(ctx, taskID, attemptNo, priorityDemoteTraceEntry(decision, candidate.PlatformID, candidate.PlatformModelID))
|
||||
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner policy", addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"demoteStep": decision.Step,
|
||||
"code": clients.ErrorCode(cause),
|
||||
}, decision.Match, decision.Info), simulated)
|
||||
}
|
||||
}
|
||||
|
||||
func effectiveRuntimePolicy(base map[string]any, override map[string]any, key string) map[string]any {
|
||||
policy := base
|
||||
if nested, ok := override[key].(map[string]any); ok && len(nested) > 0 {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -51,6 +52,7 @@ func (s *Service) ExecuteStream(ctx context.Context, task store.GatewayTask, use
|
||||
}
|
||||
|
||||
func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
|
||||
executeStartedAt := time.Now()
|
||||
body := normalizeRequest(task.Kind, task.Request)
|
||||
modelType := modelTypeFromKind(task.Kind, body)
|
||||
if err := validateRequest(task.Kind, body); err != nil {
|
||||
@@ -88,64 +90,159 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
maxAttempts := maxAttemptsForCandidates(candidates)
|
||||
runnerPolicy, err := s.store.GetActiveRunnerPolicy(ctx)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
maxPlatforms := maxPlatformsForCandidates(candidates, runnerPolicy)
|
||||
maxFailoverDuration := maxFailoverDurationForCandidates(candidates, runnerPolicy)
|
||||
attemptNo := 0
|
||||
var lastErr error
|
||||
for index, candidate := range candidates {
|
||||
if index >= maxAttempts {
|
||||
if index >= maxPlatforms {
|
||||
break
|
||||
}
|
||||
attemptNo := index + 1
|
||||
response, err := s.runCandidate(ctx, task, user, body, candidate, attemptNo, onDelta)
|
||||
if err == nil {
|
||||
billings := s.billings(ctx, user, task.Kind, body, candidate, response, isSimulation(task, candidate))
|
||||
record := buildSuccessRecord(task, user, body, candidate, response, billings, isSimulation(task, candidate))
|
||||
record.Metrics = s.withAttemptHistory(ctx, task.ID, record.Metrics)
|
||||
finished, finishErr := s.store.FinishTaskSuccess(ctx, store.FinishTaskSuccessInput{
|
||||
TaskID: task.ID,
|
||||
Result: response.Result,
|
||||
Billings: billings,
|
||||
RequestID: record.RequestID,
|
||||
ResolvedModel: record.ResolvedModel,
|
||||
Usage: record.Usage,
|
||||
Metrics: record.Metrics,
|
||||
BillingSummary: record.BillingSummary,
|
||||
FinalChargeAmount: record.FinalChargeAmount,
|
||||
ResponseStartedAt: record.ResponseStartedAt,
|
||||
ResponseFinishedAt: record.ResponseFinishedAt,
|
||||
ResponseDurationMS: record.ResponseDurationMS,
|
||||
})
|
||||
if finishErr != nil {
|
||||
return Result{}, finishErr
|
||||
}
|
||||
if settleErr := s.store.SettleTaskBilling(ctx, finished); settleErr != nil {
|
||||
return Result{}, settleErr
|
||||
}
|
||||
if finished.FinalChargeAmount > 0 {
|
||||
if err := s.emit(ctx, task.ID, "task.billing.settled", "succeeded", "billing", 0.98, "task billing settled", map[string]any{
|
||||
"amount": finished.FinalChargeAmount,
|
||||
"currency": stringFromAny(record.BillingSummary["currency"]),
|
||||
clientAttempts := clientAttemptsForCandidate(candidate)
|
||||
var candidateErr error
|
||||
for clientAttempt := 1; clientAttempt <= clientAttempts; clientAttempt++ {
|
||||
attemptNo++
|
||||
response, err := s.runCandidate(ctx, task, user, body, candidate, attemptNo, onDelta)
|
||||
if err == nil {
|
||||
billings := s.billings(ctx, user, task.Kind, body, candidate, response, isSimulation(task, candidate))
|
||||
record := buildSuccessRecord(task, user, body, candidate, response, billings, isSimulation(task, candidate))
|
||||
record.Metrics = s.withAttemptHistory(ctx, task.ID, record.Metrics)
|
||||
finished, finishErr := s.store.FinishTaskSuccess(ctx, store.FinishTaskSuccessInput{
|
||||
TaskID: task.ID,
|
||||
Result: response.Result,
|
||||
Billings: billings,
|
||||
RequestID: record.RequestID,
|
||||
ResolvedModel: record.ResolvedModel,
|
||||
Usage: record.Usage,
|
||||
Metrics: record.Metrics,
|
||||
BillingSummary: record.BillingSummary,
|
||||
FinalChargeAmount: record.FinalChargeAmount,
|
||||
ResponseStartedAt: record.ResponseStartedAt,
|
||||
ResponseFinishedAt: record.ResponseFinishedAt,
|
||||
ResponseDurationMS: record.ResponseDurationMS,
|
||||
})
|
||||
if finishErr != nil {
|
||||
return Result{}, finishErr
|
||||
}
|
||||
if settleErr := s.store.SettleTaskBilling(ctx, finished); settleErr != nil {
|
||||
return Result{}, settleErr
|
||||
}
|
||||
if finished.FinalChargeAmount > 0 {
|
||||
if err := s.emit(ctx, task.ID, "task.billing.settled", "succeeded", "billing", 0.98, "task billing settled", map[string]any{
|
||||
"amount": finished.FinalChargeAmount,
|
||||
"currency": stringFromAny(record.BillingSummary["currency"]),
|
||||
}, isSimulation(task, candidate)); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
if err := s.emit(ctx, task.ID, "task.completed", "succeeded", "completed", 1, "task completed", map[string]any{
|
||||
"result": response.Result,
|
||||
"billings": billings,
|
||||
"usage": record.Usage,
|
||||
"metrics": record.Metrics,
|
||||
"billingSummary": record.BillingSummary,
|
||||
"requestId": record.RequestID,
|
||||
}, isSimulation(task, candidate)); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{Task: finished, Output: response.Result}, nil
|
||||
}
|
||||
if err := s.emit(ctx, task.ID, "task.completed", "succeeded", "completed", 1, "task completed", map[string]any{
|
||||
"result": response.Result,
|
||||
"billings": billings,
|
||||
"usage": record.Usage,
|
||||
"metrics": record.Metrics,
|
||||
"billingSummary": record.BillingSummary,
|
||||
"requestId": record.RequestID,
|
||||
lastErr = err
|
||||
candidateErr = err
|
||||
retryDecision := retryDecisionForCandidate(candidate, err)
|
||||
retryAction := "retry"
|
||||
if !retryDecision.Retry {
|
||||
retryAction = "stop"
|
||||
}
|
||||
if clientAttempt >= clientAttempts {
|
||||
retryDecision.Retry = false
|
||||
retryDecision.Reason = "same_client_max_attempts"
|
||||
retryDecision.Match = policyRuleMatch{
|
||||
Source: "model_runtime_policy_sets.retry_policy",
|
||||
Policy: "retryPolicy",
|
||||
Rule: "maxAttempts",
|
||||
Value: strconv.Itoa(clientAttempts),
|
||||
}
|
||||
retryDecision.Info = failureInfoFromError(err)
|
||||
retryAction = "stop"
|
||||
}
|
||||
if failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration) {
|
||||
retryDecision.Retry = false
|
||||
retryDecision.Reason = "failover_time_budget_exceeded"
|
||||
retryDecision.Match = policyRuleMatch{
|
||||
Source: "gateway_runner_policies.failover_policy",
|
||||
Policy: "failoverPolicy",
|
||||
Rule: "maxDurationSeconds",
|
||||
Value: strconv.Itoa(int(maxFailoverDuration.Seconds())),
|
||||
}
|
||||
retryDecision.Info = failureInfoFromError(err)
|
||||
retryAction = "stop"
|
||||
}
|
||||
s.recordAttemptTrace(ctx, task.ID, attemptNo, retryTraceEntry(retryDecision, retryAction, clientAttempt, clientAttempts))
|
||||
if !retryDecision.Retry {
|
||||
break
|
||||
}
|
||||
if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.45, "retrying same client", addPolicyTracePayload(map[string]any{
|
||||
"attempt": attemptNo,
|
||||
"clientAttempt": clientAttempt,
|
||||
"clientId": candidate.ClientID,
|
||||
"error": err.Error(),
|
||||
"reason": retryDecision.Reason,
|
||||
"scope": "same_client",
|
||||
}, retryDecision.Match, retryDecision.Info), isSimulation(task, candidate)); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
if candidateErr == nil || index+1 >= len(candidates) || index+1 >= maxPlatforms {
|
||||
if candidateErr != nil {
|
||||
s.applyPriorityDemotePolicy(ctx, task.ID, attemptNo, runnerPolicy, candidate, candidateErr, isSimulation(task, candidate))
|
||||
decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
|
||||
if decision.Retry {
|
||||
decision.Retry = false
|
||||
decision.Action = "stop"
|
||||
decision.Reason = "no_next_platform"
|
||||
decision.Match = policyRuleMatch{Source: "runner_candidates", Policy: "candidateSelection", Rule: "candidateCount", Value: strconv.Itoa(len(candidates))}
|
||||
if index+1 >= maxPlatforms {
|
||||
decision.Reason = "max_platforms_reached"
|
||||
decision.Match = policyRuleMatch{Source: "gateway_runner_policies.failover_policy", Policy: "failoverPolicy", Rule: "maxPlatforms", Value: strconv.Itoa(maxPlatforms)}
|
||||
}
|
||||
}
|
||||
s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision))
|
||||
}
|
||||
break
|
||||
}
|
||||
s.applyPriorityDemotePolicy(ctx, task.ID, attemptNo, runnerPolicy, candidate, candidateErr, isSimulation(task, candidate))
|
||||
if failoverTimeBudgetExceeded(executeStartedAt, maxFailoverDuration) {
|
||||
elapsedSeconds := int(time.Since(executeStartedAt).Seconds())
|
||||
maxDurationSeconds := int(maxFailoverDuration.Seconds())
|
||||
s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTimeBudgetTraceEntry(elapsedSeconds, maxDurationSeconds, failureInfoFromError(candidateErr)))
|
||||
if err := s.emit(ctx, task.ID, "task.failover.stopped", "running", "retry", 0.55, "failover time budget exceeded", map[string]any{
|
||||
"elapsedSeconds": elapsedSeconds,
|
||||
"maxDurationSeconds": maxDurationSeconds,
|
||||
"scope": "next_platform",
|
||||
"statusCode": clients.ErrorResponseMetadata(candidateErr).StatusCode,
|
||||
}, isSimulation(task, candidate)); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{Task: finished, Output: response.Result}, nil
|
||||
}
|
||||
lastErr = err
|
||||
retryable := clients.IsRetryable(err)
|
||||
if !retryable || !retryEnabled(candidate) || attemptNo >= maxAttempts {
|
||||
break
|
||||
}
|
||||
if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.55, "retrying next client", map[string]any{"attempt": attemptNo, "error": err.Error()}, isSimulation(task, candidate)); err != nil {
|
||||
decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
|
||||
s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision))
|
||||
if !decision.Retry {
|
||||
break
|
||||
}
|
||||
s.applyFailoverAction(ctx, task.ID, candidate, decision, isSimulation(task, candidate))
|
||||
if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.55, "retrying next client", addPolicyTracePayload(map[string]any{
|
||||
"attempt": attemptNo,
|
||||
"action": decision.Action,
|
||||
"error": candidateErr.Error(),
|
||||
"reason": decision.Reason,
|
||||
"scope": "next_platform",
|
||||
}, decision.Match, decision.Info), isSimulation(task, candidate)); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
@@ -184,15 +281,16 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
|
||||
reservations := s.rateLimitReservations(ctx, user, candidate, body)
|
||||
limitResult, err := s.store.ReserveRateLimits(ctx, task.ID, reservations)
|
||||
if err != nil {
|
||||
clientErr := &clients.ClientError{Code: "rate_limit", Message: err.Error(), Retryable: false}
|
||||
_ = s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{
|
||||
AttemptID: attemptID,
|
||||
Status: "failed",
|
||||
Retryable: false,
|
||||
Metrics: mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), map[string]any{"error": err.Error(), "retryable": false}),
|
||||
Metrics: mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), map[string]any{"error": err.Error(), "retryable": false, "trace": []any{failureTraceEntry(clientErr, false)}}),
|
||||
ErrorCode: "rate_limit",
|
||||
ErrorMessage: err.Error(),
|
||||
})
|
||||
return clients.Response{}, &clients.ClientError{Code: "rate_limit", Message: err.Error(), Retryable: false}
|
||||
return clients.Response{}, clientErr
|
||||
}
|
||||
defer s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.LeaseIDs)
|
||||
|
||||
@@ -207,7 +305,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
|
||||
AttemptID: attemptID,
|
||||
Status: "failed",
|
||||
Retryable: false,
|
||||
Metrics: mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), map[string]any{"error": err.Error(), "retryable": false}),
|
||||
Metrics: mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), map[string]any{"error": err.Error(), "retryable": false, "trace": []any{failureTraceEntry(err, false)}}),
|
||||
ErrorCode: clients.ErrorCode(err),
|
||||
ErrorMessage: err.Error(),
|
||||
})
|
||||
@@ -266,7 +364,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
|
||||
ErrorCode: clients.ErrorCode(err),
|
||||
ErrorMessage: err.Error(),
|
||||
})
|
||||
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "metrics": metrics}, simulated)
|
||||
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
|
||||
s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated)
|
||||
return clients.Response{}, err
|
||||
}
|
||||
@@ -275,6 +373,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
|
||||
metrics := mergeMetrics(taskMetrics(task, user, body, candidate, response, simulated), map[string]any{
|
||||
"error": err.Error(),
|
||||
"retryable": clients.IsRetryable(err),
|
||||
"trace": []any{failureTraceEntry(err, clients.IsRetryable(err))},
|
||||
})
|
||||
_ = s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{
|
||||
AttemptID: attemptID,
|
||||
@@ -429,20 +528,54 @@ func retryEnabled(candidate store.RuntimeModelCandidate) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func maxAttemptsForCandidates(candidates []store.RuntimeModelCandidate) int {
|
||||
func clientAttemptsForCandidate(candidate store.RuntimeModelCandidate) int {
|
||||
if !retryEnabled(candidate) {
|
||||
return 1
|
||||
}
|
||||
if value := intFromPolicy(effectiveRetryPolicy(candidate), "maxAttempts"); value > 0 {
|
||||
return value
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func maxPlatformsForCandidates(candidates []store.RuntimeModelCandidate, runnerPolicy store.RunnerPolicy) int {
|
||||
if len(candidates) == 0 {
|
||||
return 0
|
||||
}
|
||||
maxAttempts := len(candidates)
|
||||
maxPlatforms := len(candidates)
|
||||
if value := intFromPolicy(runnerPolicy.FailoverPolicy, "maxPlatforms"); value > 0 {
|
||||
if value < maxPlatforms {
|
||||
maxPlatforms = value
|
||||
}
|
||||
} else if maxPlatforms > 99 {
|
||||
maxPlatforms = 99
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if value := intFromPolicy(effectiveRetryPolicy(candidate), "maxAttempts"); value > 0 && value < maxAttempts {
|
||||
maxAttempts = value
|
||||
if value := intFromPolicy(effectiveFailoverPolicy(runnerPolicy.FailoverPolicy, candidate.RuntimePolicyOverride), "maxPlatforms"); value > 0 && value < maxPlatforms {
|
||||
maxPlatforms = value
|
||||
}
|
||||
}
|
||||
if maxAttempts <= 0 {
|
||||
if maxPlatforms <= 0 {
|
||||
return 1
|
||||
}
|
||||
return maxAttempts
|
||||
return maxPlatforms
|
||||
}
|
||||
|
||||
func maxFailoverDurationForCandidates(candidates []store.RuntimeModelCandidate, runnerPolicy store.RunnerPolicy) time.Duration {
|
||||
seconds := intFromPolicy(runnerPolicy.FailoverPolicy, "maxDurationSeconds")
|
||||
if seconds <= 0 {
|
||||
seconds = 600
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if value := intFromPolicy(effectiveFailoverPolicy(runnerPolicy.FailoverPolicy, candidate.RuntimePolicyOverride), "maxDurationSeconds"); value > 0 && value < seconds {
|
||||
seconds = value
|
||||
}
|
||||
}
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
func failoverTimeBudgetExceeded(start time.Time, maxDuration time.Duration) bool {
|
||||
return maxDuration > 0 && time.Since(start) >= maxDuration
|
||||
}
|
||||
|
||||
func normalizeRequest(kind string, body map[string]any) map[string]any {
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
func failureTraceEntry(err error, retryable bool) map[string]any {
|
||||
info := failureInfoFromError(err)
|
||||
entry := policyTraceEntry("failure", "client", "failed", "client_call_failed", policyRuleMatch{}, info)
|
||||
entry["retryable"] = retryable
|
||||
return entry
|
||||
}
|
||||
|
||||
func retryTraceEntry(decision retryDecision, action string, clientAttempt int, maxAttempts int) map[string]any {
|
||||
entry := policyTraceEntry("same_client_retry", "same_client", action, decision.Reason, decision.Match, decision.Info)
|
||||
entry["retry"] = decision.Retry
|
||||
entry["clientAttempt"] = clientAttempt
|
||||
entry["maxAttempts"] = maxAttempts
|
||||
return entry
|
||||
}
|
||||
|
||||
func failoverTraceEntry(decision failoverDecision) map[string]any {
|
||||
event := "failover_next"
|
||||
if !decision.Retry {
|
||||
event = "failover_stop"
|
||||
}
|
||||
entry := policyTraceEntry(event, "next_platform", decision.Action, decision.Reason, decision.Match, decision.Info)
|
||||
entry["retry"] = decision.Retry
|
||||
if decision.CooldownSeconds > 0 {
|
||||
entry["cooldownSeconds"] = decision.CooldownSeconds
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func priorityDemoteTraceEntry(decision priorityDemoteDecision, platformID string, platformModelID string) map[string]any {
|
||||
entry := policyTraceEntry("priority_demoted", "priority_demote", "demote", decision.Reason, decision.Match, decision.Info)
|
||||
entry["demote"] = decision.Demote
|
||||
entry["demoteStep"] = decision.Step
|
||||
entry["platformId"] = platformID
|
||||
entry["platformModelId"] = platformModelID
|
||||
return entry
|
||||
}
|
||||
|
||||
func failoverTimeBudgetTraceEntry(elapsedSeconds int, maxDurationSeconds int, info failureInfo) map[string]any {
|
||||
entry := policyTraceEntry("failover_stop", "next_platform", "stop", "failover_time_budget_exceeded", policyRuleMatch{
|
||||
Source: "gateway_runner_policies.failover_policy",
|
||||
Policy: "failoverPolicy",
|
||||
Rule: "maxDurationSeconds",
|
||||
Value: fmt.Sprintf("%d", maxDurationSeconds),
|
||||
}, info)
|
||||
entry["elapsedSeconds"] = elapsedSeconds
|
||||
entry["maxDurationSeconds"] = maxDurationSeconds
|
||||
return entry
|
||||
}
|
||||
|
||||
func policyTraceEntry(event string, scope string, action string, reason string, match policyRuleMatch, info failureInfo) map[string]any {
|
||||
entry := map[string]any{
|
||||
"event": event,
|
||||
"scope": scope,
|
||||
"action": action,
|
||||
"reason": reason,
|
||||
"createdAt": time.Now().Format(time.RFC3339Nano),
|
||||
}
|
||||
if info.Code != "" {
|
||||
entry["errorCode"] = info.Code
|
||||
}
|
||||
if info.Message != "" {
|
||||
entry["message"] = info.Message
|
||||
}
|
||||
if info.Status > 0 {
|
||||
entry["statusCode"] = info.Status
|
||||
}
|
||||
if info.Category != "" {
|
||||
entry["category"] = info.Category
|
||||
}
|
||||
if match.Source != "" {
|
||||
entry["policySource"] = match.Source
|
||||
}
|
||||
if match.Policy != "" {
|
||||
entry["policy"] = match.Policy
|
||||
}
|
||||
if match.Rule != "" {
|
||||
entry["policyRule"] = match.Rule
|
||||
}
|
||||
if match.Value != "" {
|
||||
entry["matchedValue"] = match.Value
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func addPolicyTracePayload(payload map[string]any, match policyRuleMatch, info failureInfo) map[string]any {
|
||||
if payload == nil {
|
||||
payload = map[string]any{}
|
||||
}
|
||||
if info.Code != "" {
|
||||
payload["errorCode"] = info.Code
|
||||
}
|
||||
if info.Status > 0 {
|
||||
payload["statusCode"] = info.Status
|
||||
}
|
||||
if info.Category != "" {
|
||||
payload["category"] = info.Category
|
||||
}
|
||||
if match.Source != "" {
|
||||
payload["policySource"] = match.Source
|
||||
}
|
||||
if match.Policy != "" {
|
||||
payload["policy"] = match.Policy
|
||||
}
|
||||
if match.Rule != "" {
|
||||
payload["policyRule"] = match.Rule
|
||||
}
|
||||
if match.Value != "" {
|
||||
payload["matchedValue"] = match.Value
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func (s *Service) recordAttemptTrace(ctx context.Context, taskID string, attemptNo int, entry map[string]any) {
|
||||
if attemptNo <= 0 || len(entry) == 0 {
|
||||
return
|
||||
}
|
||||
if err := s.store.AppendTaskAttemptTrace(ctx, taskID, attemptNo, entry); err != nil {
|
||||
s.logger.Warn("append task attempt trace failed", "taskID", taskID, "attempt", attemptNo, "error", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user