feat: add runner failover rules and cache affinity
This commit is contained in:
@@ -3,6 +3,7 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
|
||||
const runnerPolicyColumns = `
|
||||
id::text, policy_key, name, COALESCE(description, ''), failover_policy,
|
||||
hard_stop_policy, priority_demote_policy, single_source_policy, metadata, status, created_at, updated_at`
|
||||
hard_stop_policy, priority_demote_policy, single_source_policy, cache_affinity_policy, metadata, status, created_at, updated_at`
|
||||
|
||||
type RunnerPolicyInput struct {
|
||||
PolicyKey string `json:"policyKey"`
|
||||
@@ -21,6 +22,7 @@ type RunnerPolicyInput struct {
|
||||
HardStopPolicy map[string]any `json:"hardStopPolicy"`
|
||||
PriorityDemotePolicy map[string]any `json:"priorityDemotePolicy"`
|
||||
SingleSourcePolicy map[string]any `json:"singleSourcePolicy"`
|
||||
CacheAffinityPolicy map[string]any `json:"cacheAffinityPolicy"`
|
||||
Metadata map[string]any `json:"metadata"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
@@ -52,24 +54,26 @@ func (s *Store) UpsertDefaultRunnerPolicy(ctx context.Context, input RunnerPolic
|
||||
hardStopPolicy, _ := json.Marshal(emptyObjectIfNil(input.HardStopPolicy))
|
||||
priorityDemotePolicy, _ := json.Marshal(emptyObjectIfNil(input.PriorityDemotePolicy))
|
||||
singleSourcePolicy, _ := json.Marshal(emptyObjectIfNil(input.SingleSourcePolicy))
|
||||
cacheAffinityPolicy, _ := json.Marshal(emptyObjectIfNil(input.CacheAffinityPolicy))
|
||||
metadata, _ := json.Marshal(emptyObjectIfNil(input.Metadata))
|
||||
return scanRunnerPolicy(s.pool.QueryRow(ctx, `
|
||||
INSERT INTO gateway_runner_policies (
|
||||
policy_key, name, description, failover_policy, hard_stop_policy, priority_demote_policy, single_source_policy, metadata, status
|
||||
)
|
||||
VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (policy_key) DO UPDATE
|
||||
SET name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
failover_policy = EXCLUDED.failover_policy,
|
||||
hard_stop_policy = EXCLUDED.hard_stop_policy,
|
||||
priority_demote_policy = EXCLUDED.priority_demote_policy,
|
||||
single_source_policy = EXCLUDED.single_source_policy,
|
||||
metadata = EXCLUDED.metadata,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = now()
|
||||
RETURNING `+runnerPolicyColumns,
|
||||
input.PolicyKey, input.Name, input.Description, failoverPolicy, hardStopPolicy, priorityDemotePolicy, singleSourcePolicy, metadata, input.Status,
|
||||
INSERT INTO gateway_runner_policies (
|
||||
policy_key, name, description, failover_policy, hard_stop_policy, priority_demote_policy, single_source_policy, cache_affinity_policy, metadata, status
|
||||
)
|
||||
VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT (policy_key) DO UPDATE
|
||||
SET name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
failover_policy = EXCLUDED.failover_policy,
|
||||
hard_stop_policy = EXCLUDED.hard_stop_policy,
|
||||
priority_demote_policy = EXCLUDED.priority_demote_policy,
|
||||
single_source_policy = EXCLUDED.single_source_policy,
|
||||
cache_affinity_policy = EXCLUDED.cache_affinity_policy,
|
||||
metadata = EXCLUDED.metadata,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = now()
|
||||
RETURNING `+runnerPolicyColumns,
|
||||
input.PolicyKey, input.Name, input.Description, failoverPolicy, hardStopPolicy, priorityDemotePolicy, singleSourcePolicy, cacheAffinityPolicy, metadata, input.Status,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -79,6 +83,7 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
|
||||
var hardStopPolicy []byte
|
||||
var priorityDemotePolicy []byte
|
||||
var singleSourcePolicy []byte
|
||||
var cacheAffinityPolicy []byte
|
||||
var metadata []byte
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
@@ -89,6 +94,7 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
|
||||
&hardStopPolicy,
|
||||
&priorityDemotePolicy,
|
||||
&singleSourcePolicy,
|
||||
&cacheAffinityPolicy,
|
||||
&metadata,
|
||||
&item.Status,
|
||||
&item.CreatedAt,
|
||||
@@ -100,7 +106,17 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
|
||||
item.HardStopPolicy = decodeObject(hardStopPolicy)
|
||||
item.PriorityDemotePolicy = decodeObject(priorityDemotePolicy)
|
||||
item.SingleSourcePolicy = decodeObject(singleSourcePolicy)
|
||||
item.CacheAffinityPolicy = decodeObject(cacheAffinityPolicy)
|
||||
item.Metadata = decodeObject(metadata)
|
||||
if len(item.CacheAffinityPolicy) == 0 {
|
||||
item.CacheAffinityPolicy = defaultRunnerCacheAffinityPolicy()
|
||||
}
|
||||
item.FailoverPolicy = markRunnerFailoverLegacyFieldsDeprecated(item.FailoverPolicy)
|
||||
item.HardStopPolicy = markRunnerLegacyPolicyDeprecated(item.HardStopPolicy)
|
||||
item.PriorityDemotePolicy = markRunnerLegacyPolicyDeprecated(item.PriorityDemotePolicy)
|
||||
if item.PolicyKey == "default-runner-v1" && runnerActionRulesLookLikeLegacyDefaults(item.FailoverPolicy["actionRules"]) {
|
||||
item.FailoverPolicy["actionRules"] = defaultRunnerFailoverPolicy()["actionRules"]
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
@@ -118,9 +134,155 @@ func normalizeRunnerPolicyInput(input RunnerPolicyInput) RunnerPolicyInput {
|
||||
if input.Status == "" {
|
||||
input.Status = "active"
|
||||
}
|
||||
if len(input.CacheAffinityPolicy) == 0 {
|
||||
input.CacheAffinityPolicy = defaultRunnerCacheAffinityPolicy()
|
||||
}
|
||||
input.FailoverPolicy = markRunnerFailoverLegacyFieldsDeprecated(input.FailoverPolicy)
|
||||
input.HardStopPolicy = markRunnerLegacyPolicyDeprecated(input.HardStopPolicy)
|
||||
input.PriorityDemotePolicy = markRunnerLegacyPolicyDeprecated(input.PriorityDemotePolicy)
|
||||
return input
|
||||
}
|
||||
|
||||
func markRunnerLegacyPolicyDeprecated(policy map[string]any) map[string]any {
|
||||
if policy == nil {
|
||||
policy = map[string]any{}
|
||||
}
|
||||
policy["deprecated"] = true
|
||||
policy["replacedBy"] = "failoverPolicy.actionRules"
|
||||
return policy
|
||||
}
|
||||
|
||||
func markRunnerFailoverLegacyFieldsDeprecated(policy map[string]any) map[string]any {
|
||||
if policy == nil {
|
||||
policy = map[string]any{}
|
||||
}
|
||||
policy["legacyFieldsDeprecated"] = true
|
||||
policy["deprecatedFields"] = []any{"allowCategories", "denyCategories", "allowCodes", "denyCodes", "allowKeywords", "denyKeywords", "allowStatusCodes", "denyStatusCodes", "actions"}
|
||||
policy["replacedBy"] = "actionRules"
|
||||
return policy
|
||||
}
|
||||
|
||||
func runnerActionRulesLookLikeLegacyDefaults(raw any) bool {
|
||||
rawRules, ok := raw.([]any)
|
||||
if !ok || len(rawRules) != 5 {
|
||||
return false
|
||||
}
|
||||
byAction := make(map[string]map[string]any, len(rawRules))
|
||||
for _, rawRule := range rawRules {
|
||||
rule, ok := rawRule.(map[string]any)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
action := strings.TrimSpace(fmt.Sprint(rule["action"]))
|
||||
if action == "" {
|
||||
action = "next"
|
||||
}
|
||||
if _, exists := byAction[action]; exists {
|
||||
return false
|
||||
}
|
||||
byAction[action] = rule
|
||||
}
|
||||
next, nextOK := byAction["next"]
|
||||
cooldown, cooldownOK := byAction["cooldown_and_next"]
|
||||
demote, demoteOK := byAction["demote_and_next"]
|
||||
disable, disableOK := byAction["disable_and_next"]
|
||||
stop, stopOK := byAction["stop"]
|
||||
if !nextOK || !cooldownOK || !demoteOK || !disableOK || !stopOK {
|
||||
return false
|
||||
}
|
||||
if runnerRulesHaveEnhancedDefaultTerms(rawRules) {
|
||||
return false
|
||||
}
|
||||
return runnerRuleHasAll(cooldown, "categories", "rate_limit") &&
|
||||
runnerRuleHasAll(cooldown, "codes", "rate_limit") &&
|
||||
runnerRuleHasAll(cooldown, "statusCodes", "429") &&
|
||||
len(runnerRuleStrings(cooldown, "codes")) <= 1 &&
|
||||
len(runnerRuleStrings(cooldown, "keywords")) <= 3 &&
|
||||
runnerRuleHasAll(disable, "categories", "auth_error") &&
|
||||
runnerRuleHasAll(disable, "codes", "auth_failed", "invalid_api_key", "missing_credentials") &&
|
||||
runnerRuleHasAll(disable, "statusCodes", "401", "403") &&
|
||||
len(runnerRuleStrings(disable, "codes")) <= 3 &&
|
||||
len(runnerRuleStrings(disable, "keywords")) <= 8 &&
|
||||
runnerRuleHasAll(stop, "categories", "request_error", "unsupported_model", "user_permission", "insufficient_balance") &&
|
||||
len(runnerRuleStrings(stop, "codes")) <= 8 &&
|
||||
len(runnerRuleStrings(stop, "keywords")) <= 4 &&
|
||||
len(runnerRuleStrings(next, "categories")) >= 5 &&
|
||||
len(runnerRuleStrings(next, "codes")) <= 6 &&
|
||||
len(runnerRuleStrings(next, "keywords")) <= 13 &&
|
||||
(runnerRuleHasAll(demote, "keywords", "temporarily_unavailable") ||
|
||||
runnerRuleHasAll(demote, "categories", "provider_5xx") ||
|
||||
runnerRuleHasAll(demote, "codes", "server_error")) &&
|
||||
len(runnerRuleStrings(demote, "codes")) <= 6 &&
|
||||
len(runnerRuleStrings(demote, "keywords")) <= 8
|
||||
}
|
||||
|
||||
func runnerRulesHaveEnhancedDefaultTerms(rawRules []any) bool {
|
||||
markers := map[string]struct{}{
|
||||
"resource_exhausted": {},
|
||||
"rpm_limit": {},
|
||||
"tpm_limit": {},
|
||||
"account_disabled": {},
|
||||
"billing_not_active": {},
|
||||
"provider_failed": {},
|
||||
"invalid_response": {},
|
||||
"request_asset_fetch_failed": {},
|
||||
"socket hang up": {},
|
||||
"econnreset": {},
|
||||
"request_asset_input_format_unsupported": {},
|
||||
"invalid_multipart_image": {},
|
||||
}
|
||||
for _, rawRule := range rawRules {
|
||||
rule, ok := rawRule.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, field := range []string{"categories", "codes", "statusCodes", "keywords"} {
|
||||
for _, value := range runnerRuleStrings(rule, field) {
|
||||
if _, ok := markers[value]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func runnerRuleHasAll(rule map[string]any, key string, values ...string) bool {
|
||||
actual := make(map[string]struct{}, len(values))
|
||||
for _, value := range runnerRuleStrings(rule, key) {
|
||||
actual[value] = struct{}{}
|
||||
}
|
||||
for _, value := range values {
|
||||
if _, ok := actual[strings.ToLower(strings.TrimSpace(value))]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func runnerRuleStrings(rule map[string]any, key string) []string {
|
||||
raw, ok := rule[key].([]any)
|
||||
if !ok {
|
||||
if typed, ok := rule[key].([]string); ok {
|
||||
out := make([]string, 0, len(typed))
|
||||
for _, item := range typed {
|
||||
if value := strings.ToLower(strings.TrimSpace(item)); value != "" {
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, item := range raw {
|
||||
if value := strings.ToLower(strings.TrimSpace(fmt.Sprint(item))); value != "" {
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func defaultRunnerPolicy() RunnerPolicy {
|
||||
now := time.Now()
|
||||
return RunnerPolicy{
|
||||
@@ -131,6 +293,7 @@ func defaultRunnerPolicy() RunnerPolicy {
|
||||
HardStopPolicy: defaultRunnerHardStopPolicy(),
|
||||
PriorityDemotePolicy: defaultRunnerPriorityDemotePolicy(),
|
||||
SingleSourcePolicy: defaultRunnerSingleSourcePolicy(),
|
||||
CacheAffinityPolicy: defaultRunnerCacheAffinityPolicy(),
|
||||
Metadata: map[string]any{"source": "code-default"},
|
||||
Status: "active",
|
||||
CreatedAt: now,
|
||||
@@ -141,6 +304,8 @@ func defaultRunnerPolicy() RunnerPolicy {
|
||||
func defaultRunnerPriorityDemotePolicy() map[string]any {
|
||||
return map[string]any{
|
||||
"enabled": true,
|
||||
"deprecated": true,
|
||||
"replacedBy": "failoverPolicy.actionRules",
|
||||
"categories": []any{"network", "timeout", "stream_error", "rate_limit", "provider_5xx", "provider_overloaded"},
|
||||
"codes": []any{"network", "timeout", "stream_read_error", "rate_limit", "server_error", "overloaded"},
|
||||
"statusCodes": []any{408, 429, 500, 502, 503, 504},
|
||||
@@ -154,24 +319,88 @@ func defaultRunnerSingleSourcePolicy() map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRunnerCacheAffinityPolicy() map[string]any {
|
||||
return map[string]any{
|
||||
"enabled": true,
|
||||
"modelTypes": []any{"chat", "text_generate", "responses"},
|
||||
"minSamples": 3,
|
||||
"minInputTokens": 512,
|
||||
"staleAfterSeconds": 86400,
|
||||
"maxPriorityBoost": 20,
|
||||
"emaAlpha": 0.35,
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRunnerFailoverPolicy() map[string]any {
|
||||
return map[string]any{
|
||||
"enabled": true,
|
||||
"maxPlatforms": 99,
|
||||
"maxDurationSeconds": 600,
|
||||
"allowCategories": []any{"network", "timeout", "stream_error", "rate_limit", "provider_5xx", "provider_overloaded", "auth_error"},
|
||||
"denyCategories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"allowCodes": []any{"auth_failed", "invalid_api_key", "missing_credentials"},
|
||||
"allowKeywords": []any{"timeout", "network", "rate_limit", "overloaded", "temporarily_unavailable", "server_error", "auth_failed", "invalid_api_key", "missing_credentials", "unauthorized", "forbidden", "429", "5xx"},
|
||||
"denyKeywords": []any{"invalid_parameter", "missing required", "bad request"},
|
||||
"allowStatusCodes": []any{401, 403, 408, 429, 500, 502, 503, 504},
|
||||
"denyStatusCodes": []any{},
|
||||
"enabled": true,
|
||||
"maxPlatforms": 99,
|
||||
"maxDurationSeconds": 600,
|
||||
"legacyFieldsDeprecated": true,
|
||||
"deprecatedFields": []any{"allowCategories", "denyCategories", "allowCodes", "denyCodes", "allowKeywords", "denyKeywords", "allowStatusCodes", "denyStatusCodes", "actions"},
|
||||
"replacedBy": "actionRules",
|
||||
"allowCategories": []any{"network", "timeout", "stream_error", "rate_limit", "provider_5xx", "provider_overloaded", "auth_error"},
|
||||
"denyCategories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"allowCodes": []any{"auth_failed", "invalid_api_key", "missing_credentials"},
|
||||
"allowKeywords": []any{"timeout", "network", "rate_limit", "overloaded", "temporarily_unavailable", "server_error", "auth_failed", "invalid_api_key", "missing_credentials", "unauthorized", "forbidden", "429", "5xx"},
|
||||
"denyKeywords": []any{"invalid_parameter", "missing required", "bad request"},
|
||||
"allowStatusCodes": []any{401, 403, 408, 429, 500, 502, 503, 504},
|
||||
"denyStatusCodes": []any{},
|
||||
"actionRules": []any{
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"rate_limit"},
|
||||
"codes": []any{"rate_limit", "too_many_requests", "too_many_request", "rate_limit_exceeded", "requests_rate_limited", "requests_limit_exceeded", "resource_exhausted", "throttled", "quota_exceeded", "quota_exhausted", "qps_limit", "rpm_limit", "tpm_limit"},
|
||||
"statusCodes": []any{429},
|
||||
"keywords": []any{"rate limit", "rate_limit", "rate-limit", "too many requests", "too_many_requests", "429", "throttle", "throttled", "resource exhausted", "quota exceeded", "quota_exceeded", "rate exceeded", "request limit", "requests per minute", "tokens per minute", "qps", "rpm", "tpm"},
|
||||
"action": "cooldown_and_next",
|
||||
"target": "model",
|
||||
"cooldownSeconds": 300,
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"auth_error"},
|
||||
"codes": []any{"auth_failed", "invalid_api_key", "missing_credentials", "missing_credential", "unauthorized", "forbidden", "account_disabled", "billing_not_active", "permission_denied"},
|
||||
"statusCodes": []any{401, 403},
|
||||
"keywords": []any{"invalid api key", "api key is invalid", "invalid_api_key", "apikey invalid", "api key invalid", "unauthorized", "forbidden", "authentication", "auth failed", "credential", "missing credential", "permission denied", "access denied", "invalid signature", "signature mismatch", "secret key", "access key", "account disabled", "account_deactivated", "billing not active", "billing_not_active", "insufficient balance", "quota exceeded", "余额不足", "欠费"},
|
||||
"action": "disable_and_next",
|
||||
"target": "platform",
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"timeout", "provider_5xx", "provider_overloaded"},
|
||||
"codes": []any{"timeout", "server_error", "overloaded", "provider_failed", "invalid_response", "script_timeout", "request_asset_fetch_failed", "upload_failed", "upload_read_failed", "upload_source_read_failed"},
|
||||
"statusCodes": []any{408, 500, 502, 503, 504, 520, 521, 522, 523, 524, 529, 530},
|
||||
"keywords": []any{"timeout", "timed out", "deadline exceeded", "context deadline exceeded", "overloaded", "overload", "temporarily_unavailable", "temporarily unavailable", "temporary unavailable", "service unavailable", "server error", "internal server error", "bad gateway", "gateway timeout", "upstream timeout", "upstream error", "provider failed", "model is overloaded", "capacity", "try again later", "please try again later", "5xx", "500", "502", "503", "504"},
|
||||
"action": "demote_and_next",
|
||||
"target": "platform",
|
||||
"demoteSteps": 1,
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"network", "stream_error"},
|
||||
"codes": []any{"network", "stream_read_error", "cancelled", "request_asset_fetch_failed", "upload_network", "upload_config_failed", "local_static_store_failed", "upload_source_fetch_failed"},
|
||||
"statusCodes": []any{},
|
||||
"keywords": []any{"network", "socket hang up", "socket closed", "connection reset", "connection refused", "connection aborted", "broken pipe", "econnreset", "econnrefused", "eai_again", "enotfound", "no such host", "dns", "tls handshake timeout", "unexpected eof", "connection closed", "stream error", "stream_read_error", "read failed", "fetch failed"},
|
||||
"action": "next",
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "unsupported_operation", "unsupported_request_resolution", "request_asset_input_format_unsupported", "request_asset_expired", "request_asset_decode_failed", "request_asset_public_url_required", "upload_no_channel", "upload_unsupported_channel", "upload_source_too_large", "upload_source_invalid_url", "upload_source_unsupported_url", "upload_decode_failed", "invalid_proxy", "invalid_json_body", "invalid_multipart_body", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio", "missing_configuration", "script_error", "cloned_voice_not_found", "cloned_voice_expired", "cloned_voice_unavailable", "insufficient_balance", "permission_denied"},
|
||||
"statusCodes": []any{400, 404, 405, 406, 409, 410, 411, 413, 415, 422},
|
||||
"keywords": []any{"invalid_parameter", "invalid parameter", "missing required", "missing_required", "bad request", "unsupported", "not supported", "does not support", "is not supported", "insufficient balance", "permission denied", "no permission", "invalid json", "invalid multipart", "image is required", "audio must", "prompt is required", "model is required", "exceeds", "too large", "expired", "not found", "required"},
|
||||
"action": "stop",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRunnerHardStopPolicy() map[string]any {
|
||||
return map[string]any{
|
||||
"enabled": true,
|
||||
"deprecated": true,
|
||||
"replacedBy": "failoverPolicy.actionRules",
|
||||
"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "insufficient_balance", "permission_denied"},
|
||||
"statusCodes": []any{},
|
||||
|
||||
Reference in New Issue
Block a user