feat: add runner failover rules and cache affinity
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type CacheAffinityObservationInput struct {
|
||||
CacheAffinityKey string
|
||||
CacheAffinityPolicy map[string]any
|
||||
PlatformID string
|
||||
PlatformModelID string
|
||||
ClientID string
|
||||
ModelType string
|
||||
InputTokens int
|
||||
CachedInputTokens int
|
||||
CachedInputTokensKnown bool
|
||||
}
|
||||
|
||||
func (s *Store) RecordCacheAffinityObservation(ctx context.Context, input CacheAffinityObservationInput) error {
|
||||
input.CacheAffinityKey = strings.TrimSpace(input.CacheAffinityKey)
|
||||
input.ClientID = strings.TrimSpace(input.ClientID)
|
||||
input.ModelType = strings.TrimSpace(input.ModelType)
|
||||
if input.CacheAffinityKey == "" || input.ClientID == "" || input.ModelType == "" {
|
||||
return nil
|
||||
}
|
||||
if !input.CachedInputTokensKnown || !cacheAffinityPolicyEnabled(input.CacheAffinityPolicy, input.ModelType) {
|
||||
return nil
|
||||
}
|
||||
minInputTokens := cacheAffinityMinInputTokens(input.CacheAffinityPolicy)
|
||||
if input.InputTokens < minInputTokens || input.InputTokens <= 0 {
|
||||
return nil
|
||||
}
|
||||
if input.CachedInputTokens < 0 {
|
||||
input.CachedInputTokens = 0
|
||||
}
|
||||
if input.CachedInputTokens > input.InputTokens {
|
||||
input.CachedInputTokens = input.InputTokens
|
||||
}
|
||||
alpha := cacheAffinityEMAAlpha(input.CacheAffinityPolicy)
|
||||
hitRatio := float64(input.CachedInputTokens) / float64(input.InputTokens)
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO gateway_cache_affinity_stats (
|
||||
client_id, cache_affinity_key, platform_id, platform_model_id, model_type,
|
||||
request_count, input_tokens, cached_input_tokens, ema_hit_ratio, last_hit_ratio, last_observed_at
|
||||
)
|
||||
VALUES (
|
||||
$1, $2, NULLIF($3::text, '')::uuid, NULLIF($4::text, '')::uuid, $5,
|
||||
1, $6, $7, $8, $8, now()
|
||||
)
|
||||
ON CONFLICT (client_id, cache_affinity_key) DO UPDATE
|
||||
SET platform_id = COALESCE(EXCLUDED.platform_id, gateway_cache_affinity_stats.platform_id),
|
||||
platform_model_id = COALESCE(EXCLUDED.platform_model_id, gateway_cache_affinity_stats.platform_model_id),
|
||||
model_type = EXCLUDED.model_type,
|
||||
request_count = gateway_cache_affinity_stats.request_count + 1,
|
||||
input_tokens = gateway_cache_affinity_stats.input_tokens + EXCLUDED.input_tokens,
|
||||
cached_input_tokens = gateway_cache_affinity_stats.cached_input_tokens + EXCLUDED.cached_input_tokens,
|
||||
ema_hit_ratio = ($9::float8 * EXCLUDED.last_hit_ratio) + ((1 - $9::float8) * gateway_cache_affinity_stats.ema_hit_ratio),
|
||||
last_hit_ratio = EXCLUDED.last_hit_ratio,
|
||||
last_observed_at = now(),
|
||||
updated_at = now()`,
|
||||
input.ClientID,
|
||||
input.CacheAffinityKey,
|
||||
input.PlatformID,
|
||||
input.PlatformModelID,
|
||||
input.ModelType,
|
||||
input.InputTokens,
|
||||
input.CachedInputTokens,
|
||||
hitRatio,
|
||||
alpha,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
@@ -10,11 +11,17 @@ import (
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
)
|
||||
|
||||
func (s *Store) ListModelCandidates(ctx context.Context, model string, modelType string, user *auth.User) ([]RuntimeModelCandidate, error) {
|
||||
type ListModelCandidatesOptions struct {
|
||||
CacheAffinityKey string
|
||||
CacheAffinityPolicy map[string]any
|
||||
}
|
||||
|
||||
func (s *Store) ListModelCandidates(ctx context.Context, model string, modelType string, user *auth.User, options ...ListModelCandidatesOptions) ([]RuntimeModelCandidate, error) {
|
||||
exactModel := strings.TrimSpace(model)
|
||||
modelMatchKey := normalizeModelMatchKey(exactModel)
|
||||
listOptions := normalizeListModelCandidatesOptions(modelType, options...)
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT p.id::text, p.platform_key, p.name, p.provider,
|
||||
SELECT p.id::text, p.platform_key, p.name, p.provider,
|
||||
COALESCE(NULLIF(p.config->>'specType', ''), NULLIF(cp.provider_type, ''), NULLIF(p.config->>'sourceSpecType', ''), p.provider) AS spec_type,
|
||||
COALESCE(p.base_url, ''),
|
||||
p.auth_type, p.credentials, p.config, p.default_pricing_mode,
|
||||
@@ -35,17 +42,27 @@ SELECT p.id::text, p.platform_key, p.name, p.provider,
|
||||
COALESCE(queued.waiting, 0)::float8,
|
||||
COALESCE(rpm.used_value, 0)::float8, COALESCE(rpm.reserved_value, 0)::float8,
|
||||
COALESCE(tpm.used_value, 0)::float8, COALESCE(tpm.reserved_value, 0)::float8,
|
||||
COALESCE(s.running_count, 0)::float8,
|
||||
COALESCE(s.waiting_count, 0)::float8,
|
||||
COALESCE(s.limiter_ratio, 0)::float8,
|
||||
COALESCE(EXTRACT(EPOCH FROM s.last_assigned_at), 0)::float8
|
||||
FROM platform_models m
|
||||
JOIN integration_platforms p ON p.id = m.platform_id
|
||||
LEFT JOIN model_catalog_providers cp ON cp.provider_key = p.provider OR cp.provider_code = p.provider
|
||||
LEFT JOIN base_model_catalog b ON b.id = m.base_model_id
|
||||
LEFT JOIN model_runtime_policy_sets rp ON rp.id = COALESCE(m.runtime_policy_set_id, b.runtime_policy_set_id)
|
||||
LEFT JOIN runtime_client_states s
|
||||
ON s.client_id = p.platform_key || ':' || $2::text || ':' || COALESCE(NULLIF(m.provider_model_name, ''), m.model_name)
|
||||
COALESCE(s.running_count, 0)::float8,
|
||||
COALESCE(s.waiting_count, 0)::float8,
|
||||
COALESCE(s.limiter_ratio, 0)::float8,
|
||||
COALESCE(EXTRACT(EPOCH FROM s.last_assigned_at), 0)::float8,
|
||||
COALESCE(ca.request_count, 0)::float8,
|
||||
COALESCE(ca.input_tokens, 0)::float8,
|
||||
COALESCE(ca.cached_input_tokens, 0)::float8,
|
||||
COALESCE(ca.ema_hit_ratio, 0)::float8,
|
||||
COALESCE(ca.last_hit_ratio, 0)::float8,
|
||||
COALESCE(EXTRACT(EPOCH FROM ca.last_observed_at), 0)::float8
|
||||
FROM platform_models m
|
||||
JOIN integration_platforms p ON p.id = m.platform_id
|
||||
LEFT JOIN model_catalog_providers cp ON cp.provider_key = p.provider OR cp.provider_code = p.provider
|
||||
LEFT JOIN base_model_catalog b ON b.id = m.base_model_id
|
||||
LEFT JOIN model_runtime_policy_sets rp ON rp.id = COALESCE(m.runtime_policy_set_id, b.runtime_policy_set_id)
|
||||
LEFT JOIN runtime_client_states s
|
||||
ON s.client_id = p.platform_key || ':' || $2::text || ':' || COALESCE(NULLIF(m.provider_model_name, ''), m.model_name)
|
||||
LEFT JOIN gateway_cache_affinity_stats ca
|
||||
ON ca.client_id = p.platform_key || ':' || $2::text || ':' || COALESCE(NULLIF(m.provider_model_name, ''), m.model_name)
|
||||
AND ca.cache_affinity_key = NULLIF($4::text, '')
|
||||
AND ($5::int <= 0 OR ca.last_observed_at >= now() - ($5::int * interval '1 second'))
|
||||
LEFT JOIN (
|
||||
SELECT scope_key, SUM(lease_value) AS active
|
||||
FROM gateway_concurrency_leases
|
||||
@@ -154,11 +171,11 @@ WHERE p.status = 'enabled'
|
||||
)
|
||||
)
|
||||
)
|
||||
ORDER BY effective_priority ASC,
|
||||
COALESCE(s.running_count, 0) ASC,
|
||||
COALESCE(s.waiting_count, 0) ASC,
|
||||
COALESCE(s.last_assigned_at, to_timestamp(0)) ASC,
|
||||
m.created_at ASC`, exactModel, modelType, modelMatchKey)
|
||||
ORDER BY effective_priority ASC,
|
||||
COALESCE(s.running_count, 0) ASC,
|
||||
COALESCE(s.waiting_count, 0) ASC,
|
||||
COALESCE(s.last_assigned_at, to_timestamp(0)) ASC,
|
||||
m.created_at ASC`, exactModel, modelType, modelMatchKey, listOptions.CacheAffinityKey, cacheAffinityStaleAfterSeconds(listOptions.CacheAffinityPolicy))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -194,6 +211,12 @@ ORDER BY effective_priority ASC,
|
||||
var stateWaitingCount float64
|
||||
var stateLimiterRatio float64
|
||||
var lastAssignedUnix float64
|
||||
var cacheRequestCount float64
|
||||
var cacheInputTokens float64
|
||||
var cacheCachedInputTokens float64
|
||||
var cacheEMAHitRatio float64
|
||||
var cacheLastHitRatio float64
|
||||
var cacheLastObservedUnix float64
|
||||
if err := rows.Scan(
|
||||
&item.PlatformID,
|
||||
&item.PlatformKey,
|
||||
@@ -246,6 +269,12 @@ ORDER BY effective_priority ASC,
|
||||
&stateWaitingCount,
|
||||
&stateLimiterRatio,
|
||||
&lastAssignedUnix,
|
||||
&cacheRequestCount,
|
||||
&cacheInputTokens,
|
||||
&cacheCachedInputTokens,
|
||||
&cacheEMAHitRatio,
|
||||
&cacheLastHitRatio,
|
||||
&cacheLastObservedUnix,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -272,6 +301,14 @@ ORDER BY effective_priority ASC,
|
||||
item.RunningCount = stateRunningCount
|
||||
item.WaitingCount = maxFloat(queuedWaiting, stateWaitingCount)
|
||||
item.LastAssignedUnix = lastAssignedUnix
|
||||
applyRuntimeCandidateCacheAffinity(&item, listOptions, runtimeCandidateCacheAffinityInput{
|
||||
RequestCount: int(cacheRequestCount),
|
||||
InputTokens: int(cacheInputTokens),
|
||||
CachedInputTokens: int(cacheCachedInputTokens),
|
||||
EMAHitRatio: cacheEMAHitRatio,
|
||||
LastHitRatio: cacheLastHitRatio,
|
||||
LastObservedUnix: cacheLastObservedUnix,
|
||||
})
|
||||
applyRuntimeCandidateLoad(&item, runtimeCandidateLoadInput{
|
||||
Policy: effectiveModelRateLimitPolicy(item.PlatformRateLimitPolicy, item.RuntimeRateLimitPolicy, item.RuntimePolicySetID, item.RuntimePolicyOverride, item.ModelRateLimitPolicy),
|
||||
ConcurrentActive: concurrentActive,
|
||||
@@ -383,6 +420,182 @@ type runtimeCandidateLoadInput struct {
|
||||
StateLimiterRatio float64
|
||||
}
|
||||
|
||||
type runtimeCandidateCacheAffinityInput struct {
|
||||
RequestCount int
|
||||
InputTokens int
|
||||
CachedInputTokens int
|
||||
EMAHitRatio float64
|
||||
LastHitRatio float64
|
||||
LastObservedUnix float64
|
||||
}
|
||||
|
||||
func normalizeListModelCandidatesOptions(modelType string, options ...ListModelCandidatesOptions) ListModelCandidatesOptions {
|
||||
if len(options) == 0 {
|
||||
return ListModelCandidatesOptions{}
|
||||
}
|
||||
out := options[0]
|
||||
out.CacheAffinityKey = strings.TrimSpace(out.CacheAffinityKey)
|
||||
if out.CacheAffinityKey == "" || !cacheAffinityPolicyEnabled(out.CacheAffinityPolicy, modelType) {
|
||||
out.CacheAffinityKey = ""
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func applyRuntimeCandidateCacheAffinity(candidate *RuntimeModelCandidate, options ListModelCandidatesOptions, input runtimeCandidateCacheAffinityInput) {
|
||||
affinity := RuntimeCandidateCacheAffinity{
|
||||
Key: options.CacheAffinityKey,
|
||||
RequestCount: input.RequestCount,
|
||||
InputTokens: input.InputTokens,
|
||||
CachedInputTokens: input.CachedInputTokens,
|
||||
EMAHitRatio: boundedRatio(input.EMAHitRatio),
|
||||
LastHitRatio: boundedRatio(input.LastHitRatio),
|
||||
LastObservedUnix: input.LastObservedUnix,
|
||||
AdjustedPriority: float64(candidate.PlatformPriority),
|
||||
}
|
||||
minSamples := cacheAffinityMinSamples(options.CacheAffinityPolicy)
|
||||
hasObservedCachedHit := affinity.CachedInputTokens > 0 || affinity.LastHitRatio > 0
|
||||
hasSampledAffinity := input.RequestCount >= minSamples && affinity.EMAHitRatio > 0
|
||||
if options.CacheAffinityKey == "" || input.RequestCount <= 0 || (!hasObservedCachedHit && !hasSampledAffinity) {
|
||||
candidate.CacheAffinity = affinity
|
||||
return
|
||||
}
|
||||
affinity.Confidence = float64(input.RequestCount) / float64(minSamples)
|
||||
if affinity.Confidence > 1 {
|
||||
affinity.Confidence = 1
|
||||
}
|
||||
if hasObservedCachedHit && affinity.Confidence < 1 {
|
||||
affinity.Confidence = 1
|
||||
}
|
||||
affinity.Score = runtimeCandidateCacheAffinityScore(affinity)
|
||||
maxBoost := cacheAffinityMaxPriorityBoost(options.CacheAffinityPolicy)
|
||||
affinity.Boost = affinity.Score * maxBoost
|
||||
if affinity.Boost > maxBoost {
|
||||
affinity.Boost = maxBoost
|
||||
}
|
||||
if hasObservedCachedHit || affinity.Score > 0 {
|
||||
affinity.Applied = true
|
||||
}
|
||||
if affinity.Boost > 0 {
|
||||
affinity.AdjustedPriority = float64(candidate.PlatformPriority) - affinity.Boost
|
||||
}
|
||||
candidate.CacheAffinity = affinity
|
||||
}
|
||||
|
||||
func cacheAffinityPolicyEnabled(policy map[string]any, modelType string) bool {
|
||||
if enabled, ok := policy["enabled"].(bool); ok && !enabled {
|
||||
return false
|
||||
}
|
||||
modelTypes := stringListFromAny(policy["modelTypes"])
|
||||
if len(modelTypes) == 0 {
|
||||
modelTypes = []string{"chat", "text_generate", "responses"}
|
||||
}
|
||||
modelType = strings.TrimSpace(modelType)
|
||||
for _, candidate := range modelTypes {
|
||||
if strings.EqualFold(strings.TrimSpace(candidate), modelType) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cacheAffinityMinSamples(policy map[string]any) int {
|
||||
value := intFromStorePolicy(policy, "minSamples", 3)
|
||||
if value <= 0 {
|
||||
return 1
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func cacheAffinityMinInputTokens(policy map[string]any) int {
|
||||
value := intFromStorePolicy(policy, "minInputTokens", 512)
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func cacheAffinityStaleAfterSeconds(policy map[string]any) int {
|
||||
return intFromStorePolicy(policy, "staleAfterSeconds", 86400)
|
||||
}
|
||||
|
||||
func cacheAffinityMaxPriorityBoost(policy map[string]any) float64 {
|
||||
value := floatFromStorePolicy(policy, "maxPriorityBoost", 20)
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func cacheAffinityEMAAlpha(policy map[string]any) float64 {
|
||||
value := floatFromStorePolicy(policy, "emaAlpha", 0.35)
|
||||
if value <= 0 || value > 1 {
|
||||
return 0.35
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func runtimeCandidateCacheAffinityScore(affinity RuntimeCandidateCacheAffinity) float64 {
|
||||
hitRatio := affinity.EMAHitRatio
|
||||
if affinity.LastHitRatio > hitRatio {
|
||||
hitRatio = affinity.LastHitRatio
|
||||
}
|
||||
hitRatio = boundedRatio(hitRatio)
|
||||
confidence := boundedRatio(affinity.Confidence)
|
||||
if hitRatio <= 0 || confidence <= 0 {
|
||||
return 0
|
||||
}
|
||||
tokenFactor := 0.0
|
||||
if affinity.CachedInputTokens > 0 {
|
||||
tokenFactor = float64(affinity.CachedInputTokens) / 8192
|
||||
if tokenFactor > 1 {
|
||||
tokenFactor = 1
|
||||
}
|
||||
}
|
||||
return math.Round(hitRatio*confidence*(1+tokenFactor)*1_000_000) / 1_000_000
|
||||
}
|
||||
|
||||
func intFromStorePolicy(policy map[string]any, key string, fallback int) int {
|
||||
if policy == nil {
|
||||
return fallback
|
||||
}
|
||||
switch value := policy[key].(type) {
|
||||
case int:
|
||||
return value
|
||||
case int64:
|
||||
return int(value)
|
||||
case float64:
|
||||
return int(value)
|
||||
default:
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
func floatFromStorePolicy(policy map[string]any, key string, fallback float64) float64 {
|
||||
if policy == nil {
|
||||
return fallback
|
||||
}
|
||||
switch value := policy[key].(type) {
|
||||
case int:
|
||||
return float64(value)
|
||||
case int64:
|
||||
return float64(value)
|
||||
case float64:
|
||||
return value
|
||||
default:
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
func boundedRatio(value float64) float64 {
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
if value > 1 {
|
||||
return 1
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func applyRuntimeCandidateLoad(candidate *RuntimeModelCandidate, input runtimeCandidateLoadInput) {
|
||||
rpmLimit := rateLimitForMetric(input.Policy, "rpm")
|
||||
tpmLimitValue := tpmLimit(input.Policy)
|
||||
@@ -422,6 +635,9 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
|
||||
hasNonFull := false
|
||||
for index := range items {
|
||||
items[index].LoadAvoided = false
|
||||
if !items[index].CacheAffinity.Applied && items[index].CacheAffinity.AdjustedPriority == 0 {
|
||||
items[index].CacheAffinity.AdjustedPriority = float64(items[index].PlatformPriority)
|
||||
}
|
||||
if runtimeCandidateFull(items[index]) {
|
||||
hasFull = true
|
||||
} else {
|
||||
@@ -439,8 +655,25 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
|
||||
if aFull != bFull {
|
||||
return !aFull
|
||||
}
|
||||
if items[i].PlatformPriority != items[j].PlatformPriority {
|
||||
return items[i].PlatformPriority < items[j].PlatformPriority
|
||||
if items[i].CacheAffinity.Applied != items[j].CacheAffinity.Applied {
|
||||
return items[i].CacheAffinity.Applied
|
||||
}
|
||||
if items[i].CacheAffinity.Applied && items[j].CacheAffinity.Applied {
|
||||
if items[i].CacheAffinity.Score != items[j].CacheAffinity.Score {
|
||||
return items[i].CacheAffinity.Score > items[j].CacheAffinity.Score
|
||||
}
|
||||
if items[i].CacheAffinity.EMAHitRatio != items[j].CacheAffinity.EMAHitRatio {
|
||||
return items[i].CacheAffinity.EMAHitRatio > items[j].CacheAffinity.EMAHitRatio
|
||||
}
|
||||
if items[i].CacheAffinity.LastHitRatio != items[j].CacheAffinity.LastHitRatio {
|
||||
return items[i].CacheAffinity.LastHitRatio > items[j].CacheAffinity.LastHitRatio
|
||||
}
|
||||
if items[i].CacheAffinity.CachedInputTokens != items[j].CacheAffinity.CachedInputTokens {
|
||||
return items[i].CacheAffinity.CachedInputTokens > items[j].CacheAffinity.CachedInputTokens
|
||||
}
|
||||
}
|
||||
if items[i].CacheAffinity.AdjustedPriority != items[j].CacheAffinity.AdjustedPriority {
|
||||
return items[i].CacheAffinity.AdjustedPriority < items[j].CacheAffinity.AdjustedPriority
|
||||
}
|
||||
if items[i].LoadRatio != items[j].LoadRatio {
|
||||
return items[i].LoadRatio < items[j].LoadRatio
|
||||
|
||||
@@ -106,6 +106,156 @@ func TestRuntimeCandidateSortingAvoidsFullCandidatesButKeepsFallback(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeCandidateSortingPrefersCacheAffinityWithinAvailableCandidates(t *testing.T) {
|
||||
candidates := []RuntimeModelCandidate{
|
||||
{
|
||||
PlatformID: "base-priority",
|
||||
PlatformPriority: 10,
|
||||
},
|
||||
{
|
||||
PlatformID: "cache-affinity",
|
||||
PlatformPriority: 20,
|
||||
},
|
||||
}
|
||||
applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
|
||||
CacheAffinityKey: "explicit:test",
|
||||
CacheAffinityPolicy: map[string]any{"enabled": true, "minSamples": 3, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}},
|
||||
}, runtimeCandidateCacheAffinityInput{
|
||||
RequestCount: 1,
|
||||
EMAHitRatio: 1,
|
||||
})
|
||||
applyRuntimeCandidateCacheAffinity(&candidates[1], ListModelCandidatesOptions{
|
||||
CacheAffinityKey: "explicit:test",
|
||||
CacheAffinityPolicy: map[string]any{"enabled": true, "minSamples": 3, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}},
|
||||
}, runtimeCandidateCacheAffinityInput{
|
||||
RequestCount: 3,
|
||||
EMAHitRatio: 0.9,
|
||||
})
|
||||
|
||||
sortRuntimeModelCandidates(candidates)
|
||||
|
||||
if candidates[0].PlatformID != "cache-affinity" || !candidates[0].CacheAffinity.Applied {
|
||||
t.Fatalf("expected cache affinity candidate first, got %+v", candidates)
|
||||
}
|
||||
if candidates[1].PlatformID != "base-priority" || candidates[1].CacheAffinity.Applied {
|
||||
t.Fatalf("expected low sample candidate to keep base priority, got %+v", candidates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeCandidateSortingPromotesObservedCacheHitAbovePriority(t *testing.T) {
|
||||
candidates := []RuntimeModelCandidate{
|
||||
{
|
||||
PlatformID: "volces-priority",
|
||||
PlatformPriority: 1,
|
||||
},
|
||||
{
|
||||
PlatformID: "deepseek-cache-hit",
|
||||
PlatformPriority: 100,
|
||||
},
|
||||
}
|
||||
policy := map[string]any{"enabled": true, "minSamples": 3, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}}
|
||||
applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
|
||||
CacheAffinityKey: "conversation:test",
|
||||
CacheAffinityPolicy: policy,
|
||||
}, runtimeCandidateCacheAffinityInput{})
|
||||
applyRuntimeCandidateCacheAffinity(&candidates[1], ListModelCandidatesOptions{
|
||||
CacheAffinityKey: "conversation:test",
|
||||
CacheAffinityPolicy: policy,
|
||||
}, runtimeCandidateCacheAffinityInput{
|
||||
RequestCount: 1,
|
||||
InputTokens: 7945,
|
||||
CachedInputTokens: 7296,
|
||||
EMAHitRatio: 0.918,
|
||||
LastHitRatio: 0.918,
|
||||
})
|
||||
|
||||
sortRuntimeModelCandidates(candidates)
|
||||
|
||||
if candidates[0].PlatformID != "deepseek-cache-hit" || !candidates[0].CacheAffinity.Applied {
|
||||
t.Fatalf("expected observed cache hit to outrank platform priority, got %+v", candidates)
|
||||
}
|
||||
if candidates[0].CacheAffinity.Score <= 0 || candidates[0].CacheAffinity.Boost <= 0 {
|
||||
t.Fatalf("expected observed cache hit to carry score and boost, got %+v", candidates[0].CacheAffinity)
|
||||
}
|
||||
if candidates[1].PlatformID != "volces-priority" || candidates[1].CacheAffinity.Applied {
|
||||
t.Fatalf("expected priority-only candidate second, got %+v", candidates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeCandidateSortingOrdersMultipleCacheAffinityCandidatesByScore(t *testing.T) {
|
||||
candidates := []RuntimeModelCandidate{
|
||||
{
|
||||
PlatformID: "lower-cache-score",
|
||||
PlatformPriority: 1,
|
||||
},
|
||||
{
|
||||
PlatformID: "higher-cache-score",
|
||||
PlatformPriority: 50,
|
||||
},
|
||||
}
|
||||
policy := map[string]any{"enabled": true, "minSamples": 1, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}}
|
||||
applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
|
||||
CacheAffinityKey: "conversation:test",
|
||||
CacheAffinityPolicy: policy,
|
||||
}, runtimeCandidateCacheAffinityInput{
|
||||
RequestCount: 2,
|
||||
InputTokens: 8000,
|
||||
CachedInputTokens: 1000,
|
||||
EMAHitRatio: 0.5,
|
||||
LastHitRatio: 0.5,
|
||||
})
|
||||
applyRuntimeCandidateCacheAffinity(&candidates[1], ListModelCandidatesOptions{
|
||||
CacheAffinityKey: "conversation:test",
|
||||
CacheAffinityPolicy: policy,
|
||||
}, runtimeCandidateCacheAffinityInput{
|
||||
RequestCount: 1,
|
||||
InputTokens: 8000,
|
||||
CachedInputTokens: 7000,
|
||||
EMAHitRatio: 0.75,
|
||||
LastHitRatio: 0.75,
|
||||
})
|
||||
|
||||
sortRuntimeModelCandidates(candidates)
|
||||
|
||||
if candidates[0].PlatformID != "higher-cache-score" {
|
||||
t.Fatalf("expected higher cache score candidate first, got %+v", candidates)
|
||||
}
|
||||
if candidates[0].CacheAffinity.Score <= candidates[1].CacheAffinity.Score {
|
||||
t.Fatalf("expected score to drive cache-affinity order, got %+v", candidates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeCandidateSortingKeepsFullCacheAffinityCandidateAvoided(t *testing.T) {
|
||||
candidates := []RuntimeModelCandidate{
|
||||
{
|
||||
PlatformID: "cache-affinity-full",
|
||||
PlatformPriority: 50,
|
||||
LoadLimited: true,
|
||||
LoadRatio: 1.2,
|
||||
},
|
||||
{
|
||||
PlatformID: "available",
|
||||
PlatformPriority: 10,
|
||||
},
|
||||
}
|
||||
applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
|
||||
CacheAffinityKey: "explicit:test",
|
||||
CacheAffinityPolicy: map[string]any{"enabled": true, "minSamples": 1, "maxPriorityBoost": 100, "modelTypes": []any{"text_generate"}},
|
||||
}, runtimeCandidateCacheAffinityInput{
|
||||
RequestCount: 10,
|
||||
EMAHitRatio: 1,
|
||||
})
|
||||
|
||||
sortRuntimeModelCandidates(candidates)
|
||||
|
||||
if candidates[0].PlatformID != "available" {
|
||||
t.Fatalf("expected available candidate before full cache-affinity candidate, got %+v", candidates)
|
||||
}
|
||||
if candidates[1].PlatformID != "cache-affinity-full" || !candidates[1].LoadAvoided {
|
||||
t.Fatalf("expected full cache-affinity candidate to remain avoided fallback, got %+v", candidates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultRunnerPriorityDemotePolicyUsesAutoMode(t *testing.T) {
|
||||
policy := defaultRunnerPriorityDemotePolicy()
|
||||
|
||||
@@ -115,4 +265,190 @@ func TestDefaultRunnerPriorityDemotePolicyUsesAutoMode(t *testing.T) {
|
||||
if policy["enabled"] != true {
|
||||
t.Fatalf("expected default priority demotion to stay enabled, got %+v", policy["enabled"])
|
||||
}
|
||||
if policy["deprecated"] != true || policy["replacedBy"] != "failoverPolicy.actionRules" {
|
||||
t.Fatalf("expected legacy priority demotion policy to be marked deprecated, got %+v", policy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultRunnerFailoverPolicyCarriesComprehensiveDefaultsInActionRules(t *testing.T) {
|
||||
policy := defaultRunnerFailoverPolicy()
|
||||
if policy["legacyFieldsDeprecated"] != true || policy["replacedBy"] != "actionRules" {
|
||||
t.Fatalf("expected legacy failover summary fields to be marked deprecated, got %+v", policy)
|
||||
}
|
||||
rules, ok := policy["actionRules"].([]any)
|
||||
if !ok || len(rules) < 5 {
|
||||
t.Fatalf("expected default action rules to be present, got %+v", policy["actionRules"])
|
||||
}
|
||||
|
||||
cooldownRule := defaultActionRuleByAction(t, rules, "cooldown_and_next")
|
||||
for _, code := range []string{"rate_limit", "too_many_requests", "resource_exhausted", "throttled", "quota_exceeded", "rpm_limit", "tpm_limit"} {
|
||||
if !actionRuleHasString(cooldownRule, "codes", code) {
|
||||
t.Fatalf("expected cooldown rule to include rate-limit code %q, got %+v", code, cooldownRule["codes"])
|
||||
}
|
||||
}
|
||||
for _, keyword := range []string{"rate limit", "too many requests", "resource exhausted", "request limit", "requests per minute", "tokens per minute"} {
|
||||
if !actionRuleHasString(cooldownRule, "keywords", keyword) {
|
||||
t.Fatalf("expected cooldown rule to include rate-limit keyword %q, got %+v", keyword, cooldownRule["keywords"])
|
||||
}
|
||||
}
|
||||
if !actionRuleHasInt(cooldownRule, "statusCodes", 429) {
|
||||
t.Fatalf("expected cooldown rule to include 429, got %+v", cooldownRule["statusCodes"])
|
||||
}
|
||||
|
||||
disableRule := defaultActionRuleByAction(t, rules, "disable_and_next")
|
||||
for _, code := range []string{"auth_failed", "invalid_api_key", "missing_credentials", "account_disabled", "billing_not_active", "permission_denied"} {
|
||||
if !actionRuleHasString(disableRule, "codes", code) {
|
||||
t.Fatalf("expected disable rule to include auth/platform code %q, got %+v", code, disableRule["codes"])
|
||||
}
|
||||
}
|
||||
for _, keyword := range []string{"invalid api key", "unauthorized", "credential", "permission denied", "account disabled", "billing_not_active", "余额不足"} {
|
||||
if !actionRuleHasString(disableRule, "keywords", keyword) {
|
||||
t.Fatalf("expected disable rule to include auth/platform keyword %q, got %+v", keyword, disableRule["keywords"])
|
||||
}
|
||||
}
|
||||
for _, status := range []int{401, 403} {
|
||||
if !actionRuleHasInt(disableRule, "statusCodes", status) {
|
||||
t.Fatalf("expected disable rule to include status %d, got %+v", status, disableRule["statusCodes"])
|
||||
}
|
||||
}
|
||||
|
||||
demoteRule := defaultActionRuleByAction(t, rules, "demote_and_next")
|
||||
for _, category := range []string{"timeout", "provider_5xx", "provider_overloaded"} {
|
||||
if !actionRuleHasString(demoteRule, "categories", category) {
|
||||
t.Fatalf("expected demote rule to include transient category %q, got %+v", category, demoteRule["categories"])
|
||||
}
|
||||
}
|
||||
for _, code := range []string{"timeout", "server_error", "overloaded", "provider_failed", "invalid_response", "script_timeout", "upload_failed"} {
|
||||
if !actionRuleHasString(demoteRule, "codes", code) {
|
||||
t.Fatalf("expected demote rule to include transient code %q, got %+v", code, demoteRule["codes"])
|
||||
}
|
||||
}
|
||||
for _, status := range []int{408, 500, 502, 503, 504, 520, 522, 524, 529, 530} {
|
||||
if !actionRuleHasInt(demoteRule, "statusCodes", status) {
|
||||
t.Fatalf("expected demote rule to include transient status %d, got %+v", status, demoteRule["statusCodes"])
|
||||
}
|
||||
}
|
||||
for _, keyword := range []string{"deadline exceeded", "temporarily unavailable", "service unavailable", "bad gateway", "upstream timeout", "try again later"} {
|
||||
if !actionRuleHasString(demoteRule, "keywords", keyword) {
|
||||
t.Fatalf("expected demote rule to include transient keyword %q, got %+v", keyword, demoteRule["keywords"])
|
||||
}
|
||||
}
|
||||
|
||||
nextRule := defaultActionRuleByAction(t, rules, "next")
|
||||
for _, category := range []string{"network", "stream_error"} {
|
||||
if !actionRuleHasString(nextRule, "categories", category) {
|
||||
t.Fatalf("expected next rule to include network category %q, got %+v", category, nextRule["categories"])
|
||||
}
|
||||
}
|
||||
for _, code := range []string{"network", "stream_read_error", "upload_network", "upload_config_failed", "upload_source_fetch_failed"} {
|
||||
if !actionRuleHasString(nextRule, "codes", code) {
|
||||
t.Fatalf("expected next rule to include network code %q, got %+v", code, nextRule["codes"])
|
||||
}
|
||||
}
|
||||
for _, keyword := range []string{"socket hang up", "connection reset", "econnreset", "enotfound", "unexpected eof", "stream error", "fetch failed"} {
|
||||
if !actionRuleHasString(nextRule, "keywords", keyword) {
|
||||
t.Fatalf("expected next rule to include network keyword %q, got %+v", keyword, nextRule["keywords"])
|
||||
}
|
||||
}
|
||||
|
||||
stopRule := defaultActionRuleByAction(t, rules, "stop")
|
||||
for _, code := range []string{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "request_asset_input_format_unsupported", "upload_source_too_large", "invalid_multipart_image", "script_error", "insufficient_balance", "permission_denied"} {
|
||||
if !actionRuleHasString(stopRule, "codes", code) {
|
||||
t.Fatalf("expected stop rule to include legacy hard-stop code %q, got %+v", code, stopRule["codes"])
|
||||
}
|
||||
}
|
||||
for _, status := range []int{400, 404, 405, 409, 413, 415, 422} {
|
||||
if !actionRuleHasInt(stopRule, "statusCodes", status) {
|
||||
t.Fatalf("expected stop rule to include request-error status %d, got %+v", status, stopRule["statusCodes"])
|
||||
}
|
||||
}
|
||||
for _, keyword := range []string{"invalid parameter", "missing required", "not supported", "image is required", "too large", "not found"} {
|
||||
if !actionRuleHasString(stopRule, "keywords", keyword) {
|
||||
t.Fatalf("expected stop rule to include hard-stop keyword %q, got %+v", keyword, stopRule["keywords"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyDefaultRunnerActionRulesAreDetectedForUpgrade(t *testing.T) {
|
||||
legacy := []any{
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"network", "timeout", "stream_error", "provider_5xx", "provider_overloaded"},
|
||||
"action": "next",
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"rate_limit"},
|
||||
"codes": []any{"rate_limit"},
|
||||
"statusCodes": []any{429},
|
||||
"keywords": []any{"rate_limit", "429", "too many requests"},
|
||||
"action": "cooldown_and_next",
|
||||
"target": "model",
|
||||
"cooldownSeconds": 300,
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"keywords": []any{"rate limit", "too many requests", "temporarily_unavailable"},
|
||||
"action": "demote_and_next",
|
||||
"target": "platform",
|
||||
"demoteSteps": 1,
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"auth_error"},
|
||||
"codes": []any{"auth_failed", "invalid_api_key", "missing_credentials"},
|
||||
"statusCodes": []any{401, 403},
|
||||
"keywords": []any{"invalid api key", "api key is invalid", "unauthorized", "forbidden", "permission denied"},
|
||||
"action": "disable_and_next",
|
||||
"target": "platform",
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "insufficient_balance", "permission_denied"},
|
||||
"keywords": []any{"invalid_parameter", "missing required", "bad request", "insufficient balance"},
|
||||
"action": "stop",
|
||||
},
|
||||
}
|
||||
if !runnerActionRulesLookLikeLegacyDefaults(legacy) {
|
||||
t.Fatal("expected old compact default action rules to be detected")
|
||||
}
|
||||
if runnerActionRulesLookLikeLegacyDefaults(defaultRunnerFailoverPolicy()["actionRules"]) {
|
||||
t.Fatal("enhanced defaults should not be treated as legacy")
|
||||
}
|
||||
}
|
||||
|
||||
func defaultActionRuleByAction(t *testing.T, rules []any, action string) map[string]any {
|
||||
t.Helper()
|
||||
for _, raw := range rules {
|
||||
rule, ok := raw.(map[string]any)
|
||||
if ok && rule["action"] == action {
|
||||
return rule
|
||||
}
|
||||
}
|
||||
t.Fatalf("expected action rule %q in %+v", action, rules)
|
||||
return nil
|
||||
}
|
||||
|
||||
func actionRuleHasString(rule map[string]any, key string, want string) bool {
|
||||
for _, raw := range ruleSlice(rule, key) {
|
||||
if raw == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func actionRuleHasInt(rule map[string]any, key string, want int) bool {
|
||||
for _, raw := range ruleSlice(rule, key) {
|
||||
if raw == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ruleSlice(rule map[string]any, key string) []any {
|
||||
values, _ := rule[key].([]any)
|
||||
return values
|
||||
}
|
||||
|
||||
@@ -278,6 +278,7 @@ type RunnerPolicy struct {
|
||||
HardStopPolicy map[string]any `json:"hardStopPolicy,omitempty"`
|
||||
PriorityDemotePolicy map[string]any `json:"priorityDemotePolicy,omitempty"`
|
||||
SingleSourcePolicy map[string]any `json:"singleSourcePolicy,omitempty"`
|
||||
CacheAffinityPolicy map[string]any `json:"cacheAffinityPolicy,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Status string `json:"status"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
|
||||
@@ -3,6 +3,7 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
|
||||
const runnerPolicyColumns = `
|
||||
id::text, policy_key, name, COALESCE(description, ''), failover_policy,
|
||||
hard_stop_policy, priority_demote_policy, single_source_policy, metadata, status, created_at, updated_at`
|
||||
hard_stop_policy, priority_demote_policy, single_source_policy, cache_affinity_policy, metadata, status, created_at, updated_at`
|
||||
|
||||
type RunnerPolicyInput struct {
|
||||
PolicyKey string `json:"policyKey"`
|
||||
@@ -21,6 +22,7 @@ type RunnerPolicyInput struct {
|
||||
HardStopPolicy map[string]any `json:"hardStopPolicy"`
|
||||
PriorityDemotePolicy map[string]any `json:"priorityDemotePolicy"`
|
||||
SingleSourcePolicy map[string]any `json:"singleSourcePolicy"`
|
||||
CacheAffinityPolicy map[string]any `json:"cacheAffinityPolicy"`
|
||||
Metadata map[string]any `json:"metadata"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
@@ -52,24 +54,26 @@ func (s *Store) UpsertDefaultRunnerPolicy(ctx context.Context, input RunnerPolic
|
||||
hardStopPolicy, _ := json.Marshal(emptyObjectIfNil(input.HardStopPolicy))
|
||||
priorityDemotePolicy, _ := json.Marshal(emptyObjectIfNil(input.PriorityDemotePolicy))
|
||||
singleSourcePolicy, _ := json.Marshal(emptyObjectIfNil(input.SingleSourcePolicy))
|
||||
cacheAffinityPolicy, _ := json.Marshal(emptyObjectIfNil(input.CacheAffinityPolicy))
|
||||
metadata, _ := json.Marshal(emptyObjectIfNil(input.Metadata))
|
||||
return scanRunnerPolicy(s.pool.QueryRow(ctx, `
|
||||
INSERT INTO gateway_runner_policies (
|
||||
policy_key, name, description, failover_policy, hard_stop_policy, priority_demote_policy, single_source_policy, metadata, status
|
||||
)
|
||||
VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (policy_key) DO UPDATE
|
||||
SET name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
failover_policy = EXCLUDED.failover_policy,
|
||||
hard_stop_policy = EXCLUDED.hard_stop_policy,
|
||||
priority_demote_policy = EXCLUDED.priority_demote_policy,
|
||||
single_source_policy = EXCLUDED.single_source_policy,
|
||||
metadata = EXCLUDED.metadata,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = now()
|
||||
RETURNING `+runnerPolicyColumns,
|
||||
input.PolicyKey, input.Name, input.Description, failoverPolicy, hardStopPolicy, priorityDemotePolicy, singleSourcePolicy, metadata, input.Status,
|
||||
INSERT INTO gateway_runner_policies (
|
||||
policy_key, name, description, failover_policy, hard_stop_policy, priority_demote_policy, single_source_policy, cache_affinity_policy, metadata, status
|
||||
)
|
||||
VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT (policy_key) DO UPDATE
|
||||
SET name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
failover_policy = EXCLUDED.failover_policy,
|
||||
hard_stop_policy = EXCLUDED.hard_stop_policy,
|
||||
priority_demote_policy = EXCLUDED.priority_demote_policy,
|
||||
single_source_policy = EXCLUDED.single_source_policy,
|
||||
cache_affinity_policy = EXCLUDED.cache_affinity_policy,
|
||||
metadata = EXCLUDED.metadata,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = now()
|
||||
RETURNING `+runnerPolicyColumns,
|
||||
input.PolicyKey, input.Name, input.Description, failoverPolicy, hardStopPolicy, priorityDemotePolicy, singleSourcePolicy, cacheAffinityPolicy, metadata, input.Status,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -79,6 +83,7 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
|
||||
var hardStopPolicy []byte
|
||||
var priorityDemotePolicy []byte
|
||||
var singleSourcePolicy []byte
|
||||
var cacheAffinityPolicy []byte
|
||||
var metadata []byte
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
@@ -89,6 +94,7 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
|
||||
&hardStopPolicy,
|
||||
&priorityDemotePolicy,
|
||||
&singleSourcePolicy,
|
||||
&cacheAffinityPolicy,
|
||||
&metadata,
|
||||
&item.Status,
|
||||
&item.CreatedAt,
|
||||
@@ -100,7 +106,17 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
|
||||
item.HardStopPolicy = decodeObject(hardStopPolicy)
|
||||
item.PriorityDemotePolicy = decodeObject(priorityDemotePolicy)
|
||||
item.SingleSourcePolicy = decodeObject(singleSourcePolicy)
|
||||
item.CacheAffinityPolicy = decodeObject(cacheAffinityPolicy)
|
||||
item.Metadata = decodeObject(metadata)
|
||||
if len(item.CacheAffinityPolicy) == 0 {
|
||||
item.CacheAffinityPolicy = defaultRunnerCacheAffinityPolicy()
|
||||
}
|
||||
item.FailoverPolicy = markRunnerFailoverLegacyFieldsDeprecated(item.FailoverPolicy)
|
||||
item.HardStopPolicy = markRunnerLegacyPolicyDeprecated(item.HardStopPolicy)
|
||||
item.PriorityDemotePolicy = markRunnerLegacyPolicyDeprecated(item.PriorityDemotePolicy)
|
||||
if item.PolicyKey == "default-runner-v1" && runnerActionRulesLookLikeLegacyDefaults(item.FailoverPolicy["actionRules"]) {
|
||||
item.FailoverPolicy["actionRules"] = defaultRunnerFailoverPolicy()["actionRules"]
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
@@ -118,9 +134,155 @@ func normalizeRunnerPolicyInput(input RunnerPolicyInput) RunnerPolicyInput {
|
||||
if input.Status == "" {
|
||||
input.Status = "active"
|
||||
}
|
||||
if len(input.CacheAffinityPolicy) == 0 {
|
||||
input.CacheAffinityPolicy = defaultRunnerCacheAffinityPolicy()
|
||||
}
|
||||
input.FailoverPolicy = markRunnerFailoverLegacyFieldsDeprecated(input.FailoverPolicy)
|
||||
input.HardStopPolicy = markRunnerLegacyPolicyDeprecated(input.HardStopPolicy)
|
||||
input.PriorityDemotePolicy = markRunnerLegacyPolicyDeprecated(input.PriorityDemotePolicy)
|
||||
return input
|
||||
}
|
||||
|
||||
func markRunnerLegacyPolicyDeprecated(policy map[string]any) map[string]any {
|
||||
if policy == nil {
|
||||
policy = map[string]any{}
|
||||
}
|
||||
policy["deprecated"] = true
|
||||
policy["replacedBy"] = "failoverPolicy.actionRules"
|
||||
return policy
|
||||
}
|
||||
|
||||
func markRunnerFailoverLegacyFieldsDeprecated(policy map[string]any) map[string]any {
|
||||
if policy == nil {
|
||||
policy = map[string]any{}
|
||||
}
|
||||
policy["legacyFieldsDeprecated"] = true
|
||||
policy["deprecatedFields"] = []any{"allowCategories", "denyCategories", "allowCodes", "denyCodes", "allowKeywords", "denyKeywords", "allowStatusCodes", "denyStatusCodes", "actions"}
|
||||
policy["replacedBy"] = "actionRules"
|
||||
return policy
|
||||
}
|
||||
|
||||
func runnerActionRulesLookLikeLegacyDefaults(raw any) bool {
|
||||
rawRules, ok := raw.([]any)
|
||||
if !ok || len(rawRules) != 5 {
|
||||
return false
|
||||
}
|
||||
byAction := make(map[string]map[string]any, len(rawRules))
|
||||
for _, rawRule := range rawRules {
|
||||
rule, ok := rawRule.(map[string]any)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
action := strings.TrimSpace(fmt.Sprint(rule["action"]))
|
||||
if action == "" {
|
||||
action = "next"
|
||||
}
|
||||
if _, exists := byAction[action]; exists {
|
||||
return false
|
||||
}
|
||||
byAction[action] = rule
|
||||
}
|
||||
next, nextOK := byAction["next"]
|
||||
cooldown, cooldownOK := byAction["cooldown_and_next"]
|
||||
demote, demoteOK := byAction["demote_and_next"]
|
||||
disable, disableOK := byAction["disable_and_next"]
|
||||
stop, stopOK := byAction["stop"]
|
||||
if !nextOK || !cooldownOK || !demoteOK || !disableOK || !stopOK {
|
||||
return false
|
||||
}
|
||||
if runnerRulesHaveEnhancedDefaultTerms(rawRules) {
|
||||
return false
|
||||
}
|
||||
return runnerRuleHasAll(cooldown, "categories", "rate_limit") &&
|
||||
runnerRuleHasAll(cooldown, "codes", "rate_limit") &&
|
||||
runnerRuleHasAll(cooldown, "statusCodes", "429") &&
|
||||
len(runnerRuleStrings(cooldown, "codes")) <= 1 &&
|
||||
len(runnerRuleStrings(cooldown, "keywords")) <= 3 &&
|
||||
runnerRuleHasAll(disable, "categories", "auth_error") &&
|
||||
runnerRuleHasAll(disable, "codes", "auth_failed", "invalid_api_key", "missing_credentials") &&
|
||||
runnerRuleHasAll(disable, "statusCodes", "401", "403") &&
|
||||
len(runnerRuleStrings(disable, "codes")) <= 3 &&
|
||||
len(runnerRuleStrings(disable, "keywords")) <= 8 &&
|
||||
runnerRuleHasAll(stop, "categories", "request_error", "unsupported_model", "user_permission", "insufficient_balance") &&
|
||||
len(runnerRuleStrings(stop, "codes")) <= 8 &&
|
||||
len(runnerRuleStrings(stop, "keywords")) <= 4 &&
|
||||
len(runnerRuleStrings(next, "categories")) >= 5 &&
|
||||
len(runnerRuleStrings(next, "codes")) <= 6 &&
|
||||
len(runnerRuleStrings(next, "keywords")) <= 13 &&
|
||||
(runnerRuleHasAll(demote, "keywords", "temporarily_unavailable") ||
|
||||
runnerRuleHasAll(demote, "categories", "provider_5xx") ||
|
||||
runnerRuleHasAll(demote, "codes", "server_error")) &&
|
||||
len(runnerRuleStrings(demote, "codes")) <= 6 &&
|
||||
len(runnerRuleStrings(demote, "keywords")) <= 8
|
||||
}
|
||||
|
||||
func runnerRulesHaveEnhancedDefaultTerms(rawRules []any) bool {
|
||||
markers := map[string]struct{}{
|
||||
"resource_exhausted": {},
|
||||
"rpm_limit": {},
|
||||
"tpm_limit": {},
|
||||
"account_disabled": {},
|
||||
"billing_not_active": {},
|
||||
"provider_failed": {},
|
||||
"invalid_response": {},
|
||||
"request_asset_fetch_failed": {},
|
||||
"socket hang up": {},
|
||||
"econnreset": {},
|
||||
"request_asset_input_format_unsupported": {},
|
||||
"invalid_multipart_image": {},
|
||||
}
|
||||
for _, rawRule := range rawRules {
|
||||
rule, ok := rawRule.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, field := range []string{"categories", "codes", "statusCodes", "keywords"} {
|
||||
for _, value := range runnerRuleStrings(rule, field) {
|
||||
if _, ok := markers[value]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func runnerRuleHasAll(rule map[string]any, key string, values ...string) bool {
|
||||
actual := make(map[string]struct{}, len(values))
|
||||
for _, value := range runnerRuleStrings(rule, key) {
|
||||
actual[value] = struct{}{}
|
||||
}
|
||||
for _, value := range values {
|
||||
if _, ok := actual[strings.ToLower(strings.TrimSpace(value))]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func runnerRuleStrings(rule map[string]any, key string) []string {
|
||||
raw, ok := rule[key].([]any)
|
||||
if !ok {
|
||||
if typed, ok := rule[key].([]string); ok {
|
||||
out := make([]string, 0, len(typed))
|
||||
for _, item := range typed {
|
||||
if value := strings.ToLower(strings.TrimSpace(item)); value != "" {
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, item := range raw {
|
||||
if value := strings.ToLower(strings.TrimSpace(fmt.Sprint(item))); value != "" {
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func defaultRunnerPolicy() RunnerPolicy {
|
||||
now := time.Now()
|
||||
return RunnerPolicy{
|
||||
@@ -131,6 +293,7 @@ func defaultRunnerPolicy() RunnerPolicy {
|
||||
HardStopPolicy: defaultRunnerHardStopPolicy(),
|
||||
PriorityDemotePolicy: defaultRunnerPriorityDemotePolicy(),
|
||||
SingleSourcePolicy: defaultRunnerSingleSourcePolicy(),
|
||||
CacheAffinityPolicy: defaultRunnerCacheAffinityPolicy(),
|
||||
Metadata: map[string]any{"source": "code-default"},
|
||||
Status: "active",
|
||||
CreatedAt: now,
|
||||
@@ -141,6 +304,8 @@ func defaultRunnerPolicy() RunnerPolicy {
|
||||
func defaultRunnerPriorityDemotePolicy() map[string]any {
|
||||
return map[string]any{
|
||||
"enabled": true,
|
||||
"deprecated": true,
|
||||
"replacedBy": "failoverPolicy.actionRules",
|
||||
"categories": []any{"network", "timeout", "stream_error", "rate_limit", "provider_5xx", "provider_overloaded"},
|
||||
"codes": []any{"network", "timeout", "stream_read_error", "rate_limit", "server_error", "overloaded"},
|
||||
"statusCodes": []any{408, 429, 500, 502, 503, 504},
|
||||
@@ -154,24 +319,88 @@ func defaultRunnerSingleSourcePolicy() map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRunnerCacheAffinityPolicy() map[string]any {
|
||||
return map[string]any{
|
||||
"enabled": true,
|
||||
"modelTypes": []any{"chat", "text_generate", "responses"},
|
||||
"minSamples": 3,
|
||||
"minInputTokens": 512,
|
||||
"staleAfterSeconds": 86400,
|
||||
"maxPriorityBoost": 20,
|
||||
"emaAlpha": 0.35,
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRunnerFailoverPolicy() map[string]any {
|
||||
return map[string]any{
|
||||
"enabled": true,
|
||||
"maxPlatforms": 99,
|
||||
"maxDurationSeconds": 600,
|
||||
"allowCategories": []any{"network", "timeout", "stream_error", "rate_limit", "provider_5xx", "provider_overloaded", "auth_error"},
|
||||
"denyCategories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"allowCodes": []any{"auth_failed", "invalid_api_key", "missing_credentials"},
|
||||
"allowKeywords": []any{"timeout", "network", "rate_limit", "overloaded", "temporarily_unavailable", "server_error", "auth_failed", "invalid_api_key", "missing_credentials", "unauthorized", "forbidden", "429", "5xx"},
|
||||
"denyKeywords": []any{"invalid_parameter", "missing required", "bad request"},
|
||||
"allowStatusCodes": []any{401, 403, 408, 429, 500, 502, 503, 504},
|
||||
"denyStatusCodes": []any{},
|
||||
"enabled": true,
|
||||
"maxPlatforms": 99,
|
||||
"maxDurationSeconds": 600,
|
||||
"legacyFieldsDeprecated": true,
|
||||
"deprecatedFields": []any{"allowCategories", "denyCategories", "allowCodes", "denyCodes", "allowKeywords", "denyKeywords", "allowStatusCodes", "denyStatusCodes", "actions"},
|
||||
"replacedBy": "actionRules",
|
||||
"allowCategories": []any{"network", "timeout", "stream_error", "rate_limit", "provider_5xx", "provider_overloaded", "auth_error"},
|
||||
"denyCategories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"allowCodes": []any{"auth_failed", "invalid_api_key", "missing_credentials"},
|
||||
"allowKeywords": []any{"timeout", "network", "rate_limit", "overloaded", "temporarily_unavailable", "server_error", "auth_failed", "invalid_api_key", "missing_credentials", "unauthorized", "forbidden", "429", "5xx"},
|
||||
"denyKeywords": []any{"invalid_parameter", "missing required", "bad request"},
|
||||
"allowStatusCodes": []any{401, 403, 408, 429, 500, 502, 503, 504},
|
||||
"denyStatusCodes": []any{},
|
||||
"actionRules": []any{
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"rate_limit"},
|
||||
"codes": []any{"rate_limit", "too_many_requests", "too_many_request", "rate_limit_exceeded", "requests_rate_limited", "requests_limit_exceeded", "resource_exhausted", "throttled", "quota_exceeded", "quota_exhausted", "qps_limit", "rpm_limit", "tpm_limit"},
|
||||
"statusCodes": []any{429},
|
||||
"keywords": []any{"rate limit", "rate_limit", "rate-limit", "too many requests", "too_many_requests", "429", "throttle", "throttled", "resource exhausted", "quota exceeded", "quota_exceeded", "rate exceeded", "request limit", "requests per minute", "tokens per minute", "qps", "rpm", "tpm"},
|
||||
"action": "cooldown_and_next",
|
||||
"target": "model",
|
||||
"cooldownSeconds": 300,
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"auth_error"},
|
||||
"codes": []any{"auth_failed", "invalid_api_key", "missing_credentials", "missing_credential", "unauthorized", "forbidden", "account_disabled", "billing_not_active", "permission_denied"},
|
||||
"statusCodes": []any{401, 403},
|
||||
"keywords": []any{"invalid api key", "api key is invalid", "invalid_api_key", "apikey invalid", "api key invalid", "unauthorized", "forbidden", "authentication", "auth failed", "credential", "missing credential", "permission denied", "access denied", "invalid signature", "signature mismatch", "secret key", "access key", "account disabled", "account_deactivated", "billing not active", "billing_not_active", "insufficient balance", "quota exceeded", "余额不足", "欠费"},
|
||||
"action": "disable_and_next",
|
||||
"target": "platform",
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"timeout", "provider_5xx", "provider_overloaded"},
|
||||
"codes": []any{"timeout", "server_error", "overloaded", "provider_failed", "invalid_response", "script_timeout", "request_asset_fetch_failed", "upload_failed", "upload_read_failed", "upload_source_read_failed"},
|
||||
"statusCodes": []any{408, 500, 502, 503, 504, 520, 521, 522, 523, 524, 529, 530},
|
||||
"keywords": []any{"timeout", "timed out", "deadline exceeded", "context deadline exceeded", "overloaded", "overload", "temporarily_unavailable", "temporarily unavailable", "temporary unavailable", "service unavailable", "server error", "internal server error", "bad gateway", "gateway timeout", "upstream timeout", "upstream error", "provider failed", "model is overloaded", "capacity", "try again later", "please try again later", "5xx", "500", "502", "503", "504"},
|
||||
"action": "demote_and_next",
|
||||
"target": "platform",
|
||||
"demoteSteps": 1,
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"network", "stream_error"},
|
||||
"codes": []any{"network", "stream_read_error", "cancelled", "request_asset_fetch_failed", "upload_network", "upload_config_failed", "local_static_store_failed", "upload_source_fetch_failed"},
|
||||
"statusCodes": []any{},
|
||||
"keywords": []any{"network", "socket hang up", "socket closed", "connection reset", "connection refused", "connection aborted", "broken pipe", "econnreset", "econnrefused", "eai_again", "enotfound", "no such host", "dns", "tls handshake timeout", "unexpected eof", "connection closed", "stream error", "stream_read_error", "read failed", "fetch failed"},
|
||||
"action": "next",
|
||||
},
|
||||
map[string]any{
|
||||
"enabled": true,
|
||||
"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "unsupported_operation", "unsupported_request_resolution", "request_asset_input_format_unsupported", "request_asset_expired", "request_asset_decode_failed", "request_asset_public_url_required", "upload_no_channel", "upload_unsupported_channel", "upload_source_too_large", "upload_source_invalid_url", "upload_source_unsupported_url", "upload_decode_failed", "invalid_proxy", "invalid_json_body", "invalid_multipart_body", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio", "missing_configuration", "script_error", "cloned_voice_not_found", "cloned_voice_expired", "cloned_voice_unavailable", "insufficient_balance", "permission_denied"},
|
||||
"statusCodes": []any{400, 404, 405, 406, 409, 410, 411, 413, 415, 422},
|
||||
"keywords": []any{"invalid_parameter", "invalid parameter", "missing required", "missing_required", "bad request", "unsupported", "not supported", "does not support", "is not supported", "insufficient balance", "permission denied", "no permission", "invalid json", "invalid multipart", "image is required", "audio must", "prompt is required", "model is required", "exceeds", "too large", "expired", "not found", "required"},
|
||||
"action": "stop",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRunnerHardStopPolicy() map[string]any {
|
||||
return map[string]any{
|
||||
"enabled": true,
|
||||
"deprecated": true,
|
||||
"replacedBy": "failoverPolicy.actionRules",
|
||||
"categories": []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
|
||||
"codes": []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "insufficient_balance", "permission_denied"},
|
||||
"statusCodes": []any{},
|
||||
|
||||
@@ -129,6 +129,18 @@ WHERE id = $1::uuid`, platformID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) DisableCandidatePlatformModel(ctx context.Context, platformModelID string) error {
|
||||
if strings.TrimSpace(platformModelID) == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
UPDATE platform_models
|
||||
SET enabled = false,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid`, platformModelID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) CooldownCandidatePlatform(ctx context.Context, platformID string, cooldownSeconds int) error {
|
||||
if strings.TrimSpace(platformID) == "" {
|
||||
return nil
|
||||
@@ -160,9 +172,16 @@ WHERE id = $1::uuid`, platformModelID, cooldownSeconds)
|
||||
}
|
||||
|
||||
func (s *Store) DemoteCandidatePlatformPriority(ctx context.Context, platformID string, platformModelID string, requestedModel string, modelType string) (int, error) {
|
||||
return s.DemoteCandidatePlatformPriorityBySteps(ctx, platformID, platformModelID, requestedModel, modelType, 99)
|
||||
}
|
||||
|
||||
func (s *Store) DemoteCandidatePlatformPriorityBySteps(ctx context.Context, platformID string, platformModelID string, requestedModel string, modelType string, demoteSteps int) (int, error) {
|
||||
if strings.TrimSpace(platformID) == "" || strings.TrimSpace(platformModelID) == "" || strings.TrimSpace(requestedModel) == "" {
|
||||
return 0, nil
|
||||
}
|
||||
if demoteSteps <= 0 {
|
||||
demoteSteps = 1
|
||||
}
|
||||
var dynamicPriority int
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
WITH current_model AS (
|
||||
@@ -171,10 +190,13 @@ func (s *Store) DemoteCandidatePlatformPriority(ctx context.Context, platformID
|
||||
WHERE id = $2::uuid
|
||||
AND platform_id = $1::uuid
|
||||
),
|
||||
peer_priority AS (
|
||||
SELECT MAX(COALESCE(peer.dynamic_priority, peer.priority)) AS max_priority
|
||||
eligible_raw AS (
|
||||
SELECT peer.id AS platform_id,
|
||||
peer.priority,
|
||||
COALESCE(peer.dynamic_priority, peer.priority) AS effective_priority,
|
||||
peer.created_at
|
||||
FROM current_model current
|
||||
JOIN platform_models peer_model ON peer_model.platform_id <> current.platform_id
|
||||
JOIN platform_models peer_model ON TRUE
|
||||
JOIN integration_platforms peer ON peer.id = peer_model.platform_id
|
||||
LEFT JOIN base_model_catalog peer_base ON peer_base.id = peer_model.base_model_id
|
||||
WHERE peer.status = 'enabled'
|
||||
@@ -194,14 +216,43 @@ func (s *Store) DemoteCandidatePlatformPriority(ctx context.Context, platformID
|
||||
)
|
||||
)
|
||||
)
|
||||
),
|
||||
eligible AS (
|
||||
SELECT DISTINCT ON (platform_id) platform_id, priority, effective_priority, created_at
|
||||
FROM eligible_raw
|
||||
ORDER BY platform_id, effective_priority ASC, priority ASC, created_at ASC
|
||||
),
|
||||
ordered AS (
|
||||
SELECT platform_id, effective_priority,
|
||||
row_number() OVER (ORDER BY effective_priority ASC, priority ASC, created_at ASC, platform_id ASC) AS rn
|
||||
FROM eligible
|
||||
),
|
||||
current_position AS (
|
||||
SELECT rn, effective_priority
|
||||
FROM ordered
|
||||
WHERE platform_id = $1::uuid
|
||||
),
|
||||
next_position AS (
|
||||
SELECT ordered.effective_priority
|
||||
FROM ordered, current_position
|
||||
WHERE ordered.rn >= current_position.rn + $5::int
|
||||
ORDER BY ordered.rn ASC
|
||||
LIMIT 1
|
||||
),
|
||||
target_priority AS (
|
||||
SELECT COALESCE(
|
||||
(SELECT effective_priority FROM next_position),
|
||||
(SELECT MAX(effective_priority) FROM ordered),
|
||||
(SELECT effective_priority + $5::int FROM current_position)
|
||||
) AS value
|
||||
)
|
||||
UPDATE integration_platforms target
|
||||
SET dynamic_priority = COALESCE((SELECT max_priority FROM peer_priority), target.priority) + 1,
|
||||
SET dynamic_priority = COALESCE((SELECT value FROM target_priority), target.priority) + 1,
|
||||
updated_at = now()
|
||||
WHERE target.id = $1::uuid
|
||||
AND target.deleted_at IS NULL
|
||||
AND EXISTS (SELECT 1 FROM current_model)
|
||||
RETURNING dynamic_priority`, platformID, platformModelID, requestedModel, modelType).Scan(&dynamicPriority)
|
||||
RETURNING dynamic_priority`, platformID, platformModelID, requestedModel, modelType, demoteSteps).Scan(&dynamicPriority)
|
||||
return dynamicPriority, err
|
||||
}
|
||||
|
||||
|
||||
@@ -163,11 +163,27 @@ type RuntimeModelCandidate struct {
|
||||
LoadLimited bool
|
||||
LoadAvoided bool
|
||||
LoadMetrics RuntimeCandidateLoadMetrics
|
||||
CacheAffinity RuntimeCandidateCacheAffinity
|
||||
RunningCount float64
|
||||
WaitingCount float64
|
||||
LastAssignedUnix float64
|
||||
}
|
||||
|
||||
type RuntimeCandidateCacheAffinity struct {
|
||||
Key string
|
||||
RequestCount int
|
||||
InputTokens int
|
||||
CachedInputTokens int
|
||||
EMAHitRatio float64
|
||||
LastHitRatio float64
|
||||
LastObservedUnix float64
|
||||
Confidence float64
|
||||
Score float64
|
||||
Boost float64
|
||||
AdjustedPriority float64
|
||||
Applied bool
|
||||
}
|
||||
|
||||
type RuntimeCandidateLoadMetrics struct {
|
||||
RPMCurrent float64
|
||||
RPMLimit float64
|
||||
|
||||
Reference in New Issue
Block a user