BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(routing): 对齐缓存亲和力硬规则与审计

Browse Source
This commit is contained in:
wangbo
2026-07-24 23:51:27 +08:00
parent d7a0ec56f5
commit 152885bbb6
10 changed files with 540 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/store/candidates.go
+64 -8
View File
@@ -500,14 +500,15 @@ func applyRuntimeCandidateCacheAffinity(candidate *RuntimeModelCandidate, option
key = options.CacheAffinityKey
}
affinity := RuntimeCandidateCacheAffinity{
Key: key,
RequestCount: input.RequestCount,
InputTokens: input.InputTokens,
CachedInputTokens: input.CachedInputTokens,
EMAHitRatio: boundedRatio(input.EMAHitRatio),
LastHitRatio: boundedRatio(input.LastHitRatio),
LastObservedUnix: input.LastObservedUnix,
AdjustedPriority: float64(candidate.PlatformPriority),
Key: key,
RequestCount: input.RequestCount,
InputTokens: input.InputTokens,
CachedInputTokens: input.CachedInputTokens,
EMAHitRatio: boundedRatio(input.EMAHitRatio),
LastHitRatio: boundedRatio(input.LastHitRatio),
LastObservedUnix: input.LastObservedUnix,
AdjustedPriority: float64(candidate.PlatformPriority),
MatchedPrefixDepth: cacheAffinityMatchedPrefixDepth(key, options.CacheAffinityKeys),
}
minSamples := cacheAffinityMinSamples(options.CacheAffinityPolicy)
hasObservedCachedHit := affinity.CachedInputTokens > 0 || affinity.LastHitRatio > 0
@@ -538,6 +539,32 @@ func applyRuntimeCandidateCacheAffinity(candidate *RuntimeModelCandidate, option
candidate.CacheAffinity = affinity
}
func cacheAffinityMatchedPrefixDepth(key string, lookup []string) int {
prefix := ""
switch {
case strings.HasPrefix(key, "prompt_lcp_v2:"):
prefix = "prompt_lcp_v2:"
case strings.HasPrefix(key, "prompt_lcp:"):
prefix = "prompt_lcp:"
default:
return 0
}
matchingKeys := make([]string, 0, len(lookup))
for _, candidateKey := range lookup {
if strings.HasPrefix(candidateKey, prefix) {
matchingKeys = append(matchingKeys, candidateKey)
}
}
for index, candidateKey := range matchingKeys {
if candidateKey == key {
return len(matchingKeys) - index + cacheAffinityMinimumPrefixDepth - 1
}
}
return 0
}
const cacheAffinityMinimumPrefixDepth = 2
func cacheAffinityPolicyEnabled(policy map[string]any, modelType string) bool {
if enabled, ok := policy["enabled"].(bool); ok && !enabled {
return false
@@ -712,6 +739,9 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
if aFull != bFull {
return !aFull
}
if items[i].PlatformPriority != items[j].PlatformPriority {
return items[i].PlatformPriority < items[j].PlatformPriority
}
if items[i].CacheAffinity.Applied != items[j].CacheAffinity.Applied {
return items[i].CacheAffinity.Applied
}
@@ -746,6 +776,32 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
}
return false
})
appliedCount := 0
for index := range items {
if items[index].CacheAffinity.Applied {
appliedCount++
}
}
for index := range items {
items[index].CacheAffinity.CandidateCount = appliedCount
}
if len(items) == 0 || items[0].CacheAffinity.Applied {
return
}
for index := 1; index < len(items); index++ {
if !items[index].CacheAffinity.Applied {
continue
}
switch {
case runtimeCandidateFull(items[index]) && !runtimeCandidateFull(items[0]):
items[0].CacheAffinity.OverrideReason = "capacity_tier_unavailable"
case items[index].PlatformPriority != items[0].PlatformPriority:
items[0].CacheAffinity.OverrideReason = "different_effective_priority_tier"
}
if items[0].CacheAffinity.OverrideReason != "" {
break
}
}
}
func runtimeCandidateFull(candidate RuntimeModelCandidate) bool {
apps/api/internal/store/candidates_test.go
+35 -9
View File
@@ -120,7 +120,7 @@ func TestRuntimeCandidateSortingPrefersCacheAffinityWithinAvailableCandidates(t
},
{
PlatformID: "cache-affinity",
PlatformPriority: 20,
PlatformPriority: 10,
},
}
applyRuntimeCandidateCacheAffinity(&candidates[0], ListModelCandidatesOptions{
@@ -148,7 +148,7 @@ func TestRuntimeCandidateSortingPrefersCacheAffinityWithinAvailableCandidates(t
}
}
func TestRuntimeCandidateSortingPromotesObservedCacheHitAbovePriority(t *testing.T) {
func TestRuntimeCandidateSortingKeepsEffectivePriorityAheadOfCacheAffinity(t *testing.T) {
candidates := []RuntimeModelCandidate{
{
PlatformID: "volces-priority",
@@ -177,14 +177,17 @@ func TestRuntimeCandidateSortingPromotesObservedCacheHitAbovePriority(t *testing
sortRuntimeModelCandidates(candidates)
if candidates[0].PlatformID != "deepseek-cache-hit" || !candidates[0].CacheAffinity.Applied {
t.Fatalf("expected observed cache hit to outrank platform priority, got %+v", candidates)
if candidates[0].PlatformID != "volces-priority" || candidates[0].CacheAffinity.Applied {
t.Fatalf("expected effective priority tier to remain first, got %+v", candidates)
}
if candidates[0].CacheAffinity.Score <= 0 || candidates[0].CacheAffinity.Boost <= 0 {
t.Fatalf("expected observed cache hit to carry score and boost, got %+v", candidates[0].CacheAffinity)
if candidates[0].CacheAffinity.OverrideReason != "different_effective_priority_tier" {
t.Fatalf("expected priority override reason on selected candidate, got %+v", candidates[0].CacheAffinity)
}
if candidates[1].PlatformID != "volces-priority" || candidates[1].CacheAffinity.Applied {
t.Fatalf("expected priority-only candidate second, got %+v", candidates)
if candidates[1].PlatformID != "deepseek-cache-hit" || !candidates[1].CacheAffinity.Applied {
t.Fatalf("expected cache affinity candidate to remain visible as fallback, got %+v", candidates)
}
if candidates[1].CacheAffinity.Score <= 0 || candidates[1].CacheAffinity.Boost <= 0 {
t.Fatalf("expected observed cache hit to retain score and boost diagnostics, got %+v", candidates[1].CacheAffinity)
}
}
@@ -196,7 +199,7 @@ func TestRuntimeCandidateSortingOrdersMultipleCacheAffinityCandidatesByScore(t *
},
{
PlatformID: "higher-cache-score",
PlatformPriority: 50,
PlatformPriority: 1,
},
}
policy := map[string]any{"enabled": true, "minSamples": 1, "maxPriorityBoost": 20, "modelTypes": []any{"text_generate"}}
@@ -260,6 +263,29 @@ func TestRuntimeCandidateSortingKeepsFullCacheAffinityCandidateAvoided(t *testin
if candidates[1].PlatformID != "cache-affinity-full" || !candidates[1].LoadAvoided {
t.Fatalf("expected full cache-affinity candidate to remain avoided fallback, got %+v", candidates)
}
if candidates[0].CacheAffinity.OverrideReason != "capacity_tier_unavailable" {
t.Fatalf("expected capacity override reason on selected candidate, got %+v", candidates[0].CacheAffinity)
}
}
func TestCacheAffinityMatchedPrefixDepthIgnoresLegacyCompatibilityKeys(t *testing.T) {
lookup := []string{
"prompt_lcp_v2:depth-four",
"prompt_lcp_v2:depth-three",
"prompt_lcp_v2:depth-two",
"prompt_lcp:depth-four",
"prompt_lcp:depth-three",
"prompt_lcp:depth-two",
}
if depth := cacheAffinityMatchedPrefixDepth("prompt_lcp_v2:depth-three", lookup); depth != 3 {
t.Fatalf("V2 matched prefix depth=%d want=3", depth)
}
if depth := cacheAffinityMatchedPrefixDepth("prompt_lcp:depth-two", lookup); depth != 2 {
t.Fatalf("legacy matched prefix depth=%d want=2", depth)
}
if depth := cacheAffinityMatchedPrefixDepth("explicit:key", lookup); depth != 0 {
t.Fatalf("explicit key prefix depth=%d want=0", depth)
}
}
func TestDefaultRunnerPriorityDemotePolicyUsesAutoMode(t *testing.T) {
apps/api/internal/store/runtime_types.go
+15 -12
View File
@@ -178,18 +178,21 @@ type RuntimeModelCandidate struct {
}
type RuntimeCandidateCacheAffinity struct {
Key string
RequestCount int
InputTokens int
CachedInputTokens int
EMAHitRatio float64
LastHitRatio float64
LastObservedUnix float64
Confidence float64
Score float64
Boost float64
AdjustedPriority float64
Applied bool
Key string
RequestCount int
InputTokens int
CachedInputTokens int
EMAHitRatio float64
LastHitRatio float64
LastObservedUnix float64
Confidence float64
Score float64
Boost float64
AdjustedPriority float64
MatchedPrefixDepth int
CandidateCount int
OverrideReason string
Applied bool
}
type RuntimeCandidateLoadMetrics struct {
apps/api/internal/store/task_history_test.go
+13 -6
View File
@@ -128,7 +128,7 @@ WHERE task_id=$1::uuid
}
}
func TestTaskAttemptDropsDuplicateJSONSnapshots(t *testing.T) {
func TestTaskAttemptKeepsOnlyCompactDiagnostics(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
user := &auth.User{ID: "attempt-minimal-" + uuid.NewString()}
@@ -157,8 +157,8 @@ func TestTaskAttemptDropsDuplicateJSONSnapshots(t *testing.T) {
Status: "failed",
Retryable: true,
StatusCode: 503,
Usage: map[string]any{"output_tokens": 100},
Metrics: map[string]any{"trace": []any{map[string]any{"large": "duplicate"}}},
Usage: map[string]any{"output_tokens": 100, "raw": strings.Repeat("duplicate", 100)},
Metrics: map[string]any{"trace": []any{map[string]any{"large": "duplicate"}}, "cacheAffinityMatched": true, "cacheAffinityMatchedPrefixDepth": 3},
ResponseSnapshot: map[string]any{"result": "duplicate"},
ErrorCode: "upstream_error",
ErrorMessage: strings.Repeat("错误", 4096),
@@ -173,9 +173,16 @@ func TestTaskAttemptDropsDuplicateJSONSnapshots(t *testing.T) {
t.Fatalf("attempts=%d", len(attempts))
}
attempt := attempts[0]
if len(attempt.RequestSnapshot) != 0 || len(attempt.ResponseSnapshot) != 0 ||
len(attempt.Usage) != 0 || len(attempt.Metrics) != 0 || len(attempt.PricingSnapshot) != 0 {
t.Fatalf("attempt contains duplicate JSON: %+v", attempt)
if len(attempt.RequestSnapshot) != 0 || len(attempt.ResponseSnapshot) != 0 || len(attempt.PricingSnapshot) != 0 {
t.Fatalf("attempt contains duplicate snapshots: %+v", attempt)
}
if len(attempt.Usage) != 1 || taskAttemptMetricInt(attempt.Usage, "output_tokens") != 100 {
t.Fatalf("attempt compact usage mismatch: %+v", attempt.Usage)
}
if len(attempt.Metrics) != 2 ||
attempt.Metrics["cacheAffinityMatched"] != true ||
taskAttemptMetricInt(attempt.Metrics, "cacheAffinityMatchedPrefixDepth") != 3 {
t.Fatalf("attempt compact metrics mismatch: %+v", attempt.Metrics)
}
if attempt.StatusCode != 503 || len(attempt.ErrorMessage) > 2048 {
t.Fatalf("attempt status/error not preserved safely: %+v", attempt)
apps/api/internal/store/tasks_runtime.go
+44 -4
View File
@@ -996,14 +996,16 @@ func (s *Store) FinishTaskAttempt(ctx context.Context, input FinishTaskAttemptIn
if statusCode == 0 {
statusCode = taskAttemptMetricInt(input.Metrics, "statusCode")
}
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptUsage(input.Usage)))
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptMetrics(input.Metrics)))
_, err := s.pool.Exec(ctx, `
UPDATE gateway_task_attempts
SET status = $2::text,
retryable = $3,
request_id = NULLIF($4::text, ''),
status_code = NULLIF($5::int, 0),
usage = '{}'::jsonb,
metrics = '{}'::jsonb,
usage = $11::jsonb,
metrics = $12::jsonb,
response_snapshot = '{}'::jsonb,
pricing_snapshot = '{}'::jsonb,
request_fingerprint = NULL,
@@ -1024,10 +1026,45 @@ WHERE id = $1::uuid`,
input.ResponseDurationMS,
input.ErrorCode,
truncateUTF8Bytes(input.ErrorMessage, 2048),
usageJSON,
metricsJSON,
)
return err
}
func minimalTaskAttemptUsage(usage map[string]any) map[string]any {
return whitelistedTaskAttemptMap(usage, []string{
"inputTokens", "promptTokens", "input_tokens", "prompt_tokens",
"cachedInputTokens", "cachedPromptTokens", "cached_input_tokens", "cached_tokens",
"cachedInputTokensKnown",
"outputTokens", "completionTokens", "output_tokens", "completion_tokens",
"totalTokens", "total_tokens",
})
}
func minimalTaskAttemptMetrics(metrics map[string]any) map[string]any {
return whitelistedTaskAttemptMap(metrics, []string{
"platformPriority", "currentPriority", "loadRatio", "loadAvoided",
"cacheAffinityKey", "cacheAdjustedPriority", "cacheAffinitySamples",
"cacheAffinityScore", "cacheAffinityConfidence", "cacheAffinityHitRatio",
"cacheAffinityEMAHitRatio", "cacheAffinityLastHitRatio",
"cacheAffinityCachedInputTokens", "cacheAffinityBoost",
"cacheAffinityApplied", "cacheAffinityMatched",
"cacheAffinityCandidateCount", "cacheAffinityMatchedPrefixDepth",
"cacheAffinityOverrideReason",
})
}
func whitelistedTaskAttemptMap(input map[string]any, keys []string) map[string]any {
out := map[string]any{}
for _, key := range keys {
if value, ok := input[key]; ok {
out[key] = value
}
}
return out
}
func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessInput) (GatewayTask, error) {
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(input.Result))
if resultReport.BinaryCount > 0 {
@@ -1041,6 +1078,8 @@ func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessIn
billingsJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Billings))
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Usage)))
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
attemptUsageJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptUsage(input.Usage)))
attemptMetricsJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptMetrics(input.Metrics)))
billingSummaryJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.BillingSummary)))
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
finalChargeAmount := strings.TrimSpace(input.FinalChargeAmountText)
@@ -1056,8 +1095,8 @@ SET status = 'succeeded',
retryable = false,
request_id = NULLIF($2, ''),
status_code = NULL,
usage = '{}'::jsonb,
metrics = '{}'::jsonb,
usage = $6::jsonb,
metrics = $7::jsonb,
response_snapshot = '{}'::jsonb,
pricing_snapshot = '{}'::jsonb,
request_fingerprint = NULL,
@@ -1072,6 +1111,7 @@ SET status = 'succeeded',
WHERE id = $1::uuid`,
input.AttemptID, input.RequestID,
nullableTime(input.ResponseStartedAt), nullableTime(input.ResponseFinishedAt), input.ResponseDurationMS,
attemptUsageJSON, attemptMetricsJSON,
); err != nil {
return err
}