fix(routing): 对齐缓存亲和力硬规则与审计
This commit is contained in:
@@ -500,14 +500,15 @@ func applyRuntimeCandidateCacheAffinity(candidate *RuntimeModelCandidate, option
|
||||
key = options.CacheAffinityKey
|
||||
}
|
||||
affinity := RuntimeCandidateCacheAffinity{
|
||||
Key: key,
|
||||
RequestCount: input.RequestCount,
|
||||
InputTokens: input.InputTokens,
|
||||
CachedInputTokens: input.CachedInputTokens,
|
||||
EMAHitRatio: boundedRatio(input.EMAHitRatio),
|
||||
LastHitRatio: boundedRatio(input.LastHitRatio),
|
||||
LastObservedUnix: input.LastObservedUnix,
|
||||
AdjustedPriority: float64(candidate.PlatformPriority),
|
||||
Key: key,
|
||||
RequestCount: input.RequestCount,
|
||||
InputTokens: input.InputTokens,
|
||||
CachedInputTokens: input.CachedInputTokens,
|
||||
EMAHitRatio: boundedRatio(input.EMAHitRatio),
|
||||
LastHitRatio: boundedRatio(input.LastHitRatio),
|
||||
LastObservedUnix: input.LastObservedUnix,
|
||||
AdjustedPriority: float64(candidate.PlatformPriority),
|
||||
MatchedPrefixDepth: cacheAffinityMatchedPrefixDepth(key, options.CacheAffinityKeys),
|
||||
}
|
||||
minSamples := cacheAffinityMinSamples(options.CacheAffinityPolicy)
|
||||
hasObservedCachedHit := affinity.CachedInputTokens > 0 || affinity.LastHitRatio > 0
|
||||
@@ -538,6 +539,32 @@ func applyRuntimeCandidateCacheAffinity(candidate *RuntimeModelCandidate, option
|
||||
candidate.CacheAffinity = affinity
|
||||
}
|
||||
|
||||
func cacheAffinityMatchedPrefixDepth(key string, lookup []string) int {
|
||||
prefix := ""
|
||||
switch {
|
||||
case strings.HasPrefix(key, "prompt_lcp_v2:"):
|
||||
prefix = "prompt_lcp_v2:"
|
||||
case strings.HasPrefix(key, "prompt_lcp:"):
|
||||
prefix = "prompt_lcp:"
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
matchingKeys := make([]string, 0, len(lookup))
|
||||
for _, candidateKey := range lookup {
|
||||
if strings.HasPrefix(candidateKey, prefix) {
|
||||
matchingKeys = append(matchingKeys, candidateKey)
|
||||
}
|
||||
}
|
||||
for index, candidateKey := range matchingKeys {
|
||||
if candidateKey == key {
|
||||
return len(matchingKeys) - index + cacheAffinityMinimumPrefixDepth - 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
const cacheAffinityMinimumPrefixDepth = 2
|
||||
|
||||
func cacheAffinityPolicyEnabled(policy map[string]any, modelType string) bool {
|
||||
if enabled, ok := policy["enabled"].(bool); ok && !enabled {
|
||||
return false
|
||||
@@ -712,6 +739,9 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
|
||||
if aFull != bFull {
|
||||
return !aFull
|
||||
}
|
||||
if items[i].PlatformPriority != items[j].PlatformPriority {
|
||||
return items[i].PlatformPriority < items[j].PlatformPriority
|
||||
}
|
||||
if items[i].CacheAffinity.Applied != items[j].CacheAffinity.Applied {
|
||||
return items[i].CacheAffinity.Applied
|
||||
}
|
||||
@@ -746,6 +776,32 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
|
||||
}
|
||||
return false
|
||||
})
|
||||
appliedCount := 0
|
||||
for index := range items {
|
||||
if items[index].CacheAffinity.Applied {
|
||||
appliedCount++
|
||||
}
|
||||
}
|
||||
for index := range items {
|
||||
items[index].CacheAffinity.CandidateCount = appliedCount
|
||||
}
|
||||
if len(items) == 0 || items[0].CacheAffinity.Applied {
|
||||
return
|
||||
}
|
||||
for index := 1; index < len(items); index++ {
|
||||
if !items[index].CacheAffinity.Applied {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case runtimeCandidateFull(items[index]) && !runtimeCandidateFull(items[0]):
|
||||
items[0].CacheAffinity.OverrideReason = "capacity_tier_unavailable"
|
||||
case items[index].PlatformPriority != items[0].PlatformPriority:
|
||||
items[0].CacheAffinity.OverrideReason = "different_effective_priority_tier"
|
||||
}
|
||||
if items[0].CacheAffinity.OverrideReason != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runtimeCandidateFull(candidate RuntimeModelCandidate) bool {
|
||||
|
||||
Reference in New Issue
Block a user