feat: add runner failover rules and cache affinity
This commit is contained in:
@@ -139,6 +139,19 @@ func attemptMetrics(candidate store.RuntimeModelCandidate, attemptNo int, simula
|
||||
"queued": candidate.LoadMetrics.QueuedCount,
|
||||
}
|
||||
}
|
||||
if candidate.CacheAffinity.Key != "" {
|
||||
metrics["cacheAffinityKey"] = candidate.CacheAffinity.Key
|
||||
metrics["cacheAdjustedPriority"] = candidate.CacheAffinity.AdjustedPriority
|
||||
metrics["cacheAffinitySamples"] = candidate.CacheAffinity.RequestCount
|
||||
metrics["cacheAffinityScore"] = candidate.CacheAffinity.Score
|
||||
metrics["cacheAffinityConfidence"] = candidate.CacheAffinity.Confidence
|
||||
metrics["cacheAffinityHitRatio"] = candidate.CacheAffinity.EMAHitRatio
|
||||
metrics["cacheAffinityEMAHitRatio"] = candidate.CacheAffinity.EMAHitRatio
|
||||
metrics["cacheAffinityLastHitRatio"] = candidate.CacheAffinity.LastHitRatio
|
||||
metrics["cacheAffinityCachedInputTokens"] = candidate.CacheAffinity.CachedInputTokens
|
||||
metrics["cacheAffinityBoost"] = candidate.CacheAffinity.Boost
|
||||
metrics["cacheAffinityApplied"] = candidate.CacheAffinity.Applied
|
||||
}
|
||||
if attemptNo > 0 {
|
||||
metrics["attempt"] = attemptNo
|
||||
}
|
||||
@@ -151,9 +164,10 @@ func usageToMap(usage clients.Usage) map[string]any {
|
||||
out["inputTokens"] = usage.InputTokens
|
||||
out["promptTokens"] = usage.InputTokens
|
||||
}
|
||||
if usage.CachedInputTokens > 0 {
|
||||
if usage.CachedInputTokensKnown || usage.CachedInputTokens > 0 {
|
||||
out["cachedInputTokens"] = usage.CachedInputTokens
|
||||
out["cachedPromptTokens"] = usage.CachedInputTokens
|
||||
out["cachedInputTokensKnown"] = usage.CachedInputTokensKnown
|
||||
}
|
||||
if usage.OutputTokens > 0 {
|
||||
out["outputTokens"] = usage.OutputTokens
|
||||
|
||||
Reference in New Issue
Block a user