feat(api): add load-aware client fallback
This commit is contained in:
@@ -112,8 +112,30 @@ func attemptMetrics(candidate store.RuntimeModelCandidate, attemptNo int, simula
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"clientId": candidate.ClientID,
|
||||
"queueKey": candidate.QueueKey,
|
||||
"loadRatio": candidate.LoadRatio,
|
||||
"loadAvoided": candidate.LoadAvoided,
|
||||
"simulated": simulated,
|
||||
}
|
||||
if candidate.LoadLimited {
|
||||
metrics["loadMetrics"] = map[string]any{
|
||||
"rpm": map[string]any{
|
||||
"current": candidate.LoadMetrics.RPMCurrent,
|
||||
"limit": candidate.LoadMetrics.RPMLimit,
|
||||
"ratio": candidate.LoadMetrics.RPMRatio,
|
||||
},
|
||||
"tpm": map[string]any{
|
||||
"current": candidate.LoadMetrics.TPMCurrent,
|
||||
"limit": candidate.LoadMetrics.TPMLimit,
|
||||
"ratio": candidate.LoadMetrics.TPMRatio,
|
||||
},
|
||||
"concurrent": map[string]any{
|
||||
"current": candidate.LoadMetrics.ConcurrentCurrent,
|
||||
"limit": candidate.LoadMetrics.ConcurrentLimit,
|
||||
"ratio": candidate.LoadMetrics.ConcurrentRatio,
|
||||
},
|
||||
"queued": candidate.LoadMetrics.QueuedCount,
|
||||
}
|
||||
}
|
||||
if attemptNo > 0 {
|
||||
metrics["attempt"] = attemptNo
|
||||
}
|
||||
|
||||
@@ -46,6 +46,24 @@ func TestFailoverTimeBudgetExceeded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAvoidanceFallbackContinuesToAvoidedCandidate(t *testing.T) {
|
||||
candidates := []store.RuntimeModelCandidate{
|
||||
{PlatformID: "available-candidate"},
|
||||
{PlatformID: "avoided-full-candidate", LoadAvoided: true},
|
||||
}
|
||||
|
||||
if !hasLoadAvoidanceFallback(candidates, 0, 99) {
|
||||
t.Fatal("expected non-avoided candidate to fall back to later avoided candidate")
|
||||
}
|
||||
if hasLoadAvoidanceFallback(candidates, 1, 99) {
|
||||
t.Fatal("avoided candidate should not force another load-avoidance fallback")
|
||||
}
|
||||
decision := loadAvoidanceFallbackDecision(&clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: false})
|
||||
if !decision.Retry || decision.Reason != "load_avoidance_fallback" || decision.Action != "next" {
|
||||
t.Fatalf("expected active load avoidance fallback to force next candidate, got %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverHardStopBeatsModelOverride(t *testing.T) {
|
||||
runnerPolicy := store.RunnerPolicy{
|
||||
Status: "active",
|
||||
|
||||
@@ -290,6 +290,9 @@ candidatesLoop:
|
||||
break
|
||||
}
|
||||
decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
|
||||
if !decision.Retry && hasLoadAvoidanceFallback(candidates, index, maxPlatforms) {
|
||||
decision = loadAvoidanceFallbackDecision(candidateErr)
|
||||
}
|
||||
s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision))
|
||||
if !decision.Retry {
|
||||
break
|
||||
@@ -792,6 +795,37 @@ func failoverTimeBudgetExceeded(start time.Time, maxDuration time.Duration) bool
|
||||
return maxDuration > 0 && time.Since(start) >= maxDuration
|
||||
}
|
||||
|
||||
func hasLoadAvoidanceFallback(candidates []store.RuntimeModelCandidate, index int, maxPlatforms int) bool {
|
||||
if index < 0 || index >= len(candidates) || candidates[index].LoadAvoided {
|
||||
return false
|
||||
}
|
||||
limit := len(candidates)
|
||||
if maxPlatforms > 0 && maxPlatforms < limit {
|
||||
limit = maxPlatforms
|
||||
}
|
||||
for next := index + 1; next < limit; next++ {
|
||||
if candidates[next].LoadAvoided {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func loadAvoidanceFallbackDecision(err error) failoverDecision {
|
||||
return failoverDecision{
|
||||
Retry: true,
|
||||
Action: "next",
|
||||
Reason: "load_avoidance_fallback",
|
||||
Match: policyRuleMatch{
|
||||
Source: "runtime_client_load",
|
||||
Policy: "loadAvoidance",
|
||||
Rule: "fallback",
|
||||
Value: "loadRatio>=1",
|
||||
},
|
||||
Info: failureInfoFromError(err),
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRequest(kind string, body map[string]any) map[string]any {
|
||||
out := cloneMap(body)
|
||||
if kind == "responses" && out["messages"] == nil && out["input"] != nil {
|
||||
|
||||
Reference in New Issue
Block a user