feat(api): add load-aware client fallback

This commit is contained in:
2026-05-12 16:59:51 +08:00
parent c5cede2359
commit c2696e7bbe
7 changed files with 558 additions and 5 deletions
+22
View File
@@ -112,8 +112,30 @@ func attemptMetrics(candidate store.RuntimeModelCandidate, attemptNo int, simula
"platformModelId": candidate.PlatformModelID,
"clientId": candidate.ClientID,
"queueKey": candidate.QueueKey,
"loadRatio": candidate.LoadRatio,
"loadAvoided": candidate.LoadAvoided,
"simulated": simulated,
}
if candidate.LoadLimited {
metrics["loadMetrics"] = map[string]any{
"rpm": map[string]any{
"current": candidate.LoadMetrics.RPMCurrent,
"limit": candidate.LoadMetrics.RPMLimit,
"ratio": candidate.LoadMetrics.RPMRatio,
},
"tpm": map[string]any{
"current": candidate.LoadMetrics.TPMCurrent,
"limit": candidate.LoadMetrics.TPMLimit,
"ratio": candidate.LoadMetrics.TPMRatio,
},
"concurrent": map[string]any{
"current": candidate.LoadMetrics.ConcurrentCurrent,
"limit": candidate.LoadMetrics.ConcurrentLimit,
"ratio": candidate.LoadMetrics.ConcurrentRatio,
},
"queued": candidate.LoadMetrics.QueuedCount,
}
}
if attemptNo > 0 {
metrics["attempt"] = attemptNo
}
@@ -46,6 +46,24 @@ func TestFailoverTimeBudgetExceeded(t *testing.T) {
}
}
func TestLoadAvoidanceFallbackContinuesToAvoidedCandidate(t *testing.T) {
candidates := []store.RuntimeModelCandidate{
{PlatformID: "available-candidate"},
{PlatformID: "avoided-full-candidate", LoadAvoided: true},
}
if !hasLoadAvoidanceFallback(candidates, 0, 99) {
t.Fatal("expected non-avoided candidate to fall back to later avoided candidate")
}
if hasLoadAvoidanceFallback(candidates, 1, 99) {
t.Fatal("avoided candidate should not force another load-avoidance fallback")
}
decision := loadAvoidanceFallbackDecision(&clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: false})
if !decision.Retry || decision.Reason != "load_avoidance_fallback" || decision.Action != "next" {
t.Fatalf("expected active load avoidance fallback to force next candidate, got %+v", decision)
}
}
func TestFailoverHardStopBeatsModelOverride(t *testing.T) {
runnerPolicy := store.RunnerPolicy{
Status: "active",
+34
View File
@@ -290,6 +290,9 @@ candidatesLoop:
break
}
decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
if !decision.Retry && hasLoadAvoidanceFallback(candidates, index, maxPlatforms) {
decision = loadAvoidanceFallbackDecision(candidateErr)
}
s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision))
if !decision.Retry {
break
@@ -792,6 +795,37 @@ func failoverTimeBudgetExceeded(start time.Time, maxDuration time.Duration) bool
return maxDuration > 0 && time.Since(start) >= maxDuration
}
func hasLoadAvoidanceFallback(candidates []store.RuntimeModelCandidate, index int, maxPlatforms int) bool {
if index < 0 || index >= len(candidates) || candidates[index].LoadAvoided {
return false
}
limit := len(candidates)
if maxPlatforms > 0 && maxPlatforms < limit {
limit = maxPlatforms
}
for next := index + 1; next < limit; next++ {
if candidates[next].LoadAvoided {
return true
}
}
return false
}
func loadAvoidanceFallbackDecision(err error) failoverDecision {
return failoverDecision{
Retry: true,
Action: "next",
Reason: "load_avoidance_fallback",
Match: policyRuleMatch{
Source: "runtime_client_load",
Policy: "loadAvoidance",
Rule: "fallback",
Value: "loadRatio>=1",
},
Info: failureInfoFromError(err),
}
}
func normalizeRequest(kind string, body map[string]any) map[string]any {
out := cloneMap(body)
if kind == "responses" && out["messages"] == nil && out["input"] != nil {