feat(api): add load-aware client fallback
This commit is contained in:
@@ -290,6 +290,9 @@ candidatesLoop:
|
||||
break
|
||||
}
|
||||
decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
|
||||
if !decision.Retry && hasLoadAvoidanceFallback(candidates, index, maxPlatforms) {
|
||||
decision = loadAvoidanceFallbackDecision(candidateErr)
|
||||
}
|
||||
s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision))
|
||||
if !decision.Retry {
|
||||
break
|
||||
@@ -792,6 +795,37 @@ func failoverTimeBudgetExceeded(start time.Time, maxDuration time.Duration) bool
|
||||
return maxDuration > 0 && time.Since(start) >= maxDuration
|
||||
}
|
||||
|
||||
func hasLoadAvoidanceFallback(candidates []store.RuntimeModelCandidate, index int, maxPlatforms int) bool {
|
||||
if index < 0 || index >= len(candidates) || candidates[index].LoadAvoided {
|
||||
return false
|
||||
}
|
||||
limit := len(candidates)
|
||||
if maxPlatforms > 0 && maxPlatforms < limit {
|
||||
limit = maxPlatforms
|
||||
}
|
||||
for next := index + 1; next < limit; next++ {
|
||||
if candidates[next].LoadAvoided {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func loadAvoidanceFallbackDecision(err error) failoverDecision {
|
||||
return failoverDecision{
|
||||
Retry: true,
|
||||
Action: "next",
|
||||
Reason: "load_avoidance_fallback",
|
||||
Match: policyRuleMatch{
|
||||
Source: "runtime_client_load",
|
||||
Policy: "loadAvoidance",
|
||||
Rule: "fallback",
|
||||
Value: "loadRatio>=1",
|
||||
},
|
||||
Info: failureInfoFromError(err),
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRequest(kind string, body map[string]any) map[string]any {
|
||||
out := cloneMap(body)
|
||||
if kind == "responses" && out["messages"] == nil && out["input"] != nil {
|
||||
|
||||
Reference in New Issue
Block a user