feat(routing): 完善平台满载避让与故障轮转

为未配置并发上限的平台增加基于运行和等待任务数的软负载,并按非满载、有效优先级、缓存亲和与负载稳定排序。\n\n平台模型 RPM、TPM 和并发额度竞争失败时只轮转候选,不触发冷却、禁用或降级;用户组额度保持不可绕过。补齐异步冷却排队恢复、满载原因、选择原因与低基数指标。\n\n验证:go test ./...;go vet ./...;PostgreSQL 原子额度、准入队列、Worker 容量回收及故障策略 HTTP acceptance。
This commit is contained in:
2026-08-03 17:42:34 +08:00
parent b125599354
commit 8362d6d27a
15 changed files with 468 additions and 152 deletions
+17 -14
View File
@@ -123,11 +123,12 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
candidates, _ = pinCandidatesToTaskAdmission(candidates, admission)
var candidateRateLimitErr error
for _, candidate := range candidates {
available, availabilityErr := s.store.RuntimeCandidateAvailable(ctx, candidate.PlatformID, candidate.PlatformModelID)
available, unavailableReason, availabilityErr := s.store.RuntimeCandidateAvailability(ctx, candidate.PlatformID, candidate.PlatformModelID)
if availabilityErr != nil {
return taskAdmissionPlan{}, availabilityErr
}
if !available {
s.observeCandidateRouting(unavailableReason)
continue
}
scopes, groupID, scopeErr := s.taskAdmissionScopes(ctx, task, user, candidate)
@@ -141,13 +142,6 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
break
}
}
if !hasConcurrentLimit {
return taskAdmissionPlan{
Candidate: candidate,
Body: body,
ModelType: modelType,
}, nil
}
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidate, body),
@@ -162,6 +156,13 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
}
return taskAdmissionPlan{}, err
}
if !hasConcurrentLimit {
return taskAdmissionPlan{
Candidate: candidate,
Body: body,
ModelType: modelType,
}, nil
}
return taskAdmissionPlan{
Candidate: candidate,
Body: body,
@@ -172,6 +173,7 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
}, nil
}
if candidateRateLimitErr != nil {
s.observeCandidateRouting("all_full_queued")
return taskAdmissionPlan{}, candidateRateLimitErr
}
return taskAdmissionPlan{}, store.ErrNoModelCandidate
@@ -475,12 +477,6 @@ func (s *Service) ensureCandidateAdmission(
break
}
}
if !hasConcurrentLimit {
if err := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
return store.TaskAdmissionResult{}, false, err
}
return store.TaskAdmissionResult{}, false, nil
}
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidate, body),
@@ -488,6 +484,12 @@ func (s *Service) ensureCandidateAdmission(
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
return store.TaskAdmissionResult{}, true, err
}
if !hasConcurrentLimit {
if err := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
return store.TaskAdmissionResult{}, false, err
}
return store.TaskAdmissionResult{}, false, nil
}
plan := taskAdmissionPlan{
Candidate: candidate,
Body: body,
@@ -809,6 +811,7 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
return false, outcome.Err
}
if !outcome.Result.Admitted {
s.observeCandidateRouting("quota_race_rotated")
platformModelID := outcome.Result.Admission.PlatformModelID
if platformModelID == "" {
for _, input := range inputs {
+23 -13
View File
@@ -50,21 +50,18 @@ func isLocalRateLimitError(err error) bool {
return errors.As(err, &limitErr)
}
func platformModelRateLimitError(err error) (*store.RateLimitExceededError, bool) {
var limitErr *store.RateLimitExceededError
if !errors.As(err, &limitErr) || limitErr.ScopeType != "platform_model" {
return nil, false
}
return limitErr, true
}
func (s *Service) rateLimitReservations(ctx context.Context, user *auth.User, candidate store.RuntimeModelCandidate, body map[string]any) []store.RateLimitReservation {
out := make([]store.RateLimitReservation, 0)
out = append(out, reservationsFromPolicy(
"platform_model",
candidate.PlatformModelID,
firstNonEmptyString(candidate.DisplayName, candidate.ModelAlias, candidate.ModelName),
map[string]any{
"platformId": candidate.PlatformID,
"platformName": candidate.PlatformName,
"modelAlias": candidate.ModelAlias,
"modelName": candidate.ModelName,
},
effectiveRateLimitPolicy(candidate),
body,
)...)
// Cross-platform scopes must be evaluated first so a user-group limit can
// never be bypassed by rotating to another platform model.
if group, err := s.store.ResolveUserGroupPolicy(ctx, user); err == nil && group.ID != "" {
out = append(out, reservationsFromPolicy(
"user_group",
@@ -78,6 +75,19 @@ func (s *Service) rateLimitReservations(ctx context.Context, user *auth.User, ca
body,
)...)
}
out = append(out, reservationsFromPolicy(
"platform_model",
candidate.PlatformModelID,
firstNonEmptyString(candidate.DisplayName, candidate.ModelAlias, candidate.ModelName),
map[string]any{
"platformId": candidate.PlatformID,
"platformName": candidate.PlatformName,
"modelAlias": candidate.ModelAlias,
"modelName": candidate.ModelName,
},
effectiveRateLimitPolicy(candidate),
body,
)...)
return out
}
+3
View File
@@ -50,6 +50,9 @@ func (s *Service) Estimate(ctx context.Context, kind string, model string, body
// using stale routing or capacity state.
func (s *Service) ValidateModelAccess(ctx context.Context, kind string, model string, body map[string]any, user *auth.User) error {
_, _, err := s.candidatesForRequest(ctx, kind, model, body, user)
if store.ModelCandidateRetryAfter(err) > 0 {
s.observeCandidateRouting("cooldown_skipped")
}
return err
}
+21 -19
View File
@@ -126,6 +126,8 @@ func attemptMetrics(candidate store.RuntimeModelCandidate, attemptNo int, simula
"currentPriority": candidate.PlatformPriority,
"loadRatio": candidate.LoadRatio,
"loadAvoided": candidate.LoadAvoided,
"fullReasons": candidate.FullReasons,
"selectionReason": candidate.SelectionReason,
"simulated": simulated,
}
if candidate.ResponseProtocol != "" {
@@ -138,25 +140,25 @@ func attemptMetrics(candidate store.RuntimeModelCandidate, attemptNo int, simula
metrics["responseConverted"] = true
}
}
if candidate.LoadLimited {
metrics["loadMetrics"] = map[string]any{
"rpm": map[string]any{
"current": candidate.LoadMetrics.RPMCurrent,
"limit": candidate.LoadMetrics.RPMLimit,
"ratio": candidate.LoadMetrics.RPMRatio,
},
"tpm": map[string]any{
"current": candidate.LoadMetrics.TPMCurrent,
"limit": candidate.LoadMetrics.TPMLimit,
"ratio": candidate.LoadMetrics.TPMRatio,
},
"concurrent": map[string]any{
"current": candidate.LoadMetrics.ConcurrentCurrent,
"limit": candidate.LoadMetrics.ConcurrentLimit,
"ratio": candidate.LoadMetrics.ConcurrentRatio,
},
"queued": candidate.LoadMetrics.QueuedCount,
}
metrics["loadMetrics"] = map[string]any{
"rpm": map[string]any{
"current": candidate.LoadMetrics.RPMCurrent,
"limit": candidate.LoadMetrics.RPMLimit,
"ratio": candidate.LoadMetrics.RPMRatio,
},
"tpm": map[string]any{
"current": candidate.LoadMetrics.TPMCurrent,
"limit": candidate.LoadMetrics.TPMLimit,
"ratio": candidate.LoadMetrics.TPMRatio,
},
"concurrent": map[string]any{
"current": candidate.LoadMetrics.ConcurrentCurrent,
"limit": candidate.LoadMetrics.ConcurrentLimit,
"ratio": candidate.LoadMetrics.ConcurrentRatio,
"softCurrent": candidate.LoadMetrics.SoftCurrent,
"softRatio": candidate.LoadMetrics.SoftRatio,
},
"queued": candidate.LoadMetrics.QueuedCount,
}
if candidate.CacheAffinity.Key != "" {
metrics["cacheAffinityKey"] = candidate.CacheAffinity.Key
@@ -160,6 +160,15 @@ func resolveCandidateFailure(input resolveCandidateFailureInput) failureDecision
}
}
if errors.Is(input.Err, store.ErrRateLimited) {
if _, platformLimited := platformModelRateLimitError(input.Err); platformLimited && input.HasNextCandidate {
return failureDecision{
Route: "next",
Effect: "none",
Reason: "quota_race_rotated",
Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "platformModelCapacity", Value: "saturated"},
Info: info,
}
}
route := "stop"
if input.Async && store.RateLimitRetryable(input.Err) {
route = "requeue"
+32 -19
View File
@@ -80,24 +80,6 @@ func TestFailoverTimeBudgetExceeded(t *testing.T) {
}
}
func TestLoadAvoidanceFallbackContinuesToAvoidedCandidate(t *testing.T) {
candidates := []store.RuntimeModelCandidate{
{PlatformID: "available-candidate"},
{PlatformID: "avoided-full-candidate", LoadAvoided: true},
}
if !hasLoadAvoidanceFallback(candidates, 0, 99) {
t.Fatal("expected non-avoided candidate to fall back to later avoided candidate")
}
if hasLoadAvoidanceFallback(candidates, 1, 99) {
t.Fatal("avoided candidate should not force another load-avoidance fallback")
}
decision := loadAvoidanceFallbackDecision(&clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: false})
if !decision.Retry || decision.Reason != "load_avoidance_fallback" || decision.Action != "next" {
t.Fatalf("expected active load avoidance fallback to force next candidate, got %+v", decision)
}
}
func TestFailoverHardStopBeatsModelOverride(t *testing.T) {
runnerPolicy := store.RunnerPolicy{
Status: "active",
@@ -148,7 +130,7 @@ func TestFailoverPolicyAllowsModelOverride(t *testing.T) {
func TestLocalRateLimitWaitsInQueueWithoutRetryOrFailover(t *testing.T) {
err := &localRateLimitError{
clientErr: &clients.ClientError{Code: "rate_limit", Message: "local capacity exceeded", Retryable: true},
cause: &store.RateLimitExceededError{Metric: "concurrent", Retryable: true},
cause: &store.RateLimitExceededError{ScopeType: "user_group", Metric: "concurrent", Retryable: true},
}
retryDecision := retryDecisionForCandidate(store.RuntimeModelCandidate{}, err)
if retryDecision.Retry || retryDecision.Reason != "local_rate_limit_wait_queue" {
@@ -164,6 +146,37 @@ func TestLocalRateLimitWaitsInQueueWithoutRetryOrFailover(t *testing.T) {
}
}
func TestPlatformModelRateLimitRaceRotatesWithoutFailureEffect(t *testing.T) {
err := &localRateLimitError{
clientErr: &clients.ClientError{Code: "rate_limit", Message: "platform capacity raced", Retryable: true},
cause: &store.RateLimitExceededError{
ScopeType: "platform_model",
Metric: "concurrent",
Retryable: true,
RetryAfter: time.Second,
},
}
decision := resolveCandidateFailure(resolveCandidateFailureInput{
RunnerPolicy: store.RunnerPolicy{Status: "active"},
Err: err,
HasNextCandidate: true,
Async: true,
})
if decision.Route != "next" || decision.Effect != "none" || decision.Reason != "quota_race_rotated" {
t.Fatalf("platform quota race should rotate without mutation: %+v", decision)
}
last := resolveCandidateFailure(resolveCandidateFailureInput{
RunnerPolicy: store.RunnerPolicy{Status: "active"},
Err: err,
Async: true,
})
if last.Route != "requeue" || last.Effect != "none" {
t.Fatalf("last saturated platform should requeue: %+v", last)
}
}
func TestProviderAuthErrorsFailOverInsteadOfHardStop(t *testing.T) {
runnerPolicy := store.RunnerPolicy{
Status: "active",
+82 -55
View File
@@ -382,6 +382,7 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
err = responseChainUnavailableError()
}
if task.AsyncMode && store.ModelCandidateRetryAfter(err) > 0 {
s.observeCandidateRouting("cooldown_skipped")
queued, delay, queueErr := s.requeueModelCoolingTask(ctx, task, err)
if queueErr != nil {
return Result{}, queueErr
@@ -389,6 +390,7 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
}
if store.ModelCandidateRetryAfter(err) > 0 {
s.observeCandidateRouting("cooldown_skipped")
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err)
if finishErr != nil {
return Result{}, finishErr
@@ -481,6 +483,11 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
return Result{Task: failed, Output: failed.Result}, err
}
}
for _, candidate := range candidates {
if candidate.LoadAvoided {
s.observeCandidateRouting("full_avoided")
}
}
var asyncAdmission *store.TaskAdmission
if distributedAdmission && task.AsyncMode {
asyncAdmission, err = s.loadAsyncTaskAdmission(ctx, task)
@@ -614,25 +621,35 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
break
}
}
if hasConcurrentLimit {
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidates[0], body),
)
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
if queueErr != nil {
return Result{}, queueErr
deferInitialAdmission := false
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidates[0], body),
)
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
if _, platformLimited := platformModelRateLimitError(err); platformLimited && len(candidates) > 1 {
if task.AsyncMode {
if deleteErr := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); deleteErr != nil {
return Result{}, deleteErr
}
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
asyncAdmission = nil
}
deferInitialAdmission = true
} else if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
if queueErr != nil {
return Result{}, queueErr
}
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
} else {
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err)
if finishErr != nil {
return Result{}, finishErr
}
return Result{Task: failed, Output: failed.Result}, err
}
}
if hasConcurrentLimit && !deferInitialAdmission {
plan := taskAdmissionPlan{
Candidate: candidates[0],
Body: body,
@@ -799,13 +816,17 @@ candidatesLoop:
if platformsVisited >= maxPlatforms {
break
}
available, availabilityErr := s.store.RuntimeCandidateAvailable(ctx, candidate.PlatformID, candidate.PlatformModelID)
available, unavailableReason, availabilityErr := s.store.RuntimeCandidateAvailability(ctx, candidate.PlatformID, candidate.PlatformModelID)
if availabilityErr != nil {
return Result{}, availabilityErr
}
if !available {
s.observeCandidateRouting(unavailableReason)
continue
}
if candidate.SelectionReason == "normal_rotation" || candidate.SelectionReason == "" {
s.observeCandidateRouting("normal_rotation")
}
if distributedAdmission && candidate.PlatformModelID != admittedPlatformModelID {
admissionResult, candidateLimited, admissionErr := s.ensureCandidateAdmission(ctx, task, user, body, candidate)
if admissionErr != nil {
@@ -814,9 +835,17 @@ candidatesLoop:
if errors.Is(admissionErr, store.ErrQueueTimeout) {
break
}
if candidateLimited && errors.Is(admissionErr, store.ErrRateLimited) {
if _, platformLimited := platformModelRateLimitError(admissionErr); platformLimited {
s.observeCandidateRouting("quota_race_rotated")
continue
}
if task.AsyncMode && errors.Is(admissionErr, store.ErrRateLimited) && store.RateLimitRetryable(admissionErr) {
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, admissionErr, candidate)
if queueErr != nil {
return Result{}, queueErr
}
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
}
return Result{}, admissionErr
}
if candidateLimited {
@@ -1062,6 +1091,9 @@ candidatesLoop:
DownstreamStarted: downstreamStarted.Load(),
})
if candidateDecision.Route == "requeue" {
if _, platformLimited := platformModelRateLimitError(err); platformLimited {
s.observeCandidateRouting("all_full_queued")
}
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidate)
if queueErr != nil {
return Result{}, queueErr
@@ -1069,21 +1101,35 @@ candidatesLoop:
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
}
attemptNo = s.recordFailedAttempt(ctx, failedAttemptRecord{
Task: task,
Body: candidateBody,
Candidate: &candidate,
AttemptNo: nextAttemptNo,
Code: clients.ErrorCode(err),
Cause: err,
Simulated: isSimulation(task, candidate),
Scope: "rate_limit",
Reason: "local_rate_limit_blocked",
ExtraMetrics: []map[string]any{parameterPreprocessingMetrics(preprocessing.Log)},
ModelType: candidate.ModelType,
Task: task,
Body: candidateBody,
Candidate: &candidate,
AttemptNo: nextAttemptNo,
Code: clients.ErrorCode(err),
Cause: err,
Simulated: isSimulation(task, candidate),
Scope: "rate_limit",
Reason: "local_rate_limit_blocked",
ExtraMetrics: []map[string]any{
parameterPreprocessingMetrics(preprocessing.Log),
{"selectionReason": candidateDecision.Reason},
},
ModelType: candidate.ModelType,
})
candidateDecisionAttempt = attemptNo
candidateClientAttempt = clientAttempt
s.recordAttemptTrace(ctx, task.ID, attemptNo, failureDecisionTraceEntry(candidateDecision, candidate, clientAttempt, clientAttempts, false, ""))
if candidateDecision.Route == "next" {
s.observeCandidateRouting("quota_race_rotated")
if admittedPlatformModelID == candidate.PlatformModelID {
if deleteErr := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); deleteErr != nil {
return Result{}, deleteErr
}
admittedPlatformModelID = ""
admittedLeases = nil
}
break
}
break candidatesLoop
}
attemptNo = nextAttemptNo
@@ -1179,6 +1225,9 @@ candidatesLoop:
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: 0}
}
if task.AsyncMode && errors.Is(lastErr, store.ErrRateLimited) && store.RateLimitRetryable(lastErr) {
if _, platformLimited := platformModelRateLimitError(lastErr); platformLimited {
s.observeCandidateRouting("all_full_queued")
}
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, lastErr, lastCandidate)
if queueErr != nil {
return Result{}, queueErr
@@ -1754,6 +1803,15 @@ func (s *Service) observeProviderQuotaWait(metric string) {
}
}
func (s *Service) observeCandidateRouting(reason string) {
observer, ok := s.billingMetrics.(interface {
ObserveCandidateRouting(string)
})
if ok {
observer.ObserveCandidateRouting(reason)
}
}
func minimalRemoteTaskCheckpoint(provider string, specType string, payload map[string]any) map[string]any {
const maxBytes = 8192
provider = strings.ToLower(strings.TrimSpace(provider))
@@ -2545,37 +2603,6 @@ func failoverTimeBudgetExceeded(start time.Time, maxDuration time.Duration) bool
return maxDuration > 0 && time.Since(start) >= maxDuration
}
func hasLoadAvoidanceFallback(candidates []store.RuntimeModelCandidate, index int, maxPlatforms int) bool {
if index < 0 || index >= len(candidates) || candidates[index].LoadAvoided {
return false
}
limit := len(candidates)
if maxPlatforms > 0 && maxPlatforms < limit {
limit = maxPlatforms
}
for next := index + 1; next < limit; next++ {
if candidates[next].LoadAvoided {
return true
}
}
return false
}
func loadAvoidanceFallbackDecision(err error) failoverDecision {
return failoverDecision{
Retry: true,
Action: "next",
Reason: "load_avoidance_fallback",
Match: policyRuleMatch{
Source: "runtime_client_load",
Policy: "loadAvoidance",
Rule: "fallback",
Value: "loadRatio>=1",
},
Info: failureInfoFromError(err),
}
}
func normalizeRequest(kind string, body map[string]any) map[string]any {
out := cloneMap(body)
return out