feat(routing): 完善平台满载避让与故障轮转
为未配置并发上限的平台增加基于运行和等待任务数的软负载,并按非满载、有效优先级、缓存亲和与负载稳定排序。\n\n平台模型 RPM、TPM 和并发额度竞争失败时只轮转候选,不触发冷却、禁用或降级;用户组额度保持不可绕过。补齐异步冷却排队恢复、满载原因、选择原因与低基数指标。\n\n验证:go test ./...;go vet ./...;PostgreSQL 原子额度、准入队列、Worker 容量回收及故障策略 HTTP acceptance。
This commit is contained in:
@@ -123,11 +123,12 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
candidates, _ = pinCandidatesToTaskAdmission(candidates, admission)
|
||||
var candidateRateLimitErr error
|
||||
for _, candidate := range candidates {
|
||||
available, availabilityErr := s.store.RuntimeCandidateAvailable(ctx, candidate.PlatformID, candidate.PlatformModelID)
|
||||
available, unavailableReason, availabilityErr := s.store.RuntimeCandidateAvailability(ctx, candidate.PlatformID, candidate.PlatformModelID)
|
||||
if availabilityErr != nil {
|
||||
return taskAdmissionPlan{}, availabilityErr
|
||||
}
|
||||
if !available {
|
||||
s.observeCandidateRouting(unavailableReason)
|
||||
continue
|
||||
}
|
||||
scopes, groupID, scopeErr := s.taskAdmissionScopes(ctx, task, user, candidate)
|
||||
@@ -141,13 +142,6 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasConcurrentLimit {
|
||||
return taskAdmissionPlan{
|
||||
Candidate: candidate,
|
||||
Body: body,
|
||||
ModelType: modelType,
|
||||
}, nil
|
||||
}
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidate, body),
|
||||
@@ -162,6 +156,13 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
}
|
||||
return taskAdmissionPlan{}, err
|
||||
}
|
||||
if !hasConcurrentLimit {
|
||||
return taskAdmissionPlan{
|
||||
Candidate: candidate,
|
||||
Body: body,
|
||||
ModelType: modelType,
|
||||
}, nil
|
||||
}
|
||||
return taskAdmissionPlan{
|
||||
Candidate: candidate,
|
||||
Body: body,
|
||||
@@ -172,6 +173,7 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
}, nil
|
||||
}
|
||||
if candidateRateLimitErr != nil {
|
||||
s.observeCandidateRouting("all_full_queued")
|
||||
return taskAdmissionPlan{}, candidateRateLimitErr
|
||||
}
|
||||
return taskAdmissionPlan{}, store.ErrNoModelCandidate
|
||||
@@ -475,12 +477,6 @@ func (s *Service) ensureCandidateAdmission(
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasConcurrentLimit {
|
||||
if err := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return store.TaskAdmissionResult{}, false, err
|
||||
}
|
||||
return store.TaskAdmissionResult{}, false, nil
|
||||
}
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidate, body),
|
||||
@@ -488,6 +484,12 @@ func (s *Service) ensureCandidateAdmission(
|
||||
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
|
||||
return store.TaskAdmissionResult{}, true, err
|
||||
}
|
||||
if !hasConcurrentLimit {
|
||||
if err := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return store.TaskAdmissionResult{}, false, err
|
||||
}
|
||||
return store.TaskAdmissionResult{}, false, nil
|
||||
}
|
||||
plan := taskAdmissionPlan{
|
||||
Candidate: candidate,
|
||||
Body: body,
|
||||
@@ -809,6 +811,7 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
|
||||
return false, outcome.Err
|
||||
}
|
||||
if !outcome.Result.Admitted {
|
||||
s.observeCandidateRouting("quota_race_rotated")
|
||||
platformModelID := outcome.Result.Admission.PlatformModelID
|
||||
if platformModelID == "" {
|
||||
for _, input := range inputs {
|
||||
|
||||
@@ -50,21 +50,18 @@ func isLocalRateLimitError(err error) bool {
|
||||
return errors.As(err, &limitErr)
|
||||
}
|
||||
|
||||
func platformModelRateLimitError(err error) (*store.RateLimitExceededError, bool) {
|
||||
var limitErr *store.RateLimitExceededError
|
||||
if !errors.As(err, &limitErr) || limitErr.ScopeType != "platform_model" {
|
||||
return nil, false
|
||||
}
|
||||
return limitErr, true
|
||||
}
|
||||
|
||||
func (s *Service) rateLimitReservations(ctx context.Context, user *auth.User, candidate store.RuntimeModelCandidate, body map[string]any) []store.RateLimitReservation {
|
||||
out := make([]store.RateLimitReservation, 0)
|
||||
out = append(out, reservationsFromPolicy(
|
||||
"platform_model",
|
||||
candidate.PlatformModelID,
|
||||
firstNonEmptyString(candidate.DisplayName, candidate.ModelAlias, candidate.ModelName),
|
||||
map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformName": candidate.PlatformName,
|
||||
"modelAlias": candidate.ModelAlias,
|
||||
"modelName": candidate.ModelName,
|
||||
},
|
||||
effectiveRateLimitPolicy(candidate),
|
||||
body,
|
||||
)...)
|
||||
// Cross-platform scopes must be evaluated first so a user-group limit can
|
||||
// never be bypassed by rotating to another platform model.
|
||||
if group, err := s.store.ResolveUserGroupPolicy(ctx, user); err == nil && group.ID != "" {
|
||||
out = append(out, reservationsFromPolicy(
|
||||
"user_group",
|
||||
@@ -78,6 +75,19 @@ func (s *Service) rateLimitReservations(ctx context.Context, user *auth.User, ca
|
||||
body,
|
||||
)...)
|
||||
}
|
||||
out = append(out, reservationsFromPolicy(
|
||||
"platform_model",
|
||||
candidate.PlatformModelID,
|
||||
firstNonEmptyString(candidate.DisplayName, candidate.ModelAlias, candidate.ModelName),
|
||||
map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformName": candidate.PlatformName,
|
||||
"modelAlias": candidate.ModelAlias,
|
||||
"modelName": candidate.ModelName,
|
||||
},
|
||||
effectiveRateLimitPolicy(candidate),
|
||||
body,
|
||||
)...)
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,9 @@ func (s *Service) Estimate(ctx context.Context, kind string, model string, body
|
||||
// using stale routing or capacity state.
|
||||
func (s *Service) ValidateModelAccess(ctx context.Context, kind string, model string, body map[string]any, user *auth.User) error {
|
||||
_, _, err := s.candidatesForRequest(ctx, kind, model, body, user)
|
||||
if store.ModelCandidateRetryAfter(err) > 0 {
|
||||
s.observeCandidateRouting("cooldown_skipped")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -126,6 +126,8 @@ func attemptMetrics(candidate store.RuntimeModelCandidate, attemptNo int, simula
|
||||
"currentPriority": candidate.PlatformPriority,
|
||||
"loadRatio": candidate.LoadRatio,
|
||||
"loadAvoided": candidate.LoadAvoided,
|
||||
"fullReasons": candidate.FullReasons,
|
||||
"selectionReason": candidate.SelectionReason,
|
||||
"simulated": simulated,
|
||||
}
|
||||
if candidate.ResponseProtocol != "" {
|
||||
@@ -138,25 +140,25 @@ func attemptMetrics(candidate store.RuntimeModelCandidate, attemptNo int, simula
|
||||
metrics["responseConverted"] = true
|
||||
}
|
||||
}
|
||||
if candidate.LoadLimited {
|
||||
metrics["loadMetrics"] = map[string]any{
|
||||
"rpm": map[string]any{
|
||||
"current": candidate.LoadMetrics.RPMCurrent,
|
||||
"limit": candidate.LoadMetrics.RPMLimit,
|
||||
"ratio": candidate.LoadMetrics.RPMRatio,
|
||||
},
|
||||
"tpm": map[string]any{
|
||||
"current": candidate.LoadMetrics.TPMCurrent,
|
||||
"limit": candidate.LoadMetrics.TPMLimit,
|
||||
"ratio": candidate.LoadMetrics.TPMRatio,
|
||||
},
|
||||
"concurrent": map[string]any{
|
||||
"current": candidate.LoadMetrics.ConcurrentCurrent,
|
||||
"limit": candidate.LoadMetrics.ConcurrentLimit,
|
||||
"ratio": candidate.LoadMetrics.ConcurrentRatio,
|
||||
},
|
||||
"queued": candidate.LoadMetrics.QueuedCount,
|
||||
}
|
||||
metrics["loadMetrics"] = map[string]any{
|
||||
"rpm": map[string]any{
|
||||
"current": candidate.LoadMetrics.RPMCurrent,
|
||||
"limit": candidate.LoadMetrics.RPMLimit,
|
||||
"ratio": candidate.LoadMetrics.RPMRatio,
|
||||
},
|
||||
"tpm": map[string]any{
|
||||
"current": candidate.LoadMetrics.TPMCurrent,
|
||||
"limit": candidate.LoadMetrics.TPMLimit,
|
||||
"ratio": candidate.LoadMetrics.TPMRatio,
|
||||
},
|
||||
"concurrent": map[string]any{
|
||||
"current": candidate.LoadMetrics.ConcurrentCurrent,
|
||||
"limit": candidate.LoadMetrics.ConcurrentLimit,
|
||||
"ratio": candidate.LoadMetrics.ConcurrentRatio,
|
||||
"softCurrent": candidate.LoadMetrics.SoftCurrent,
|
||||
"softRatio": candidate.LoadMetrics.SoftRatio,
|
||||
},
|
||||
"queued": candidate.LoadMetrics.QueuedCount,
|
||||
}
|
||||
if candidate.CacheAffinity.Key != "" {
|
||||
metrics["cacheAffinityKey"] = candidate.CacheAffinity.Key
|
||||
|
||||
@@ -160,6 +160,15 @@ func resolveCandidateFailure(input resolveCandidateFailureInput) failureDecision
|
||||
}
|
||||
}
|
||||
if errors.Is(input.Err, store.ErrRateLimited) {
|
||||
if _, platformLimited := platformModelRateLimitError(input.Err); platformLimited && input.HasNextCandidate {
|
||||
return failureDecision{
|
||||
Route: "next",
|
||||
Effect: "none",
|
||||
Reason: "quota_race_rotated",
|
||||
Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "platformModelCapacity", Value: "saturated"},
|
||||
Info: info,
|
||||
}
|
||||
}
|
||||
route := "stop"
|
||||
if input.Async && store.RateLimitRetryable(input.Err) {
|
||||
route = "requeue"
|
||||
|
||||
@@ -80,24 +80,6 @@ func TestFailoverTimeBudgetExceeded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAvoidanceFallbackContinuesToAvoidedCandidate(t *testing.T) {
|
||||
candidates := []store.RuntimeModelCandidate{
|
||||
{PlatformID: "available-candidate"},
|
||||
{PlatformID: "avoided-full-candidate", LoadAvoided: true},
|
||||
}
|
||||
|
||||
if !hasLoadAvoidanceFallback(candidates, 0, 99) {
|
||||
t.Fatal("expected non-avoided candidate to fall back to later avoided candidate")
|
||||
}
|
||||
if hasLoadAvoidanceFallback(candidates, 1, 99) {
|
||||
t.Fatal("avoided candidate should not force another load-avoidance fallback")
|
||||
}
|
||||
decision := loadAvoidanceFallbackDecision(&clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: false})
|
||||
if !decision.Retry || decision.Reason != "load_avoidance_fallback" || decision.Action != "next" {
|
||||
t.Fatalf("expected active load avoidance fallback to force next candidate, got %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverHardStopBeatsModelOverride(t *testing.T) {
|
||||
runnerPolicy := store.RunnerPolicy{
|
||||
Status: "active",
|
||||
@@ -148,7 +130,7 @@ func TestFailoverPolicyAllowsModelOverride(t *testing.T) {
|
||||
func TestLocalRateLimitWaitsInQueueWithoutRetryOrFailover(t *testing.T) {
|
||||
err := &localRateLimitError{
|
||||
clientErr: &clients.ClientError{Code: "rate_limit", Message: "local capacity exceeded", Retryable: true},
|
||||
cause: &store.RateLimitExceededError{Metric: "concurrent", Retryable: true},
|
||||
cause: &store.RateLimitExceededError{ScopeType: "user_group", Metric: "concurrent", Retryable: true},
|
||||
}
|
||||
retryDecision := retryDecisionForCandidate(store.RuntimeModelCandidate{}, err)
|
||||
if retryDecision.Retry || retryDecision.Reason != "local_rate_limit_wait_queue" {
|
||||
@@ -164,6 +146,37 @@ func TestLocalRateLimitWaitsInQueueWithoutRetryOrFailover(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformModelRateLimitRaceRotatesWithoutFailureEffect(t *testing.T) {
|
||||
err := &localRateLimitError{
|
||||
clientErr: &clients.ClientError{Code: "rate_limit", Message: "platform capacity raced", Retryable: true},
|
||||
cause: &store.RateLimitExceededError{
|
||||
ScopeType: "platform_model",
|
||||
Metric: "concurrent",
|
||||
Retryable: true,
|
||||
RetryAfter: time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
decision := resolveCandidateFailure(resolveCandidateFailureInput{
|
||||
RunnerPolicy: store.RunnerPolicy{Status: "active"},
|
||||
Err: err,
|
||||
HasNextCandidate: true,
|
||||
Async: true,
|
||||
})
|
||||
if decision.Route != "next" || decision.Effect != "none" || decision.Reason != "quota_race_rotated" {
|
||||
t.Fatalf("platform quota race should rotate without mutation: %+v", decision)
|
||||
}
|
||||
|
||||
last := resolveCandidateFailure(resolveCandidateFailureInput{
|
||||
RunnerPolicy: store.RunnerPolicy{Status: "active"},
|
||||
Err: err,
|
||||
Async: true,
|
||||
})
|
||||
if last.Route != "requeue" || last.Effect != "none" {
|
||||
t.Fatalf("last saturated platform should requeue: %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderAuthErrorsFailOverInsteadOfHardStop(t *testing.T) {
|
||||
runnerPolicy := store.RunnerPolicy{
|
||||
Status: "active",
|
||||
|
||||
@@ -382,6 +382,7 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
|
||||
err = responseChainUnavailableError()
|
||||
}
|
||||
if task.AsyncMode && store.ModelCandidateRetryAfter(err) > 0 {
|
||||
s.observeCandidateRouting("cooldown_skipped")
|
||||
queued, delay, queueErr := s.requeueModelCoolingTask(ctx, task, err)
|
||||
if queueErr != nil {
|
||||
return Result{}, queueErr
|
||||
@@ -389,6 +390,7 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
|
||||
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
|
||||
}
|
||||
if store.ModelCandidateRetryAfter(err) > 0 {
|
||||
s.observeCandidateRouting("cooldown_skipped")
|
||||
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err)
|
||||
if finishErr != nil {
|
||||
return Result{}, finishErr
|
||||
@@ -481,6 +483,11 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
|
||||
return Result{Task: failed, Output: failed.Result}, err
|
||||
}
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if candidate.LoadAvoided {
|
||||
s.observeCandidateRouting("full_avoided")
|
||||
}
|
||||
}
|
||||
var asyncAdmission *store.TaskAdmission
|
||||
if distributedAdmission && task.AsyncMode {
|
||||
asyncAdmission, err = s.loadAsyncTaskAdmission(ctx, task)
|
||||
@@ -614,25 +621,35 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasConcurrentLimit {
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidates[0], body),
|
||||
)
|
||||
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
|
||||
if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
|
||||
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
|
||||
if queueErr != nil {
|
||||
return Result{}, queueErr
|
||||
deferInitialAdmission := false
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidates[0], body),
|
||||
)
|
||||
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
|
||||
if _, platformLimited := platformModelRateLimitError(err); platformLimited && len(candidates) > 1 {
|
||||
if task.AsyncMode {
|
||||
if deleteErr := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); deleteErr != nil {
|
||||
return Result{}, deleteErr
|
||||
}
|
||||
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
|
||||
asyncAdmission = nil
|
||||
}
|
||||
deferInitialAdmission = true
|
||||
} else if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
|
||||
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
|
||||
if queueErr != nil {
|
||||
return Result{}, queueErr
|
||||
}
|
||||
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
|
||||
} else {
|
||||
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err)
|
||||
if finishErr != nil {
|
||||
return Result{}, finishErr
|
||||
}
|
||||
return Result{Task: failed, Output: failed.Result}, err
|
||||
}
|
||||
}
|
||||
if hasConcurrentLimit && !deferInitialAdmission {
|
||||
plan := taskAdmissionPlan{
|
||||
Candidate: candidates[0],
|
||||
Body: body,
|
||||
@@ -799,13 +816,17 @@ candidatesLoop:
|
||||
if platformsVisited >= maxPlatforms {
|
||||
break
|
||||
}
|
||||
available, availabilityErr := s.store.RuntimeCandidateAvailable(ctx, candidate.PlatformID, candidate.PlatformModelID)
|
||||
available, unavailableReason, availabilityErr := s.store.RuntimeCandidateAvailability(ctx, candidate.PlatformID, candidate.PlatformModelID)
|
||||
if availabilityErr != nil {
|
||||
return Result{}, availabilityErr
|
||||
}
|
||||
if !available {
|
||||
s.observeCandidateRouting(unavailableReason)
|
||||
continue
|
||||
}
|
||||
if candidate.SelectionReason == "normal_rotation" || candidate.SelectionReason == "" {
|
||||
s.observeCandidateRouting("normal_rotation")
|
||||
}
|
||||
if distributedAdmission && candidate.PlatformModelID != admittedPlatformModelID {
|
||||
admissionResult, candidateLimited, admissionErr := s.ensureCandidateAdmission(ctx, task, user, body, candidate)
|
||||
if admissionErr != nil {
|
||||
@@ -814,9 +835,17 @@ candidatesLoop:
|
||||
if errors.Is(admissionErr, store.ErrQueueTimeout) {
|
||||
break
|
||||
}
|
||||
if candidateLimited && errors.Is(admissionErr, store.ErrRateLimited) {
|
||||
if _, platformLimited := platformModelRateLimitError(admissionErr); platformLimited {
|
||||
s.observeCandidateRouting("quota_race_rotated")
|
||||
continue
|
||||
}
|
||||
if task.AsyncMode && errors.Is(admissionErr, store.ErrRateLimited) && store.RateLimitRetryable(admissionErr) {
|
||||
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, admissionErr, candidate)
|
||||
if queueErr != nil {
|
||||
return Result{}, queueErr
|
||||
}
|
||||
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
|
||||
}
|
||||
return Result{}, admissionErr
|
||||
}
|
||||
if candidateLimited {
|
||||
@@ -1062,6 +1091,9 @@ candidatesLoop:
|
||||
DownstreamStarted: downstreamStarted.Load(),
|
||||
})
|
||||
if candidateDecision.Route == "requeue" {
|
||||
if _, platformLimited := platformModelRateLimitError(err); platformLimited {
|
||||
s.observeCandidateRouting("all_full_queued")
|
||||
}
|
||||
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidate)
|
||||
if queueErr != nil {
|
||||
return Result{}, queueErr
|
||||
@@ -1069,21 +1101,35 @@ candidatesLoop:
|
||||
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
|
||||
}
|
||||
attemptNo = s.recordFailedAttempt(ctx, failedAttemptRecord{
|
||||
Task: task,
|
||||
Body: candidateBody,
|
||||
Candidate: &candidate,
|
||||
AttemptNo: nextAttemptNo,
|
||||
Code: clients.ErrorCode(err),
|
||||
Cause: err,
|
||||
Simulated: isSimulation(task, candidate),
|
||||
Scope: "rate_limit",
|
||||
Reason: "local_rate_limit_blocked",
|
||||
ExtraMetrics: []map[string]any{parameterPreprocessingMetrics(preprocessing.Log)},
|
||||
ModelType: candidate.ModelType,
|
||||
Task: task,
|
||||
Body: candidateBody,
|
||||
Candidate: &candidate,
|
||||
AttemptNo: nextAttemptNo,
|
||||
Code: clients.ErrorCode(err),
|
||||
Cause: err,
|
||||
Simulated: isSimulation(task, candidate),
|
||||
Scope: "rate_limit",
|
||||
Reason: "local_rate_limit_blocked",
|
||||
ExtraMetrics: []map[string]any{
|
||||
parameterPreprocessingMetrics(preprocessing.Log),
|
||||
{"selectionReason": candidateDecision.Reason},
|
||||
},
|
||||
ModelType: candidate.ModelType,
|
||||
})
|
||||
candidateDecisionAttempt = attemptNo
|
||||
candidateClientAttempt = clientAttempt
|
||||
s.recordAttemptTrace(ctx, task.ID, attemptNo, failureDecisionTraceEntry(candidateDecision, candidate, clientAttempt, clientAttempts, false, ""))
|
||||
if candidateDecision.Route == "next" {
|
||||
s.observeCandidateRouting("quota_race_rotated")
|
||||
if admittedPlatformModelID == candidate.PlatformModelID {
|
||||
if deleteErr := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); deleteErr != nil {
|
||||
return Result{}, deleteErr
|
||||
}
|
||||
admittedPlatformModelID = ""
|
||||
admittedLeases = nil
|
||||
}
|
||||
break
|
||||
}
|
||||
break candidatesLoop
|
||||
}
|
||||
attemptNo = nextAttemptNo
|
||||
@@ -1179,6 +1225,9 @@ candidatesLoop:
|
||||
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: 0}
|
||||
}
|
||||
if task.AsyncMode && errors.Is(lastErr, store.ErrRateLimited) && store.RateLimitRetryable(lastErr) {
|
||||
if _, platformLimited := platformModelRateLimitError(lastErr); platformLimited {
|
||||
s.observeCandidateRouting("all_full_queued")
|
||||
}
|
||||
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, lastErr, lastCandidate)
|
||||
if queueErr != nil {
|
||||
return Result{}, queueErr
|
||||
@@ -1754,6 +1803,15 @@ func (s *Service) observeProviderQuotaWait(metric string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) observeCandidateRouting(reason string) {
|
||||
observer, ok := s.billingMetrics.(interface {
|
||||
ObserveCandidateRouting(string)
|
||||
})
|
||||
if ok {
|
||||
observer.ObserveCandidateRouting(reason)
|
||||
}
|
||||
}
|
||||
|
||||
func minimalRemoteTaskCheckpoint(provider string, specType string, payload map[string]any) map[string]any {
|
||||
const maxBytes = 8192
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
@@ -2545,37 +2603,6 @@ func failoverTimeBudgetExceeded(start time.Time, maxDuration time.Duration) bool
|
||||
return maxDuration > 0 && time.Since(start) >= maxDuration
|
||||
}
|
||||
|
||||
func hasLoadAvoidanceFallback(candidates []store.RuntimeModelCandidate, index int, maxPlatforms int) bool {
|
||||
if index < 0 || index >= len(candidates) || candidates[index].LoadAvoided {
|
||||
return false
|
||||
}
|
||||
limit := len(candidates)
|
||||
if maxPlatforms > 0 && maxPlatforms < limit {
|
||||
limit = maxPlatforms
|
||||
}
|
||||
for next := index + 1; next < limit; next++ {
|
||||
if candidates[next].LoadAvoided {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func loadAvoidanceFallbackDecision(err error) failoverDecision {
|
||||
return failoverDecision{
|
||||
Retry: true,
|
||||
Action: "next",
|
||||
Reason: "load_avoidance_fallback",
|
||||
Match: policyRuleMatch{
|
||||
Source: "runtime_client_load",
|
||||
Policy: "loadAvoidance",
|
||||
Rule: "fallback",
|
||||
Value: "loadRatio>=1",
|
||||
},
|
||||
Info: failureInfoFromError(err),
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRequest(kind string, body map[string]any) map[string]any {
|
||||
out := cloneMap(body)
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user