feat(routing): 完善平台满载避让与故障轮转
为未配置并发上限的平台增加基于运行和等待任务数的软负载,并按非满载、有效优先级、缓存亲和与负载稳定排序。\n\n平台模型 RPM、TPM 和并发额度竞争失败时只轮转候选,不触发冷却、禁用或降级;用户组额度保持不可绕过。补齐异步冷却排队恢复、满载原因、选择原因与低基数指标。\n\n验证:go test ./...;go vet ./...;PostgreSQL 原子额度、准入队列、Worker 容量回收及故障策略 HTTP acceptance。
This commit is contained in:
@@ -123,11 +123,12 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
candidates, _ = pinCandidatesToTaskAdmission(candidates, admission)
|
||||
var candidateRateLimitErr error
|
||||
for _, candidate := range candidates {
|
||||
available, availabilityErr := s.store.RuntimeCandidateAvailable(ctx, candidate.PlatformID, candidate.PlatformModelID)
|
||||
available, unavailableReason, availabilityErr := s.store.RuntimeCandidateAvailability(ctx, candidate.PlatformID, candidate.PlatformModelID)
|
||||
if availabilityErr != nil {
|
||||
return taskAdmissionPlan{}, availabilityErr
|
||||
}
|
||||
if !available {
|
||||
s.observeCandidateRouting(unavailableReason)
|
||||
continue
|
||||
}
|
||||
scopes, groupID, scopeErr := s.taskAdmissionScopes(ctx, task, user, candidate)
|
||||
@@ -141,13 +142,6 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasConcurrentLimit {
|
||||
return taskAdmissionPlan{
|
||||
Candidate: candidate,
|
||||
Body: body,
|
||||
ModelType: modelType,
|
||||
}, nil
|
||||
}
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidate, body),
|
||||
@@ -162,6 +156,13 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
}
|
||||
return taskAdmissionPlan{}, err
|
||||
}
|
||||
if !hasConcurrentLimit {
|
||||
return taskAdmissionPlan{
|
||||
Candidate: candidate,
|
||||
Body: body,
|
||||
ModelType: modelType,
|
||||
}, nil
|
||||
}
|
||||
return taskAdmissionPlan{
|
||||
Candidate: candidate,
|
||||
Body: body,
|
||||
@@ -172,6 +173,7 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
}, nil
|
||||
}
|
||||
if candidateRateLimitErr != nil {
|
||||
s.observeCandidateRouting("all_full_queued")
|
||||
return taskAdmissionPlan{}, candidateRateLimitErr
|
||||
}
|
||||
return taskAdmissionPlan{}, store.ErrNoModelCandidate
|
||||
@@ -475,12 +477,6 @@ func (s *Service) ensureCandidateAdmission(
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasConcurrentLimit {
|
||||
if err := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return store.TaskAdmissionResult{}, false, err
|
||||
}
|
||||
return store.TaskAdmissionResult{}, false, nil
|
||||
}
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidate, body),
|
||||
@@ -488,6 +484,12 @@ func (s *Service) ensureCandidateAdmission(
|
||||
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
|
||||
return store.TaskAdmissionResult{}, true, err
|
||||
}
|
||||
if !hasConcurrentLimit {
|
||||
if err := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return store.TaskAdmissionResult{}, false, err
|
||||
}
|
||||
return store.TaskAdmissionResult{}, false, nil
|
||||
}
|
||||
plan := taskAdmissionPlan{
|
||||
Candidate: candidate,
|
||||
Body: body,
|
||||
@@ -809,6 +811,7 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
|
||||
return false, outcome.Err
|
||||
}
|
||||
if !outcome.Result.Admitted {
|
||||
s.observeCandidateRouting("quota_race_rotated")
|
||||
platformModelID := outcome.Result.Admission.PlatformModelID
|
||||
if platformModelID == "" {
|
||||
for _, input := range inputs {
|
||||
|
||||
Reference in New Issue
Block a user