feat(routing): 完善平台满载避让与故障轮转

为未配置并发上限的平台增加基于运行和等待任务数的软负载,并按非满载、有效优先级、缓存亲和与负载稳定排序。\n\n平台模型 RPM、TPM 和并发额度竞争失败时只轮转候选,不触发冷却、禁用或降级;用户组额度保持不可绕过。补齐异步冷却排队恢复、满载原因、选择原因与低基数指标。\n\n验证:go test ./...;go vet ./...;PostgreSQL 原子额度、准入队列、Worker 容量回收及故障策略 HTTP acceptance。
This commit is contained in:
2026-08-03 17:42:34 +08:00
parent b125599354
commit 8362d6d27a
15 changed files with 468 additions and 152 deletions
+17 -14
View File
@@ -123,11 +123,12 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
candidates, _ = pinCandidatesToTaskAdmission(candidates, admission)
var candidateRateLimitErr error
for _, candidate := range candidates {
available, availabilityErr := s.store.RuntimeCandidateAvailable(ctx, candidate.PlatformID, candidate.PlatformModelID)
available, unavailableReason, availabilityErr := s.store.RuntimeCandidateAvailability(ctx, candidate.PlatformID, candidate.PlatformModelID)
if availabilityErr != nil {
return taskAdmissionPlan{}, availabilityErr
}
if !available {
s.observeCandidateRouting(unavailableReason)
continue
}
scopes, groupID, scopeErr := s.taskAdmissionScopes(ctx, task, user, candidate)
@@ -141,13 +142,6 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
break
}
}
if !hasConcurrentLimit {
return taskAdmissionPlan{
Candidate: candidate,
Body: body,
ModelType: modelType,
}, nil
}
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidate, body),
@@ -162,6 +156,13 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
}
return taskAdmissionPlan{}, err
}
if !hasConcurrentLimit {
return taskAdmissionPlan{
Candidate: candidate,
Body: body,
ModelType: modelType,
}, nil
}
return taskAdmissionPlan{
Candidate: candidate,
Body: body,
@@ -172,6 +173,7 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
}, nil
}
if candidateRateLimitErr != nil {
s.observeCandidateRouting("all_full_queued")
return taskAdmissionPlan{}, candidateRateLimitErr
}
return taskAdmissionPlan{}, store.ErrNoModelCandidate
@@ -475,12 +477,6 @@ func (s *Service) ensureCandidateAdmission(
break
}
}
if !hasConcurrentLimit {
if err := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
return store.TaskAdmissionResult{}, false, err
}
return store.TaskAdmissionResult{}, false, nil
}
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidate, body),
@@ -488,6 +484,12 @@ func (s *Service) ensureCandidateAdmission(
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
return store.TaskAdmissionResult{}, true, err
}
if !hasConcurrentLimit {
if err := s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
return store.TaskAdmissionResult{}, false, err
}
return store.TaskAdmissionResult{}, false, nil
}
plan := taskAdmissionPlan{
Candidate: candidate,
Body: body,
@@ -809,6 +811,7 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
return false, outcome.Err
}
if !outcome.Result.Admitted {
s.observeCandidateRouting("quota_race_rotated")
platformModelID := outcome.Result.Admission.PlatformModelID
if platformModelID == "" {
for _, input := range inputs {