feat(worker): 实现集群限流与自适应负载

保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
2026-08-03 00:13:46 +08:00
parent 9a01fd4657
commit c28bf74230
52 changed files with 3700 additions and 272 deletions
+32 -19
View File
@@ -205,23 +205,12 @@ func (s *Service) taskAdmissionScopes(
}
func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionScope) []store.AdmissionScope {
if task.RunMode != "acceptance" && task.RunMode != "acceptance_canary" {
if task.RunMode != "acceptance" {
return scopes
}
out := append([]store.AdmissionScope(nil), scopes...)
for index := range out {
// Protocol-emulated acceptance measures Gateway and Worker capacity, so
// the isolated Run is bounded by the worker_capacity scope instead of a
// production supplier quota. The real acceptance_canary path deliberately
// retains the production platform-model concurrency limit.
if task.RunMode == "acceptance" && out[index].ScopeType == "platform_model" {
out[index].ConcurrentLimit = 0
}
if out[index].ConcurrentLimit <= 0 {
if task.RunMode != "acceptance" || out[index].ScopeType != "platform_model" {
continue
}
}
out[index].ScopeKey = acceptanceScopeKey(task, out[index].ScopeKey)
out[index].QueueLimit = acceptanceQueueLimit
out[index].MaxWaitSeconds = acceptanceQueueMaxWait
}
@@ -235,16 +224,21 @@ func acceptanceInfrastructureReservations(
if task.RunMode != "acceptance" {
return reservations
}
out := make([]store.RateLimitReservation, 0, len(reservations))
for _, reservation := range reservations {
if reservation.ScopeType == "platform_model" && reservation.Metric == "concurrent" {
continue
}
out = append(out, reservation)
out := append([]store.RateLimitReservation(nil), reservations...)
for index := range out {
out[index].ScopeKey = acceptanceScopeKey(task, out[index].ScopeKey)
}
return out
}
func acceptanceScopeKey(task store.GatewayTask, scopeKey string) string {
runID := strings.TrimSpace(task.AcceptanceRunID)
if runID == "" {
runID = "unbound"
}
return "acceptance:" + runID + ":" + scopeKey
}
func (s *Service) loadAsyncTaskAdmission(ctx context.Context, task store.GatewayTask) (*store.TaskAdmission, error) {
if !task.AsyncMode {
return nil, nil
@@ -265,6 +259,7 @@ func pinCandidatesToTaskAdmission(
) ([]store.RuntimeModelCandidate, bool) {
if admission == nil ||
(admission.Status != "waiting" && admission.Status != "admitted") ||
!admission.ReselectRequestedAt.IsZero() ||
len(candidates) < 2 {
return candidates, false
}
@@ -785,6 +780,24 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
return false, outcome.Err
}
if !outcome.Result.Admitted {
platformModelID := outcome.Result.Admission.PlatformModelID
if platformModelID == "" {
for _, input := range inputs {
if input.TaskID == outcome.TaskID {
platformModelID = input.PlatformModelID
break
}
}
}
if platformModelID != "" {
marked, markErr := s.store.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
if markErr != nil {
return false, markErr
}
if marked > 0 {
s.observeTaskAdmission("candidate_reselect_requested")
}
}
return true, nil
}
}