feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -205,23 +205,12 @@ func (s *Service) taskAdmissionScopes(
|
||||
}
|
||||
|
||||
func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionScope) []store.AdmissionScope {
|
||||
if task.RunMode != "acceptance" && task.RunMode != "acceptance_canary" {
|
||||
if task.RunMode != "acceptance" {
|
||||
return scopes
|
||||
}
|
||||
out := append([]store.AdmissionScope(nil), scopes...)
|
||||
for index := range out {
|
||||
// Protocol-emulated acceptance measures Gateway and Worker capacity, so
|
||||
// the isolated Run is bounded by the worker_capacity scope instead of a
|
||||
// production supplier quota. The real acceptance_canary path deliberately
|
||||
// retains the production platform-model concurrency limit.
|
||||
if task.RunMode == "acceptance" && out[index].ScopeType == "platform_model" {
|
||||
out[index].ConcurrentLimit = 0
|
||||
}
|
||||
if out[index].ConcurrentLimit <= 0 {
|
||||
if task.RunMode != "acceptance" || out[index].ScopeType != "platform_model" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out[index].ScopeKey = acceptanceScopeKey(task, out[index].ScopeKey)
|
||||
out[index].QueueLimit = acceptanceQueueLimit
|
||||
out[index].MaxWaitSeconds = acceptanceQueueMaxWait
|
||||
}
|
||||
@@ -235,16 +224,21 @@ func acceptanceInfrastructureReservations(
|
||||
if task.RunMode != "acceptance" {
|
||||
return reservations
|
||||
}
|
||||
out := make([]store.RateLimitReservation, 0, len(reservations))
|
||||
for _, reservation := range reservations {
|
||||
if reservation.ScopeType == "platform_model" && reservation.Metric == "concurrent" {
|
||||
continue
|
||||
}
|
||||
out = append(out, reservation)
|
||||
out := append([]store.RateLimitReservation(nil), reservations...)
|
||||
for index := range out {
|
||||
out[index].ScopeKey = acceptanceScopeKey(task, out[index].ScopeKey)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func acceptanceScopeKey(task store.GatewayTask, scopeKey string) string {
|
||||
runID := strings.TrimSpace(task.AcceptanceRunID)
|
||||
if runID == "" {
|
||||
runID = "unbound"
|
||||
}
|
||||
return "acceptance:" + runID + ":" + scopeKey
|
||||
}
|
||||
|
||||
func (s *Service) loadAsyncTaskAdmission(ctx context.Context, task store.GatewayTask) (*store.TaskAdmission, error) {
|
||||
if !task.AsyncMode {
|
||||
return nil, nil
|
||||
@@ -265,6 +259,7 @@ func pinCandidatesToTaskAdmission(
|
||||
) ([]store.RuntimeModelCandidate, bool) {
|
||||
if admission == nil ||
|
||||
(admission.Status != "waiting" && admission.Status != "admitted") ||
|
||||
!admission.ReselectRequestedAt.IsZero() ||
|
||||
len(candidates) < 2 {
|
||||
return candidates, false
|
||||
}
|
||||
@@ -785,6 +780,24 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
|
||||
return false, outcome.Err
|
||||
}
|
||||
if !outcome.Result.Admitted {
|
||||
platformModelID := outcome.Result.Admission.PlatformModelID
|
||||
if platformModelID == "" {
|
||||
for _, input := range inputs {
|
||||
if input.TaskID == outcome.TaskID {
|
||||
platformModelID = input.PlatformModelID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if platformModelID != "" {
|
||||
marked, markErr := s.store.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
|
||||
if markErr != nil {
|
||||
return false, markErr
|
||||
}
|
||||
if marked > 0 {
|
||||
s.observeTaskAdmission("candidate_reselect_requested")
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user