feat(worker): 实现集群限流与自适应负载

保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
2026-08-03 00:13:46 +08:00
parent 9a01fd4657
commit c28bf74230
52 changed files with 3700 additions and 272 deletions
@@ -451,6 +451,20 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID, queuedSyntheticRiverJobID)
if listedSnapshot == nil || len(listedSnapshot.Scopes) != len(queuedAdmission.Scopes) {
t.Fatalf("listed admission snapshot=%+v, want %d scopes", listedSnapshot, len(queuedAdmission.Scopes))
}
markedForReselect, err := first.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
if err != nil {
t.Fatalf("request waiting admission reselection: %v", err)
}
if markedForReselect < 1 {
t.Fatalf("marked admissions=%d, want at least the queued task", markedForReselect)
}
reselectAdmission, err := first.GetTaskAdmission(ctx, queuedAtomicTask.ID)
if err != nil {
t.Fatalf("read admission reselection marker: %v", err)
}
if reselectAdmission.ReselectRequestedAt.IsZero() {
t.Fatal("queued admission was not marked for candidate reselection")
}
var riverJobID int64
if err := first.pool.QueryRow(ctx, `
SELECT
@@ -1296,4 +1310,35 @@ WHERE instance_id = $1`, secondID, (workerHeartbeatStaleAfter + time.Second).Str
if err != nil || second.Allocated != 2 || second.GlobalAllocated != 5 || second.ActiveInstances != 2 {
t.Fatalf("bounded two-worker allocation = %+v, err=%v", second, err)
}
first, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
InstanceID: firstID, DesiredCapacity: 100, CapacityLimit: 10,
LoadMode: "adaptive", SafeCapacity: 0, HeavyCapacity: 1,
ActiveTasks: 2, WaitingUpstreamTasks: 2,
PressureState: "critical", PressureReason: "memory",
})
if err != nil || first.Allocated != 0 {
t.Fatalf("critical worker allocation = %+v, err=%v", first, err)
}
second, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
InstanceID: secondID, DesiredCapacity: 100, CapacityLimit: 10,
LoadMode: "adaptive", SafeCapacity: 5, HeavyCapacity: 2,
ActiveTasks: 3, PreparingTasks: 1, WaitingUpstreamTasks: 1, FinalizingTasks: 1,
PressureState: "normal",
})
if err != nil || second.Allocated != 5 || second.GlobalAllocated != 5 {
t.Fatalf("adaptive redistribution allocation = %+v, err=%v", second, err)
}
instances, err := db.ListWorkerInstanceRuntime(ctx)
if err != nil {
t.Fatalf("list adaptive worker runtime: %v", err)
}
foundCritical := false
for _, instance := range instances {
if instance.InstanceID == firstID {
foundCritical = instance.SafeCapacity == 0 && instance.ReportedActiveTasks == 2 && instance.WaitingUpstreamTasks == 2 && instance.PressureState == "critical"
}
}
if !foundCritical {
t.Fatalf("adaptive runtime did not expose the critical worker: %+v", instances)
}
}