feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -451,6 +451,20 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID, queuedSyntheticRiverJobID)
|
||||
if listedSnapshot == nil || len(listedSnapshot.Scopes) != len(queuedAdmission.Scopes) {
|
||||
t.Fatalf("listed admission snapshot=%+v, want %d scopes", listedSnapshot, len(queuedAdmission.Scopes))
|
||||
}
|
||||
markedForReselect, err := first.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
|
||||
if err != nil {
|
||||
t.Fatalf("request waiting admission reselection: %v", err)
|
||||
}
|
||||
if markedForReselect < 1 {
|
||||
t.Fatalf("marked admissions=%d, want at least the queued task", markedForReselect)
|
||||
}
|
||||
reselectAdmission, err := first.GetTaskAdmission(ctx, queuedAtomicTask.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("read admission reselection marker: %v", err)
|
||||
}
|
||||
if reselectAdmission.ReselectRequestedAt.IsZero() {
|
||||
t.Fatal("queued admission was not marked for candidate reselection")
|
||||
}
|
||||
var riverJobID int64
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
@@ -1296,4 +1310,35 @@ WHERE instance_id = $1`, secondID, (workerHeartbeatStaleAfter + time.Second).Str
|
||||
if err != nil || second.Allocated != 2 || second.GlobalAllocated != 5 || second.ActiveInstances != 2 {
|
||||
t.Fatalf("bounded two-worker allocation = %+v, err=%v", second, err)
|
||||
}
|
||||
first, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
|
||||
InstanceID: firstID, DesiredCapacity: 100, CapacityLimit: 10,
|
||||
LoadMode: "adaptive", SafeCapacity: 0, HeavyCapacity: 1,
|
||||
ActiveTasks: 2, WaitingUpstreamTasks: 2,
|
||||
PressureState: "critical", PressureReason: "memory",
|
||||
})
|
||||
if err != nil || first.Allocated != 0 {
|
||||
t.Fatalf("critical worker allocation = %+v, err=%v", first, err)
|
||||
}
|
||||
second, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
|
||||
InstanceID: secondID, DesiredCapacity: 100, CapacityLimit: 10,
|
||||
LoadMode: "adaptive", SafeCapacity: 5, HeavyCapacity: 2,
|
||||
ActiveTasks: 3, PreparingTasks: 1, WaitingUpstreamTasks: 1, FinalizingTasks: 1,
|
||||
PressureState: "normal",
|
||||
})
|
||||
if err != nil || second.Allocated != 5 || second.GlobalAllocated != 5 {
|
||||
t.Fatalf("adaptive redistribution allocation = %+v, err=%v", second, err)
|
||||
}
|
||||
instances, err := db.ListWorkerInstanceRuntime(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list adaptive worker runtime: %v", err)
|
||||
}
|
||||
foundCritical := false
|
||||
for _, instance := range instances {
|
||||
if instance.InstanceID == firstID {
|
||||
foundCritical = instance.SafeCapacity == 0 && instance.ReportedActiveTasks == 2 && instance.WaitingUpstreamTasks == 2 && instance.PressureState == "critical"
|
||||
}
|
||||
}
|
||||
if !foundCritical {
|
||||
t.Fatalf("adaptive runtime did not expose the critical worker: %+v", instances)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user