perf(worker): 批量填充异步执行槽
跨地域同步复制下逐任务准入会为每个 River Job 支付一次事务提交 RTT，导致 P24 实际只能维持少量运行任务。调度器现在按全局容量窗口准备 FIFO 批次，在同一 PostgreSQL 事务内逐项校验队首、创建租约并插入唯一 River Job，一次提交即可填满可用槽；任一 Hook 失败时整批原子回滚。\n\n验收压力采样同时排除正在终止的 Ready Pod，避免滚动切换期间误连已移除容器。\n\n验证：真实 PostgreSQL 批量提交及整批回滚集成测试、Go 全量测试、go vet、runner/store race、gofmt、bash -n、ShellCheck、迁移安全检查通过。
This commit is contained in:
@@ -501,6 +501,140 @@ WHERE id = $1::uuid`, recoveryGraceTask.ID); err != nil {
|
||||
if !recoveryGraceFound {
|
||||
t.Fatal("generic River recovery did not claim a task beyond the recovery grace period")
|
||||
}
|
||||
|
||||
batchScope := AdmissionScope{
|
||||
ScopeType: "worker_capacity",
|
||||
ScopeKey: "batch-" + suffix,
|
||||
ScopeName: "batch capacity",
|
||||
ConcurrentLimit: 3,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 120,
|
||||
QueueLimit: 100,
|
||||
MaxWaitSeconds: 600,
|
||||
}
|
||||
batchTasks := []GatewayTask{createTask(true), createTask(true), createTask(true)}
|
||||
batchInputs := make([]TaskAdmissionInput, 0, len(batchTasks))
|
||||
batchJobIDs := make(map[string]int64, len(batchTasks))
|
||||
for index, task := range batchTasks {
|
||||
input := inputFor(task, 100+index, "")
|
||||
input.Scopes = []AdmissionScope{batchScope}
|
||||
if _, err := first.QueueTaskAdmissionWithHook(ctx, input, nil); err != nil {
|
||||
t.Fatalf("queue batch task %d: %v", index, err)
|
||||
}
|
||||
batchInputs = append(batchInputs, input)
|
||||
batchJobIDs[task.ID] = int64(987654400 + index)
|
||||
}
|
||||
batchOutcomes, err := first.TryTaskAdmissionBatchWithAdmittedHook(
|
||||
ctx,
|
||||
batchInputs,
|
||||
func(tx pgx.Tx, input TaskAdmissionInput) error {
|
||||
_, hookErr := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET river_job_id = $2
|
||||
WHERE id = $1::uuid`, input.TaskID, batchJobIDs[input.TaskID])
|
||||
return hookErr
|
||||
},
|
||||
)
|
||||
if err != nil || len(batchOutcomes) != len(batchTasks) {
|
||||
t.Fatalf("batch admission outcomes=%+v err=%v", batchOutcomes, err)
|
||||
}
|
||||
for index, outcome := range batchOutcomes {
|
||||
if outcome.Err != nil || !outcome.Result.Admitted || !outcome.Result.NewlyAdmitted ||
|
||||
len(outcome.Result.Leases) != 1 {
|
||||
t.Fatalf("batch outcome %d=%+v", index, outcome)
|
||||
}
|
||||
}
|
||||
var batchAdmissions, batchLeases, batchRiverJobs int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT count(*)
|
||||
FROM gateway_task_admissions
|
||||
WHERE task_id = ANY($1::uuid[]) AND status = 'admitted'),
|
||||
(SELECT count(*)
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE task_id = ANY($1::uuid[]) AND released_at IS NULL AND expires_at > now()),
|
||||
(SELECT count(*)
|
||||
FROM gateway_tasks
|
||||
WHERE id = ANY($1::uuid[]) AND river_job_id IS NOT NULL)`,
|
||||
[]string{batchTasks[0].ID, batchTasks[1].ID, batchTasks[2].ID},
|
||||
).Scan(&batchAdmissions, &batchLeases, &batchRiverJobs); err != nil {
|
||||
t.Fatalf("read committed batch admission: %v", err)
|
||||
}
|
||||
if batchAdmissions != 3 || batchLeases != 3 || batchRiverJobs != 3 {
|
||||
t.Fatalf(
|
||||
"batch admissions=%d leases=%d River jobs=%d, want 3/3/3",
|
||||
batchAdmissions,
|
||||
batchLeases,
|
||||
batchRiverJobs,
|
||||
)
|
||||
}
|
||||
for _, task := range batchTasks {
|
||||
if err := first.DeleteTaskAdmission(ctx, task.ID); err != nil {
|
||||
t.Fatalf("release batch task %s: %v", task.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
rollbackScope := batchScope
|
||||
rollbackScope.ScopeKey = "batch-rollback-" + suffix
|
||||
rollbackScope.ConcurrentLimit = 2
|
||||
rollbackTasks := []GatewayTask{createTask(true), createTask(true)}
|
||||
rollbackInputs := make([]TaskAdmissionInput, 0, len(rollbackTasks))
|
||||
for index, task := range rollbackTasks {
|
||||
input := inputFor(task, 200+index, "")
|
||||
input.Scopes = []AdmissionScope{rollbackScope}
|
||||
if _, err := first.QueueTaskAdmissionWithHook(ctx, input, nil); err != nil {
|
||||
t.Fatalf("queue rollback batch task %d: %v", index, err)
|
||||
}
|
||||
rollbackInputs = append(rollbackInputs, input)
|
||||
}
|
||||
batchHookFailure := errors.New("synthetic batch hook failure")
|
||||
_, err = first.TryTaskAdmissionBatchWithAdmittedHook(
|
||||
ctx,
|
||||
rollbackInputs,
|
||||
func(tx pgx.Tx, input TaskAdmissionInput) error {
|
||||
if input.TaskID == rollbackTasks[1].ID {
|
||||
return batchHookFailure
|
||||
}
|
||||
_, hookErr := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET river_job_id = 987654499
|
||||
WHERE id = $1::uuid`, input.TaskID)
|
||||
return hookErr
|
||||
},
|
||||
)
|
||||
if !errors.Is(err, batchHookFailure) {
|
||||
t.Fatalf("batch hook failure=%v, want synthetic failure", err)
|
||||
}
|
||||
var rollbackWaiting, rollbackLeases, rollbackRiverJobs int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT count(*)
|
||||
FROM gateway_task_admissions
|
||||
WHERE task_id = ANY($1::uuid[]) AND status = 'waiting'),
|
||||
(SELECT count(*)
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE task_id = ANY($1::uuid[]) AND released_at IS NULL),
|
||||
(SELECT count(*)
|
||||
FROM gateway_tasks
|
||||
WHERE id = ANY($1::uuid[]) AND river_job_id IS NOT NULL)`,
|
||||
[]string{rollbackTasks[0].ID, rollbackTasks[1].ID},
|
||||
).Scan(&rollbackWaiting, &rollbackLeases, &rollbackRiverJobs); err != nil {
|
||||
t.Fatalf("read rolled back batch admission: %v", err)
|
||||
}
|
||||
if rollbackWaiting != 2 || rollbackLeases != 0 || rollbackRiverJobs != 0 {
|
||||
t.Fatalf(
|
||||
"rolled back batch waiting=%d leases=%d River jobs=%d, want 2/0/0",
|
||||
rollbackWaiting,
|
||||
rollbackLeases,
|
||||
rollbackRiverJobs,
|
||||
)
|
||||
}
|
||||
for _, task := range rollbackTasks {
|
||||
if err := first.DeleteTaskAdmission(ctx, task.ID); err != nil {
|
||||
t.Fatalf("release rollback batch task %s: %v", task.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
result, err = second.TryTaskAdmission(ctx, inputFor(queuedAtomicTask, 100, ""))
|
||||
if err != nil || !result.Admitted || len(result.Leases) != 1 {
|
||||
t.Fatalf("worker-time queued admission result=%+v err=%v", result, err)
|
||||
|
||||
Reference in New Issue
Block a user