fix(acceptance): 解除八任务准入瓶颈

线上 P24 验收证明 Worker 注册容量为 48,但异步 admission 始终只有 8 条活跃租约。将单事务调度窗口覆盖当前 2 x P32 的 64 槽,避免多 Worker 在同一 FIFO 小批次上竞争形成隐性全局上限。\n\nGemini 流式请求补充可回放 GetBody 与精确 Content-Length,使带幂等键的 POST 遇到 HTTP/2 可重试传输错误时可安全重放;同时提前部署验收回调收集器,保证旧 Run side effects 能在接管前收口。\n\n已通过 Go 全量测试、go vet、gofmt、Shell bash -n、ShellCheck、验收与人工发布脚本测试、迁移安全检查,以及独立 PostgreSQL 48 条 admission/租约/River job 原子集成测试。
This commit is contained in:
2026-08-01 16:59:28 +08:00
parent 7623449e33
commit 215b6b607f
7 changed files with 105 additions and 27 deletions
+13 -8
View File
@@ -33,7 +33,11 @@ const (
asyncWorkerCapacityScopeKey = "global"
asyncWorkerQueueLimit = 10000
asyncWorkerMaxWaitSeconds = 24 * 60 * 60
asyncAdmissionBatchMax = 8
// One transaction must be able to fill the certified 2 x P32 global
// capacity. Smaller windows let multiple Worker dispatchers repeatedly race
// on the same FIFO head and can turn a batch size into an accidental cluster
// concurrency ceiling.
asyncAdmissionBatchMax = 64
acceptanceQueueLimit = 10000
acceptanceQueueMaxWait = 15 * 60
synchronousAdmissionWakeMax = 256
@@ -737,13 +741,7 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
case <-ticker.C:
}
for {
batchLimit := s.cfg.AsyncWorkerHardLimit
if batchLimit <= 0 {
batchLimit = 64
}
if batchLimit > asyncAdmissionBatchMax {
batchLimit = asyncAdmissionBatchMax
}
batchLimit := asyncAdmissionBatchLimit(s.cfg.AsyncWorkerHardLimit)
taskIDs, err := s.store.ListWaitingAsyncAdmissionTaskIDs(ctx, batchLimit)
if err != nil {
s.logger.Warn("list waiting async admissions failed", "error", err)
@@ -789,6 +787,13 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
}
}
func asyncAdmissionBatchLimit(globalHardLimit int) int {
if globalHardLimit <= 0 || globalHardLimit > asyncAdmissionBatchMax {
return asyncAdmissionBatchMax
}
return globalHardLimit
}
func (s *Service) reapExpiredTaskAdmissions(ctx context.Context) {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
@@ -62,6 +62,22 @@ func TestAcceptanceAdmissionScopesLeaveProductionPolicyUnchanged(t *testing.T) {
}
}
func TestAsyncAdmissionBatchCoversCertifiedGlobalCapacity(t *testing.T) {
for _, testCase := range []struct {
globalHardLimit int
want int
}{
{globalHardLimit: 48, want: 48},
{globalHardLimit: 64, want: 64},
{globalHardLimit: 128, want: 64},
{globalHardLimit: 0, want: 64},
} {
if got := asyncAdmissionBatchLimit(testCase.globalHardLimit); got != testCase.want {
t.Fatalf("async admission batch limit for %d = %d, want %d", testCase.globalHardLimit, got, testCase.want)
}
}
}
func TestPinCandidatesToTaskAdmissionPreservesAdmittedCandidate(t *testing.T) {
input := []store.RuntimeModelCandidate{
{PlatformID: "platform-a", PlatformModelID: "model-a"},
@@ -506,13 +506,16 @@ WHERE id = $1::uuid`, recoveryGraceTask.ID); err != nil {
ScopeType: "worker_capacity",
ScopeKey: "batch-" + suffix,
ScopeName: "batch capacity",
ConcurrentLimit: 3,
ConcurrentLimit: 48,
Amount: 1,
LeaseTTLSeconds: 120,
QueueLimit: 100,
MaxWaitSeconds: 600,
}
batchTasks := []GatewayTask{createTask(true), createTask(true), createTask(true)}
batchTasks := make([]GatewayTask, 48)
for index := range batchTasks {
batchTasks[index] = createTask(true)
}
batchInputs := make([]TaskAdmissionInput, 0, len(batchTasks))
batchJobIDs := make(map[string]int64, len(batchTasks))
for index, task := range batchTasks {
@@ -544,6 +547,10 @@ WHERE id = $1::uuid`, input.TaskID, batchJobIDs[input.TaskID])
t.Fatalf("batch outcome %d=%+v", index, outcome)
}
}
batchTaskIDs := make([]string, 0, len(batchTasks))
for _, task := range batchTasks {
batchTaskIDs = append(batchTaskIDs, task.ID)
}
var batchAdmissions, batchLeases, batchRiverJobs int
if err := first.pool.QueryRow(ctx, `
SELECT
@@ -556,16 +563,19 @@ SELECT
(SELECT count(*)
FROM gateway_tasks
WHERE id = ANY($1::uuid[]) AND river_job_id IS NOT NULL)`,
[]string{batchTasks[0].ID, batchTasks[1].ID, batchTasks[2].ID},
batchTaskIDs,
).Scan(&batchAdmissions, &batchLeases, &batchRiverJobs); err != nil {
t.Fatalf("read committed batch admission: %v", err)
}
if batchAdmissions != 3 || batchLeases != 3 || batchRiverJobs != 3 {
if batchAdmissions != len(batchTasks) || batchLeases != len(batchTasks) || batchRiverJobs != len(batchTasks) {
t.Fatalf(
"batch admissions=%d leases=%d River jobs=%d, want 3/3/3",
"batch admissions=%d leases=%d River jobs=%d, want %d/%d/%d",
batchAdmissions,
batchLeases,
batchRiverJobs,
len(batchTasks),
len(batchTasks),
len(batchTasks),
)
}
for _, task := range batchTasks {