fix(acceptance): 解除八任务准入瓶颈
线上 P24 验收证明 Worker 注册容量为 48,但异步 admission 始终只有 8 条活跃租约。将单事务调度窗口覆盖当前 2 x P32 的 64 槽,避免多 Worker 在同一 FIFO 小批次上竞争形成隐性全局上限。\n\nGemini 流式请求补充可回放 GetBody 与精确 Content-Length,使带幂等键的 POST 遇到 HTTP/2 可重试传输错误时可安全重放;同时提前部署验收回调收集器,保证旧 Run side effects 能在接管前收口。\n\n已通过 Go 全量测试、go vet、gofmt、Shell bash -n、ShellCheck、验收与人工发布脚本测试、迁移安全检查,以及独立 PostgreSQL 48 条 admission/租约/River job 原子集成测试。
This commit is contained in:
@@ -33,7 +33,11 @@ const (
|
||||
asyncWorkerCapacityScopeKey = "global"
|
||||
asyncWorkerQueueLimit = 10000
|
||||
asyncWorkerMaxWaitSeconds = 24 * 60 * 60
|
||||
asyncAdmissionBatchMax = 8
|
||||
// One transaction must be able to fill the certified 2 x P32 global
|
||||
// capacity. Smaller windows let multiple Worker dispatchers repeatedly race
|
||||
// on the same FIFO head and can turn a batch size into an accidental cluster
|
||||
// concurrency ceiling.
|
||||
asyncAdmissionBatchMax = 64
|
||||
acceptanceQueueLimit = 10000
|
||||
acceptanceQueueMaxWait = 15 * 60
|
||||
synchronousAdmissionWakeMax = 256
|
||||
@@ -737,13 +741,7 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
|
||||
case <-ticker.C:
|
||||
}
|
||||
for {
|
||||
batchLimit := s.cfg.AsyncWorkerHardLimit
|
||||
if batchLimit <= 0 {
|
||||
batchLimit = 64
|
||||
}
|
||||
if batchLimit > asyncAdmissionBatchMax {
|
||||
batchLimit = asyncAdmissionBatchMax
|
||||
}
|
||||
batchLimit := asyncAdmissionBatchLimit(s.cfg.AsyncWorkerHardLimit)
|
||||
taskIDs, err := s.store.ListWaitingAsyncAdmissionTaskIDs(ctx, batchLimit)
|
||||
if err != nil {
|
||||
s.logger.Warn("list waiting async admissions failed", "error", err)
|
||||
@@ -789,6 +787,13 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
func asyncAdmissionBatchLimit(globalHardLimit int) int {
|
||||
if globalHardLimit <= 0 || globalHardLimit > asyncAdmissionBatchMax {
|
||||
return asyncAdmissionBatchMax
|
||||
}
|
||||
return globalHardLimit
|
||||
}
|
||||
|
||||
func (s *Service) reapExpiredTaskAdmissions(ctx context.Context) {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
@@ -62,6 +62,22 @@ func TestAcceptanceAdmissionScopesLeaveProductionPolicyUnchanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAsyncAdmissionBatchCoversCertifiedGlobalCapacity(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
globalHardLimit int
|
||||
want int
|
||||
}{
|
||||
{globalHardLimit: 48, want: 48},
|
||||
{globalHardLimit: 64, want: 64},
|
||||
{globalHardLimit: 128, want: 64},
|
||||
{globalHardLimit: 0, want: 64},
|
||||
} {
|
||||
if got := asyncAdmissionBatchLimit(testCase.globalHardLimit); got != testCase.want {
|
||||
t.Fatalf("async admission batch limit for %d = %d, want %d", testCase.globalHardLimit, got, testCase.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPinCandidatesToTaskAdmissionPreservesAdmittedCandidate(t *testing.T) {
|
||||
input := []store.RuntimeModelCandidate{
|
||||
{PlatformID: "platform-a", PlatformModelID: "model-a"},
|
||||
|
||||
@@ -506,13 +506,16 @@ WHERE id = $1::uuid`, recoveryGraceTask.ID); err != nil {
|
||||
ScopeType: "worker_capacity",
|
||||
ScopeKey: "batch-" + suffix,
|
||||
ScopeName: "batch capacity",
|
||||
ConcurrentLimit: 3,
|
||||
ConcurrentLimit: 48,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 120,
|
||||
QueueLimit: 100,
|
||||
MaxWaitSeconds: 600,
|
||||
}
|
||||
batchTasks := []GatewayTask{createTask(true), createTask(true), createTask(true)}
|
||||
batchTasks := make([]GatewayTask, 48)
|
||||
for index := range batchTasks {
|
||||
batchTasks[index] = createTask(true)
|
||||
}
|
||||
batchInputs := make([]TaskAdmissionInput, 0, len(batchTasks))
|
||||
batchJobIDs := make(map[string]int64, len(batchTasks))
|
||||
for index, task := range batchTasks {
|
||||
@@ -544,6 +547,10 @@ WHERE id = $1::uuid`, input.TaskID, batchJobIDs[input.TaskID])
|
||||
t.Fatalf("batch outcome %d=%+v", index, outcome)
|
||||
}
|
||||
}
|
||||
batchTaskIDs := make([]string, 0, len(batchTasks))
|
||||
for _, task := range batchTasks {
|
||||
batchTaskIDs = append(batchTaskIDs, task.ID)
|
||||
}
|
||||
var batchAdmissions, batchLeases, batchRiverJobs int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
@@ -556,16 +563,19 @@ SELECT
|
||||
(SELECT count(*)
|
||||
FROM gateway_tasks
|
||||
WHERE id = ANY($1::uuid[]) AND river_job_id IS NOT NULL)`,
|
||||
[]string{batchTasks[0].ID, batchTasks[1].ID, batchTasks[2].ID},
|
||||
batchTaskIDs,
|
||||
).Scan(&batchAdmissions, &batchLeases, &batchRiverJobs); err != nil {
|
||||
t.Fatalf("read committed batch admission: %v", err)
|
||||
}
|
||||
if batchAdmissions != 3 || batchLeases != 3 || batchRiverJobs != 3 {
|
||||
if batchAdmissions != len(batchTasks) || batchLeases != len(batchTasks) || batchRiverJobs != len(batchTasks) {
|
||||
t.Fatalf(
|
||||
"batch admissions=%d leases=%d River jobs=%d, want 3/3/3",
|
||||
"batch admissions=%d leases=%d River jobs=%d, want %d/%d/%d",
|
||||
batchAdmissions,
|
||||
batchLeases,
|
||||
batchRiverJobs,
|
||||
len(batchTasks),
|
||||
len(batchTasks),
|
||||
len(batchTasks),
|
||||
)
|
||||
}
|
||||
for _, task := range batchTasks {
|
||||
|
||||
Reference in New Issue
Block a user