perf(worker): 以有界微批次提升准入吞吐
原因:线上 P24 同构验收中,单任务同步复制提交与全局容量锁串行化,使两个 Worker 的 48 个执行槽只能维持约 10–14 个运行任务,最老等待超过 15 分钟。 影响:新增可配置的 1–32 条准入微批次,默认 8;租约、任务准入与唯一 River job 在一个有界事务内原子提交,并按确定顺序预锁任务和容量范围,避免整窗 48 条大事务和多 Dispatcher 死锁。并容忍 rebind 已被其他 Dispatcher 完成的幂等竞态。 验证:Go 全量测试、go vet、真实 PostgreSQL 跨 Store 集成测试、ShellCheck、迁移安全检查、OpenAPI、前端 lint/test/build、Compose/Kubernetes 渲染及人工发布脚本均通过。
This commit is contained in:
@@ -354,6 +354,13 @@ func (s *Service) prepareTaskAdmissionInput(
|
||||
if currentErr == nil && current.Status == "waiting" &&
|
||||
(current.PlatformID != input.PlatformID || current.PlatformModelID != input.PlatformModelID || current.UserGroupID != input.UserGroupID) {
|
||||
if _, rebindErr := s.store.RebindWaitingTaskAdmission(ctx, input); rebindErr != nil {
|
||||
// Another dispatcher may admit or remove the same FIFO row between
|
||||
// the read above and the conditional rebind. Let the atomic admission
|
||||
// transaction observe the committed state instead of aborting an
|
||||
// entire dispatcher batch on that benign race.
|
||||
if errors.Is(rebindErr, pgx.ErrNoRows) {
|
||||
return input, nil
|
||||
}
|
||||
return store.TaskAdmissionInput{}, rebindErr
|
||||
}
|
||||
s.observeTaskAdmission("candidate_migrated")
|
||||
@@ -715,33 +722,40 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, tasks []store.G
|
||||
inputs = append(inputs, input)
|
||||
tasksByID[task.ID] = task
|
||||
}
|
||||
outcomes, err := s.store.TryTaskAdmissionBatchWithAdmittedHook(
|
||||
ctx,
|
||||
inputs,
|
||||
func(tx pgx.Tx, input store.TaskAdmissionInput) error {
|
||||
task, ok := tasksByID[input.TaskID]
|
||||
if !ok {
|
||||
return fmt.Errorf("async admission batch task %s was not prepared", input.TaskID)
|
||||
}
|
||||
return s.enqueueAsyncTaskTx(ctx, tx, task.ID, asyncTaskInsertOpts(task))
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
saturated := false
|
||||
for _, outcome := range outcomes {
|
||||
s.observeTaskAdmissionAttempt(outcome.Result, outcome.Err)
|
||||
if outcome.Err != nil {
|
||||
if errors.Is(outcome.Err, store.ErrTaskExecutionFinished) ||
|
||||
errors.Is(outcome.Err, store.ErrQueueTimeout) {
|
||||
continue
|
||||
}
|
||||
return false, outcome.Err
|
||||
microbatchSize := s.cfg.AsyncAdmissionMicrobatchSize
|
||||
if microbatchSize <= 0 {
|
||||
microbatchSize = 1
|
||||
}
|
||||
for start := 0; start < len(inputs) && !saturated; start += microbatchSize {
|
||||
end := min(start+microbatchSize, len(inputs))
|
||||
outcomes, err := s.store.TryTaskAdmissionAtomicBatchWithAdmittedHook(
|
||||
ctx,
|
||||
inputs[start:end],
|
||||
func(tx pgx.Tx, input store.TaskAdmissionInput) error {
|
||||
task, ok := tasksByID[input.TaskID]
|
||||
if !ok {
|
||||
return fmt.Errorf("async admission batch task %s was not prepared", input.TaskID)
|
||||
}
|
||||
return s.enqueueAsyncTaskTx(ctx, tx, task.ID, asyncTaskInsertOpts(task))
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !outcome.Result.Admitted {
|
||||
saturated = true
|
||||
break
|
||||
for _, outcome := range outcomes {
|
||||
s.observeTaskAdmissionAttempt(outcome.Result, outcome.Err)
|
||||
if outcome.Err != nil {
|
||||
if errors.Is(outcome.Err, store.ErrTaskExecutionFinished) ||
|
||||
errors.Is(outcome.Err, store.ErrQueueTimeout) {
|
||||
continue
|
||||
}
|
||||
return false, outcome.Err
|
||||
}
|
||||
if !outcome.Result.Admitted {
|
||||
saturated = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return saturated, nil
|
||||
|
||||
Reference in New Issue
Block a user