perf(worker): 以有界微批次提升准入吞吐
原因:线上 P24 同构验收中,单任务同步复制提交与全局容量锁串行化,使两个 Worker 的 48 个执行槽只能维持约 10–14 个运行任务,最老等待超过 15 分钟。 影响:新增可配置的 1–32 条准入微批次,默认 8;租约、任务准入与唯一 River job 在一个有界事务内原子提交,并按确定顺序预锁任务和容量范围,避免整窗 48 条大事务和多 Dispatcher 死锁。并容忍 rebind 已被其他 Dispatcher 完成的幂等竞态。 验证:Go 全量测试、go vet、真实 PostgreSQL 跨 Store 集成测试、ShellCheck、迁移安全检查、OpenAPI、前端 lint/test/build、Compose/Kubernetes 渲染及人工发布脚本均通过。
This commit is contained in:
@@ -502,6 +502,99 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
||||
return outcomes, nil
|
||||
}
|
||||
|
||||
// TryTaskAdmissionAtomicBatchWithAdmittedHook admits a deliberately small
|
||||
// group of tasks in one transaction. The caller bounds the group size so the
|
||||
// global capacity lock spans one synchronous commit without returning to the
|
||||
// old failure mode where a full capacity window locked dozens of task rows.
|
||||
// Admission leases and unique River jobs either commit together for the whole
|
||||
// microbatch or roll back together.
|
||||
func (s *Store) TryTaskAdmissionAtomicBatchWithAdmittedHook(
|
||||
ctx context.Context,
|
||||
inputs []TaskAdmissionInput,
|
||||
onAdmitted func(pgx.Tx, TaskAdmissionInput) error,
|
||||
) ([]TaskAdmissionBatchOutcome, error) {
|
||||
if len(inputs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
for _, input := range inputs {
|
||||
if err := validateTaskAdmissionInput(input); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
lockKeys := make([]string, 0, len(inputs)*3)
|
||||
for _, input := range inputs {
|
||||
lockKeys = append(lockKeys, admissionOperationLockKeys(input)...)
|
||||
}
|
||||
return retryAdmissionOperation(ctx, lockKeys, func() ([]TaskAdmissionBatchOutcome, error) {
|
||||
return s.tryTaskAdmissionAtomicBatchOnce(ctx, inputs, onAdmitted, lockKeys)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) tryTaskAdmissionAtomicBatchOnce(
|
||||
ctx context.Context,
|
||||
inputs []TaskAdmissionInput,
|
||||
onAdmitted func(pgx.Tx, TaskAdmissionInput) error,
|
||||
lockKeys []string,
|
||||
) ([]TaskAdmissionBatchOutcome, error) {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
// Lock every task and scope in deterministic order. Without this pre-lock,
|
||||
// two dispatchers could each hold a different task lock while waiting for
|
||||
// the shared worker-capacity lock.
|
||||
for _, key := range normalizedAdmissionLockKeys(lockKeys) {
|
||||
if err := tryAdmissionTransactionLock(ctx, tx, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
|
||||
notify := false
|
||||
for _, input := range inputs {
|
||||
hook := func(tx pgx.Tx) error {
|
||||
if onAdmitted == nil {
|
||||
return nil
|
||||
}
|
||||
return onAdmitted(tx, input)
|
||||
}
|
||||
outcome, admissionErr := s.tryTaskAdmissionTx(ctx, tx, input, hook)
|
||||
if errors.Is(admissionErr, ErrTaskExecutionFinished) {
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Err: admissionErr,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if admissionErr != nil {
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Err: admissionErr,
|
||||
})
|
||||
return outcomes, admissionErr
|
||||
}
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Result: outcome.Result,
|
||||
Err: outcome.PostCommitErr,
|
||||
})
|
||||
if outcome.NotifyTaskID != "" {
|
||||
notify = true
|
||||
}
|
||||
if !outcome.Result.Admitted {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return outcomes, err
|
||||
}
|
||||
if notify {
|
||||
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||
}
|
||||
return outcomes, nil
|
||||
}
|
||||
|
||||
func validateNewAdmissionCapacity(states []admissionScopeState) error {
|
||||
mustWait := false
|
||||
for _, state := range states {
|
||||
|
||||
Reference in New Issue
Block a user