fix(worker): 拆分准入事务并收敛验收失败状态
将批量准入改为每任务独立事务,避免 P24 压测时 48 条任务共享长事务造成 transactionid 锁潮与 Worker 槽位空转。失败验收在保持 validation 的同时清理未提交任务,并记录精确压力门禁原因。统一生产 0+2 Worker 拓扑、运行时 ConfigMap 与发布配置,补充镜像预检诊断和回归测试。\n\n验证:Go 全量测试、go vet、真实 PostgreSQL 48 任务集成测试、迁移安全检查、发布脚本测试、bash -n、ShellCheck、Compose 配置均通过。
This commit is contained in:
@@ -455,10 +455,12 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TryTaskAdmissionBatchWithAdmittedHook fills a FIFO capacity window in one
|
||||
// transaction. The hook inserts every unique River job in that same
|
||||
// transaction, preserving the admission/job crash boundary while avoiding one
|
||||
// synchronous replication round trip per task.
|
||||
// TryTaskAdmissionBatchWithAdmittedHook fills a FIFO capacity window with one
|
||||
// transaction per task. Each transaction still atomically reserves admission
|
||||
// and inserts its unique River job, but a large global capacity window no
|
||||
// longer holds every task row and scope lock until the entire batch commits.
|
||||
// This is deliberately a batch at the dispatcher boundary, not a database
|
||||
// transaction boundary.
|
||||
func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
||||
ctx context.Context,
|
||||
inputs []TaskAdmissionInput,
|
||||
@@ -467,68 +469,37 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
||||
if len(inputs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
lockKeys := make([]string, 0, len(inputs)*3)
|
||||
for _, input := range inputs {
|
||||
if err := validateTaskAdmissionInput(input); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lockKeys = append(lockKeys, admissionOperationLockKeys(input)...)
|
||||
}
|
||||
lockKeys = normalizedAdmissionLockKeys(lockKeys)
|
||||
return retryAdmissionOperation(ctx, lockKeys, func() ([]TaskAdmissionBatchOutcome, error) {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
// A batch touches one shared capacity scope and multiple task keys. Lock
|
||||
// the complete union in one global order before processing any task. If
|
||||
// two API processes dispatch overlapping FIFO windows, neither can hold a
|
||||
// scope while waiting on a task key already owned by the other batch.
|
||||
for _, lockKey := range lockKeys {
|
||||
if err := tryAdmissionTransactionLock(ctx, tx, lockKey); err != nil {
|
||||
return nil, err
|
||||
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
|
||||
for _, input := range inputs {
|
||||
hook := func(tx pgx.Tx) error {
|
||||
if onAdmitted == nil {
|
||||
return nil
|
||||
}
|
||||
return onAdmitted(tx, input)
|
||||
}
|
||||
|
||||
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
|
||||
notify := false
|
||||
for _, input := range inputs {
|
||||
hook := func(tx pgx.Tx) error {
|
||||
if onAdmitted == nil {
|
||||
return nil
|
||||
}
|
||||
return onAdmitted(tx, input)
|
||||
}
|
||||
outcome, admissionErr := s.tryTaskAdmissionTx(ctx, tx, input, hook)
|
||||
if errors.Is(admissionErr, ErrTaskExecutionFinished) {
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Err: admissionErr,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if admissionErr != nil {
|
||||
return nil, admissionErr
|
||||
}
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Result: outcome.Result,
|
||||
Err: outcome.PostCommitErr,
|
||||
})
|
||||
notify = notify || outcome.NotifyTaskID != ""
|
||||
if outcome.PostCommitErr == nil && !outcome.Result.Admitted {
|
||||
break
|
||||
}
|
||||
result, admissionErr := s.TryTaskAdmissionWithAdmittedHook(ctx, input, hook)
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Result: result,
|
||||
Err: admissionErr,
|
||||
})
|
||||
if errors.Is(admissionErr, ErrTaskExecutionFinished) ||
|
||||
errors.Is(admissionErr, ErrQueueTimeout) {
|
||||
continue
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, err
|
||||
if admissionErr != nil {
|
||||
return outcomes, admissionErr
|
||||
}
|
||||
if notify {
|
||||
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||
if !result.Admitted {
|
||||
break
|
||||
}
|
||||
return outcomes, nil
|
||||
})
|
||||
}
|
||||
return outcomes, nil
|
||||
}
|
||||
|
||||
func validateNewAdmissionCapacity(states []admissionScopeState) error {
|
||||
|
||||
Reference in New Issue
Block a user