fix(worker): 拆分准入事务并收敛验收失败状态

将批量准入改为每任务独立事务,避免 P24 压测时 48 条任务共享长事务造成 transactionid 锁潮与 Worker 槽位空转。失败验收在保持 validation 的同时清理未提交任务,并记录精确压力门禁原因。统一生产 0+2 Worker 拓扑、运行时 ConfigMap 与发布配置,补充镜像预检诊断和回归测试。\n\n验证:Go 全量测试、go vet、真实 PostgreSQL 48 任务集成测试、迁移安全检查、发布脚本测试、bash -n、ShellCheck、Compose 配置均通过。
This commit is contained in:
2026-08-01 18:23:00 +08:00
parent 215b6b607f
commit edcbbd6a87
11 changed files with 194 additions and 100 deletions
+23 -1
View File
@@ -258,7 +258,12 @@ func (s *Store) FinishAcceptanceRun(ctx context.Context, input FinishAcceptanceR
status = "passed"
}
report, _ := json.Marshal(sanitizeAcceptanceMetadata(sanitizeJSONForStorage(input.Report)))
return scanAcceptanceRun(s.pool.QueryRow(ctx, `
tx, err := s.pool.Begin(ctx)
if err != nil {
return AcceptanceRun{}, err
}
defer rollbackTransaction(tx)
run, err := scanAcceptanceRun(tx.QueryRow(ctx, `
UPDATE gateway_acceptance_runs
SET status = $2, report = $3::jsonb, failure_reason = NULLIF($4, ''),
finished_at = now(), updated_at = now()
@@ -287,6 +292,23 @@ RETURNING `+acceptanceRunColumns,
strings.TrimSpace(input.RunID), status, report, strings.TrimSpace(input.FailureReason),
SystemSettingGatewayTrafficMode,
))
if err != nil {
return AcceptanceRun{}, err
}
cancelled := int64(0)
if !input.Passed {
cancelled, err = cancelSafeAcceptanceTasksTx(ctx, tx, run.ID)
if err != nil {
return AcceptanceRun{}, err
}
}
if err := tx.Commit(ctx); err != nil {
return AcceptanceRun{}, err
}
if cancelled > 0 {
s.notifyTaskAdmissionBestEffort(ctx, "*")
}
return run, nil
}
func (s *Store) RetryAcceptanceRun(ctx context.Context, runID string) (AcceptanceRun, error) {
@@ -252,6 +252,20 @@ WHERE id = $1::uuid`, submittingAttemptID); err != nil {
}); err != nil || failed.Status != "failed" {
t.Fatalf("downgrade unpromoted acceptance run after external gate failure=%+v err=%v", failed, err)
}
var failedRunCancelled int
if err := db.pool.QueryRow(ctx, `
SELECT count(*)
FROM gateway_tasks
WHERE id = ANY($1::uuid[])
AND status = 'cancelled'
AND error_code = 'acceptance_run_aborted'`,
[]string{queuedTask.ID, runningTask.ID, notSubmittedTask.ID},
).Scan(&failedRunCancelled); err != nil {
t.Fatalf("read failed acceptance cleanup states: %v", err)
}
if failedRunCancelled != 3 {
t.Fatalf("failed acceptance cleanup cancelled=%d, want 3", failedRunCancelled)
}
if retried, err := db.RetryAcceptanceRun(ctx, failedRun.ID); err != nil || retried.Status != "running" {
t.Fatalf("retry acceptance run=%+v err=%v", retried, err)
}
+27 -56
View File
@@ -455,10 +455,12 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
}, nil
}
// TryTaskAdmissionBatchWithAdmittedHook fills a FIFO capacity window in one
// transaction. The hook inserts every unique River job in that same
// transaction, preserving the admission/job crash boundary while avoiding one
// synchronous replication round trip per task.
// TryTaskAdmissionBatchWithAdmittedHook fills a FIFO capacity window with one
// transaction per task. Each transaction still atomically reserves admission
// and inserts its unique River job, but a large global capacity window no
// longer holds every task row and scope lock until the entire batch commits.
// This is deliberately a batch at the dispatcher boundary, not a database
// transaction boundary.
func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
ctx context.Context,
inputs []TaskAdmissionInput,
@@ -467,68 +469,37 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
if len(inputs) == 0 {
return nil, nil
}
lockKeys := make([]string, 0, len(inputs)*3)
for _, input := range inputs {
if err := validateTaskAdmissionInput(input); err != nil {
return nil, err
}
lockKeys = append(lockKeys, admissionOperationLockKeys(input)...)
}
lockKeys = normalizedAdmissionLockKeys(lockKeys)
return retryAdmissionOperation(ctx, lockKeys, func() ([]TaskAdmissionBatchOutcome, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
defer rollbackTransaction(tx)
// A batch touches one shared capacity scope and multiple task keys. Lock
// the complete union in one global order before processing any task. If
// two API processes dispatch overlapping FIFO windows, neither can hold a
// scope while waiting on a task key already owned by the other batch.
for _, lockKey := range lockKeys {
if err := tryAdmissionTransactionLock(ctx, tx, lockKey); err != nil {
return nil, err
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
for _, input := range inputs {
hook := func(tx pgx.Tx) error {
if onAdmitted == nil {
return nil
}
return onAdmitted(tx, input)
}
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
notify := false
for _, input := range inputs {
hook := func(tx pgx.Tx) error {
if onAdmitted == nil {
return nil
}
return onAdmitted(tx, input)
}
outcome, admissionErr := s.tryTaskAdmissionTx(ctx, tx, input, hook)
if errors.Is(admissionErr, ErrTaskExecutionFinished) {
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
TaskID: input.TaskID,
Err: admissionErr,
})
continue
}
if admissionErr != nil {
return nil, admissionErr
}
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
TaskID: input.TaskID,
Result: outcome.Result,
Err: outcome.PostCommitErr,
})
notify = notify || outcome.NotifyTaskID != ""
if outcome.PostCommitErr == nil && !outcome.Result.Admitted {
break
}
result, admissionErr := s.TryTaskAdmissionWithAdmittedHook(ctx, input, hook)
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
TaskID: input.TaskID,
Result: result,
Err: admissionErr,
})
if errors.Is(admissionErr, ErrTaskExecutionFinished) ||
errors.Is(admissionErr, ErrQueueTimeout) {
continue
}
if err := tx.Commit(ctx); err != nil {
return nil, err
if admissionErr != nil {
return outcomes, admissionErr
}
if notify {
s.notifyTaskAdmissionBestEffort(ctx, "*")
if !result.Admitted {
break
}
return outcomes, nil
})
}
return outcomes, nil
}
func validateNewAdmissionCapacity(states []admissionScopeState) error {
@@ -598,7 +598,7 @@ SELECT
rollbackInputs = append(rollbackInputs, input)
}
batchHookFailure := errors.New("synthetic batch hook failure")
_, err = first.TryTaskAdmissionBatchWithAdmittedHook(
partialOutcomes, err := first.TryTaskAdmissionBatchWithAdmittedHook(
ctx,
rollbackInputs,
func(tx pgx.Tx, input TaskAdmissionInput) error {
@@ -615,6 +615,10 @@ WHERE id = $1::uuid`, input.TaskID)
if !errors.Is(err, batchHookFailure) {
t.Fatalf("batch hook failure=%v, want synthetic failure", err)
}
if len(partialOutcomes) != 2 || !partialOutcomes[0].Result.Admitted ||
!errors.Is(partialOutcomes[1].Err, batchHookFailure) {
t.Fatalf("per-task batch outcomes=%+v, want first committed and second failed", partialOutcomes)
}
var rollbackWaiting, rollbackLeases, rollbackRiverJobs int
if err := first.pool.QueryRow(ctx, `
SELECT
@@ -631,9 +635,9 @@ SELECT
).Scan(&rollbackWaiting, &rollbackLeases, &rollbackRiverJobs); err != nil {
t.Fatalf("read rolled back batch admission: %v", err)
}
if rollbackWaiting != 2 || rollbackLeases != 0 || rollbackRiverJobs != 0 {
if rollbackWaiting != 1 || rollbackLeases != 1 || rollbackRiverJobs != 1 {
t.Fatalf(
"rolled back batch waiting=%d leases=%d River jobs=%d, want 2/0/0",
"per-task rollback waiting=%d leases=%d River jobs=%d, want 1/1/1",
rollbackWaiting,
rollbackLeases,
rollbackRiverJobs,