fix(worker): 拆分准入事务并收敛验收失败状态
将批量准入改为每任务独立事务,避免 P24 压测时 48 条任务共享长事务造成 transactionid 锁潮与 Worker 槽位空转。失败验收在保持 validation 的同时清理未提交任务,并记录精确压力门禁原因。统一生产 0+2 Worker 拓扑、运行时 ConfigMap 与发布配置,补充镜像预检诊断和回归测试。\n\n验证:Go 全量测试、go vet、真实 PostgreSQL 48 任务集成测试、迁移安全检查、发布脚本测试、bash -n、ShellCheck、Compose 配置均通过。
This commit is contained in:
@@ -258,7 +258,12 @@ func (s *Store) FinishAcceptanceRun(ctx context.Context, input FinishAcceptanceR
|
||||
status = "passed"
|
||||
}
|
||||
report, _ := json.Marshal(sanitizeAcceptanceMetadata(sanitizeJSONForStorage(input.Report)))
|
||||
return scanAcceptanceRun(s.pool.QueryRow(ctx, `
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return AcceptanceRun{}, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
run, err := scanAcceptanceRun(tx.QueryRow(ctx, `
|
||||
UPDATE gateway_acceptance_runs
|
||||
SET status = $2, report = $3::jsonb, failure_reason = NULLIF($4, ''),
|
||||
finished_at = now(), updated_at = now()
|
||||
@@ -287,6 +292,23 @@ RETURNING `+acceptanceRunColumns,
|
||||
strings.TrimSpace(input.RunID), status, report, strings.TrimSpace(input.FailureReason),
|
||||
SystemSettingGatewayTrafficMode,
|
||||
))
|
||||
if err != nil {
|
||||
return AcceptanceRun{}, err
|
||||
}
|
||||
cancelled := int64(0)
|
||||
if !input.Passed {
|
||||
cancelled, err = cancelSafeAcceptanceTasksTx(ctx, tx, run.ID)
|
||||
if err != nil {
|
||||
return AcceptanceRun{}, err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return AcceptanceRun{}, err
|
||||
}
|
||||
if cancelled > 0 {
|
||||
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||
}
|
||||
return run, nil
|
||||
}
|
||||
|
||||
func (s *Store) RetryAcceptanceRun(ctx context.Context, runID string) (AcceptanceRun, error) {
|
||||
|
||||
@@ -252,6 +252,20 @@ WHERE id = $1::uuid`, submittingAttemptID); err != nil {
|
||||
}); err != nil || failed.Status != "failed" {
|
||||
t.Fatalf("downgrade unpromoted acceptance run after external gate failure=%+v err=%v", failed, err)
|
||||
}
|
||||
var failedRunCancelled int
|
||||
if err := db.pool.QueryRow(ctx, `
|
||||
SELECT count(*)
|
||||
FROM gateway_tasks
|
||||
WHERE id = ANY($1::uuid[])
|
||||
AND status = 'cancelled'
|
||||
AND error_code = 'acceptance_run_aborted'`,
|
||||
[]string{queuedTask.ID, runningTask.ID, notSubmittedTask.ID},
|
||||
).Scan(&failedRunCancelled); err != nil {
|
||||
t.Fatalf("read failed acceptance cleanup states: %v", err)
|
||||
}
|
||||
if failedRunCancelled != 3 {
|
||||
t.Fatalf("failed acceptance cleanup cancelled=%d, want 3", failedRunCancelled)
|
||||
}
|
||||
if retried, err := db.RetryAcceptanceRun(ctx, failedRun.ID); err != nil || retried.Status != "running" {
|
||||
t.Fatalf("retry acceptance run=%+v err=%v", retried, err)
|
||||
}
|
||||
|
||||
@@ -455,10 +455,12 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TryTaskAdmissionBatchWithAdmittedHook fills a FIFO capacity window in one
|
||||
// transaction. The hook inserts every unique River job in that same
|
||||
// transaction, preserving the admission/job crash boundary while avoiding one
|
||||
// synchronous replication round trip per task.
|
||||
// TryTaskAdmissionBatchWithAdmittedHook fills a FIFO capacity window with one
|
||||
// transaction per task. Each transaction still atomically reserves admission
|
||||
// and inserts its unique River job, but a large global capacity window no
|
||||
// longer holds every task row and scope lock until the entire batch commits.
|
||||
// This is deliberately a batch at the dispatcher boundary, not a database
|
||||
// transaction boundary.
|
||||
func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
||||
ctx context.Context,
|
||||
inputs []TaskAdmissionInput,
|
||||
@@ -467,68 +469,37 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
||||
if len(inputs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
lockKeys := make([]string, 0, len(inputs)*3)
|
||||
for _, input := range inputs {
|
||||
if err := validateTaskAdmissionInput(input); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lockKeys = append(lockKeys, admissionOperationLockKeys(input)...)
|
||||
}
|
||||
lockKeys = normalizedAdmissionLockKeys(lockKeys)
|
||||
return retryAdmissionOperation(ctx, lockKeys, func() ([]TaskAdmissionBatchOutcome, error) {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
// A batch touches one shared capacity scope and multiple task keys. Lock
|
||||
// the complete union in one global order before processing any task. If
|
||||
// two API processes dispatch overlapping FIFO windows, neither can hold a
|
||||
// scope while waiting on a task key already owned by the other batch.
|
||||
for _, lockKey := range lockKeys {
|
||||
if err := tryAdmissionTransactionLock(ctx, tx, lockKey); err != nil {
|
||||
return nil, err
|
||||
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
|
||||
for _, input := range inputs {
|
||||
hook := func(tx pgx.Tx) error {
|
||||
if onAdmitted == nil {
|
||||
return nil
|
||||
}
|
||||
return onAdmitted(tx, input)
|
||||
}
|
||||
|
||||
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
|
||||
notify := false
|
||||
for _, input := range inputs {
|
||||
hook := func(tx pgx.Tx) error {
|
||||
if onAdmitted == nil {
|
||||
return nil
|
||||
}
|
||||
return onAdmitted(tx, input)
|
||||
}
|
||||
outcome, admissionErr := s.tryTaskAdmissionTx(ctx, tx, input, hook)
|
||||
if errors.Is(admissionErr, ErrTaskExecutionFinished) {
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Err: admissionErr,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if admissionErr != nil {
|
||||
return nil, admissionErr
|
||||
}
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Result: outcome.Result,
|
||||
Err: outcome.PostCommitErr,
|
||||
})
|
||||
notify = notify || outcome.NotifyTaskID != ""
|
||||
if outcome.PostCommitErr == nil && !outcome.Result.Admitted {
|
||||
break
|
||||
}
|
||||
result, admissionErr := s.TryTaskAdmissionWithAdmittedHook(ctx, input, hook)
|
||||
outcomes = append(outcomes, TaskAdmissionBatchOutcome{
|
||||
TaskID: input.TaskID,
|
||||
Result: result,
|
||||
Err: admissionErr,
|
||||
})
|
||||
if errors.Is(admissionErr, ErrTaskExecutionFinished) ||
|
||||
errors.Is(admissionErr, ErrQueueTimeout) {
|
||||
continue
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, err
|
||||
if admissionErr != nil {
|
||||
return outcomes, admissionErr
|
||||
}
|
||||
if notify {
|
||||
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||
if !result.Admitted {
|
||||
break
|
||||
}
|
||||
return outcomes, nil
|
||||
})
|
||||
}
|
||||
return outcomes, nil
|
||||
}
|
||||
|
||||
func validateNewAdmissionCapacity(states []admissionScopeState) error {
|
||||
|
||||
@@ -598,7 +598,7 @@ SELECT
|
||||
rollbackInputs = append(rollbackInputs, input)
|
||||
}
|
||||
batchHookFailure := errors.New("synthetic batch hook failure")
|
||||
_, err = first.TryTaskAdmissionBatchWithAdmittedHook(
|
||||
partialOutcomes, err := first.TryTaskAdmissionBatchWithAdmittedHook(
|
||||
ctx,
|
||||
rollbackInputs,
|
||||
func(tx pgx.Tx, input TaskAdmissionInput) error {
|
||||
@@ -615,6 +615,10 @@ WHERE id = $1::uuid`, input.TaskID)
|
||||
if !errors.Is(err, batchHookFailure) {
|
||||
t.Fatalf("batch hook failure=%v, want synthetic failure", err)
|
||||
}
|
||||
if len(partialOutcomes) != 2 || !partialOutcomes[0].Result.Admitted ||
|
||||
!errors.Is(partialOutcomes[1].Err, batchHookFailure) {
|
||||
t.Fatalf("per-task batch outcomes=%+v, want first committed and second failed", partialOutcomes)
|
||||
}
|
||||
var rollbackWaiting, rollbackLeases, rollbackRiverJobs int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
@@ -631,9 +635,9 @@ SELECT
|
||||
).Scan(&rollbackWaiting, &rollbackLeases, &rollbackRiverJobs); err != nil {
|
||||
t.Fatalf("read rolled back batch admission: %v", err)
|
||||
}
|
||||
if rollbackWaiting != 2 || rollbackLeases != 0 || rollbackRiverJobs != 0 {
|
||||
if rollbackWaiting != 1 || rollbackLeases != 1 || rollbackRiverJobs != 1 {
|
||||
t.Fatalf(
|
||||
"rolled back batch waiting=%d leases=%d River jobs=%d, want 2/0/0",
|
||||
"per-task rollback waiting=%d leases=%d River jobs=%d, want 1/1/1",
|
||||
rollbackWaiting,
|
||||
rollbackLeases,
|
||||
rollbackRiverJobs,
|
||||
|
||||
Reference in New Issue
Block a user