perf(worker): 提交后发送准入唤醒通知

将准入登记、派发、释放、恢复和验收清理中的 PostgreSQL NOTIFY 从同步复制业务事务移到提交后的独立异步提交事务,避免数据库对象锁串行阻塞跨城事务。通知继续作为有轮询兜底的限时提示,不改变任务状态、租约和账务的同步提交语义。

验证:Go 全量测试、go vet、迁移安全检查、gofmt、diff check 通过;新增集成回归确认 NOTIFY 对象锁存在时准入状态仍先提交可见。
This commit is contained in:
2026-07-31 09:56:47 +08:00
parent 580f84f569
commit 447e7ed701
5 changed files with 118 additions and 39 deletions
+5 -6
View File
@@ -376,7 +376,8 @@ FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&value); err != nil {
current.WorkerImageDigest != strings.TrimSpace(input.WorkerImageDigest) {
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
}
if _, err := cancelSafeAcceptanceTasksTx(ctx, tx, current.RunID); err != nil {
cancelled, err := cancelSafeAcceptanceTasksTx(ctx, tx, current.RunID)
if err != nil {
return GatewayTrafficMode{}, err
}
next := GatewayTrafficMode{Mode: "live", Revision: current.Revision + 1}
@@ -396,6 +397,9 @@ WHERE id = $1::uuid AND status IN ('running', 'failed', 'passed')`, current.RunI
if err := tx.Commit(ctx); err != nil {
return GatewayTrafficMode{}, err
}
if cancelled > 0 {
s.notifyTaskAdmissionBestEffort(ctx, "*")
}
next.UpdatedAt = time.Now()
return next, nil
}
@@ -555,11 +559,6 @@ WHERE admission.task_id = task.id
)`, runID); err != nil {
return 0, err
}
if tag.RowsAffected() > 0 {
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return 0, err
}
}
return tag.RowsAffected(), nil
}
+49 -25
View File
@@ -16,6 +16,7 @@ import (
const (
admissionWaiterLeaseTTL = 15 * time.Second
admissionNotifyChannel = "gateway_task_admission"
admissionNotifyTimeout = 2 * time.Second
executionSlotWaitMax = 24 * time.Hour
)
@@ -216,12 +217,10 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
return TaskAdmission{}, err
}
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return TaskAdmission{}, err
}
if err := tx.Commit(ctx); err != nil {
return TaskAdmission{}, err
}
s.notifyTaskAdmissionBestEffort(ctx, "*")
return TaskAdmission{}, &QueueTimeoutError{TaskID: input.TaskID}
}
if !found {
@@ -243,12 +242,10 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
return TaskAdmission{}, err
}
}
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
return TaskAdmission{}, err
}
if err := tx.Commit(ctx); err != nil {
return TaskAdmission{}, err
}
s.notifyTaskAdmissionBestEffort(ctx, input.TaskID)
return admission, nil
}
@@ -283,12 +280,14 @@ func (s *Store) tryTaskAdmissionOnce(
if err := tx.Commit(ctx); err != nil {
return TaskAdmissionResult{}, err
}
s.notifyTaskAdmissionBestEffort(ctx, outcome.NotifyTaskID)
return outcome.Result, outcome.PostCommitErr
}
type taskAdmissionTxOutcome struct {
Result TaskAdmissionResult
PostCommitErr error
NotifyTaskID string
}
func (s *Store) tryTaskAdmissionTx(
@@ -363,11 +362,9 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
return taskAdmissionTxOutcome{}, err
}
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return taskAdmissionTxOutcome{}, err
}
return taskAdmissionTxOutcome{
PostCommitErr: &QueueTimeoutError{TaskID: input.TaskID},
NotifyTaskID: "*",
}, nil
}
@@ -442,9 +439,6 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
// Continue the FIFO chain after one task is admitted. Every API process
// only probes the global head it owns, so free capacity is filled without
// broadcasting an advisory-lock attempt to every waiter.
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return taskAdmissionTxOutcome{}, err
}
if onAdmitted != nil {
if err := onAdmitted(tx); err != nil {
return taskAdmissionTxOutcome{}, err
@@ -457,6 +451,7 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
NewlyAdmitted: true,
Leases: leases,
},
NotifyTaskID: "*",
}, nil
}
@@ -487,6 +482,7 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
defer rollbackTransaction(tx)
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
notify := false
for _, input := range inputs {
hook := func(tx pgx.Tx) error {
if onAdmitted == nil {
@@ -510,6 +506,7 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
Result: outcome.Result,
Err: outcome.PostCommitErr,
})
notify = notify || outcome.NotifyTaskID != ""
if outcome.PostCommitErr == nil && !outcome.Result.Admitted {
break
}
@@ -517,6 +514,9 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
if err := tx.Commit(ctx); err != nil {
return nil, err
}
if notify {
s.notifyTaskAdmissionBestEffort(ctx, "*")
}
return outcomes, nil
})
}
@@ -649,12 +649,10 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
if err != nil {
return TaskAdmission{}, err
}
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
return TaskAdmission{}, err
}
if err := tx.Commit(ctx); err != nil {
return TaskAdmission{}, err
}
s.notifyTaskAdmissionBestEffort(ctx, input.TaskID)
return migrated, nil
}
@@ -781,10 +779,11 @@ WHERE task_id = $1::uuid
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
return err
}
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
if err := tx.Commit(ctx); err != nil {
return err
}
return tx.Commit(ctx)
s.notifyTaskAdmissionBestEffort(ctx, "*")
return nil
}
func (s *Store) ActiveTaskAdmissionLeases(ctx context.Context, taskID string) ([]ConcurrencyLease, error) {
@@ -1026,14 +1025,12 @@ WHERE id = $1::uuid
}
reaped++
}
if reaped > 0 {
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return TaskAdmissionReapResult{}, err
}
}
if err := tx.Commit(ctx); err != nil {
return TaskAdmissionReapResult{}, err
}
if reaped > 0 {
s.notifyTaskAdmissionBestEffort(ctx, "*")
}
return result, nil
}
@@ -1413,7 +1410,34 @@ WHERE id = $1::uuid AND status = 'queued'`, taskID)
return err
}
func notifyTaskAdmissionTx(ctx context.Context, tx pgx.Tx, taskID string) error {
_, err := tx.Exec(ctx, `SELECT pg_notify($1, $2)`, admissionNotifyChannel, taskID)
return err
// notifyTaskAdmission publishes a non-durable wake-up hint only after the
// corresponding state transaction has committed. Keeping NOTIFY out of
// synchronous replication transactions prevents its database object lock from
// serializing unrelated admission, execution, and release commits.
func (s *Store) notifyTaskAdmission(ctx context.Context, taskID string) error {
taskID = strings.TrimSpace(taskID)
if taskID == "" {
return nil
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer rollbackTransaction(tx)
if _, err := tx.Exec(ctx, `SET LOCAL synchronous_commit = off`); err != nil {
return err
}
if _, err := tx.Exec(ctx, `SELECT pg_notify($1, $2)`, admissionNotifyChannel, taskID); err != nil {
return err
}
return tx.Commit(ctx)
}
func (s *Store) notifyTaskAdmissionBestEffort(ctx context.Context, taskID string) {
if strings.TrimSpace(taskID) == "" {
return
}
notifyCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), admissionNotifyTimeout)
defer cancel()
_ = s.notifyTaskAdmission(notifyCtx, taskID)
}
@@ -113,4 +113,53 @@ RETURNING id::text`, platform.ID, modelName).Scan(&platformModelID); err != nil
if registrationErr != nil {
t.Fatalf("queue registration blocked behind execution scope lock: %v", registrationErr)
}
// Simulate a legacy synchronous transaction that is still holding the
// database object lock used by NOTIFY. The admission deletion must commit
// and become visible before the best-effort wake-up hint can acquire that
// lock.
notificationLockTx, err := first.pool.Begin(ctx)
if err != nil {
t.Fatalf("begin notification lock transaction: %v", err)
}
if _, err := notificationLockTx.Exec(ctx, `SELECT pg_notify($1, '*')`, admissionNotifyChannel); err != nil {
_ = notificationLockTx.Rollback(ctx)
t.Fatalf("hold notification object lock: %v", err)
}
deleteResult := make(chan error, 1)
go func() {
deleteResult <- second.DeleteTaskAdmission(ctx, task.ID)
}()
deleteVisible := false
visibilityDeadline := time.Now().Add(time.Second)
for time.Now().Before(visibilityDeadline) {
var admissionCount int
if err := first.pool.QueryRow(ctx, `
SELECT count(*)
FROM gateway_task_admissions
WHERE task_id = $1::uuid`, task.ID).Scan(&admissionCount); err != nil {
_ = notificationLockTx.Rollback(ctx)
t.Fatalf("read admission visibility: %v", err)
}
if admissionCount == 0 {
deleteVisible = true
break
}
time.Sleep(20 * time.Millisecond)
}
if !deleteVisible {
_ = notificationLockTx.Rollback(ctx)
t.Fatal("admission state commit waited behind notification object lock")
}
if err := notificationLockTx.Rollback(ctx); err != nil {
t.Fatalf("release notification object lock: %v", err)
}
select {
case err := <-deleteResult:
if err != nil {
t.Fatalf("delete admission after notification lock release: %v", err)
}
case <-time.After(3 * time.Second):
t.Fatal("delete admission did not finish after notification lock release")
}
}
+12 -3
View File
@@ -715,11 +715,14 @@ WHERE task_id = $1::uuid
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
return err
}
return notifyTaskAdmissionTx(ctx, tx, "*")
return nil
})
if err != nil {
return GatewayTask{}, false, err
}
if changed {
s.notifyTaskAdmissionBestEffort(ctx, "*")
}
return task, changed, nil
}
@@ -877,11 +880,14 @@ WHERE task_id = $1::uuid
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
return err
}
return notifyTaskAdmissionTx(ctx, tx, "*")
return nil
})
if err != nil {
return GatewayTask{}, false, err
}
if changed {
s.notifyTaskAdmissionBestEffort(ctx, "*")
}
return task, changed, nil
}
@@ -963,11 +969,14 @@ WHERE task_id = $1::uuid
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
return err
}
return notifyTaskAdmissionTx(ctx, tx, "*")
return nil
})
if err != nil {
return GatewayTask{}, false, err
}
if changed {
s.notifyTaskAdmissionBestEffort(ctx, "*")
}
return task, changed, nil
}
+3 -5
View File
@@ -286,14 +286,12 @@ SELECT count(*)::bigint
FROM deleted`, staleAfter.String(), limit).Scan(&yielded); err != nil {
return 0, err
}
if yielded > 0 {
if _, err := tx.Exec(ctx, `SELECT pg_notify('gateway_task_admission', '*')`); err != nil {
return 0, err
}
}
if err := tx.Commit(ctx); err != nil {
return 0, err
}
if yielded > 0 {
s.notifyTaskAdmissionBestEffort(ctx, "*")
}
return yielded, nil
}