perf(worker): 提交后发送准入唤醒通知
将准入登记、派发、释放、恢复和验收清理中的 PostgreSQL NOTIFY 从同步复制业务事务移到提交后的独立异步提交事务,避免数据库对象锁串行阻塞跨城事务。通知继续作为有轮询兜底的限时提示,不改变任务状态、租约和账务的同步提交语义。 验证:Go 全量测试、go vet、迁移安全检查、gofmt、diff check 通过;新增集成回归确认 NOTIFY 对象锁存在时准入状态仍先提交可见。
This commit is contained in:
@@ -376,7 +376,8 @@ FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&value); err != nil {
|
|||||||
current.WorkerImageDigest != strings.TrimSpace(input.WorkerImageDigest) {
|
current.WorkerImageDigest != strings.TrimSpace(input.WorkerImageDigest) {
|
||||||
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
||||||
}
|
}
|
||||||
if _, err := cancelSafeAcceptanceTasksTx(ctx, tx, current.RunID); err != nil {
|
cancelled, err := cancelSafeAcceptanceTasksTx(ctx, tx, current.RunID)
|
||||||
|
if err != nil {
|
||||||
return GatewayTrafficMode{}, err
|
return GatewayTrafficMode{}, err
|
||||||
}
|
}
|
||||||
next := GatewayTrafficMode{Mode: "live", Revision: current.Revision + 1}
|
next := GatewayTrafficMode{Mode: "live", Revision: current.Revision + 1}
|
||||||
@@ -396,6 +397,9 @@ WHERE id = $1::uuid AND status IN ('running', 'failed', 'passed')`, current.RunI
|
|||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return GatewayTrafficMode{}, err
|
return GatewayTrafficMode{}, err
|
||||||
}
|
}
|
||||||
|
if cancelled > 0 {
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
|
}
|
||||||
next.UpdatedAt = time.Now()
|
next.UpdatedAt = time.Now()
|
||||||
return next, nil
|
return next, nil
|
||||||
}
|
}
|
||||||
@@ -555,11 +559,6 @@ WHERE admission.task_id = task.id
|
|||||||
)`, runID); err != nil {
|
)`, runID); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
if tag.RowsAffected() > 0 {
|
|
||||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return tag.RowsAffected(), nil
|
return tag.RowsAffected(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
const (
|
const (
|
||||||
admissionWaiterLeaseTTL = 15 * time.Second
|
admissionWaiterLeaseTTL = 15 * time.Second
|
||||||
admissionNotifyChannel = "gateway_task_admission"
|
admissionNotifyChannel = "gateway_task_admission"
|
||||||
|
admissionNotifyTimeout = 2 * time.Second
|
||||||
executionSlotWaitMax = 24 * time.Hour
|
executionSlotWaitMax = 24 * time.Hour
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -216,12 +217,10 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
|||||||
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
||||||
return TaskAdmission{}, err
|
return TaskAdmission{}, err
|
||||||
}
|
}
|
||||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
|
||||||
return TaskAdmission{}, err
|
|
||||||
}
|
|
||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return TaskAdmission{}, err
|
return TaskAdmission{}, err
|
||||||
}
|
}
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
return TaskAdmission{}, &QueueTimeoutError{TaskID: input.TaskID}
|
return TaskAdmission{}, &QueueTimeoutError{TaskID: input.TaskID}
|
||||||
}
|
}
|
||||||
if !found {
|
if !found {
|
||||||
@@ -243,12 +242,10 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
|||||||
return TaskAdmission{}, err
|
return TaskAdmission{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
|
||||||
return TaskAdmission{}, err
|
|
||||||
}
|
|
||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return TaskAdmission{}, err
|
return TaskAdmission{}, err
|
||||||
}
|
}
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, input.TaskID)
|
||||||
return admission, nil
|
return admission, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -283,12 +280,14 @@ func (s *Store) tryTaskAdmissionOnce(
|
|||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return TaskAdmissionResult{}, err
|
return TaskAdmissionResult{}, err
|
||||||
}
|
}
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, outcome.NotifyTaskID)
|
||||||
return outcome.Result, outcome.PostCommitErr
|
return outcome.Result, outcome.PostCommitErr
|
||||||
}
|
}
|
||||||
|
|
||||||
type taskAdmissionTxOutcome struct {
|
type taskAdmissionTxOutcome struct {
|
||||||
Result TaskAdmissionResult
|
Result TaskAdmissionResult
|
||||||
PostCommitErr error
|
PostCommitErr error
|
||||||
|
NotifyTaskID string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) tryTaskAdmissionTx(
|
func (s *Store) tryTaskAdmissionTx(
|
||||||
@@ -363,11 +362,9 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
|||||||
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
||||||
return taskAdmissionTxOutcome{}, err
|
return taskAdmissionTxOutcome{}, err
|
||||||
}
|
}
|
||||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
|
||||||
return taskAdmissionTxOutcome{}, err
|
|
||||||
}
|
|
||||||
return taskAdmissionTxOutcome{
|
return taskAdmissionTxOutcome{
|
||||||
PostCommitErr: &QueueTimeoutError{TaskID: input.TaskID},
|
PostCommitErr: &QueueTimeoutError{TaskID: input.TaskID},
|
||||||
|
NotifyTaskID: "*",
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -442,9 +439,6 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
|||||||
// Continue the FIFO chain after one task is admitted. Every API process
|
// Continue the FIFO chain after one task is admitted. Every API process
|
||||||
// only probes the global head it owns, so free capacity is filled without
|
// only probes the global head it owns, so free capacity is filled without
|
||||||
// broadcasting an advisory-lock attempt to every waiter.
|
// broadcasting an advisory-lock attempt to every waiter.
|
||||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
|
||||||
return taskAdmissionTxOutcome{}, err
|
|
||||||
}
|
|
||||||
if onAdmitted != nil {
|
if onAdmitted != nil {
|
||||||
if err := onAdmitted(tx); err != nil {
|
if err := onAdmitted(tx); err != nil {
|
||||||
return taskAdmissionTxOutcome{}, err
|
return taskAdmissionTxOutcome{}, err
|
||||||
@@ -457,6 +451,7 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
|||||||
NewlyAdmitted: true,
|
NewlyAdmitted: true,
|
||||||
Leases: leases,
|
Leases: leases,
|
||||||
},
|
},
|
||||||
|
NotifyTaskID: "*",
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -487,6 +482,7 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
|||||||
defer rollbackTransaction(tx)
|
defer rollbackTransaction(tx)
|
||||||
|
|
||||||
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
|
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
|
||||||
|
notify := false
|
||||||
for _, input := range inputs {
|
for _, input := range inputs {
|
||||||
hook := func(tx pgx.Tx) error {
|
hook := func(tx pgx.Tx) error {
|
||||||
if onAdmitted == nil {
|
if onAdmitted == nil {
|
||||||
@@ -510,6 +506,7 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
|||||||
Result: outcome.Result,
|
Result: outcome.Result,
|
||||||
Err: outcome.PostCommitErr,
|
Err: outcome.PostCommitErr,
|
||||||
})
|
})
|
||||||
|
notify = notify || outcome.NotifyTaskID != ""
|
||||||
if outcome.PostCommitErr == nil && !outcome.Result.Admitted {
|
if outcome.PostCommitErr == nil && !outcome.Result.Admitted {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -517,6 +514,9 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
|||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if notify {
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
|
}
|
||||||
return outcomes, nil
|
return outcomes, nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -649,12 +649,10 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return TaskAdmission{}, err
|
return TaskAdmission{}, err
|
||||||
}
|
}
|
||||||
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
|
||||||
return TaskAdmission{}, err
|
|
||||||
}
|
|
||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return TaskAdmission{}, err
|
return TaskAdmission{}, err
|
||||||
}
|
}
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, input.TaskID)
|
||||||
return migrated, nil
|
return migrated, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -781,10 +779,11 @@ WHERE task_id = $1::uuid
|
|||||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return tx.Commit(ctx)
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) ActiveTaskAdmissionLeases(ctx context.Context, taskID string) ([]ConcurrencyLease, error) {
|
func (s *Store) ActiveTaskAdmissionLeases(ctx context.Context, taskID string) ([]ConcurrencyLease, error) {
|
||||||
@@ -1026,14 +1025,12 @@ WHERE id = $1::uuid
|
|||||||
}
|
}
|
||||||
reaped++
|
reaped++
|
||||||
}
|
}
|
||||||
if reaped > 0 {
|
|
||||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
|
||||||
return TaskAdmissionReapResult{}, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return TaskAdmissionReapResult{}, err
|
return TaskAdmissionReapResult{}, err
|
||||||
}
|
}
|
||||||
|
if reaped > 0 {
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1413,7 +1410,34 @@ WHERE id = $1::uuid AND status = 'queued'`, taskID)
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func notifyTaskAdmissionTx(ctx context.Context, tx pgx.Tx, taskID string) error {
|
// notifyTaskAdmission publishes a non-durable wake-up hint only after the
|
||||||
_, err := tx.Exec(ctx, `SELECT pg_notify($1, $2)`, admissionNotifyChannel, taskID)
|
// corresponding state transaction has committed. Keeping NOTIFY out of
|
||||||
return err
|
// synchronous replication transactions prevents its database object lock from
|
||||||
|
// serializing unrelated admission, execution, and release commits.
|
||||||
|
func (s *Store) notifyTaskAdmission(ctx context.Context, taskID string) error {
|
||||||
|
taskID = strings.TrimSpace(taskID)
|
||||||
|
if taskID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
tx, err := s.pool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rollbackTransaction(tx)
|
||||||
|
if _, err := tx.Exec(ctx, `SET LOCAL synchronous_commit = off`); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `SELECT pg_notify($1, $2)`, admissionNotifyChannel, taskID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return tx.Commit(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) notifyTaskAdmissionBestEffort(ctx context.Context, taskID string) {
|
||||||
|
if strings.TrimSpace(taskID) == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
notifyCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), admissionNotifyTimeout)
|
||||||
|
defer cancel()
|
||||||
|
_ = s.notifyTaskAdmission(notifyCtx, taskID)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -113,4 +113,53 @@ RETURNING id::text`, platform.ID, modelName).Scan(&platformModelID); err != nil
|
|||||||
if registrationErr != nil {
|
if registrationErr != nil {
|
||||||
t.Fatalf("queue registration blocked behind execution scope lock: %v", registrationErr)
|
t.Fatalf("queue registration blocked behind execution scope lock: %v", registrationErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Simulate a legacy synchronous transaction that is still holding the
|
||||||
|
// database object lock used by NOTIFY. The admission deletion must commit
|
||||||
|
// and become visible before the best-effort wake-up hint can acquire that
|
||||||
|
// lock.
|
||||||
|
notificationLockTx, err := first.pool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("begin notification lock transaction: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := notificationLockTx.Exec(ctx, `SELECT pg_notify($1, '*')`, admissionNotifyChannel); err != nil {
|
||||||
|
_ = notificationLockTx.Rollback(ctx)
|
||||||
|
t.Fatalf("hold notification object lock: %v", err)
|
||||||
|
}
|
||||||
|
deleteResult := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
deleteResult <- second.DeleteTaskAdmission(ctx, task.ID)
|
||||||
|
}()
|
||||||
|
deleteVisible := false
|
||||||
|
visibilityDeadline := time.Now().Add(time.Second)
|
||||||
|
for time.Now().Before(visibilityDeadline) {
|
||||||
|
var admissionCount int
|
||||||
|
if err := first.pool.QueryRow(ctx, `
|
||||||
|
SELECT count(*)
|
||||||
|
FROM gateway_task_admissions
|
||||||
|
WHERE task_id = $1::uuid`, task.ID).Scan(&admissionCount); err != nil {
|
||||||
|
_ = notificationLockTx.Rollback(ctx)
|
||||||
|
t.Fatalf("read admission visibility: %v", err)
|
||||||
|
}
|
||||||
|
if admissionCount == 0 {
|
||||||
|
deleteVisible = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if !deleteVisible {
|
||||||
|
_ = notificationLockTx.Rollback(ctx)
|
||||||
|
t.Fatal("admission state commit waited behind notification object lock")
|
||||||
|
}
|
||||||
|
if err := notificationLockTx.Rollback(ctx); err != nil {
|
||||||
|
t.Fatalf("release notification object lock: %v", err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-deleteResult:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("delete admission after notification lock release: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("delete admission did not finish after notification lock release")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -715,11 +715,14 @@ WHERE task_id = $1::uuid
|
|||||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return notifyTaskAdmissionTx(ctx, tx, "*")
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return GatewayTask{}, false, err
|
return GatewayTask{}, false, err
|
||||||
}
|
}
|
||||||
|
if changed {
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
|
}
|
||||||
return task, changed, nil
|
return task, changed, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -877,11 +880,14 @@ WHERE task_id = $1::uuid
|
|||||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return notifyTaskAdmissionTx(ctx, tx, "*")
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return GatewayTask{}, false, err
|
return GatewayTask{}, false, err
|
||||||
}
|
}
|
||||||
|
if changed {
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
|
}
|
||||||
return task, changed, nil
|
return task, changed, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -963,11 +969,14 @@ WHERE task_id = $1::uuid
|
|||||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return notifyTaskAdmissionTx(ctx, tx, "*")
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return GatewayTask{}, false, err
|
return GatewayTask{}, false, err
|
||||||
}
|
}
|
||||||
|
if changed {
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
|
}
|
||||||
return task, changed, nil
|
return task, changed, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -286,14 +286,12 @@ SELECT count(*)::bigint
|
|||||||
FROM deleted`, staleAfter.String(), limit).Scan(&yielded); err != nil {
|
FROM deleted`, staleAfter.String(), limit).Scan(&yielded); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
if yielded > 0 {
|
|
||||||
if _, err := tx.Exec(ctx, `SELECT pg_notify('gateway_task_admission', '*')`); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
if yielded > 0 {
|
||||||
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||||
|
}
|
||||||
return yielded, nil
|
return yielded, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user