fix(acceptance): 修复快照迁移重放与派发死锁
本地同构验收在全量迁移后导入生产快照,导致本次发布的数据迁移被旧能力覆盖;增加仅允许严格本地集群标记启用的导入后迁移重放,并新增幂等 Seedance 约束校准。\n\n批量异步派发改为在事务开始按全局顺序预锁全部任务与容量作用域,同时对 PostgreSQL 死锁和序列化失败做退避重试,避免双 API 重叠批次形成环形等待。\n\n验证:Go 全量测试、gofmt、bash -n、ShellCheck、迁移安全检查。
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -153,7 +154,7 @@ func retryAdmissionOperation[T any](
|
||||
}
|
||||
result, operationErr := operation()
|
||||
release()
|
||||
if !errors.Is(operationErr, errAdmissionLockBusy) {
|
||||
if !isRetryableAdmissionTransactionError(operationErr) {
|
||||
return result, operationErr
|
||||
}
|
||||
if err := waitAdmissionLockRetry(ctx, attempt); err != nil {
|
||||
@@ -162,6 +163,17 @@ func retryAdmissionOperation[T any](
|
||||
}
|
||||
}
|
||||
|
||||
func isRetryableAdmissionTransactionError(err error) bool {
|
||||
if errors.Is(err, errAdmissionLockBusy) {
|
||||
return true
|
||||
}
|
||||
var postgresError *pgconn.PgError
|
||||
if !errors.As(err, &postgresError) {
|
||||
return false
|
||||
}
|
||||
return postgresError.Code == "40P01" || postgresError.Code == "40001"
|
||||
}
|
||||
|
||||
func waitAdmissionLockRetry(ctx context.Context, attempt int) error {
|
||||
delay := admissionLockRetryMin
|
||||
for index := 0; index < attempt && delay < admissionLockRetryMax; index++ {
|
||||
|
||||
@@ -11,8 +11,23 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
func TestAdmissionTransactionRetryClassification(t *testing.T) {
|
||||
for _, code := range []string{"40P01", "40001"} {
|
||||
if !isRetryableAdmissionTransactionError(&pgconn.PgError{Code: code}) {
|
||||
t.Fatalf("PostgreSQL error %s should be retried", code)
|
||||
}
|
||||
}
|
||||
if !isRetryableAdmissionTransactionError(errAdmissionLockBusy) {
|
||||
t.Fatal("admission lock timeout should be retried")
|
||||
}
|
||||
if isRetryableAdmissionTransactionError(&pgconn.PgError{Code: "23505"}) {
|
||||
t.Fatal("unique violations must not be retried")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdmissionLocalLockSetSerializesSharedKeys(t *testing.T) {
|
||||
lockSet := newAdmissionLocalLockSet()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
|
||||
@@ -474,12 +474,22 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
||||
}
|
||||
lockKeys = append(lockKeys, admissionOperationLockKeys(input)...)
|
||||
}
|
||||
lockKeys = normalizedAdmissionLockKeys(lockKeys)
|
||||
return retryAdmissionOperation(ctx, lockKeys, func() ([]TaskAdmissionBatchOutcome, error) {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
// A batch touches one shared capacity scope and multiple task keys. Lock
|
||||
// the complete union in one global order before processing any task. If
|
||||
// two API processes dispatch overlapping FIFO windows, neither can hold a
|
||||
// scope while waiting on a task key already owned by the other batch.
|
||||
for _, lockKey := range lockKeys {
|
||||
if err := tryAdmissionTransactionLock(ctx, tx, lockKey); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
outcomes := make([]TaskAdmissionBatchOutcome, 0, len(inputs))
|
||||
notify := false
|
||||
|
||||
Reference in New Issue
Block a user