fix(acceptance): 修复快照迁移重放与派发死锁

本地同构验收在全量迁移后导入生产快照，导致本次发布的数据迁移被旧能力覆盖；增加仅允许严格本地集群标记启用的导入后迁移重放，并新增幂等 Seedance 约束校准。\n\n批量异步派发改为在事务开始按全局顺序预锁全部任务与容量作用域，同时对 PostgreSQL 死锁和序列化失败做退避重试，避免双 API 重叠批次形成环形等待。\n\n验证：Go 全量测试、gofmt、bash -n、ShellCheck、迁移安全检查。
2026-07-31 20:39:42 +08:00
parent bfdabd3853
commit f190af00be
7 changed files with 259 additions and 17 deletions
@@ -6,6 +6,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"

@@ -14,10 +15,15 @@ import (
 )

 const (
-	noTransactionMigrationMarker = "-- easyai:migration:no-transaction"
-	migrationStatementSeparator  = "-- easyai:migration:statement"
+	noTransactionMigrationMarker      = "-- easyai:migration:no-transaction"
+	migrationStatementSeparator       = "-- easyai:migration:statement"
+	acceptanceImportReplayMarker      = "-- easyai:migration:reapply-after-acceptance-import"
+	acceptanceImportReplayEnvironment = "AI_GATEWAY_MIGRATION_REAPPLY_ACCEPTANCE_IMPORT"
+	acceptanceLocalClusterSettingKey  = "acceptance_local_cluster_id"
 )

+var acceptanceLocalClusterMarkerPattern = regexp.MustCompile(`^easyai-local-[0-9a-f]{24}$`)
+
 func main() {
 	cfg := config.Load()
 	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
@@ -42,6 +48,21 @@ CREATE TABLE IF NOT EXISTS schema_migrations (
 		logger.Error("ensure schema_migrations failed", "error", err)
 		os.Exit(1)
 	}
+	reapplyAcceptanceImport := strings.EqualFold(strings.TrimSpace(os.Getenv(acceptanceImportReplayEnvironment)), "true")
+	if reapplyAcceptanceImport {
+		var localClusterID string
+		if err := conn.QueryRow(ctx, `
+SELECT COALESCE(value->>'clusterId', '')
+FROM system_settings
+WHERE setting_key = $1`, acceptanceLocalClusterSettingKey).Scan(&localClusterID); err != nil {
+			logger.Error("verify local acceptance database failed", "error", err)
+			os.Exit(1)
+		}
+		if !acceptanceLocalClusterMarkerPattern.MatchString(localClusterID) {
+			logger.Error("refusing acceptance migration replay outside a marked local cluster")
+			os.Exit(1)
+		}
+	}

 	files, err := filepath.Glob("migrations/*.sql")
 	if err != nil {
@@ -57,16 +78,16 @@ CREATE TABLE IF NOT EXISTS schema_migrations (
 			logger.Error("check migration failed", "version", version, "error", err)
 			os.Exit(1)
 		}
-		if exists {
-			logger.Info("migration skipped", "version", version)
-			continue
-		}
-
 		sqlBytes, err := os.ReadFile(file)
 		if err != nil {
 			logger.Error("read migration file failed", "file", file, "error", err)
 			os.Exit(1)
 		}
+		replaying := exists && reapplyAcceptanceImport && hasMigrationMarker(string(sqlBytes), acceptanceImportReplayMarker)
+		if exists && !replaying {
+			logger.Info("migration skipped", "version", version)
+			continue
+		}

 		noTransaction, statements := migrationStatements(string(sqlBytes))
 		if noTransaction {
@@ -76,11 +97,17 @@ CREATE TABLE IF NOT EXISTS schema_migrations (
 					os.Exit(1)
 				}
 			}
-			if _, err := conn.Exec(ctx, "INSERT INTO schema_migrations(version) VALUES($1)", version); err != nil {
-				logger.Error("record non-transaction migration failed", "version", version, "error", err)
-				os.Exit(1)
+			if !exists {
+				if _, err := conn.Exec(ctx, "INSERT INTO schema_migrations(version) VALUES($1)", version); err != nil {
+					logger.Error("record non-transaction migration failed", "version", version, "error", err)
+					os.Exit(1)
+				}
+			}
+			if replaying {
+				logger.Info("acceptance migration replayed", "version", version, "transactional", false)
+			} else {
+				logger.Info("migration applied", "version", version, "transactional", false)
 			}
-			logger.Info("migration applied", "version", version, "transactional", false)
 			continue
 		}

@@ -102,21 +129,36 @@ CREATE TABLE IF NOT EXISTS schema_migrations (
 			logger.Error("execute migration failed", "version", version, "error", err)
 			os.Exit(1)
 		}
-		if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations(version) VALUES($1)", version); err != nil {
-			_ = tx.Rollback(ctx)
-			logger.Error("record migration failed", "version", version, "error", err)
-			os.Exit(1)
+		if !exists {
+			if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations(version) VALUES($1)", version); err != nil {
+				_ = tx.Rollback(ctx)
+				logger.Error("record migration failed", "version", version, "error", err)
+				os.Exit(1)
+			}
 		}
 		if err := tx.Commit(ctx); err != nil {
 			logger.Error("commit migration failed", "version", version, "error", err)
 			os.Exit(1)
 		}
-		logger.Info("migration applied", "version", version)
+		if replaying {
+			logger.Info("acceptance migration replayed", "version", version)
+		} else {
+			logger.Info("migration applied", "version", version)
+		}
 	}

 	fmt.Println("migrations complete")
 }

+func hasMigrationMarker(sql string, marker string) bool {
+	for _, line := range strings.Split(sql, "\n") {
+		if strings.TrimSpace(line) == marker {
+			return true
+		}
+	}
+	return false
+}
+
 func migrationStatements(sql string) (bool, []string) {
 	trimmed := strings.TrimSpace(sql)
 	if !strings.HasPrefix(trimmed, noTransactionMigrationMarker) {
@@ -32,6 +32,21 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS second_index ON second_table(id);
 	}
 }

+func TestAcceptanceImportReplayMarkerRequiresAnExactCommentLine(t *testing.T) {
+	if !hasMigrationMarker(
+		"-- preface\n"+acceptanceImportReplayMarker+"\nSELECT 1;\n",
+		acceptanceImportReplayMarker,
+	) {
+		t.Fatal("expected exact acceptance import replay marker")
+	}
+	if hasMigrationMarker(
+		"-- mentions "+acceptanceImportReplayMarker+" in prose\nSELECT 1;\n",
+		acceptanceImportReplayMarker,
+	) {
+		t.Fatal("prose mention must not enable acceptance import replay")
+	}
+}
+
 func TestSeedanceInputImageConstraintMigrationKeepsCatalogSnapshotsInSync(t *testing.T) {
 	payload, err := os.ReadFile("../../migrations/0078_seedance_input_image_constraints.sql")
 	if err != nil {
@@ -74,6 +89,27 @@ func TestVolcesSeedanceInputImageConstraintMigrationUsesDocumentedBounds(t *test
 	}
 }

+func TestVolcesSeedanceAcceptanceReconciliationIsReplayable(t *testing.T) {
+	payload, err := os.ReadFile("../../migrations/0099_reconcile_volces_seedance20_acceptance_constraints.sql")
+	if err != nil {
+		t.Fatal(err)
+	}
+	content := string(payload)
+	for _, required := range []string{
+		acceptanceImportReplayMarker,
+		"volces:doubao-seedance-2-0-260128",
+		"volces:doubao-seedance-2-0-fast-260128",
+		"volces:doubao-seedance-2-0-mini-260615",
+		`"long_edge":300`,
+		`"long_edge":6000`,
+		`'[0.4,2.5]'::jsonb`,
+	} {
+		if !strings.Contains(content, required) {
+			t.Fatalf("Volces acceptance reconciliation is missing %q", required)
+		}
+	}
+}
+
 func TestSecurityEventSchemaMigrationsDefineCurrentLifecycle(t *testing.T) {
 	streamPayload, err := os.ReadFile("../../migrations/0063_oidc_security_events.sql")
 	if err != nil {