fix(acceptance): 隔离容量压测并稳定数据库连接

将协议模拟验收的供应商并发限额与 Worker 容量门禁分离,真实金丝雀继续保留生产限流。readyz 改用关键连接池并预热关键及 River 池,延长生产连接空闲周期,降低跨地域连接抖动。失败 Run 现在可以原子替换且失败报告记录任务数量,避免 validation 之间短暂放开正式流量。\n\n验证:Go 全量测试、go vet、PostgreSQL 集成测试、ShellCheck、迁移安全、发布脚本、pnpm lint/test/build、OpenAPI 无漂移均通过。
This commit is contained in:
2026-08-01 19:29:08 +08:00
parent 89a0ff7e99
commit 92e328a575
11 changed files with 277 additions and 42 deletions
+34 -3
View File
@@ -209,9 +209,6 @@ FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&currentValue); err != nil {
if err := json.Unmarshal(currentValue, &current); err != nil {
return GatewayTrafficMode{}, err
}
if normalizeTrafficMode(current.Mode) != "live" {
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
}
run, err := scanAcceptanceRun(tx.QueryRow(ctx, `
SELECT `+acceptanceRunColumns+`
FROM gateway_acceptance_runs
@@ -223,6 +220,40 @@ FOR UPDATE`, strings.TrimSpace(runID)))
if run.Status != "pending" && run.Status != "failed" {
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
}
switch normalizeTrafficMode(current.Mode) {
case "live":
case "validation":
if current.RunID == "" || current.RunID == run.ID {
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
}
previous, previousErr := scanAcceptanceRun(tx.QueryRow(ctx, `
SELECT `+acceptanceRunColumns+`
FROM gateway_acceptance_runs
WHERE id = $1::uuid
FOR UPDATE`, current.RunID))
if previousErr != nil {
return GatewayTrafficMode{}, previousErr
}
if previous.Status != "failed" ||
previous.ReleaseSHA != current.ReleaseSHA ||
previous.APIImageDigest != current.APIImageDigest ||
previous.WorkerImageDigest != current.WorkerImageDigest {
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
}
var activeTasks int
if err := tx.QueryRow(ctx, `
SELECT count(*)
FROM gateway_tasks
WHERE acceptance_run_id = $1::uuid
AND status IN ('queued', 'running')`, previous.ID).Scan(&activeTasks); err != nil {
return GatewayTrafficMode{}, err
}
if activeTasks != 0 {
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
}
default:
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
}
next := GatewayTrafficMode{
Mode: "validation",
RunID: run.ID,
@@ -323,3 +323,77 @@ FROM gateway_tasks`,
t.Fatalf("delete acceptance cleanup billing user: %v", err)
}
}
func TestActivateAcceptanceRunAtomicallyReplacesDrainedFailedRun(t *testing.T) {
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run acceptance traffic integration test")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
applyOIDCJITTestMigrations(t, ctx, databaseURL)
db, err := Connect(ctx, databaseURL)
if err != nil {
t.Fatalf("connect store: %v", err)
}
defer db.Close()
if _, err := db.pool.Exec(ctx, `
UPDATE system_settings
SET value = '{"mode":"live","revision":0}'::jsonb, updated_at = now()
WHERE setting_key = $1`, SystemSettingGatewayTrafficMode); err != nil {
t.Fatalf("reset traffic mode: %v", err)
}
t.Cleanup(func() {
_, _ = db.pool.Exec(context.Background(), `
UPDATE system_settings
SET value = '{"mode":"live","revision":0}'::jsonb, updated_at = now()
WHERE setting_key = $1`, SystemSettingGatewayTrafficMode)
})
createRun := func(releaseCharacter, digestCharacter string) AcceptanceRun {
t.Helper()
run, createErr := db.CreateAcceptanceRun(ctx, CreateAcceptanceRunInput{
ReleaseSHA: strings.Repeat(releaseCharacter, 40),
APIImageDigest: "sha256:" + strings.Repeat(digestCharacter, 64),
WorkerImageDigest: "sha256:" + strings.Repeat(digestCharacter, 64),
APIKeyID: "atomic-replace-api-key",
UserID: "atomic-replace-user",
Token: strings.Repeat("t", 32),
EmulatorBaseURL: "http://acceptance-emulator:8090",
CallbackURL: "http://acceptance-emulator:8090/callbacks",
CapacityProfile: "P24",
})
if createErr != nil {
t.Fatalf("create acceptance run: %v", createErr)
}
return run
}
previous := createRun("1", "2")
firstMode, err := db.ActivateAcceptanceRun(ctx, previous.ID)
if err != nil {
t.Fatalf("activate previous run: %v", err)
}
if _, err := db.FinishAcceptanceRun(ctx, FinishAcceptanceRunInput{
RunID: previous.ID, Passed: false, FailureReason: "capacity gate failed",
}); err != nil {
t.Fatalf("fail previous run: %v", err)
}
nextRun := createRun("3", "4")
nextMode, err := db.ActivateAcceptanceRun(ctx, nextRun.ID)
if err != nil {
t.Fatalf("atomically replace failed run: %v", err)
}
if firstMode.Mode != "validation" || nextMode.Mode != "validation" ||
nextMode.RunID != nextRun.ID || nextMode.Revision != firstMode.Revision+1 {
t.Fatalf("unexpected replacement modes: first=%+v next=%+v", firstMode, nextMode)
}
storedPrevious, err := db.GetAcceptanceRun(ctx, previous.ID)
if err != nil || storedPrevious.Status != "failed" {
t.Fatalf("previous run changed during replacement: run=%+v err=%v", storedPrevious, err)
}
storedNext, err := db.GetAcceptanceRun(ctx, nextRun.ID)
if err != nil || storedNext.Status != "running" {
t.Fatalf("next run was not activated: run=%+v err=%v", storedNext, err)
}
}