fix(acceptance): 隔离容量压测并稳定数据库连接
将协议模拟验收的供应商并发限额与 Worker 容量门禁分离,真实金丝雀继续保留生产限流。readyz 改用关键连接池并预热关键及 River 池,延长生产连接空闲周期,降低跨地域连接抖动。失败 Run 现在可以原子替换且失败报告记录任务数量,避免 validation 之间短暂放开正式流量。\n\n验证:Go 全量测试、go vet、PostgreSQL 集成测试、ShellCheck、迁移安全、发布脚本、pnpm lint/test/build、OpenAPI 无漂移均通过。
This commit is contained in:
@@ -209,9 +209,6 @@ FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(¤tValue); err != nil {
|
||||
if err := json.Unmarshal(currentValue, ¤t); err != nil {
|
||||
return GatewayTrafficMode{}, err
|
||||
}
|
||||
if normalizeTrafficMode(current.Mode) != "live" {
|
||||
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
||||
}
|
||||
run, err := scanAcceptanceRun(tx.QueryRow(ctx, `
|
||||
SELECT `+acceptanceRunColumns+`
|
||||
FROM gateway_acceptance_runs
|
||||
@@ -223,6 +220,40 @@ FOR UPDATE`, strings.TrimSpace(runID)))
|
||||
if run.Status != "pending" && run.Status != "failed" {
|
||||
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
||||
}
|
||||
switch normalizeTrafficMode(current.Mode) {
|
||||
case "live":
|
||||
case "validation":
|
||||
if current.RunID == "" || current.RunID == run.ID {
|
||||
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
||||
}
|
||||
previous, previousErr := scanAcceptanceRun(tx.QueryRow(ctx, `
|
||||
SELECT `+acceptanceRunColumns+`
|
||||
FROM gateway_acceptance_runs
|
||||
WHERE id = $1::uuid
|
||||
FOR UPDATE`, current.RunID))
|
||||
if previousErr != nil {
|
||||
return GatewayTrafficMode{}, previousErr
|
||||
}
|
||||
if previous.Status != "failed" ||
|
||||
previous.ReleaseSHA != current.ReleaseSHA ||
|
||||
previous.APIImageDigest != current.APIImageDigest ||
|
||||
previous.WorkerImageDigest != current.WorkerImageDigest {
|
||||
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
||||
}
|
||||
var activeTasks int
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT count(*)
|
||||
FROM gateway_tasks
|
||||
WHERE acceptance_run_id = $1::uuid
|
||||
AND status IN ('queued', 'running')`, previous.ID).Scan(&activeTasks); err != nil {
|
||||
return GatewayTrafficMode{}, err
|
||||
}
|
||||
if activeTasks != 0 {
|
||||
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
||||
}
|
||||
default:
|
||||
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
||||
}
|
||||
next := GatewayTrafficMode{
|
||||
Mode: "validation",
|
||||
RunID: run.ID,
|
||||
|
||||
@@ -323,3 +323,77 @@ FROM gateway_tasks`,
|
||||
t.Fatalf("delete acceptance cleanup billing user: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateAcceptanceRunAtomicallyReplacesDrainedFailedRun(t *testing.T) {
|
||||
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run acceptance traffic integration test")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
applyOIDCJITTestMigrations(t, ctx, databaseURL)
|
||||
db, err := Connect(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect store: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.pool.Exec(ctx, `
|
||||
UPDATE system_settings
|
||||
SET value = '{"mode":"live","revision":0}'::jsonb, updated_at = now()
|
||||
WHERE setting_key = $1`, SystemSettingGatewayTrafficMode); err != nil {
|
||||
t.Fatalf("reset traffic mode: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.pool.Exec(context.Background(), `
|
||||
UPDATE system_settings
|
||||
SET value = '{"mode":"live","revision":0}'::jsonb, updated_at = now()
|
||||
WHERE setting_key = $1`, SystemSettingGatewayTrafficMode)
|
||||
})
|
||||
|
||||
createRun := func(releaseCharacter, digestCharacter string) AcceptanceRun {
|
||||
t.Helper()
|
||||
run, createErr := db.CreateAcceptanceRun(ctx, CreateAcceptanceRunInput{
|
||||
ReleaseSHA: strings.Repeat(releaseCharacter, 40),
|
||||
APIImageDigest: "sha256:" + strings.Repeat(digestCharacter, 64),
|
||||
WorkerImageDigest: "sha256:" + strings.Repeat(digestCharacter, 64),
|
||||
APIKeyID: "atomic-replace-api-key",
|
||||
UserID: "atomic-replace-user",
|
||||
Token: strings.Repeat("t", 32),
|
||||
EmulatorBaseURL: "http://acceptance-emulator:8090",
|
||||
CallbackURL: "http://acceptance-emulator:8090/callbacks",
|
||||
CapacityProfile: "P24",
|
||||
})
|
||||
if createErr != nil {
|
||||
t.Fatalf("create acceptance run: %v", createErr)
|
||||
}
|
||||
return run
|
||||
}
|
||||
|
||||
previous := createRun("1", "2")
|
||||
firstMode, err := db.ActivateAcceptanceRun(ctx, previous.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("activate previous run: %v", err)
|
||||
}
|
||||
if _, err := db.FinishAcceptanceRun(ctx, FinishAcceptanceRunInput{
|
||||
RunID: previous.ID, Passed: false, FailureReason: "capacity gate failed",
|
||||
}); err != nil {
|
||||
t.Fatalf("fail previous run: %v", err)
|
||||
}
|
||||
nextRun := createRun("3", "4")
|
||||
nextMode, err := db.ActivateAcceptanceRun(ctx, nextRun.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("atomically replace failed run: %v", err)
|
||||
}
|
||||
if firstMode.Mode != "validation" || nextMode.Mode != "validation" ||
|
||||
nextMode.RunID != nextRun.ID || nextMode.Revision != firstMode.Revision+1 {
|
||||
t.Fatalf("unexpected replacement modes: first=%+v next=%+v", firstMode, nextMode)
|
||||
}
|
||||
storedPrevious, err := db.GetAcceptanceRun(ctx, previous.ID)
|
||||
if err != nil || storedPrevious.Status != "failed" {
|
||||
t.Fatalf("previous run changed during replacement: run=%+v err=%v", storedPrevious, err)
|
||||
}
|
||||
storedNext, err := db.GetAcceptanceRun(ctx, nextRun.ID)
|
||||
if err != nil || storedNext.Status != "running" {
|
||||
t.Fatalf("next run was not activated: run=%+v err=%v", storedNext, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user