fix(acceptance): 隔离容量压测并稳定数据库连接

将协议模拟验收的供应商并发限额与 Worker 容量门禁分离，真实金丝雀继续保留生产限流。readyz 改用关键连接池并预热关键及 River 池，延长生产连接空闲周期，降低跨地域连接抖动。失败 Run 现在可以原子替换且失败报告记录任务数量，避免 validation 之间短暂放开正式流量。\n\n验证：Go 全量测试、go vet、PostgreSQL 集成测试、ShellCheck、迁移安全、发布脚本、pnpm lint/test/build、OpenAPI 无漂移均通过。

apps/api/cmd/gateway/main.go

@@ -62,7 +62,7 @@ func main() {
 	if cfg.DatabaseCriticalMaxConns > 0 {
 		coordinationDB, err = store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
 			MaxConns:                 cfg.DatabaseCriticalMaxConns,
-			MinIdleConns:             min(cfg.DatabaseCriticalMaxConns, 1),
+			MinIdleConns:             min(cfg.DatabaseCriticalMaxConns, max(cfg.DatabaseMinIdleConns, 1)),
 			MaxConnIdleTime:          time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second,
 			IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
 			LockTimeout:              time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second,
@@ -78,7 +78,7 @@ func main() {
 	if cfg.DatabaseRiverMaxConns > 0 {
 		riverDB, err = store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
 			MaxConns:                 cfg.DatabaseRiverMaxConns,
-			MinIdleConns:             min(cfg.DatabaseRiverMaxConns, 1),
+			MinIdleConns:             min(cfg.DatabaseRiverMaxConns, max(cfg.DatabaseMinIdleConns, 1)),
 			MaxConnIdleTime:          time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second,
 			IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
 			LockTimeout:              time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second,

apps/api/internal/httpapi/availability_timeout_integration_test.go

@@ -35,6 +35,24 @@ func TestReadyReturnsPostgresUnavailableWithinTwoSeconds(t *testing.T) {
 	}
 }
 
+func TestReadyUsesReservedCriticalPoolWhenExecutionPoolIsBusy(t *testing.T) {
+	executionDB := newExhaustedPostgresStore(t)
+	criticalDB := newAvailablePostgresStore(t)
+	server := &Server{
+		store:             executionDB,
+		coordinationStore: criticalDB,
+		logger:            slog.New(slog.NewJSONHandler(io.Discard, nil)),
+	}
+	request := httptest.NewRequest(http.MethodGet, "/readyz", nil)
+	recorder := httptest.NewRecorder()
+
+	server.ready(recorder, request)
+
+	if recorder.Code != http.StatusOK {
+		t.Fatalf("readiness status=%d, want 200 while critical pool is available; body=%s", recorder.Code, recorder.Body.String())
+	}
+}
+
 func TestLoginReturnsAuthStoreUnavailableWithinFiveSeconds(t *testing.T) {
 	db := newExhaustedPostgresStore(t)
 	var logs bytes.Buffer
@@ -93,6 +111,31 @@ func newExhaustedPostgresStore(t *testing.T) *store.Store {
 	return db
 }
 
+func newAvailablePostgresStore(t *testing.T) *store.Store {
+	t.Helper()
+	databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
+	if databaseURL == "" {
+		t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run PostgreSQL availability timeout tests")
+	}
+	parsed, err := url.Parse(databaseURL)
+	if err != nil {
+		t.Fatalf("parse test database URL: %v", err)
+	}
+	query := parsed.Query()
+	query.Set("pool_max_conns", "1")
+	query.Set("pool_min_conns", "0")
+	parsed.RawQuery = query.Encode()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	db, err := store.Connect(ctx, parsed.String())
+	if err != nil {
+		t.Fatalf("connect available test store: %v", err)
+	}
+	t.Cleanup(db.Close)
+	return db
+}
+
 func assertUnavailableResponse(t *testing.T, recorder *httptest.ResponseRecorder, expectedCode, expectedMessage string) {
 	t.Helper()
 	if recorder.Code != http.StatusServiceUnavailable {

apps/api/internal/httpapi/handlers.go

@@ -53,7 +53,11 @@ func (s *Server) health(w http.ResponseWriter, r *http.Request) {
 func (s *Server) ready(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(r.Context(), postgresReadinessTimeout)
 	defer cancel()
-	if err := s.store.Ping(ctx); err != nil {
+	readinessStore := s.coordinationStore
+	if readinessStore == nil {
+		readinessStore = s.store
+	}
+	if err := readinessStore.Ping(ctx); err != nil {
 		s.logPostgresUnavailable("postgres readiness check failed")
 		writeError(w, http.StatusServiceUnavailable, "postgres unavailable", errorCodePostgresDown)
 		return

apps/api/internal/runner/admission.go

@@ -146,7 +146,11 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
 				ModelType: modelType,
 			}, nil
 		}
-		if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidate, body)); err != nil {
+		reservations := acceptanceInfrastructureReservations(
+			task,
+			s.rateLimitReservations(ctx, user, candidate, body),
+		)
+		if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
 			return taskAdmissionPlan{}, err
 		}
 		return taskAdmissionPlan{
@@ -206,8 +210,17 @@ func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionS
 	}
 	out := append([]store.AdmissionScope(nil), scopes...)
 	for index := range out {
+		// Protocol-emulated acceptance measures Gateway and Worker capacity, so
+		// the isolated Run is bounded by the worker_capacity scope instead of a
+		// production supplier quota. The real acceptance_canary path deliberately
+		// retains the production platform-model concurrency limit.
+		if task.RunMode == "acceptance" && out[index].ScopeType == "platform_model" {
+			out[index].ConcurrentLimit = 0
+		}
 		if out[index].ConcurrentLimit <= 0 {
-			continue
+			if task.RunMode != "acceptance" || out[index].ScopeType != "platform_model" {
+				continue
+			}
 		}
 		out[index].QueueLimit = acceptanceQueueLimit
 		out[index].MaxWaitSeconds = acceptanceQueueMaxWait
@@ -215,6 +228,23 @@ func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionS
 	return out
 }
 
+func acceptanceInfrastructureReservations(
+	task store.GatewayTask,
+	reservations []store.RateLimitReservation,
+) []store.RateLimitReservation {
+	if task.RunMode != "acceptance" {
+		return reservations
+	}
+	out := make([]store.RateLimitReservation, 0, len(reservations))
+	for _, reservation := range reservations {
+		if reservation.ScopeType == "platform_model" && reservation.Metric == "concurrent" {
+			continue
+		}
+		out = append(out, reservation)
+	}
+	return out
+}
+
 func (s *Service) loadAsyncTaskAdmission(ctx context.Context, task store.GatewayTask) (*store.TaskAdmission, error) {
 	if !task.AsyncMode {
 		return nil, nil
@@ -420,7 +450,11 @@ func (s *Service) ensureCandidateAdmission(
 		}
 		return store.TaskAdmissionResult{}, false, nil
 	}
-	if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidate, body)); err != nil {
+	reservations := acceptanceInfrastructureReservations(
+		task,
+		s.rateLimitReservations(ctx, user, candidate, body),
+	)
+	if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
 		return store.TaskAdmissionResult{}, true, err
 	}
 	plan := taskAdmissionPlan{

apps/api/internal/runner/admission_test.go

@@ -24,28 +24,66 @@ func TestDistributedAdmissionModelTypeBoundary(t *testing.T) {
 	}
 }
 
-func TestAcceptanceAdmissionScopesEnableBoundedQueueWithoutChangingConcurrency(t *testing.T) {
+func TestAcceptanceAdmissionScopesUseWorkerCapacityInsteadOfSupplierConcurrency(t *testing.T) {
 	input := []store.AdmissionScope{{
 		ScopeType:       "platform_model",
 		ScopeKey:        "model-1",
 		ConcurrentLimit: 10,
 		QueueLimit:      0,
 		MaxWaitSeconds:  0,
+	}, {
+		ScopeType:       "worker_capacity",
+		ScopeKey:        "global",
+		ConcurrentLimit: 48,
 	}}
 
 	got := acceptanceAdmissionScopes(store.GatewayTask{RunMode: "acceptance"}, input)
 
-	if got[0].ConcurrentLimit != 10 {
-		t.Fatalf("acceptance must preserve the production concurrency limit, got %+v", got[0])
+	if got[0].ConcurrentLimit != 0 {
+		t.Fatalf("protocol-emulated acceptance must defer to worker capacity, got %+v", got[0])
 	}
 	if got[0].QueueLimit != acceptanceQueueLimit || got[0].MaxWaitSeconds != acceptanceQueueMaxWait {
 		t.Fatalf("acceptance must enable a bounded queue, got %+v", got[0])
 	}
+	if got[1].ConcurrentLimit != 48 {
+		t.Fatalf("acceptance worker capacity changed, got %+v", got[1])
+	}
 	if input[0].QueueLimit != 0 || input[0].MaxWaitSeconds != 0 {
 		t.Fatalf("acceptance queue overlay must not mutate the production scopes, got %+v", input[0])
 	}
 }
 
+func TestAcceptanceCanaryPreservesProductionConcurrency(t *testing.T) {
+	input := []store.AdmissionScope{{
+		ScopeType:       "platform_model",
+		ScopeKey:        "model-1",
+		ConcurrentLimit: 10,
+	}}
+
+	got := acceptanceAdmissionScopes(store.GatewayTask{RunMode: "acceptance_canary"}, input)
+
+	if got[0].ConcurrentLimit != 10 {
+		t.Fatalf("real acceptance canary must preserve production concurrency, got %+v", got[0])
+	}
+}
+
+func TestAcceptanceInfrastructureReservationsOnlyRemoveSupplierConcurrency(t *testing.T) {
+	input := []store.RateLimitReservation{
+		{ScopeType: "platform_model", Metric: "concurrent", Limit: 10},
+		{ScopeType: "platform_model", Metric: "rpm", Limit: 600},
+		{ScopeType: "user_group", Metric: "concurrent", Limit: 20},
+	}
+
+	got := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance"}, input)
+	if len(got) != 2 || got[0].Metric != "rpm" || got[1].ScopeType != "user_group" {
+		t.Fatalf("unexpected acceptance reservations: %+v", got)
+	}
+	canary := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance_canary"}, input)
+	if len(canary) != len(input) {
+		t.Fatalf("real canary reservations changed: %+v", canary)
+	}
+}
+
 func TestAcceptanceAdmissionScopesLeaveProductionPolicyUnchanged(t *testing.T) {
 	input := []store.AdmissionScope{{
 		ScopeType:       "platform_model",

apps/api/internal/runner/service.go

@@ -601,7 +601,11 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 			}
 		}
 		if hasConcurrentLimit {
-			if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidates[0], body)); err != nil {
+			reservations := acceptanceInfrastructureReservations(
+				task,
+				s.rateLimitReservations(ctx, user, candidates[0], body),
+			)
+			if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
 				if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
 					queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
 					if queueErr != nil {

apps/api/internal/store/acceptance.go

@@ -209,9 +209,6 @@ FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&currentValue); err != nil {
 	if err := json.Unmarshal(currentValue, &current); err != nil {
 		return GatewayTrafficMode{}, err
 	}
-	if normalizeTrafficMode(current.Mode) != "live" {
-		return GatewayTrafficMode{}, ErrAcceptanceStateConflict
-	}
 	run, err := scanAcceptanceRun(tx.QueryRow(ctx, `
 SELECT `+acceptanceRunColumns+`
 FROM gateway_acceptance_runs
@@ -223,6 +220,40 @@ FOR UPDATE`, strings.TrimSpace(runID)))
 	if run.Status != "pending" && run.Status != "failed" {
 		return GatewayTrafficMode{}, ErrAcceptanceStateConflict
 	}
+	switch normalizeTrafficMode(current.Mode) {
+	case "live":
+	case "validation":
+		if current.RunID == "" || current.RunID == run.ID {
+			return GatewayTrafficMode{}, ErrAcceptanceStateConflict
+		}
+		previous, previousErr := scanAcceptanceRun(tx.QueryRow(ctx, `
+SELECT `+acceptanceRunColumns+`
+FROM gateway_acceptance_runs
+WHERE id = $1::uuid
+FOR UPDATE`, current.RunID))
+		if previousErr != nil {
+			return GatewayTrafficMode{}, previousErr
+		}
+		if previous.Status != "failed" ||
+			previous.ReleaseSHA != current.ReleaseSHA ||
+			previous.APIImageDigest != current.APIImageDigest ||
+			previous.WorkerImageDigest != current.WorkerImageDigest {
+			return GatewayTrafficMode{}, ErrAcceptanceStateConflict
+		}
+		var activeTasks int
+		if err := tx.QueryRow(ctx, `
+SELECT count(*)
+FROM gateway_tasks
+WHERE acceptance_run_id = $1::uuid
+  AND status IN ('queued', 'running')`, previous.ID).Scan(&activeTasks); err != nil {
+			return GatewayTrafficMode{}, err
+		}
+		if activeTasks != 0 {
+			return GatewayTrafficMode{}, ErrAcceptanceStateConflict
+		}
+	default:
+		return GatewayTrafficMode{}, ErrAcceptanceStateConflict
+	}
 	next := GatewayTrafficMode{
 		Mode:              "validation",
 		RunID:             run.ID,

apps/api/internal/store/acceptance_integration_test.go

@@ -323,3 +323,77 @@ FROM gateway_tasks`,
 		t.Fatalf("delete acceptance cleanup billing user: %v", err)
 	}
 }
+
+func TestActivateAcceptanceRunAtomicallyReplacesDrainedFailedRun(t *testing.T) {
+	databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
+	if databaseURL == "" {
+		t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run acceptance traffic integration test")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	applyOIDCJITTestMigrations(t, ctx, databaseURL)
+	db, err := Connect(ctx, databaseURL)
+	if err != nil {
+		t.Fatalf("connect store: %v", err)
+	}
+	defer db.Close()
+	if _, err := db.pool.Exec(ctx, `
+UPDATE system_settings
+SET value = '{"mode":"live","revision":0}'::jsonb, updated_at = now()
+WHERE setting_key = $1`, SystemSettingGatewayTrafficMode); err != nil {
+		t.Fatalf("reset traffic mode: %v", err)
+	}
+	t.Cleanup(func() {
+		_, _ = db.pool.Exec(context.Background(), `
+UPDATE system_settings
+SET value = '{"mode":"live","revision":0}'::jsonb, updated_at = now()
+WHERE setting_key = $1`, SystemSettingGatewayTrafficMode)
+	})
+
+	createRun := func(releaseCharacter, digestCharacter string) AcceptanceRun {
+		t.Helper()
+		run, createErr := db.CreateAcceptanceRun(ctx, CreateAcceptanceRunInput{
+			ReleaseSHA:        strings.Repeat(releaseCharacter, 40),
+			APIImageDigest:    "sha256:" + strings.Repeat(digestCharacter, 64),
+			WorkerImageDigest: "sha256:" + strings.Repeat(digestCharacter, 64),
+			APIKeyID:          "atomic-replace-api-key",
+			UserID:            "atomic-replace-user",
+			Token:             strings.Repeat("t", 32),
+			EmulatorBaseURL:   "http://acceptance-emulator:8090",
+			CallbackURL:       "http://acceptance-emulator:8090/callbacks",
+			CapacityProfile:   "P24",
+		})
+		if createErr != nil {
+			t.Fatalf("create acceptance run: %v", createErr)
+		}
+		return run
+	}
+
+	previous := createRun("1", "2")
+	firstMode, err := db.ActivateAcceptanceRun(ctx, previous.ID)
+	if err != nil {
+		t.Fatalf("activate previous run: %v", err)
+	}
+	if _, err := db.FinishAcceptanceRun(ctx, FinishAcceptanceRunInput{
+		RunID: previous.ID, Passed: false, FailureReason: "capacity gate failed",
+	}); err != nil {
+		t.Fatalf("fail previous run: %v", err)
+	}
+	nextRun := createRun("3", "4")
+	nextMode, err := db.ActivateAcceptanceRun(ctx, nextRun.ID)
+	if err != nil {
+		t.Fatalf("atomically replace failed run: %v", err)
+	}
+	if firstMode.Mode != "validation" || nextMode.Mode != "validation" ||
+		nextMode.RunID != nextRun.ID || nextMode.Revision != firstMode.Revision+1 {
+		t.Fatalf("unexpected replacement modes: first=%+v next=%+v", firstMode, nextMode)
+	}
+	storedPrevious, err := db.GetAcceptanceRun(ctx, previous.ID)
+	if err != nil || storedPrevious.Status != "failed" {
+		t.Fatalf("previous run changed during replacement: run=%+v err=%v", storedPrevious, err)
+	}
+	storedNext, err := db.GetAcceptanceRun(ctx, nextRun.ID)
+	if err != nil || storedNext.Status != "running" {
+		t.Fatalf("next run was not activated: run=%+v err=%v", storedNext, err)
+	}
+}