From 2529679b67b4b27602679c2c75b037c93ca038e5 Mon Sep 17 00:00:00 2001 From: wangbo Date: Fri, 31 Jul 2026 03:28:43 +0800 Subject: [PATCH] =?UTF-8?q?perf(postgres):=20=E5=88=86=E7=A6=BB=20API=20?= =?UTF-8?q?=E6=B1=A0=E5=B9=B6=E9=99=8D=E4=BD=8E=E8=BF=9E=E6=8E=A5=E9=A2=84?= =?UTF-8?q?=E7=83=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 生产同构千并发请求在干净队列上复现 API Pool 31/32、idle=1 且 canceled acquire 持续增长，Worker Pool 仅使用少量连接，确认瓶颈位于 API 数据库连接预算而非 Worker 执行槽。 新增可配置的 AI_GATEWAY_DATABASE_MIN_IDLE_CONNS，生产仅预热 4 条连接；API Pool 独立固定为 48，Worker Pool 继续跟随 P24/P28/P32 的 32/36/40 档位。发布工具、Kubernetes 清单、验收门禁和文档同步更新。 验证：Go 全量测试、gofmt、bash -n、ShellCheck、kubectl kustomize 和 git diff --check 通过。 --- README.md | 2 +- apps/api/cmd/gateway/main.go | 1 + apps/api/internal/config/config.go | 6 ++++++ apps/api/internal/config/config_test.go | 17 +++++++++++++-- apps/api/internal/store/postgres.go | 4 ++-- .../internal/store/postgres_config_test.go | 21 ++++++++++--------- .../easyai-ai-gateway-cluster-release | 19 ++++++++++++++--- ...ai-ai-gateway-cluster-release.conf.example | 2 ++ deploy/kubernetes/production/application.yaml | 12 +++++++++-- docs/operations/production-acceptance.md | 6 ++++++ scripts/cluster/run-production-acceptance.sh | 11 +++++++--- 11 files changed, 78 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 3ae9409..faae217 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ AI_GATEWAY_DATABASE_URL=postgresql://easyai:easyai2025@localhost:5432/easyai_ai_ 如果现有 `easyai-pgvector` 没有把 `5432` 映射到宿主机，就需要补端口映射，或者把 AI Gateway 后端容器化后接入同一个 `easyai` Docker network。 -异步队列 worker 不使用固定业务并发。服务分别汇总启用平台模型和活跃用户组的有效 `concurrent` 策略，采用两者中更严格的集群容量，默认每 5 秒在线调整 River 执行容量。策略解析兼容历史 `platformLimits/modelLimits.max_concurrent_requests`，运行时统一转换为 `rules`。`AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT` 默认 `2048`，限制策略推导出的集群目标；`AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT` 默认 `32`，按单 Worker 的内存安全容量限制实例分配，即使其他实例失活也不会突破。Worker 的 `AI_GATEWAY_DATABASE_MAX_CONNS` 必须高于实例执行容量，为心跳、选主、租约续期和健康检查保留连接；生产环境按每实例执行容量 `24` 配置连接池 `32`。平台模型和用户组的 PostgreSQL concurrency lease 仍是业务并发真值。可通过 `AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS` 调整刷新周期。 +异步队列 worker 不使用固定业务并发。服务分别汇总启用平台模型和活跃用户组的有效 `concurrent` 策略，采用两者中更严格的集群容量，默认每 5 秒在线调整 River 执行容量。策略解析兼容历史 `platformLimits/modelLimits.max_concurrent_requests`，运行时统一转换为 `rules`。`AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT` 默认 `2048`，限制策略推导出的集群目标；`AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT` 默认 `32`，按单 Worker 的内存安全容量限制实例分配，即使其他实例失活也不会突破。Worker 的 `AI_GATEWAY_DATABASE_MAX_CONNS` 必须高于实例执行容量，为心跳、选主、租约续期和健康检查保留连接；生产环境按每实例执行容量 `24` 配置连接池上限 `32`。`AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 控制启动预热连接数，默认 `0`，生产 K3s 配置为 `4`，避免把连接池上限全部常驻占用。平台模型和用户组的 PostgreSQL concurrency lease 仍是业务并发真值。可通过 `AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS` 调整刷新周期。 ## 迁移原则 diff --git a/apps/api/cmd/gateway/main.go b/apps/api/cmd/gateway/main.go index 181f898..393351b 100644 --- a/apps/api/cmd/gateway/main.go +++ b/apps/api/cmd/gateway/main.go @@ -39,6 +39,7 @@ func main() { db, err := store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{ MaxConns: cfg.DatabaseMaxConns, + MinIdleConns: cfg.DatabaseMinIdleConns, IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second, LockTimeout: time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second, }) diff --git a/apps/api/internal/config/config.go b/apps/api/internal/config/config.go index 92ee0c2..528c958 100644 --- a/apps/api/internal/config/config.go +++ b/apps/api/internal/config/config.go @@ -59,6 +59,7 @@ type Config struct { BillingEngineMode string ProcessRole string DatabaseMaxConns int + DatabaseMinIdleConns int DatabaseIdleInTransactionTimeoutSeconds int DatabaseLockTimeoutSeconds int MediaRequestConcurrency int @@ -122,6 +123,7 @@ func Load() Config { BillingEngineMode: strings.ToLower(env("BILLING_ENGINE_MODE", "observe")), ProcessRole: strings.ToLower(strings.TrimSpace(env("AI_GATEWAY_PROCESS_ROLE", "all"))), DatabaseMaxConns: envInt("AI_GATEWAY_DATABASE_MAX_CONNS", 0), + DatabaseMinIdleConns: envOptionalIntValidated("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", 0), DatabaseIdleInTransactionTimeoutSeconds: envOptionalIntValidated( "AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", @@ -147,6 +149,10 @@ func (c Config) Validate() error { if c.DatabaseMaxConns < 0 || c.DatabaseMaxConns > 1000 { return errors.New("AI_GATEWAY_DATABASE_MAX_CONNS must be between 1 and 1000 when configured") } + if c.DatabaseMinIdleConns < 0 || c.DatabaseMinIdleConns > 1000 || + (c.DatabaseMaxConns > 0 && c.DatabaseMinIdleConns > c.DatabaseMaxConns) { + return errors.New("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS must be between 0 and AI_GATEWAY_DATABASE_MAX_CONNS") + } if c.DatabaseIdleInTransactionTimeoutSeconds < 0 || c.DatabaseIdleInTransactionTimeoutSeconds > 3600 { return errors.New("AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS must be between 0 and 3600") } diff --git a/apps/api/internal/config/config_test.go b/apps/api/internal/config/config_test.go index 72bc10b..75f054d 100644 --- a/apps/api/internal/config/config_test.go +++ b/apps/api/internal/config/config_test.go @@ -144,9 +144,12 @@ func TestProcessRolePrecedenceAndCompatibility(t *testing.T) { func TestValidateProcessRoleAndDatabasePool(t *testing.T) { cfg := Load() - if cfg.DatabaseIdleInTransactionTimeoutSeconds != 60 || cfg.DatabaseLockTimeoutSeconds != 30 { + if cfg.DatabaseMinIdleConns != 0 || + cfg.DatabaseIdleInTransactionTimeoutSeconds != 60 || + cfg.DatabaseLockTimeoutSeconds != 30 { t.Fatalf( - "database transaction timeouts = %d/%d, want 60/30", + "database pool minimum/transaction timeouts = %d/%d/%d, want 0/60/30", + cfg.DatabaseMinIdleConns, cfg.DatabaseIdleInTransactionTimeoutSeconds, cfg.DatabaseLockTimeoutSeconds, ) @@ -161,6 +164,16 @@ func TestValidateProcessRoleAndDatabasePool(t *testing.T) { t.Fatalf("Validate() error = %v, want invalid database max conns", err) } cfg.DatabaseMaxConns = 16 + cfg.DatabaseMinIdleConns = 17 + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MIN_IDLE_CONNS") { + t.Fatalf("Validate() error = %v, want invalid database min idle conns", err) + } + cfg.DatabaseMinIdleConns = 4 + t.Setenv("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", "not-an-integer") + if err := Load().Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MIN_IDLE_CONNS") { + t.Fatalf("Validate() error = %v, want invalid non-integer database min idle conns", err) + } + t.Setenv("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", "4") cfg.DatabaseIdleInTransactionTimeoutSeconds = 3601 if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDLE_IN_TRANSACTION") { t.Fatalf("Validate() error = %v, want invalid idle transaction timeout", err) diff --git a/apps/api/internal/store/postgres.go b/apps/api/internal/store/postgres.go index f6844b4..40c9371 100644 --- a/apps/api/internal/store/postgres.go +++ b/apps/api/internal/store/postgres.go @@ -27,11 +27,11 @@ type Store struct { const ( postgresApplicationName = "easyai-ai-gateway" postgresConnectTimeout = 5 * time.Second - postgresPoolWarmLimit = 64 ) type PostgresPoolOptions struct { MaxConns int + MinIdleConns int IdleInTransactionTimeout time.Duration LockTimeout time.Duration } @@ -130,8 +130,8 @@ func postgresPoolConfigWithOptions(databaseURL string, options PostgresPoolOptio } if options.MaxConns > 0 { config.MaxConns = int32(options.MaxConns) - config.MinIdleConns = int32(min(options.MaxConns, postgresPoolWarmLimit)) } + config.MinIdleConns = int32(min(max(options.MinIdleConns, 0), int(config.MaxConns))) return config, nil } diff --git a/apps/api/internal/store/postgres_config_test.go b/apps/api/internal/store/postgres_config_test.go index e910f40..6bdc0ec 100644 --- a/apps/api/internal/store/postgres_config_test.go +++ b/apps/api/internal/store/postgres_config_test.go @@ -27,7 +27,7 @@ func TestPostgresPoolConfigRejectsMalformedURL(t *testing.T) { } } -func TestPostgresPoolConfigWarmsBoundedIdleConnections(t *testing.T) { +func TestPostgresPoolConfigUsesExplicitMinimumIdleConnections(t *testing.T) { config, err := postgresPoolConfig( "postgresql://gateway:password@localhost:5432/gateway?sslmode=disable", 24, @@ -35,23 +35,23 @@ func TestPostgresPoolConfigWarmsBoundedIdleConnections(t *testing.T) { if err != nil { t.Fatalf("parse PostgreSQL pool config: %v", err) } - if config.MaxConns != 24 || config.MinIdleConns != 24 { + if config.MaxConns != 24 || config.MinIdleConns != 0 { t.Fatalf( - "pool bounds max=%d minIdle=%d, want 24/24", + "pool bounds max=%d minIdle=%d, want 24/0", config.MaxConns, config.MinIdleConns, ) } - small, err := postgresPoolConfig( + warmed, err := postgresPoolConfigWithOptions( "postgresql://gateway:password@localhost:5432/gateway?sslmode=disable", - 4, + PostgresPoolOptions{MaxConns: 24, MinIdleConns: 4}, ) if err != nil { - t.Fatalf("parse small PostgreSQL pool config: %v", err) + t.Fatalf("parse warmed PostgreSQL pool config: %v", err) } - if small.MinIdleConns != 4 { - t.Fatalf("small pool minIdle=%d, want 4", small.MinIdleConns) + if warmed.MinIdleConns != 4 { + t.Fatalf("warmed pool minIdle=%d, want 4", warmed.MinIdleConns) } } @@ -60,6 +60,7 @@ func TestPostgresPoolConfigSetsBoundedTransactionTimeouts(t *testing.T) { "postgresql://gateway:password@localhost:5432/gateway?sslmode=disable", PostgresPoolOptions{ MaxConns: 32, + MinIdleConns: 4, IdleInTransactionTimeout: 60 * time.Second, LockTimeout: 30 * time.Second, }, @@ -73,8 +74,8 @@ func TestPostgresPoolConfigSetsBoundedTransactionTimeouts(t *testing.T) { if got := config.ConnConfig.RuntimeParams["lock_timeout"]; got != "30000ms" { t.Fatalf("lock timeout = %q, want 30000ms", got) } - if config.MaxConns != 32 || config.MinIdleConns != 32 { - t.Fatalf("pool bounds max=%d minIdle=%d, want 32/32", config.MaxConns, config.MinIdleConns) + if config.MaxConns != 32 || config.MinIdleConns != 4 { + t.Fatalf("pool bounds max=%d minIdle=%d, want 32/4", config.MaxConns, config.MinIdleConns) } } diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index 805909e..5e61198 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -19,6 +19,8 @@ source "$config_file" : "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}" : "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}" : "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}" +: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=48}" +: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}" : "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}" : "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}" [[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || { @@ -31,6 +33,7 @@ for capacity_value in \ "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "$AI_GATEWAY_DATABASE_MAX_CONNS" \ + "$AI_GATEWAY_API_DATABASE_MAX_CONNS" \ "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \ "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do [[ $capacity_value =~ ^[1-9][0-9]*$ ]] || { @@ -38,11 +41,18 @@ for capacity_value in \ exit 1 } done +[[ $AI_GATEWAY_DATABASE_MIN_IDLE_CONNS =~ ^[0-9]+$ ]] || { + echo 'database minimum idle connection configuration must be a non-negative integer' >&2 + exit 1 +} (( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 && AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 && AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 && AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 && AI_GATEWAY_DATABASE_MAX_CONNS <= 256 && + AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 && + AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS && + AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS && AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 && AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || { echo 'worker capacity configuration exceeds the release safety bounds' >&2 @@ -127,6 +137,7 @@ rollout_worker_site() { "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \ + "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \ "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \ "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" "${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \ @@ -146,7 +157,8 @@ rollout_api_capacity_site() { local site=$1 "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \ "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ - "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" + "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \ + "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s } @@ -158,7 +170,7 @@ apply_capacity_config() { verify_site hongkong verify_site ningbo wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true' - echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" + echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" } rollout_site() { @@ -170,7 +182,8 @@ rollout_site() { if [[ $api_changed == true ]]; then "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \ "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ - "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" + "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \ + "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \ "api=$api_image" "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example index 1f2e399..d7322b4 100644 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example @@ -10,5 +10,7 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24 AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 AI_GATEWAY_DATABASE_MAX_CONNS=32 +AI_GATEWAY_API_DATABASE_MAX_CONNS=48 +AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24 AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index d3b27b3..38ef184 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -53,7 +53,9 @@ spec: - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED value: "false" - name: AI_GATEWAY_DATABASE_MAX_CONNS - value: "32" + value: "48" + - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS + value: "4" - name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY value: "16" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY @@ -182,7 +184,9 @@ spec: - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED value: "false" - name: AI_GATEWAY_DATABASE_MAX_CONNS - value: "32" + value: "48" + - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS + value: "4" - name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY value: "16" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY @@ -312,6 +316,8 @@ spec: value: "true" - name: AI_GATEWAY_DATABASE_MAX_CONNS value: "32" + - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS + value: "4" - name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT value: "24" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY @@ -443,6 +449,8 @@ spec: value: "true" - name: AI_GATEWAY_DATABASE_MAX_CONNS value: "32" + - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS + value: "4" - name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT value: "24" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY diff --git a/docs/operations/production-acceptance.md b/docs/operations/production-acceptance.md index 4342910..a53ee18 100644 --- a/docs/operations/production-acceptance.md +++ b/docs/operations/production-acceptance.md @@ -64,6 +64,7 @@ WebP 和 4K 图片。128 组输入组合避免只命中相同缓存；每四个 ```text AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT AI_GATEWAY_DATABASE_MAX_CONNS +AI_GATEWAY_DATABASE_MIN_IDLE_CONNS AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY AI_GATEWAY_WORKER_REPLICAS_NINGBO @@ -79,6 +80,11 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG | P28 | 28 | 36 | 28 | 56 | | P32 | 32 | 40 | 32 | 64 | +API Pod 使用独立发布配置 `AI_GATEWAY_API_DATABASE_MAX_CONNS`，生产同构验收固定为 `48`。 +所有 API/Worker Pool 的 `AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 固定为 `4`，仅预热必要连接， +其余连接按负载弹性创建，避免四个 Pod 在空闲时就占满 PostgreSQL 连接预算。各档表中的 +数据库池仍表示单个 Worker 的最大连接数。 + 每档依次运行全部五个模拟 Profile 三次。失败时恢复上一个已完整通过的档位，流量保持 `validation`，不会自动放开正式请求。 diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 4380c60..93edb13 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -667,6 +667,8 @@ mark_run_failed() { apply_capacity_profile() { local profile=$1 local slots pool media global + local api_pool=48 + local min_idle=4 case $profile in P24) slots=24; pool=32; media=24; global=48 ;; P28) slots=28; pool=36; media=28; global=56 ;; @@ -679,13 +681,15 @@ apply_capacity_profile() { "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$slots" \ "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \ "AI_GATEWAY_DATABASE_MAX_CONNS=$pool" \ + "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" \ "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$media" \ "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$media" >/dev/null remote_kubectl rollout status "deployment/easyai-worker-$site" \ -n "$namespace" --timeout=300s remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \ "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \ - "AI_GATEWAY_DATABASE_MAX_CONNS=$pool" >/dev/null + "AI_GATEWAY_DATABASE_MAX_CONNS=$api_pool" \ + "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" >/dev/null remote_kubectl rollout status "deployment/easyai-api-$site" \ -n "$namespace" --timeout=300s done @@ -895,6 +899,7 @@ wait_for_memory_recovery() { verify_runtime_gates() { local expected_slots expected_global expected_pool + local expected_api_pool=48 case $active_profile in P24) expected_slots=24; expected_pool=32; expected_global=48 ;; P28) expected_slots=28; expected_pool=36; expected_global=56 ;; @@ -1055,8 +1060,8 @@ WHERE status='active' pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics") acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics") idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics") - [[ ${pool_max%.*} == "$expected_pool" ]] - if [[ ${acquired%.*} == "$expected_pool" && ${idle%.*} == 0 ]]; then + [[ ${pool_max%.*} == "$expected_api_pool" ]] + if [[ ${acquired%.*} == "$expected_api_pool" && ${idle%.*} == 0 ]]; then echo "API connection pool saturated for site $site" >&2 return 1 fi