From 36546bd5c45fc99f90926d4f01cb1d8bfca90110 Mon Sep 17 00:00:00 2001 From: wangbo Date: Fri, 31 Jul 2026 03:46:41 +0800 Subject: [PATCH] =?UTF-8?q?perf(postgres):=20=E5=9B=9E=E6=94=B6=E7=AA=81?= =?UTF-8?q?=E5=8F=91=E8=BF=9E=E6=8E=A5=E5=B9=B6=E7=AD=89=E5=BE=85=E9=AA=8C?= =?UTF-8?q?=E6=94=B6=E5=89=AF=E4=BD=9C=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 失败 Run 的未提交任务已取消，但退款 outbox 尚未完成时下一轮会继承 API 连接高水位和 canceled acquire 增量，污染容量验收。 新增 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS，生产配置 30 秒回收突发空闲连接；验收中止后等待旧任务、退款、回调和数据库连接全部收敛后才允许新负载启动。 验证：Go 全量测试、gofmt、bash -n、ShellCheck、kubectl kustomize 和 git diff --check 通过。 --- README.md | 2 +- apps/api/cmd/gateway/main.go | 1 + apps/api/internal/config/config.go | 5 +++ apps/api/internal/config/config_test.go | 9 +++- apps/api/internal/store/postgres.go | 4 ++ .../internal/store/postgres_config_test.go | 4 ++ .../easyai-ai-gateway-cluster-release | 12 +++-- ...ai-ai-gateway-cluster-release.conf.example | 1 + deploy/kubernetes/production/application.yaml | 8 ++++ docs/operations/production-acceptance.md | 7 ++- scripts/cluster/run-production-acceptance.sh | 45 ++++++++++++++----- 11 files changed, 80 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index faae217..82a03b3 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ AI_GATEWAY_DATABASE_URL=postgresql://easyai:easyai2025@localhost:5432/easyai_ai_ 如果现有 `easyai-pgvector` 没有把 `5432` 映射到宿主机，就需要补端口映射，或者把 AI Gateway 后端容器化后接入同一个 `easyai` Docker network。 -异步队列 worker 不使用固定业务并发。服务分别汇总启用平台模型和活跃用户组的有效 `concurrent` 策略，采用两者中更严格的集群容量，默认每 5 秒在线调整 River 执行容量。策略解析兼容历史 `platformLimits/modelLimits.max_concurrent_requests`，运行时统一转换为 `rules`。`AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT` 默认 `2048`，限制策略推导出的集群目标；`AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT` 默认 `32`，按单 Worker 的内存安全容量限制实例分配，即使其他实例失活也不会突破。Worker 的 `AI_GATEWAY_DATABASE_MAX_CONNS` 必须高于实例执行容量，为心跳、选主、租约续期和健康检查保留连接；生产环境按每实例执行容量 `24` 配置连接池上限 `32`。`AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 控制启动预热连接数，默认 `0`，生产 K3s 配置为 `4`，避免把连接池上限全部常驻占用。平台模型和用户组的 PostgreSQL concurrency lease 仍是业务并发真值。可通过 `AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS` 调整刷新周期。 +异步队列 worker 不使用固定业务并发。服务分别汇总启用平台模型和活跃用户组的有效 `concurrent` 策略，采用两者中更严格的集群容量，默认每 5 秒在线调整 River 执行容量。策略解析兼容历史 `platformLimits/modelLimits.max_concurrent_requests`，运行时统一转换为 `rules`。`AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT` 默认 `2048`，限制策略推导出的集群目标；`AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT` 默认 `32`，按单 Worker 的内存安全容量限制实例分配，即使其他实例失活也不会突破。Worker 的 `AI_GATEWAY_DATABASE_MAX_CONNS` 必须高于实例执行容量，为心跳、选主、租约续期和健康检查保留连接；生产环境按每实例执行容量 `24` 配置连接池上限 `32`。`AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 控制启动预热连接数，默认 `0`，生产 K3s 配置为 `4`；`AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS` 可缩短突发连接的空闲回收时间，生产配置为 `30` 秒，避免把连接池高水位长期常驻。平台模型和用户组的 PostgreSQL concurrency lease 仍是业务并发真值。可通过 `AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS` 调整刷新周期。 ## 迁移原则 diff --git a/apps/api/cmd/gateway/main.go b/apps/api/cmd/gateway/main.go index 393351b..ccb3bad 100644 --- a/apps/api/cmd/gateway/main.go +++ b/apps/api/cmd/gateway/main.go @@ -40,6 +40,7 @@ func main() { db, err := store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{ MaxConns: cfg.DatabaseMaxConns, MinIdleConns: cfg.DatabaseMinIdleConns, + MaxConnIdleTime: time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second, IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second, LockTimeout: time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second, }) diff --git a/apps/api/internal/config/config.go b/apps/api/internal/config/config.go index 528c958..2867cc9 100644 --- a/apps/api/internal/config/config.go +++ b/apps/api/internal/config/config.go @@ -60,6 +60,7 @@ type Config struct { ProcessRole string DatabaseMaxConns int DatabaseMinIdleConns int + DatabaseMaxConnIdleSeconds int DatabaseIdleInTransactionTimeoutSeconds int DatabaseLockTimeoutSeconds int MediaRequestConcurrency int @@ -124,6 +125,7 @@ func Load() Config { ProcessRole: strings.ToLower(strings.TrimSpace(env("AI_GATEWAY_PROCESS_ROLE", "all"))), DatabaseMaxConns: envInt("AI_GATEWAY_DATABASE_MAX_CONNS", 0), DatabaseMinIdleConns: envOptionalIntValidated("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", 0), + DatabaseMaxConnIdleSeconds: envOptionalIntValidated("AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS", 0), DatabaseIdleInTransactionTimeoutSeconds: envOptionalIntValidated( "AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", @@ -153,6 +155,9 @@ func (c Config) Validate() error { (c.DatabaseMaxConns > 0 && c.DatabaseMinIdleConns > c.DatabaseMaxConns) { return errors.New("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS must be between 0 and AI_GATEWAY_DATABASE_MAX_CONNS") } + if c.DatabaseMaxConnIdleSeconds < 0 || c.DatabaseMaxConnIdleSeconds > 3600 { + return errors.New("AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS must be between 0 and 3600") + } if c.DatabaseIdleInTransactionTimeoutSeconds < 0 || c.DatabaseIdleInTransactionTimeoutSeconds > 3600 { return errors.New("AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS must be between 0 and 3600") } diff --git a/apps/api/internal/config/config_test.go b/apps/api/internal/config/config_test.go index 75f054d..9c8bd0b 100644 --- a/apps/api/internal/config/config_test.go +++ b/apps/api/internal/config/config_test.go @@ -145,11 +145,13 @@ func TestProcessRolePrecedenceAndCompatibility(t *testing.T) { func TestValidateProcessRoleAndDatabasePool(t *testing.T) { cfg := Load() if cfg.DatabaseMinIdleConns != 0 || + cfg.DatabaseMaxConnIdleSeconds != 0 || cfg.DatabaseIdleInTransactionTimeoutSeconds != 60 || cfg.DatabaseLockTimeoutSeconds != 30 { t.Fatalf( - "database pool minimum/transaction timeouts = %d/%d/%d, want 0/60/30", + "database pool minimum/max-idle/transaction timeouts = %d/%d/%d/%d, want 0/0/60/30", cfg.DatabaseMinIdleConns, + cfg.DatabaseMaxConnIdleSeconds, cfg.DatabaseIdleInTransactionTimeoutSeconds, cfg.DatabaseLockTimeoutSeconds, ) @@ -174,6 +176,11 @@ func TestValidateProcessRoleAndDatabasePool(t *testing.T) { t.Fatalf("Validate() error = %v, want invalid non-integer database min idle conns", err) } t.Setenv("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", "4") + cfg.DatabaseMaxConnIdleSeconds = 3601 + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MAX_CONN_IDLE_SECONDS") { + t.Fatalf("Validate() error = %v, want invalid database max connection idle seconds", err) + } + cfg.DatabaseMaxConnIdleSeconds = 30 cfg.DatabaseIdleInTransactionTimeoutSeconds = 3601 if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDLE_IN_TRANSACTION") { t.Fatalf("Validate() error = %v, want invalid idle transaction timeout", err) diff --git a/apps/api/internal/store/postgres.go b/apps/api/internal/store/postgres.go index 40c9371..7f31b48 100644 --- a/apps/api/internal/store/postgres.go +++ b/apps/api/internal/store/postgres.go @@ -32,6 +32,7 @@ const ( type PostgresPoolOptions struct { MaxConns int MinIdleConns int + MaxConnIdleTime time.Duration IdleInTransactionTimeout time.Duration LockTimeout time.Duration } @@ -132,6 +133,9 @@ func postgresPoolConfigWithOptions(databaseURL string, options PostgresPoolOptio config.MaxConns = int32(options.MaxConns) } config.MinIdleConns = int32(min(max(options.MinIdleConns, 0), int(config.MaxConns))) + if options.MaxConnIdleTime > 0 { + config.MaxConnIdleTime = options.MaxConnIdleTime + } return config, nil } diff --git a/apps/api/internal/store/postgres_config_test.go b/apps/api/internal/store/postgres_config_test.go index 6bdc0ec..a138ac8 100644 --- a/apps/api/internal/store/postgres_config_test.go +++ b/apps/api/internal/store/postgres_config_test.go @@ -61,6 +61,7 @@ func TestPostgresPoolConfigSetsBoundedTransactionTimeouts(t *testing.T) { PostgresPoolOptions{ MaxConns: 32, MinIdleConns: 4, + MaxConnIdleTime: 30 * time.Second, IdleInTransactionTimeout: 60 * time.Second, LockTimeout: 30 * time.Second, }, @@ -77,6 +78,9 @@ func TestPostgresPoolConfigSetsBoundedTransactionTimeouts(t *testing.T) { if config.MaxConns != 32 || config.MinIdleConns != 4 { t.Fatalf("pool bounds max=%d minIdle=%d, want 32/4", config.MaxConns, config.MinIdleConns) } + if config.MaxConnIdleTime != 30*time.Second { + t.Fatalf("pool max idle time=%s, want 30s", config.MaxConnIdleTime) + } } func TestIsPostgresUnavailableClassifiesConnectivityFailures(t *testing.T) { diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index 327267f..219561c 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -21,6 +21,7 @@ source "$config_file" : "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}" : "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=64}" : "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}" +: "${AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS:=30}" : "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}" : "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}" [[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || { @@ -34,6 +35,7 @@ for capacity_value in \ "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "$AI_GATEWAY_DATABASE_MAX_CONNS" \ "$AI_GATEWAY_API_DATABASE_MAX_CONNS" \ + "$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \ "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \ "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do [[ $capacity_value =~ ^[1-9][0-9]*$ ]] || { @@ -53,6 +55,7 @@ done AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 && AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS && AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS && + AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS <= 3600 && AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 && AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || { echo 'worker capacity configuration exceeds the release safety bounds' >&2 @@ -138,6 +141,7 @@ rollout_worker_site() { "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \ "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \ + "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \ "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \ "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" "${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \ @@ -158,7 +162,8 @@ rollout_api_capacity_site() { "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \ "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \ - "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" + "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \ + "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s } @@ -170,7 +175,7 @@ apply_capacity_config() { verify_site hongkong verify_site ningbo wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true' - echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" + echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" } rollout_site() { @@ -183,7 +188,8 @@ rollout_site() { "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \ "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \ - "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" + "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \ + "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \ "api=$api_image" "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example index 88cdeff..12f48d3 100644 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example @@ -12,5 +12,6 @@ AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 AI_GATEWAY_DATABASE_MAX_CONNS=32 AI_GATEWAY_API_DATABASE_MAX_CONNS=64 AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 +AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24 AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index fa52b91..569d159 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -56,6 +56,8 @@ spec: value: "64" - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS value: "4" + - name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS + value: "30" - name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY value: "16" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY @@ -187,6 +189,8 @@ spec: value: "64" - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS value: "4" + - name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS + value: "30" - name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY value: "16" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY @@ -318,6 +322,8 @@ spec: value: "32" - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS value: "4" + - name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS + value: "30" - name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT value: "24" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY @@ -451,6 +457,8 @@ spec: value: "32" - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS value: "4" + - name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS + value: "30" - name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT value: "24" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY diff --git a/docs/operations/production-acceptance.md b/docs/operations/production-acceptance.md index 8db3b73..fe66a68 100644 --- a/docs/operations/production-acceptance.md +++ b/docs/operations/production-acceptance.md @@ -65,6 +65,7 @@ WebP 和 4K 图片。128 组输入组合避免只命中相同缓存；每四个 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT AI_GATEWAY_DATABASE_MAX_CONNS AI_GATEWAY_DATABASE_MIN_IDLE_CONNS +AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY AI_GATEWAY_WORKER_REPLICAS_NINGBO @@ -83,8 +84,10 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG API Pod 使用独立发布配置 `AI_GATEWAY_API_DATABASE_MAX_CONNS`，生产同构验收默认使用 `64`； 也可通过 `AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS` 覆盖本次验收值，无需修改 Go 代码。 所有 API/Worker Pool 的 `AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 固定为 `4`，仅预热必要连接， -其余连接按负载弹性创建，避免四个 Pod 在空闲时就占满 PostgreSQL 连接预算。各档表中的 -数据库池仍表示单个 Worker 的最大连接数。 +`AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS` 为 `30`，突发扩出的空闲连接会及时回落， +避免四个 Pod 在空闲时占满 PostgreSQL 连接预算。各档表中的数据库池仍表示单个 Worker +的最大连接数。中止旧 Run 后还必须等待未提交任务、退款 outbox、回调 outbox 和连接高水位 +全部收敛，才会启动下一轮负载。 每档依次运行全部五个模拟 Profile 三次。失败时恢复上一个已完整通过的档位，流量保持 `validation`，不会自动放开正式请求。 diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index c9c0d41..5db5740 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -226,21 +226,41 @@ wait_for_existing_tasks_to_drain() { wait_for_terminal_acceptance_tasks_to_drain() { local deadline=$((SECONDS + 300)) - local active + local state active side_effects connections max_connections while (( SECONDS < deadline )); do - active=$(database_query " -SELECT count(*) -FROM gateway_tasks task -JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id -WHERE run.status IN ('failed', 'aborted') - AND task.status IN ('queued', 'running');") - if [[ $active == 0 ]]; then + state=$(database_query " +SELECT + (SELECT count(*) + FROM gateway_tasks task + JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id + WHERE run.status IN ('failed', 'aborted') + AND task.status IN ('queued', 'running'))||','|| + ( + (SELECT count(*) + FROM settlement_outbox settlement + JOIN gateway_tasks task ON task.id=settlement.task_id + JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id + WHERE run.status IN ('failed', 'aborted') + AND settlement.status IN ('pending', 'processing', 'retryable_failed')) + + + (SELECT count(*) + FROM gateway_task_callback_outbox callback + JOIN gateway_tasks task ON task.id=callback.task_id + JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id + WHERE run.status IN ('failed', 'aborted') + AND callback.status <> 'delivered') + )||','|| + (SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','|| + (SELECT setting FROM pg_settings WHERE name='max_connections');") + IFS=',' read -r active side_effects connections max_connections <<<"$state" + if [[ $active == 0 && $side_effects == 0 ]] && + (( connections * 2 < max_connections )); then return 0 fi - echo "waiting_for_terminal_acceptance_tasks=$active" + echo "waiting_for_terminal_acceptance_tasks=$active side_effects=$side_effects database_connections=$connections" sleep 2 done - echo "terminal acceptance tasks did not drain within 5 minutes: active=$active" >&2 + echo "terminal acceptance state did not drain within 5 minutes: active=$active side_effects=$side_effects database_connections=$connections" >&2 return 1 } @@ -675,6 +695,7 @@ apply_capacity_profile() { local slots pool media global local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS local min_idle=4 + local max_idle_seconds=30 case $profile in P24) slots=24; pool=32; media=24; global=48 ;; P28) slots=28; pool=36; media=28; global=56 ;; @@ -688,6 +709,7 @@ apply_capacity_profile() { "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \ "AI_GATEWAY_DATABASE_MAX_CONNS=$pool" \ "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" \ + "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$max_idle_seconds" \ "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$media" \ "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$media" >/dev/null remote_kubectl rollout status "deployment/easyai-worker-$site" \ @@ -695,7 +717,8 @@ apply_capacity_profile() { remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \ "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \ "AI_GATEWAY_DATABASE_MAX_CONNS=$api_pool" \ - "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" >/dev/null + "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" \ + "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$max_idle_seconds" >/dev/null remote_kubectl rollout status "deployment/easyai-api-$site" \ -n "$namespace" --timeout=300s done