perf(postgres): 回收突发连接并等待验收副作用

失败 Run 的未提交任务已取消,但退款 outbox 尚未完成时下一轮会继承 API 连接高水位和 canceled acquire 增量,污染容量验收。

新增 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS,生产配置 30 秒回收突发空闲连接;验收中止后等待旧任务、退款、回调和数据库连接全部收敛后才允许新负载启动。

验证:Go 全量测试、gofmt、bash -n、ShellCheck、kubectl kustomize 和 git diff --check 通过。
This commit is contained in:
2026-07-31 03:46:41 +08:00
parent 75a56b21bc
commit 36546bd5c4
11 changed files with 80 additions and 18 deletions
@@ -21,6 +21,7 @@ source "$config_file"
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=64}"
: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS:=30}"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
@@ -34,6 +35,7 @@ for capacity_value in \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
@@ -53,6 +55,7 @@ done
AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS <= 3600 &&
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
echo 'worker capacity configuration exceeds the release safety bounds' >&2
@@ -138,6 +141,7 @@ rollout_worker_site() {
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
@@ -158,7 +162,8 @@ rollout_api_capacity_site() {
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS"
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
}
@@ -170,7 +175,7 @@ apply_capacity_config() {
verify_site hongkong
verify_site ningbo
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
}
rollout_site() {
@@ -183,7 +188,8 @@ rollout_site() {
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS"
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
"api=$api_image"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
@@ -12,5 +12,6 @@ AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
AI_GATEWAY_DATABASE_MAX_CONNS=32
AI_GATEWAY_API_DATABASE_MAX_CONNS=64
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
@@ -56,6 +56,8 @@ spec:
value: "64"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
value: "30"
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
value: "16"
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
@@ -187,6 +189,8 @@ spec:
value: "64"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
value: "30"
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
value: "16"
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
@@ -318,6 +322,8 @@ spec:
value: "32"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
value: "30"
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
value: "24"
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
@@ -451,6 +457,8 @@ spec:
value: "32"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
value: "30"
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
value: "24"
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY