perf(postgres): 回收突发连接并等待验收副作用
失败 Run 的未提交任务已取消,但退款 outbox 尚未完成时下一轮会继承 API 连接高水位和 canceled acquire 增量,污染容量验收。 新增 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS,生产配置 30 秒回收突发空闲连接;验收中止后等待旧任务、退款、回调和数据库连接全部收敛后才允许新负载启动。 验证:Go 全量测试、gofmt、bash -n、ShellCheck、kubectl kustomize 和 git diff --check 通过。
This commit is contained in:
@@ -21,6 +21,7 @@ source "$config_file"
|
||||
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
|
||||
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=64}"
|
||||
: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}"
|
||||
: "${AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS:=30}"
|
||||
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
||||
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
||||
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
||||
@@ -34,6 +35,7 @@ for capacity_value in \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||
"$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
|
||||
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
|
||||
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
|
||||
@@ -53,6 +55,7 @@ done
|
||||
AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 &&
|
||||
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS &&
|
||||
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS &&
|
||||
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS <= 3600 &&
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
|
||||
echo 'worker capacity configuration exceeds the release safety bounds' >&2
|
||||
@@ -138,6 +141,7 @@ rollout_worker_site() {
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
|
||||
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
|
||||
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
@@ -158,7 +162,8 @@ rollout_api_capacity_site() {
|
||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS"
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
}
|
||||
|
||||
@@ -170,7 +175,7 @@ apply_capacity_config() {
|
||||
verify_site hongkong
|
||||
verify_site ningbo
|
||||
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
||||
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
|
||||
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
|
||||
}
|
||||
|
||||
rollout_site() {
|
||||
@@ -183,7 +188,8 @@ rollout_site() {
|
||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS"
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
|
||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"api=$api_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
|
||||
@@ -12,5 +12,6 @@ AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
||||
AI_GATEWAY_API_DATABASE_MAX_CONNS=64
|
||||
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4
|
||||
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
||||
|
||||
@@ -56,6 +56,8 @@ spec:
|
||||
value: "64"
|
||||
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||
value: "4"
|
||||
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
|
||||
value: "30"
|
||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||
value: "16"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
@@ -187,6 +189,8 @@ spec:
|
||||
value: "64"
|
||||
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||
value: "4"
|
||||
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
|
||||
value: "30"
|
||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||
value: "16"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
@@ -318,6 +322,8 @@ spec:
|
||||
value: "32"
|
||||
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||
value: "4"
|
||||
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
|
||||
value: "30"
|
||||
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
||||
value: "24"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
@@ -451,6 +457,8 @@ spec:
|
||||
value: "32"
|
||||
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||
value: "4"
|
||||
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
|
||||
value: "30"
|
||||
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
||||
value: "24"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
|
||||
Reference in New Issue
Block a user