perf(postgres): 分离 API 池并降低连接预热
生产同构千并发请求在干净队列上复现 API Pool 31/32、idle=1 且 canceled acquire 持续增长,Worker Pool 仅使用少量连接,确认瓶颈位于 API 数据库连接预算而非 Worker 执行槽。 新增可配置的 AI_GATEWAY_DATABASE_MIN_IDLE_CONNS,生产仅预热 4 条连接;API Pool 独立固定为 48,Worker Pool 继续跟随 P24/P28/P32 的 32/36/40 档位。发布工具、Kubernetes 清单、验收门禁和文档同步更新。 验证:Go 全量测试、gofmt、bash -n、ShellCheck、kubectl kustomize 和 git diff --check 通过。
This commit is contained in:
@@ -19,6 +19,8 @@ source "$config_file"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
|
||||
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
|
||||
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=48}"
|
||||
: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}"
|
||||
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
||||
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
||||
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
||||
@@ -31,6 +33,7 @@ for capacity_value in \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
|
||||
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
|
||||
@@ -38,11 +41,18 @@ for capacity_value in \
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
[[ $AI_GATEWAY_DATABASE_MIN_IDLE_CONNS =~ ^[0-9]+$ ]] || {
|
||||
echo 'database minimum idle connection configuration must be a non-negative integer' >&2
|
||||
exit 1
|
||||
}
|
||||
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
|
||||
AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 &&
|
||||
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS &&
|
||||
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS &&
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
|
||||
echo 'worker capacity configuration exceeds the release safety bounds' >&2
|
||||
@@ -127,6 +137,7 @@ rollout_worker_site() {
|
||||
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
|
||||
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
|
||||
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
@@ -146,7 +157,8 @@ rollout_api_capacity_site() {
|
||||
local site=$1
|
||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS"
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
}
|
||||
|
||||
@@ -158,7 +170,7 @@ apply_capacity_config() {
|
||||
verify_site hongkong
|
||||
verify_site ningbo
|
||||
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
||||
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
|
||||
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
|
||||
}
|
||||
|
||||
rollout_site() {
|
||||
@@ -170,7 +182,8 @@ rollout_site() {
|
||||
if [[ $api_changed == true ]]; then
|
||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS"
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS"
|
||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"api=$api_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
|
||||
@@ -10,5 +10,7 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
||||
AI_GATEWAY_API_DATABASE_MAX_CONNS=48
|
||||
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
||||
|
||||
@@ -53,7 +53,9 @@ spec:
|
||||
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
||||
value: "false"
|
||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||
value: "32"
|
||||
value: "48"
|
||||
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||
value: "4"
|
||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||
value: "16"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
@@ -182,7 +184,9 @@ spec:
|
||||
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
||||
value: "false"
|
||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||
value: "32"
|
||||
value: "48"
|
||||
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||
value: "4"
|
||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||
value: "16"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
@@ -312,6 +316,8 @@ spec:
|
||||
value: "true"
|
||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||
value: "32"
|
||||
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||
value: "4"
|
||||
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
||||
value: "24"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
@@ -443,6 +449,8 @@ spec:
|
||||
value: "true"
|
||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||
value: "32"
|
||||
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||
value: "4"
|
||||
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
||||
value: "24"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
|
||||
Reference in New Issue
Block a user