fix(acceptance): 纳入 API 数据库池容量门禁
高并发 Gemini 验收暴露两地 API 数据库池仍固定为 16,导致连接池满载、readiness 失败且压力曲线只监测 Worker,无法及时识别真实瓶颈。\n\n将容量档位的数据库连接池同步应用到 API,发布工具和生产清单保持同一配置;压力采样及日志门禁覆盖 API 与 Worker 四个连接池,并在失败时保留验收 JSON 报告。\n\n验证:bash -n、ShellCheck、kubectl kustomize、git diff --check。
This commit is contained in:
@@ -145,7 +145,8 @@ rollout_worker_site() {
|
|||||||
rollout_api_capacity_site() {
|
rollout_api_capacity_site() {
|
||||||
local site=$1
|
local site=$1
|
||||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
|
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||||
|
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS"
|
||||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -168,7 +169,8 @@ rollout_site() {
|
|||||||
local web_changed=$5
|
local web_changed=$5
|
||||||
if [[ $api_changed == true ]]; then
|
if [[ $api_changed == true ]]; then
|
||||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
|
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||||
|
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS"
|
||||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||||
"api=$api_image"
|
"api=$api_image"
|
||||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ spec:
|
|||||||
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
||||||
value: "false"
|
value: "false"
|
||||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||||
value: "16"
|
value: "32"
|
||||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||||
value: "16"
|
value: "16"
|
||||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
@@ -182,7 +182,7 @@ spec:
|
|||||||
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
||||||
value: "false"
|
value: "false"
|
||||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||||
value: "16"
|
value: "32"
|
||||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||||
value: "16"
|
value: "16"
|
||||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
|
|||||||
@@ -663,7 +663,8 @@ apply_capacity_profile() {
|
|||||||
remote_kubectl rollout status "deployment/easyai-worker-$site" \
|
remote_kubectl rollout status "deployment/easyai-worker-$site" \
|
||||||
-n "$namespace" --timeout=300s
|
-n "$namespace" --timeout=300s
|
||||||
remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \
|
remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \
|
||||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" >/dev/null
|
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
|
||||||
|
"AI_GATEWAY_DATABASE_MAX_CONNS=$pool" >/dev/null
|
||||||
remote_kubectl rollout status "deployment/easyai-api-$site" \
|
remote_kubectl rollout status "deployment/easyai-api-$site" \
|
||||||
-n "$namespace" --timeout=300s
|
-n "$namespace" --timeout=300s
|
||||||
done
|
done
|
||||||
@@ -709,7 +710,14 @@ export AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=$(decode "$run_token_b64")
|
|||||||
export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(decode "$gemini_model_b64")
|
export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(decode "$gemini_model_b64")
|
||||||
export AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$(decode "$video_model_b64")
|
export AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$(decode "$video_model_b64")
|
||||||
export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(decode "$image_urls_b64")
|
export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(decode "$image_urls_b64")
|
||||||
exec /app/easyai-ai-gateway-acceptance-load -profile "$1"
|
report_file=$(mktemp)
|
||||||
|
trap '\''rm -f -- "$report_file"'\'' EXIT
|
||||||
|
set +e
|
||||||
|
/app/easyai-ai-gateway-acceptance-load -profile "$1" >"$report_file"
|
||||||
|
status=$?
|
||||||
|
set -e
|
||||||
|
cat "$report_file"
|
||||||
|
exit "$status"
|
||||||
'
|
'
|
||||||
{
|
{
|
||||||
printf '%s' "$AI_GATEWAY_ACCEPTANCE_GATEWAYS" | openssl base64 -A
|
printf '%s' "$AI_GATEWAY_ACCEPTANCE_GATEWAYS" | openssl base64 -A
|
||||||
@@ -805,23 +813,25 @@ verify_cluster_links() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
verify_control_plane_and_recent_logs() {
|
verify_control_plane_and_recent_logs() {
|
||||||
local host ready_output pod error_count
|
local host ready_output workload pod error_count
|
||||||
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
|
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
|
||||||
ready_output=$(cluster_ssh "$host" "k3s kubectl get --raw='/readyz?verbose'")
|
ready_output=$(cluster_ssh "$host" "k3s kubectl get --raw='/readyz?verbose'")
|
||||||
grep -Fq '[+]etcd ok' <<<"$ready_output"
|
grep -Fq '[+]etcd ok' <<<"$ready_output"
|
||||||
done
|
done
|
||||||
while read -r pod; do
|
for workload in api worker; do
|
||||||
error_count=$(remote_kubectl logs "$pod" -n "$namespace" --since=10m |
|
while read -r pod; do
|
||||||
awk 'BEGIN {IGNORECASE=1}
|
error_count=$(remote_kubectl logs "$pod" -n "$namespace" --since=10m |
|
||||||
/postgres_unavailable/ ||
|
awk 'BEGIN {IGNORECASE=1}
|
||||||
/postgres readiness.*(fail|unavailable|error)/ ||
|
/postgres_unavailable/ ||
|
||||||
/leadership elector.*(error|fail)/ ||
|
/postgres readiness.*(fail|unavailable|error)/ ||
|
||||||
/concurrency lease.*(renew.*(error|fail)|lost)/ {count++}
|
/leadership elector.*(error|fail)/ ||
|
||||||
END {print count+0}')
|
/concurrency lease.*(renew.*(error|fail)|lost)/ {count++}
|
||||||
[[ $error_count == 0 ]]
|
END {print count+0}')
|
||||||
done < <(remote_kubectl get pods -n "$namespace" \
|
[[ $error_count == 0 ]]
|
||||||
-l 'app.kubernetes.io/name=easyai-worker' \
|
done < <(remote_kubectl get pods -n "$namespace" \
|
||||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
-l "app.kubernetes.io/name=easyai-$workload" \
|
||||||
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||||
|
done
|
||||||
while read -r pod; do
|
while read -r pod; do
|
||||||
error_count=$(remote_kubectl logs "$pod" -n "$namespace" -c postgres --since=10m |
|
error_count=$(remote_kubectl logs "$pod" -n "$namespace" -c postgres --since=10m |
|
||||||
awk 'BEGIN {IGNORECASE=1}
|
awk 'BEGIN {IGNORECASE=1}
|
||||||
@@ -1017,6 +1027,21 @@ WHERE status='active'
|
|||||||
|
|
||||||
local pod metrics desired current global allocated active_instances pool_max acquired idle
|
local pod metrics desired current global allocated active_instances pool_max acquired idle
|
||||||
local canceled_acquires lease_failures lease_lost
|
local canceled_acquires lease_failures lease_lost
|
||||||
|
for site in ningbo hongkong; do
|
||||||
|
pod=$(remote_kubectl get pods -n "$namespace" \
|
||||||
|
-l "app.kubernetes.io/name=easyai-api,easyai.io/site=$site" \
|
||||||
|
-o 'jsonpath={.items[0].metadata.name}')
|
||||||
|
metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \
|
||||||
|
wget -qO- http://127.0.0.1:8088/metrics)
|
||||||
|
pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics")
|
||||||
|
acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics")
|
||||||
|
idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics")
|
||||||
|
[[ ${pool_max%.*} == "$expected_pool" ]]
|
||||||
|
if [[ ${acquired%.*} == "$expected_pool" && ${idle%.*} == 0 ]]; then
|
||||||
|
echo "API connection pool saturated for site $site" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
for _ in 1 2 3 4 5 6; do
|
for _ in 1 2 3 4 5 6; do
|
||||||
for site in ningbo hongkong; do
|
for site in ningbo hongkong; do
|
||||||
pod=$(remote_kubectl get pods -n "$namespace" \
|
pod=$(remote_kubectl get pods -n "$namespace" \
|
||||||
@@ -1070,27 +1095,37 @@ WHERE acceptance_run_id='$run_id'::uuid;") || break
|
|||||||
awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break
|
awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break
|
||||||
pod_memory_mib=$(current_max_pod_memory_mib) || break
|
pod_memory_mib=$(current_max_pod_memory_mib) || break
|
||||||
pool_state=$(
|
pool_state=$(
|
||||||
for site in ningbo hongkong; do
|
for workload in api worker; do
|
||||||
pod=$(remote_kubectl get pods -n "$namespace" \
|
for site in ningbo hongkong; do
|
||||||
-l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" -o json |
|
pod=$(remote_kubectl get pods -n "$namespace" \
|
||||||
jq -r '[.items[]
|
-l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json |
|
||||||
| select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))
|
jq -r '[.items[]
|
||||||
| .metadata.name][0] // empty') || exit
|
| select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))
|
||||||
[[ -n $pod ]] || continue
|
| .metadata.name][0] // empty') || exit
|
||||||
metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \
|
[[ -n $pod ]] || continue
|
||||||
wget -qO- http://127.0.0.1:8088/metrics) || exit
|
metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \
|
||||||
awk '
|
wget -qO- http://127.0.0.1:8088/metrics) || exit
|
||||||
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
|
awk -v workload="$workload" '
|
||||||
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
|
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
|
||||||
$1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2}
|
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
|
||||||
$1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2}
|
$1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2}
|
||||||
$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2}
|
$1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2}
|
||||||
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2}
|
$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2}
|
||||||
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2}
|
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2}
|
||||||
$1=="easyai_gateway_worker_active_instances" {active=$2}
|
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2}
|
||||||
$1=="easyai_gateway_worker_allocated_capacity" {allocated=$2}
|
$1=="easyai_gateway_worker_active_instances" {active=$2}
|
||||||
END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated}
|
$1=="easyai_gateway_worker_allocated_capacity" {allocated=$2}
|
||||||
' <<<"$metrics"
|
END {
|
||||||
|
if (workload=="api") {
|
||||||
|
failure=0
|
||||||
|
lost=0
|
||||||
|
active=999999
|
||||||
|
allocated=0
|
||||||
|
}
|
||||||
|
print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated
|
||||||
|
}
|
||||||
|
' <<<"$metrics"
|
||||||
|
done
|
||||||
done |
|
done |
|
||||||
awk -F',' '
|
awk -F',' '
|
||||||
NR==1 {
|
NR==1 {
|
||||||
|
|||||||
Reference in New Issue
Block a user