fix(acceptance): 纳入 API 数据库池容量门禁
高并发 Gemini 验收暴露两地 API 数据库池仍固定为 16，导致连接池满载、readiness 失败且压力曲线只监测 Worker，无法及时识别真实瓶颈。\n\n将容量档位的数据库连接池同步应用到 API，发布工具和生产清单保持同一配置；压力采样及日志门禁覆盖 API 与 Worker 四个连接池，并在失败时保留验收 JSON 报告。\n\n验证：bash -n、ShellCheck、kubectl kustomize、git diff --check。
This commit is contained in:
@@ -663,7 +663,8 @@ apply_capacity_profile() {
|
||||
remote_kubectl rollout status "deployment/easyai-worker-$site" \
|
||||
-n "$namespace" --timeout=300s
|
||||
remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" >/dev/null
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$pool" >/dev/null
|
||||
remote_kubectl rollout status "deployment/easyai-api-$site" \
|
||||
-n "$namespace" --timeout=300s
|
||||
done
|
||||
@@ -709,7 +710,14 @@ export AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=$(decode "$run_token_b64")
|
||||
export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(decode "$gemini_model_b64")
|
||||
export AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$(decode "$video_model_b64")
|
||||
export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(decode "$image_urls_b64")
|
||||
exec /app/easyai-ai-gateway-acceptance-load -profile "$1"
|
||||
report_file=$(mktemp)
|
||||
trap '\''rm -f -- "$report_file"'\'' EXIT
|
||||
set +e
|
||||
/app/easyai-ai-gateway-acceptance-load -profile "$1" >"$report_file"
|
||||
status=$?
|
||||
set -e
|
||||
cat "$report_file"
|
||||
exit "$status"
|
||||
'
|
||||
{
|
||||
printf '%s' "$AI_GATEWAY_ACCEPTANCE_GATEWAYS" | openssl base64 -A
|
||||
@@ -805,23 +813,25 @@ verify_cluster_links() {
|
||||
}
|
||||
|
||||
verify_control_plane_and_recent_logs() {
|
||||
local host ready_output pod error_count
|
||||
local host ready_output workload pod error_count
|
||||
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
|
||||
ready_output=$(cluster_ssh "$host" "k3s kubectl get --raw='/readyz?verbose'")
|
||||
grep -Fq '[+]etcd ok' <<<"$ready_output"
|
||||
done
|
||||
while read -r pod; do
|
||||
error_count=$(remote_kubectl logs "$pod" -n "$namespace" --since=10m |
|
||||
awk 'BEGIN {IGNORECASE=1}
|
||||
/postgres_unavailable/ ||
|
||||
/postgres readiness.*(fail|unavailable|error)/ ||
|
||||
/leadership elector.*(error|fail)/ ||
|
||||
/concurrency lease.*(renew.*(error|fail)|lost)/ {count++}
|
||||
END {print count+0}')
|
||||
[[ $error_count == 0 ]]
|
||||
done < <(remote_kubectl get pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/name=easyai-worker' \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
for workload in api worker; do
|
||||
while read -r pod; do
|
||||
error_count=$(remote_kubectl logs "$pod" -n "$namespace" --since=10m |
|
||||
awk 'BEGIN {IGNORECASE=1}
|
||||
/postgres_unavailable/ ||
|
||||
/postgres readiness.*(fail|unavailable|error)/ ||
|
||||
/leadership elector.*(error|fail)/ ||
|
||||
/concurrency lease.*(renew.*(error|fail)|lost)/ {count++}
|
||||
END {print count+0}')
|
||||
[[ $error_count == 0 ]]
|
||||
done < <(remote_kubectl get pods -n "$namespace" \
|
||||
-l "app.kubernetes.io/name=easyai-$workload" \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
done
|
||||
while read -r pod; do
|
||||
error_count=$(remote_kubectl logs "$pod" -n "$namespace" -c postgres --since=10m |
|
||||
awk 'BEGIN {IGNORECASE=1}
|
||||
@@ -1017,6 +1027,21 @@ WHERE status='active'
|
||||
|
||||
local pod metrics desired current global allocated active_instances pool_max acquired idle
|
||||
local canceled_acquires lease_failures lease_lost
|
||||
for site in ningbo hongkong; do
|
||||
pod=$(remote_kubectl get pods -n "$namespace" \
|
||||
-l "app.kubernetes.io/name=easyai-api,easyai.io/site=$site" \
|
||||
-o 'jsonpath={.items[0].metadata.name}')
|
||||
metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \
|
||||
wget -qO- http://127.0.0.1:8088/metrics)
|
||||
pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics")
|
||||
acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics")
|
||||
idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics")
|
||||
[[ ${pool_max%.*} == "$expected_pool" ]]
|
||||
if [[ ${acquired%.*} == "$expected_pool" && ${idle%.*} == 0 ]]; then
|
||||
echo "API connection pool saturated for site $site" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
for _ in 1 2 3 4 5 6; do
|
||||
for site in ningbo hongkong; do
|
||||
pod=$(remote_kubectl get pods -n "$namespace" \
|
||||
@@ -1070,27 +1095,37 @@ WHERE acceptance_run_id='$run_id'::uuid;") || break
|
||||
awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break
|
||||
pod_memory_mib=$(current_max_pod_memory_mib) || break
|
||||
pool_state=$(
|
||||
for site in ningbo hongkong; do
|
||||
pod=$(remote_kubectl get pods -n "$namespace" \
|
||||
-l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" -o json |
|
||||
jq -r '[.items[]
|
||||
| select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))
|
||||
| .metadata.name][0] // empty') || exit
|
||||
[[ -n $pod ]] || continue
|
||||
metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \
|
||||
wget -qO- http://127.0.0.1:8088/metrics) || exit
|
||||
awk '
|
||||
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
|
||||
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
|
||||
$1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2}
|
||||
$1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2}
|
||||
$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2}
|
||||
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2}
|
||||
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2}
|
||||
$1=="easyai_gateway_worker_active_instances" {active=$2}
|
||||
$1=="easyai_gateway_worker_allocated_capacity" {allocated=$2}
|
||||
END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated}
|
||||
' <<<"$metrics"
|
||||
for workload in api worker; do
|
||||
for site in ningbo hongkong; do
|
||||
pod=$(remote_kubectl get pods -n "$namespace" \
|
||||
-l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json |
|
||||
jq -r '[.items[]
|
||||
| select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))
|
||||
| .metadata.name][0] // empty') || exit
|
||||
[[ -n $pod ]] || continue
|
||||
metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \
|
||||
wget -qO- http://127.0.0.1:8088/metrics) || exit
|
||||
awk -v workload="$workload" '
|
||||
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
|
||||
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
|
||||
$1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2}
|
||||
$1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2}
|
||||
$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2}
|
||||
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2}
|
||||
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2}
|
||||
$1=="easyai_gateway_worker_active_instances" {active=$2}
|
||||
$1=="easyai_gateway_worker_allocated_capacity" {allocated=$2}
|
||||
END {
|
||||
if (workload=="api") {
|
||||
failure=0
|
||||
lost=0
|
||||
active=999999
|
||||
allocated=0
|
||||
}
|
||||
print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated
|
||||
}
|
||||
' <<<"$metrics"
|
||||
done
|
||||
done |
|
||||
awk -F',' '
|
||||
NR==1 {
|
||||
|
||||
Reference in New Issue
Block a user