From 489208cef989f2d0c3b67b0edcd49af3266bd573 Mon Sep 17 00:00:00 2001 From: wangbo Date: Fri, 31 Jul 2026 02:46:10 +0800 Subject: [PATCH] =?UTF-8?q?fix(acceptance):=20=E7=BA=B3=E5=85=A5=20API=20?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E6=B1=A0=E5=AE=B9=E9=87=8F=E9=97=A8?= =?UTF-8?q?=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 高并发 Gemini 验收暴露两地 API 数据库池仍固定为 16，导致连接池满载、readiness 失败且压力曲线只监测 Worker，无法及时识别真实瓶颈。\n\n将容量档位的数据库连接池同步应用到 API，发布工具和生产清单保持同一配置；压力采样及日志门禁覆盖 API 与 Worker 四个连接池，并在失败时保留验收 JSON 报告。\n\n验证：bash -n、ShellCheck、kubectl kustomize、git diff --check。 --- .../easyai-ai-gateway-cluster-release | 6 +- deploy/kubernetes/production/application.yaml | 4 +- scripts/cluster/run-production-acceptance.sh | 107 ++++++++++++------ 3 files changed, 77 insertions(+), 40 deletions(-) diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index 95922e8..805909e 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -145,7 +145,8 @@ rollout_worker_site() { rollout_api_capacity_site() { local site=$1 "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \ - "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" + "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ + "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s } @@ -168,7 +169,8 @@ rollout_site() { local web_changed=$5 if [[ $api_changed == true ]]; then "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \ - "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" + "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ + "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \ "api=$api_image" "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index eddd98e..d3b27b3 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -53,7 +53,7 @@ spec: - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED value: "false" - name: AI_GATEWAY_DATABASE_MAX_CONNS - value: "16" + value: "32" - name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY value: "16" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY @@ -182,7 +182,7 @@ spec: - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED value: "false" - name: AI_GATEWAY_DATABASE_MAX_CONNS - value: "16" + value: "32" - name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY value: "16" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index a0baf42..efa0568 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -663,7 +663,8 @@ apply_capacity_profile() { remote_kubectl rollout status "deployment/easyai-worker-$site" \ -n "$namespace" --timeout=300s remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \ - "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" >/dev/null + "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \ + "AI_GATEWAY_DATABASE_MAX_CONNS=$pool" >/dev/null remote_kubectl rollout status "deployment/easyai-api-$site" \ -n "$namespace" --timeout=300s done @@ -709,7 +710,14 @@ export AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=$(decode "$run_token_b64") export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(decode "$gemini_model_b64") export AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$(decode "$video_model_b64") export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(decode "$image_urls_b64") -exec /app/easyai-ai-gateway-acceptance-load -profile "$1" +report_file=$(mktemp) +trap '\''rm -f -- "$report_file"'\'' EXIT +set +e +/app/easyai-ai-gateway-acceptance-load -profile "$1" >"$report_file" +status=$? +set -e +cat "$report_file" +exit "$status" ' { printf '%s' "$AI_GATEWAY_ACCEPTANCE_GATEWAYS" | openssl base64 -A @@ -805,23 +813,25 @@ verify_cluster_links() { } verify_control_plane_and_recent_logs() { - local host ready_output pod error_count + local host ready_output workload pod error_count for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do ready_output=$(cluster_ssh "$host" "k3s kubectl get --raw='/readyz?verbose'") grep -Fq '[+]etcd ok' <<<"$ready_output" done - while read -r pod; do - error_count=$(remote_kubectl logs "$pod" -n "$namespace" --since=10m | - awk 'BEGIN {IGNORECASE=1} - /postgres_unavailable/ || - /postgres readiness.*(fail|unavailable|error)/ || - /leadership elector.*(error|fail)/ || - /concurrency lease.*(renew.*(error|fail)|lost)/ {count++} - END {print count+0}') - [[ $error_count == 0 ]] - done < <(remote_kubectl get pods -n "$namespace" \ - -l 'app.kubernetes.io/name=easyai-worker' \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + for workload in api worker; do + while read -r pod; do + error_count=$(remote_kubectl logs "$pod" -n "$namespace" --since=10m | + awk 'BEGIN {IGNORECASE=1} + /postgres_unavailable/ || + /postgres readiness.*(fail|unavailable|error)/ || + /leadership elector.*(error|fail)/ || + /concurrency lease.*(renew.*(error|fail)|lost)/ {count++} + END {print count+0}') + [[ $error_count == 0 ]] + done < <(remote_kubectl get pods -n "$namespace" \ + -l "app.kubernetes.io/name=easyai-$workload" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + done while read -r pod; do error_count=$(remote_kubectl logs "$pod" -n "$namespace" -c postgres --since=10m | awk 'BEGIN {IGNORECASE=1} @@ -1017,6 +1027,21 @@ WHERE status='active' local pod metrics desired current global allocated active_instances pool_max acquired idle local canceled_acquires lease_failures lease_lost + for site in ningbo hongkong; do + pod=$(remote_kubectl get pods -n "$namespace" \ + -l "app.kubernetes.io/name=easyai-api,easyai.io/site=$site" \ + -o 'jsonpath={.items[0].metadata.name}') + metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \ + wget -qO- http://127.0.0.1:8088/metrics) + pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics") + acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics") + idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics") + [[ ${pool_max%.*} == "$expected_pool" ]] + if [[ ${acquired%.*} == "$expected_pool" && ${idle%.*} == 0 ]]; then + echo "API connection pool saturated for site $site" >&2 + return 1 + fi + done for _ in 1 2 3 4 5 6; do for site in ningbo hongkong; do pod=$(remote_kubectl get pods -n "$namespace" \ @@ -1070,27 +1095,37 @@ WHERE acceptance_run_id='$run_id'::uuid;") || break awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break pod_memory_mib=$(current_max_pod_memory_mib) || break pool_state=$( - for site in ningbo hongkong; do - pod=$(remote_kubectl get pods -n "$namespace" \ - -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" -o json | - jq -r '[.items[] - | select(any(.status.conditions[]?; .type=="Ready" and .status=="True")) - | .metadata.name][0] // empty') || exit - [[ -n $pod ]] || continue - metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \ - wget -qO- http://127.0.0.1:8088/metrics) || exit - awk ' - $1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2} - $1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2} - $1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2} - $1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2} - $1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2} - index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2} - index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2} - $1=="easyai_gateway_worker_active_instances" {active=$2} - $1=="easyai_gateway_worker_allocated_capacity" {allocated=$2} - END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated} - ' <<<"$metrics" + for workload in api worker; do + for site in ningbo hongkong; do + pod=$(remote_kubectl get pods -n "$namespace" \ + -l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json | + jq -r '[.items[] + | select(any(.status.conditions[]?; .type=="Ready" and .status=="True")) + | .metadata.name][0] // empty') || exit + [[ -n $pod ]] || continue + metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \ + wget -qO- http://127.0.0.1:8088/metrics) || exit + awk -v workload="$workload" ' + $1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2} + $1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2} + $1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2} + $1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2} + $1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2} + index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2} + index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2} + $1=="easyai_gateway_worker_active_instances" {active=$2} + $1=="easyai_gateway_worker_allocated_capacity" {allocated=$2} + END { + if (workload=="api") { + failure=0 + lost=0 + active=999999 + allocated=0 + } + print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated + } + ' <<<"$metrics" + done done | awk -F',' ' NR==1 {