fix(acceptance): 隔离容量压测并稳定数据库连接

将协议模拟验收的供应商并发限额与 Worker 容量门禁分离，真实金丝雀继续保留生产限流。readyz 改用关键连接池并预热关键及 River 池，延长生产连接空闲周期，降低跨地域连接抖动。失败 Run 现在可以原子替换且失败报告记录任务数量，避免 validation 之间短暂放开正式流量。\n\n验证：Go 全量测试、go vet、PostgreSQL 集成测试、ShellCheck、迁移安全、发布脚本、pnpm lint/test/build、OpenAPI 无漂移均通过。

scripts/cluster/run-production-acceptance.sh

@@ -88,6 +88,7 @@ require_commands curl git go jq node openssl sed shasum
 : "${AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS:=31}"
 : "${AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS:=8}"
 : "${AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS:=4}"
+: "${AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS:=300}"
 : "${AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY:=128}"
 : "${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:=32}"
 : "${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB:=1536}"
@@ -122,6 +123,11 @@ fi
   echo 'AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS must be between 1 and 256' >&2
   exit 1
 }
+[[ $AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS =~ ^[1-9][0-9]*$ &&
+   $AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS -le 3600 ]] || {
+  echo 'AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS must be between 1 and 3600' >&2
+  exit 1
+}
 [[ $AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS =~ ^[1-9][0-9]*$ &&
    $AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS =~ ^[1-9][0-9]*$ &&
    $AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS -lt 28 &&
@@ -1344,7 +1350,6 @@ create_and_activate_run() {
   local activate_response=$temporary_root/activate-run.json
   local traffic_response=$temporary_root/pre-activate-traffic.json
   local previous_run_response=$temporary_root/previous-run.json
-  local abort_response=$temporary_root/abort-previous-run.json
   local body
   body=$(jq -cn \
     --arg releaseSha "$release_sha" \
@@ -1421,24 +1426,9 @@ create_and_activate_run() {
       echo 'refusing to replace validation mode owned by a non-ancestor release' >&2
       return 1
     fi
-    body=$(jq -cn \
-      --argjson revision "$(jq -r '.revision' "$traffic_response")" \
-      --arg releaseSha "$traffic_release" \
-      --arg apiDigest "$traffic_api_digest" \
-      --arg workerDigest "$traffic_worker_digest" \
-      '{
-        revision:$revision,
-        releaseSha:$releaseSha,
-        apiImageDigest:$apiDigest,
-        workerImageDigest:$workerDigest
-      }')
-    admin_request POST \
-      "/api/admin/system/acceptance/runs/$previous_run_id/abort" \
-      "$body" "$abort_response"
-    [[ $(jq -r '.mode // empty' "$abort_response") == live ]] || {
-      echo 'previous failed acceptance Run did not return traffic to live' >&2
-      return 1
-    }
+    # ActivateAcceptanceRun replaces a drained failed Run while holding the
+    # traffic-mode row lock. Formal traffic therefore never observes a brief
+    # live interval between two validation Runs.
     wait_for_terminal_acceptance_tasks_to_drain
   elif [[ $traffic_mode != live ]]; then
     echo "unsupported production traffic mode before acceptance: $traffic_mode" >&2
@@ -1455,13 +1445,23 @@ mark_run_failed() {
   local response=$temporary_root/failed-run.json
   local body
   [[ -n $run_id ]] || return 0
-  local artifact_names
+  local artifact_names task_count gemini_count video_count queue_state
+  task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;" 2>/dev/null || echo 0)
+  gemini_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND model='$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL';" 2>/dev/null || echo 0)
+  video_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND model='$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL';" 2>/dev/null || echo 0)
+  queue_state=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;" 2>/dev/null || echo '0:0')
+  [[ $task_count =~ ^[0-9]+$ ]] || task_count=0
+  [[ $gemini_count =~ ^[0-9]+$ ]] || gemini_count=0
+  [[ $video_count =~ ^[0-9]+$ ]] || video_count=0
+  [[ $queue_state =~ ^[0-9]+:[0-9]+$ ]] || queue_state=0:0
   artifact_names=$(find "$report_root" -maxdepth 1 -type f -exec basename {} \; 2>/dev/null |
     LC_ALL=C sort | jq -Rsc 'split("\n") | map(select(length > 0))')
   body=$(jq -cn \
     --arg reason "$reason" \
     --arg gateId "${failure_gate_id:-unknown_gate}" \
     --arg activeProfile "$active_profile" \
+    --arg queueState "$queue_state" \
+    --argjson taskCount "$task_count" \
     --argjson artifacts "$artifact_names" \
     '{
       passed:false,
@@ -1469,7 +1469,8 @@ mark_run_failed() {
       report:{
         gateId:$gateId,
         activeCapacityProfile:$activeProfile,
-        queueFinal:null,
+        tasks:$taskCount,
+        queueAtFailure:$queueState,
         completedArtifacts:$artifacts
       }
     }')
@@ -1482,6 +1483,10 @@ mark_run_failed() {
     --arg stableCapacityProfile "$stable_profile" \
     --arg gateId "${failure_gate_id:-unknown_gate}" \
     --arg reason "$reason" \
+    --arg queueState "$queue_state" \
+    --argjson taskCount "$task_count" \
+    --argjson geminiCount "$gemini_count" \
+    --argjson videoCount "$video_count" \
     --argjson artifacts "$artifact_names" \
     '{
       runId:$runId,
@@ -1490,9 +1495,10 @@ mark_run_failed() {
       snapshotConfigHash:$snapshotConfigHash,
       snapshotSha256:$snapshotSha256,
       stableCapacityProfile:$stableCapacityProfile,
-      tasks:null,
-      geminiTasks:null,
-      videoTasks:null,
+      tasks:$taskCount,
+      geminiTasks:$geminiCount,
+      videoTasks:$videoCount,
+      queueAtFailure:$queueState,
       realCanaryPassed:false,
       failureReason:$reason,
       completedArtifacts:$artifacts,
@@ -1530,7 +1536,7 @@ apply_capacity_profile() {
   local slots pool media global baseline_replicas
   local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
   local min_idle=4
-  local max_idle_seconds=30
+  local max_idle_seconds=$AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS
   case $profile in
     P24) slots=24; pool=32; media=24 ;;
     P28) slots=28; pool=36; media=28 ;;
@@ -1715,7 +1721,7 @@ apply_autoscaling_profile() {
   global=$((slots * (max_replicas_ningbo + max_replicas_hongkong)))
   local command_text capacity_output
   printf -v command_text \
-    'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
+    'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
     "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
     "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
     "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
@@ -1726,6 +1732,7 @@ apply_autoscaling_profile() {
     "$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS" \
     "$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" \
     "$AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS" \
+    "$AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS" \
     "$media" "$AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY" "$((slots * 2))" \
     "$AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB" \
     "$AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES" \
@@ -2560,7 +2567,7 @@ workload_metrics_for_site() {
         fi
         ;;
     esac
-    cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics" \
+    cluster_ssh "$host" "curl -fsS --max-time 10 http://$pod_ip:8088/metrics" \
       </dev/null || return 1
   done <<<"$targets"
 }