From 1bd9e618cac0606a1109f55094cb4cb2620e58b1 Mon Sep 17 00:00:00 2001
From: wangbo <wangbolhr@163.com>
Date: Fri, 31 Jul 2026 03:03:11 +0800
Subject: [PATCH] =?UTF-8?q?fix(acceptance):=20=E9=87=8D=E8=AF=95=E7=9E=AC?=
 =?UTF-8?q?=E6=97=B6=E5=8E=8B=E5=8A=9B=E9=87=87=E6=A0=B7=E5=A4=B1=E8=B4=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pod 滚动后的首次 metrics 读取可能短暂失败，原采样器会直接退出且主流程忽略后台状态，导致验收缺少资源曲线。\n\n采样器现在允许两次瞬时失败，连续三次失败才中止；主流程同时检查采样进程状态，确保任何曲线中断都会明确阻断验收。\n\n验证：bash -n、ShellCheck、git diff --check。
---
 scripts/cluster/run-production-acceptance.sh | 43 +++++++++++++++-----
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh
index 2da29a7..c441cf7 100755
--- a/scripts/cluster/run-production-acceptance.sh
+++ b/scripts/cluster/run-production-acceptance.sh
@@ -1078,9 +1078,10 @@ sample_pressure() {
   local output=$1
   local stop_file=$2
   local database_state node_memory_percent pod_memory_mib pool_state pod metrics
+  local consecutive_failures=0
   echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max' >"$output"
   while [[ ! -f $stop_file ]]; do
-    database_state=$(database_query "
+    if ! database_state=$(database_query "
 SELECT
   count(*) FILTER (WHERE status='queued')||','||
   count(*) FILTER (WHERE status='running')||','||
@@ -1088,11 +1089,17 @@ SELECT
   (SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
   (SELECT setting FROM pg_settings WHERE name='max_connections')
 FROM gateway_tasks
-WHERE acceptance_run_id='$run_id'::uuid;") || break
-    node_memory_percent=$(remote_kubectl top nodes --no-headers |
-      awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break
-    pod_memory_mib=$(current_max_pod_memory_mib) || break
-    pool_state=$(
+WHERE acceptance_run_id='$run_id'::uuid;"); then
+      database_state=
+    fi
+    if ! node_memory_percent=$(remote_kubectl top nodes --no-headers 2>/dev/null |
+      awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}'); then
+      node_memory_percent=
+    fi
+    if ! pod_memory_mib=$(current_max_pod_memory_mib 2>/dev/null); then
+      pod_memory_mib=
+    fi
+    if ! pool_state=$(
       for workload in api worker; do
         for site in ningbo hongkong; do
           pod=$(remote_kubectl get pods -n "$namespace" \
@@ -1144,7 +1151,20 @@ WHERE acceptance_run_id='$run_id'::uuid;") || break
           }
           END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated}
         '
-    ) || break
+    ) 2>/dev/null; then
+      pool_state=
+    fi
+    if [[ -z $database_state || -z $node_memory_percent ||
+          -z $pod_memory_mib || -z $pool_state ]]; then
+      ((consecutive_failures += 1))
+      if (( consecutive_failures >= 3 )); then
+        echo 'pressure sampling failed three consecutive times' >&2
+        return 1
+      fi
+      sleep 2
+      continue
+    fi
+    consecutive_failures=0
     printf '%s,%s,%s,%s,%s\n' \
       "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$database_state" "$node_memory_percent" "$pod_memory_mib" "$pool_state" >>"$output"
     sleep 5
@@ -1201,7 +1221,7 @@ run_capacity_round() {
   local prefix=$report_root/$profile_slug-$repetition
   local pressure_report=$prefix-pressure.csv
   local pressure_stop=$temporary_root/pressure-stop
-  local pressure_pid status=0
+  local pressure_pid status=0 pressure_status=0
   local baseline_memory_mib
   local expected_slots
   case $profile in
@@ -1219,8 +1239,11 @@ run_capacity_round() {
   if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi
   if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi
   touch "$pressure_stop"
-  wait "$pressure_pid" || true
-  (( status == 0 )) || return "$status"
+  wait "$pressure_pid" || pressure_status=$?
+  (( status == 0 && pressure_status == 0 )) || {
+    (( status != 0 )) && return "$status"
+    return "$pressure_status"
+  }
   verify_pressure_report "$pressure_report" "$expected_slots"
   verify_runtime_gates
   wait_for_memory_recovery "$baseline_memory_mib"