fix(acceptance): 增强生产验收控制面容错

原因：资源前置检查中的一次瞬时 SSH 中断会直接终止 Run，失败收尾又可能因 partial 报告已存在而无法完成。\n\n影响：节点内存采集增加有界重试并区分传输故障；验收管理 GET 与 finish 支持香港本地入口安全回退，并对响应丢失后的 409 反查 Run 终态；已有 partial 报告会验证复用。\n\n验证：bash -n、ShellCheck、production-acceptance-script-test.sh、manual-release-test.sh 通过。

scripts/cluster/run-production-acceptance.sh

@@ -266,6 +266,7 @@ failure_reason=
 failure_gate_id=
 failure_recorded=false
 acceptance_admin_base=
+acceptance_admin_fallback_base=
 acceptance_group_id=
 acceptance_participants_json='[]'
 AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
@@ -612,17 +613,24 @@ admin_request() {
   local body=${3:-}
   local output=$4
   local status response payload token_encoded body_encoded has_body=false
-  local service_ip service_port remote_script remote_command
+  local service_ip service_port fallback_service_ip fallback_service_port
+  local remote_script remote_command fallback_used=false
   if [[ -z $acceptance_admin_base ]]; then
     service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
       -o 'jsonpath={.spec.clusterIP}')
     service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
       -o 'jsonpath={.spec.ports[0].port}')
-    [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ ]] || {
+    fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
+      -o 'jsonpath={.spec.clusterIP}')
+    fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
+      -o 'jsonpath={.spec.ports[0].port}')
+    [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ &&
+       $fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || {
       echo 'acceptance admin Service has an invalid cluster address' >&2
       return 1
     }
     acceptance_admin_base=http://$service_ip:$service_port
+    acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port
   fi
   token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" |
     openssl base64 -A)
@@ -651,11 +659,38 @@ fi
     "$remote_script" "$acceptance_admin_base" "$method" "$path" \
     "$token_encoded" "$body_encoded" "$has_body"
   if ! response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_command"); then
-    echo "acceptance control API transport failed: method=$method path=$path" >&2
+    if [[ $method != GET && $path != /api/admin/system/acceptance/runs/*/finish ]]; then
-    return 1
+      echo "acceptance control API transport failed without safe retry: method=$method path=$path" >&2
+      return 1
+    fi
+    printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
+      "$remote_script" "$acceptance_admin_fallback_base" "$method" "$path" \
+      "$token_encoded" "$body_encoded" "$has_body"
+    if ! response=$(cluster_ssh "$CLUSTER_HONGKONG_HOST" "$remote_command"); then
+      echo "acceptance control API transport failed on both sites: method=$method path=$path" >&2
+      return 1
+    fi
+    fallback_used=true
   fi
   status=${response##*$'\n'}
   payload=${response%$'\n'*}
+  if [[ $fallback_used == true && $status == 409 &&
+        $path == /api/admin/system/acceptance/runs/*/finish ]]; then
+    local expected_status verification_path verification_response verification_status
+    expected_status=$(jq -r 'if .passed == true then "passed" else "failed" end' <<<"$body")
+    verification_path=${path%/finish}
+    printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
+      "$remote_script" "$acceptance_admin_fallback_base" GET "$verification_path" \
+      "$token_encoded" '' false
+    if verification_response=$(cluster_ssh "$CLUSTER_HONGKONG_HOST" "$remote_command"); then
+      verification_status=${verification_response##*$'\n'}
+      payload=${verification_response%$'\n'*}
+      if [[ $verification_status == 200 &&
+            $(jq -r '.status // empty' <<<"$payload") == "$expected_status" ]]; then
+        status=200
+      fi
+    fi
+  fi
   printf '%s' "$payload" >"$output"
   [[ $status =~ ^2[0-9][0-9]$ ]] || {
     echo "acceptance control API failed: method=$method path=$path status=$status" >&2
@@ -721,6 +756,23 @@ verify_release_cas() {
   fi
 }
 
+read_node_memory_percent() {
+  local node=$1
+  local attempt output memory_percent
+  for attempt in 1 2 3; do
+    if output=$(remote_kubectl top node "$node" --no-headers 2>/dev/null); then
+      memory_percent=$(awk '{value=$5; sub(/%$/, "", value); print value}' <<<"$output")
+      if [[ $memory_percent =~ ^[0-9]+$ ]]; then
+        printf '%s\n' "$memory_percent"
+        return 0
+      fi
+    fi
+    (( attempt == 3 )) || sleep 2
+  done
+  echo "Worker node memory collection failed after retries: node=$node attempts=3" >&2
+  return 2
+}
+
 verify_resource_preconditions() {
   local report=$temporary_root/resource-preconditions.csv
   local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
@@ -772,11 +824,11 @@ verify_resource_preconditions() {
   for sample in $(seq 1 90); do
     while read -r node; do
       [[ -n $node ]] || continue
-      memory_percent=$(remote_kubectl top node "$node" --no-headers |
+      if ! memory_percent=$(read_node_memory_percent "$node"); then
-        awk '{value=$5; sub(/%$/, "", value); print value}')
+        return 2
-      [[ $memory_percent =~ ^[0-9]+$ &&
+      fi
-         $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
+      [[ $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
-        echo "Worker node memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
+        echo "Worker node memory precondition failed: node=$node memory_percent=$memory_percent limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
         return 1
       }
       printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
@@ -1446,6 +1498,24 @@ create_and_activate_run() {
      $(jq -r '.runId' "$activate_response") == "$run_id" ]]
 }
 
+merge_partial_acceptance_report() {
+  local summary=$1
+  local output=$2
+  if [[ -e $output ]]; then
+    [[ -f $output && ! -L $output ]] || {
+      echo "existing partial acceptance report is not a regular file: $output" >&2
+      return 1
+    }
+    node "$cluster_root/scripts/acceptance/report.mjs" validate \
+      --input "$output" >/dev/null
+    return 0
+  fi
+  node "$cluster_root/scripts/acceptance/report.mjs" merge-production \
+    --local-report "$local_acceptance_report" \
+    --production-summary "$summary" \
+    --output "$output" >/dev/null
+}
+
 mark_run_failed() {
   local reason=$1
   local response=$temporary_root/failed-run.json
@@ -1518,10 +1588,9 @@ mark_run_failed() {
       passed:false
     }' >"$report_root/summary.partial.json"
   chmod 0600 "$report_root/summary.partial.json"
-  node "$cluster_root/scripts/acceptance/report.mjs" merge-production \
+  merge_partial_acceptance_report \
-    --local-report "$local_acceptance_report" \
+    "$report_root/summary.partial.json" \
-    --production-summary "$report_root/summary.partial.json" \
+    "$report_root/acceptance-report.partial.json" || true
-    --output "$report_root/acceptance-report.partial.json" >/dev/null || true
   if admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$body" "$response"; then
     failure_recorded=true
     if ! wait_for_terminal_acceptance_tasks_to_drain; then
@@ -3602,9 +3671,16 @@ if ! wait_for_post_rollout_stability P24; then
   exit 1
 fi
 verify_release_cas
-if ! verify_resource_preconditions; then
+resource_precondition_status=0
-  failure_gate_id=resource_preconditions
+verify_resource_preconditions || resource_precondition_status=$?
-  failure_reason='production resource preconditions failed'
+if (( resource_precondition_status != 0 )); then
+  if (( resource_precondition_status == 2 )); then
+    failure_gate_id=resource_precondition_transport
+    failure_reason='production resource precondition metrics could not be collected after bounded retries'
+  else
+    failure_gate_id=resource_preconditions
+    failure_reason='production resource preconditions failed'
+  fi
   mark_run_failed "$failure_reason"
   exit 1
 fi

tests/release/production-acceptance-script-test.sh

@@ -40,6 +40,33 @@ awk '
 # shellcheck source=/dev/null
 source "$tmp/pressure-monitor.sh"
 
+awk '
+  /^read_node_memory_percent\(\)/ { capture = 1 }
+  capture && /^verify_resource_preconditions\(\)/ { exit }
+  capture { print }
+' "$script" >"$tmp/resource-precondition.sh"
+
+# shellcheck source=/dev/null
+source "$tmp/resource-precondition.sh"
+
+awk '
+  /^merge_partial_acceptance_report\(\)/ { capture = 1 }
+  capture && /^mark_run_failed\(\)/ { exit }
+  capture { print }
+' "$script" >"$tmp/partial-report.sh"
+
+# shellcheck source=/dev/null
+source "$tmp/partial-report.sh"
+
+awk '
+  /^admin_request\(\)/ { capture = 1 }
+  capture && /^verify_release_cas\(\)/ { exit }
+  capture { print }
+' "$script" >"$tmp/admin-request.sh"
+
+# shellcheck source=/dev/null
+source "$tmp/admin-request.sh"
+
 cluster_root=$tmp/repository
 git init -q "$cluster_root"
 git -C "$cluster_root" config user.name 'Acceptance Script Test'
@@ -108,6 +135,87 @@ grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics"
 [[ $(metric_max test_metric <<<"$metrics") == 1 ]]
 [[ $(metric_sum test_metric <<<"$metrics") == 2 ]]
 
+memory_attempts=$tmp/memory-attempts
+printf '0\n' >"$memory_attempts"
+remote_kubectl() {
+  local attempts
+  attempts=$(<"$memory_attempts")
+  attempts=$((attempts + 1))
+  printf '%s\n' "$attempts" >"$memory_attempts"
+  if (( attempts == 1 )); then
+    return 1
+  fi
+  printf 'easyai-hongkong 100m 5%% 1024Mi 64%%\n'
+}
+sleep() { :; }
+[[ $(read_node_memory_percent easyai-hongkong) == 64 ]]
+[[ $(<"$memory_attempts") == 2 ]]
+
+remote_kubectl() { return 1; }
+if read_node_memory_percent easyai-hongkong >/dev/null 2>&1; then
+  echo 'unavailable node metrics passed resource precondition collection' >&2
+  exit 1
+else
+  resource_status=$?
+fi
+[[ $resource_status == 2 ]]
+
+cluster_root=$tmp/repository
+local_acceptance_report=$tmp/local-report.json
+summary=$tmp/summary.partial.json
+partial=$tmp/acceptance-report.partial.json
+printf '{}\n' >"$local_acceptance_report"
+printf '{}\n' >"$summary"
+printf '{}\n' >"$partial"
+node_calls=$tmp/node-calls
+node() { printf '%s\n' "$*" >>"$node_calls"; }
+merge_partial_acceptance_report "$summary" "$partial"
+[[ $(wc -l <"$node_calls" | tr -d ' ') == 1 ]]
+grep -Fq ' validate --input ' "$node_calls"
+unlink "$partial"
+merge_partial_acceptance_report "$summary" "$partial"
+[[ $(wc -l <"$node_calls" | tr -d ' ') == 2 ]]
+grep -Fq ' merge-production --local-report ' "$node_calls"
+
+acceptance_admin_base=http://10.43.1.1:8088
+acceptance_admin_fallback_base=http://10.43.2.2:8088
+AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN=test-token
+CLUSTER_NINGBO_HOST=root@ningbo.invalid
+CLUSTER_HONGKONG_HOST=root@hongkong.invalid
+admin_attempts=$tmp/admin-attempts
+printf '0\n' >"$admin_attempts"
+cluster_ssh() {
+  local attempts
+  attempts=$(<"$admin_attempts")
+  attempts=$((attempts + 1))
+  printf '%s\n' "$attempts" >"$admin_attempts"
+  if (( attempts == 1 )); then
+    return 1
+  fi
+  printf '{"status":"running"}\n200\n'
+}
+admin_output=$tmp/admin-output.json
+admin_request GET /api/admin/system/acceptance/runs/test-run '' "$admin_output"
+[[ $(jq -r '.status' "$admin_output") == running ]]
+[[ $(<"$admin_attempts") == 2 ]]
+
+printf '0\n' >"$admin_attempts"
+cluster_ssh() {
+  local attempts
+  attempts=$(<"$admin_attempts")
+  attempts=$((attempts + 1))
+  printf '%s\n' "$attempts" >"$admin_attempts"
+  case $attempts in
+    1) return 1 ;;
+    2) printf '{"error":"acceptance run is not running"}\n409\n' ;;
+    3) printf '{"status":"failed"}\n200\n' ;;
+  esac
+}
+admin_body='{"passed":false}'
+admin_request POST /api/admin/system/acceptance/runs/test-run/finish "$admin_body" "$admin_output"
+[[ $(jq -r '.status' "$admin_output") == failed ]]
+[[ $(<"$admin_attempts") == 3 ]]
+
 passing_pressure_row='2026-08-01T00:00:00Z,0,0,899,149,200,84,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100'
 pressure_row_passes_live_gates "$passing_pressure_row"
 [[ $(pressure_row_live_gate_reason '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100') == \