fix(acceptance): 增强生产验收控制面容错

原因：资源前置检查中的一次瞬时 SSH 中断会直接终止 Run，失败收尾又可能因 partial 报告已存在而无法完成。\n\n影响：节点内存采集增加有界重试并区分传输故障；验收管理 GET 与 finish 支持香港本地入口安全回退，并对响应丢失后的 409 反查 Run 终态；已有 partial 报告会验证复用。\n\n验证：bash -n、ShellCheck、production-acceptance-script-test.sh、manual-release-test.sh 通过。

scripts/cluster/run-production-acceptance.sh

@@ -266,6 +266,7 @@ failure_reason=
 failure_gate_id=
 failure_recorded=false
 acceptance_admin_base=
+acceptance_admin_fallback_base=
 acceptance_group_id=
 acceptance_participants_json='[]'
 AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
@@ -612,17 +613,24 @@ admin_request() {
   local body=${3:-}
   local output=$4
   local status response payload token_encoded body_encoded has_body=false
-  local service_ip service_port remote_script remote_command
+  local service_ip service_port fallback_service_ip fallback_service_port
+  local remote_script remote_command fallback_used=false
   if [[ -z $acceptance_admin_base ]]; then
     service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
       -o 'jsonpath={.spec.clusterIP}')
     service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
       -o 'jsonpath={.spec.ports[0].port}')
-    [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ ]] || {
+    fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
+      -o 'jsonpath={.spec.clusterIP}')
+    fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
+      -o 'jsonpath={.spec.ports[0].port}')
+    [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ &&
+       $fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || {
       echo 'acceptance admin Service has an invalid cluster address' >&2
       return 1
     }
     acceptance_admin_base=http://$service_ip:$service_port
+    acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port
   fi
   token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" |
     openssl base64 -A)
@@ -651,11 +659,38 @@ fi
     "$remote_script" "$acceptance_admin_base" "$method" "$path" \
     "$token_encoded" "$body_encoded" "$has_body"
   if ! response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_command"); then
-    echo "acceptance control API transport failed: method=$method path=$path" >&2
-    return 1
+    if [[ $method != GET && $path != /api/admin/system/acceptance/runs/*/finish ]]; then
+      echo "acceptance control API transport failed without safe retry: method=$method path=$path" >&2
+      return 1
+    fi
+    printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
+      "$remote_script" "$acceptance_admin_fallback_base" "$method" "$path" \
+      "$token_encoded" "$body_encoded" "$has_body"
+    if ! response=$(cluster_ssh "$CLUSTER_HONGKONG_HOST" "$remote_command"); then
+      echo "acceptance control API transport failed on both sites: method=$method path=$path" >&2
+      return 1
+    fi
+    fallback_used=true
   fi
   status=${response##*$'\n'}
   payload=${response%$'\n'*}
+  if [[ $fallback_used == true && $status == 409 &&
+        $path == /api/admin/system/acceptance/runs/*/finish ]]; then
+    local expected_status verification_path verification_response verification_status
+    expected_status=$(jq -r 'if .passed == true then "passed" else "failed" end' <<<"$body")
+    verification_path=${path%/finish}
+    printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
+      "$remote_script" "$acceptance_admin_fallback_base" GET "$verification_path" \
+      "$token_encoded" '' false
+    if verification_response=$(cluster_ssh "$CLUSTER_HONGKONG_HOST" "$remote_command"); then
+      verification_status=${verification_response##*$'\n'}
+      payload=${verification_response%$'\n'*}
+      if [[ $verification_status == 200 &&
+            $(jq -r '.status // empty' <<<"$payload") == "$expected_status" ]]; then
+        status=200
+      fi
+    fi
+  fi
   printf '%s' "$payload" >"$output"
   [[ $status =~ ^2[0-9][0-9]$ ]] || {
     echo "acceptance control API failed: method=$method path=$path status=$status" >&2
@@ -721,6 +756,23 @@ verify_release_cas() {
   fi
 }
 
+read_node_memory_percent() {
+  local node=$1
+  local attempt output memory_percent
+  for attempt in 1 2 3; do
+    if output=$(remote_kubectl top node "$node" --no-headers 2>/dev/null); then
+      memory_percent=$(awk '{value=$5; sub(/%$/, "", value); print value}' <<<"$output")
+      if [[ $memory_percent =~ ^[0-9]+$ ]]; then
+        printf '%s\n' "$memory_percent"
+        return 0
+      fi
+    fi
+    (( attempt == 3 )) || sleep 2
+  done
+  echo "Worker node memory collection failed after retries: node=$node attempts=3" >&2
+  return 2
+}
+
 verify_resource_preconditions() {
   local report=$temporary_root/resource-preconditions.csv
   local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
@@ -772,11 +824,11 @@ verify_resource_preconditions() {
   for sample in $(seq 1 90); do
     while read -r node; do
       [[ -n $node ]] || continue
-      memory_percent=$(remote_kubectl top node "$node" --no-headers |
-        awk '{value=$5; sub(/%$/, "", value); print value}')
-      [[ $memory_percent =~ ^[0-9]+$ &&
-         $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
-        echo "Worker node memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
+      if ! memory_percent=$(read_node_memory_percent "$node"); then
+        return 2
+      fi
+      [[ $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
+        echo "Worker node memory precondition failed: node=$node memory_percent=$memory_percent limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
         return 1
       }
       printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
@@ -1446,6 +1498,24 @@ create_and_activate_run() {
      $(jq -r '.runId' "$activate_response") == "$run_id" ]]
 }
 
+merge_partial_acceptance_report() {
+  local summary=$1
+  local output=$2
+  if [[ -e $output ]]; then
+    [[ -f $output && ! -L $output ]] || {
+      echo "existing partial acceptance report is not a regular file: $output" >&2
+      return 1
+    }
+    node "$cluster_root/scripts/acceptance/report.mjs" validate \
+      --input "$output" >/dev/null
+    return 0
+  fi
+  node "$cluster_root/scripts/acceptance/report.mjs" merge-production \
+    --local-report "$local_acceptance_report" \
+    --production-summary "$summary" \
+    --output "$output" >/dev/null
+}
+
 mark_run_failed() {
   local reason=$1
   local response=$temporary_root/failed-run.json
@@ -1518,10 +1588,9 @@ mark_run_failed() {
       passed:false
     }' >"$report_root/summary.partial.json"
   chmod 0600 "$report_root/summary.partial.json"
-  node "$cluster_root/scripts/acceptance/report.mjs" merge-production \
-    --local-report "$local_acceptance_report" \
-    --production-summary "$report_root/summary.partial.json" \
-    --output "$report_root/acceptance-report.partial.json" >/dev/null || true
+  merge_partial_acceptance_report \
+    "$report_root/summary.partial.json" \
+    "$report_root/acceptance-report.partial.json" || true
   if admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$body" "$response"; then
     failure_recorded=true
     if ! wait_for_terminal_acceptance_tasks_to_drain; then
@@ -3602,9 +3671,16 @@ if ! wait_for_post_rollout_stability P24; then
   exit 1
 fi
 verify_release_cas
-if ! verify_resource_preconditions; then
-  failure_gate_id=resource_preconditions
-  failure_reason='production resource preconditions failed'
+resource_precondition_status=0
+verify_resource_preconditions || resource_precondition_status=$?
+if (( resource_precondition_status != 0 )); then
+  if (( resource_precondition_status == 2 )); then
+    failure_gate_id=resource_precondition_transport
+    failure_reason='production resource precondition metrics could not be collected after bounded retries'
+  else
+    failure_gate_id=resource_preconditions
+    failure_reason='production resource preconditions failed'
+  fi
   mark_run_failed "$failure_reason"
   exit 1
 fi