fix(acceptance): 复用集群 SSH 控制连接

线上验收在宁波 SSH 建连超时时于负载前失败。为集群脚本增加受限 ControlMaster 复用和仅建连阶段的 ConnectionAttempts，避免重放远程命令。\n\n生产快照导出增加三次只读重试、原子落盘和空 Pod 清理防护。已通过 bash -n、ShellCheck、production acceptance 脚本测试、人工发布脚本测试，并验证宁波真实 SSH 控制连接可复用。

scripts/cluster/common.sh

@@ -5,6 +5,12 @@ set -euo pipefail
 cluster_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
 cluster_env_file=${AI_GATEWAY_CLUSTER_ENV_FILE:-"$cluster_root/.env.local"}
 cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-"$HOME/.ssh/id_ed25519_easyai_gateway_cluster"}
+cluster_ssh_control_dir=${AI_GATEWAY_CLUSTER_SSH_CONTROL_DIR:-"/tmp/easyai-gateway-ssh-${UID:-$(id -u)}-$$"}
+
+prepare_cluster_ssh_control_dir() {
+  (umask 077 && mkdir -p -- "$cluster_ssh_control_dir")
+  chmod 0700 "$cluster_ssh_control_dir"
+}
 
 load_cluster_env() {
   [[ -f $cluster_env_file && ! -L $cluster_env_file ]] || {
@@ -91,20 +97,30 @@ load_cluster_env() {
 cluster_ssh() {
   local host=$1
   shift
+  prepare_cluster_ssh_control_dir
   ssh \
     -i "$cluster_ssh_key" \
     -o BatchMode=yes \
+    -o ConnectionAttempts=3 \
     -o ConnectTimeout=10 \
+    -o ControlMaster=auto \
+    -o ControlPersist=120 \
+    -o "ControlPath=$cluster_ssh_control_dir/%C" \
     -o ServerAliveInterval=15 \
     -o ServerAliveCountMax=3 \
     "$host" "$@"
 }
 
 cluster_scp() {
+  prepare_cluster_ssh_control_dir
   scp \
     -i "$cluster_ssh_key" \
     -o BatchMode=yes \
+    -o ConnectionAttempts=3 \
     -o ConnectTimeout=10 \
+    -o ControlMaster=auto \
+    -o ControlPersist=120 \
+    -o "ControlPath=$cluster_ssh_control_dir/%C" \
     "$@"
 }
 

scripts/cluster/run-production-acceptance.sh

@@ -359,7 +359,11 @@ export_production_snapshot() {
         --release-sha "$release_sha" \
         --output "$output" >/dev/null
   else
-    cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$output" <<'REMOTE'
+    local attempt temporary_output
+    temporary_output=$output.remote
+    rm -f -- "$temporary_output"
+    for attempt in 1 2 3; do
+      if cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$temporary_output" <<'REMOTE'
 set -euo pipefail
 release_sha=$1
 namespace=easyai
@@ -367,12 +371,14 @@ pod=$(k3s kubectl get pods -n "$namespace" \
   -l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
   --field-selector status.phase=Running \
   -o 'jsonpath={.items[0].metadata.name}')
-[[ $pod == easyai-api-ningbo-* ]]
+[[ -n $pod && $pod == easyai-api-ningbo-* ]]
 [[ $(k3s kubectl get pod "$pod" -n "$namespace" \
   -o 'jsonpath={.status.containerStatuses[0].ready}') == true ]]
 snapshot=/tmp/easyai-production-acceptance-snapshot-$release_sha.json
 cleanup() {
-  k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
+  if [[ -n ${pod:-} ]]; then
+    k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
+  fi
 }
 trap cleanup EXIT
 k3s kubectl exec -n "$namespace" "$pod" -- \
@@ -380,6 +386,18 @@ k3s kubectl exec -n "$namespace" "$pod" -- \
     --release-sha "$release_sha" --output "$snapshot" >/dev/null
 k3s kubectl exec -n "$namespace" "$pod" -- cat "$snapshot"
 REMOTE
+      then
+        mv -- "$temporary_output" "$output"
+        break
+      fi
+      rm -f -- "$temporary_output"
+      if (( attempt == 3 )); then
+        echo 'production snapshot export failed after three read-only attempts' >&2
+        return 1
+      fi
+      echo "production snapshot export attempt $attempt failed; retrying read-only export" >&2
+      sleep "$attempt"
+    done
   fi
   chmod 0600 "$output"
   "$acceptance_snapshot_binary" validate --input "$output" >/dev/null

tests/release/production-acceptance-script-test.sh

@@ -7,6 +7,21 @@ tmp=$(mktemp -d)
 trap 'rm -rf "$tmp"' EXIT
 
 script=$root/scripts/cluster/run-production-acceptance.sh
+
+# shellcheck source=scripts/cluster/common.sh
+source "$root/scripts/cluster/common.sh"
+ssh() {
+  printf '%s\n' "$@" >"$tmp/ssh-arguments"
+}
+cluster_ssh_key=$tmp/test-key
+cluster_ssh_control_dir=$tmp/ssh-control
+cluster_ssh root@cluster.invalid true
+grep -Fxq 'ConnectionAttempts=3' "$tmp/ssh-arguments"
+grep -Fxq 'ControlMaster=auto' "$tmp/ssh-arguments"
+grep -Fxq 'ControlPersist=120' "$tmp/ssh-arguments"
+grep -Fxq "ControlPath=$cluster_ssh_control_dir/%C" "$tmp/ssh-arguments"
+[[ $(stat -f '%Lp' "$cluster_ssh_control_dir") == 700 ]]
+
 awk '
   /^is_release_ancestor\(\)/ { capture = 1 }
   capture && /^create_and_activate_run\(\)/ { exit }