From 7623449e3392f686a8065d2a512165480c8febcd Mon Sep 17 00:00:00 2001 From: wangbo Date: Sat, 1 Aug 2026 15:59:00 +0800 Subject: [PATCH] =?UTF-8?q?fix(acceptance):=20=E5=A4=8D=E7=94=A8=E9=9B=86?= =?UTF-8?q?=E7=BE=A4=20SSH=20=E6=8E=A7=E5=88=B6=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 线上验收在宁波 SSH 建连超时时于负载前失败。为集群脚本增加受限 ControlMaster 复用和仅建连阶段的 ConnectionAttempts，避免重放远程命令。\n\n生产快照导出增加三次只读重试、原子落盘和空 Pod 清理防护。已通过 bash -n、ShellCheck、production acceptance 脚本测试、人工发布脚本测试，并验证宁波真实 SSH 控制连接可复用。 --- scripts/cluster/common.sh | 16 +++++++++++++ scripts/cluster/run-production-acceptance.sh | 24 ++++++++++++++++--- .../production-acceptance-script-test.sh | 15 ++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/scripts/cluster/common.sh b/scripts/cluster/common.sh index b203648..f929c3b 100755 --- a/scripts/cluster/common.sh +++ b/scripts/cluster/common.sh @@ -5,6 +5,12 @@ set -euo pipefail cluster_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) cluster_env_file=${AI_GATEWAY_CLUSTER_ENV_FILE:-"$cluster_root/.env.local"} cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-"$HOME/.ssh/id_ed25519_easyai_gateway_cluster"} +cluster_ssh_control_dir=${AI_GATEWAY_CLUSTER_SSH_CONTROL_DIR:-"/tmp/easyai-gateway-ssh-${UID:-$(id -u)}-$$"} + +prepare_cluster_ssh_control_dir() { + (umask 077 && mkdir -p -- "$cluster_ssh_control_dir") + chmod 0700 "$cluster_ssh_control_dir" +} load_cluster_env() { [[ -f $cluster_env_file && ! -L $cluster_env_file ]] || { @@ -91,20 +97,30 @@ load_cluster_env() { cluster_ssh() { local host=$1 shift + prepare_cluster_ssh_control_dir ssh \ -i "$cluster_ssh_key" \ -o BatchMode=yes \ + -o ConnectionAttempts=3 \ -o ConnectTimeout=10 \ + -o ControlMaster=auto \ + -o ControlPersist=120 \ + -o "ControlPath=$cluster_ssh_control_dir/%C" \ -o ServerAliveInterval=15 \ -o ServerAliveCountMax=3 \ "$host" "$@" } cluster_scp() { + prepare_cluster_ssh_control_dir scp \ -i "$cluster_ssh_key" \ -o BatchMode=yes \ + -o ConnectionAttempts=3 \ -o ConnectTimeout=10 \ + -o ControlMaster=auto \ + -o ControlPersist=120 \ + -o "ControlPath=$cluster_ssh_control_dir/%C" \ "$@" } diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 2917111..5c812ef 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -359,7 +359,11 @@ export_production_snapshot() { --release-sha "$release_sha" \ --output "$output" >/dev/null else - cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$output" <<'REMOTE' + local attempt temporary_output + temporary_output=$output.remote + rm -f -- "$temporary_output" + for attempt in 1 2 3; do + if cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$temporary_output" <<'REMOTE' set -euo pipefail release_sha=$1 namespace=easyai @@ -367,12 +371,14 @@ pod=$(k3s kubectl get pods -n "$namespace" \ -l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \ --field-selector status.phase=Running \ -o 'jsonpath={.items[0].metadata.name}') -[[ $pod == easyai-api-ningbo-* ]] +[[ -n $pod && $pod == easyai-api-ningbo-* ]] [[ $(k3s kubectl get pod "$pod" -n "$namespace" \ -o 'jsonpath={.status.containerStatuses[0].ready}') == true ]] snapshot=/tmp/easyai-production-acceptance-snapshot-$release_sha.json cleanup() { - k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true + if [[ -n ${pod:-} ]]; then + k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true + fi } trap cleanup EXIT k3s kubectl exec -n "$namespace" "$pod" -- \ @@ -380,6 +386,18 @@ k3s kubectl exec -n "$namespace" "$pod" -- \ --release-sha "$release_sha" --output "$snapshot" >/dev/null k3s kubectl exec -n "$namespace" "$pod" -- cat "$snapshot" REMOTE + then + mv -- "$temporary_output" "$output" + break + fi + rm -f -- "$temporary_output" + if (( attempt == 3 )); then + echo 'production snapshot export failed after three read-only attempts' >&2 + return 1 + fi + echo "production snapshot export attempt $attempt failed; retrying read-only export" >&2 + sleep "$attempt" + done fi chmod 0600 "$output" "$acceptance_snapshot_binary" validate --input "$output" >/dev/null diff --git a/tests/release/production-acceptance-script-test.sh b/tests/release/production-acceptance-script-test.sh index 9d35dea..8d4dae0 100755 --- a/tests/release/production-acceptance-script-test.sh +++ b/tests/release/production-acceptance-script-test.sh @@ -7,6 +7,21 @@ tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT script=$root/scripts/cluster/run-production-acceptance.sh + +# shellcheck source=scripts/cluster/common.sh +source "$root/scripts/cluster/common.sh" +ssh() { + printf '%s\n' "$@" >"$tmp/ssh-arguments" +} +cluster_ssh_key=$tmp/test-key +cluster_ssh_control_dir=$tmp/ssh-control +cluster_ssh root@cluster.invalid true +grep -Fxq 'ConnectionAttempts=3' "$tmp/ssh-arguments" +grep -Fxq 'ControlMaster=auto' "$tmp/ssh-arguments" +grep -Fxq 'ControlPersist=120' "$tmp/ssh-arguments" +grep -Fxq "ControlPath=$cluster_ssh_control_dir/%C" "$tmp/ssh-arguments" +[[ $(stat -f '%Lp' "$cluster_ssh_control_dir") == 700 ]] + awk ' /^is_release_ancestor\(\)/ { capture = 1 } capture && /^create_and_activate_run\(\)/ { exit }