fix(acceptance): 复用集群 SSH 控制连接

线上验收在宁波 SSH 建连超时时于负载前失败。为集群脚本增加受限 ControlMaster 复用和仅建连阶段的 ConnectionAttempts,避免重放远程命令。\n\n生产快照导出增加三次只读重试、原子落盘和空 Pod 清理防护。已通过 bash -n、ShellCheck、production acceptance 脚本测试、人工发布脚本测试,并验证宁波真实 SSH 控制连接可复用。
This commit is contained in:
2026-08-01 15:59:00 +08:00
parent 8ca3e2dbc2
commit 7623449e33
3 changed files with 52 additions and 3 deletions
+16
View File
@@ -5,6 +5,12 @@ set -euo pipefail
cluster_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
cluster_env_file=${AI_GATEWAY_CLUSTER_ENV_FILE:-"$cluster_root/.env.local"}
cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-"$HOME/.ssh/id_ed25519_easyai_gateway_cluster"}
cluster_ssh_control_dir=${AI_GATEWAY_CLUSTER_SSH_CONTROL_DIR:-"/tmp/easyai-gateway-ssh-${UID:-$(id -u)}-$$"}
prepare_cluster_ssh_control_dir() {
(umask 077 && mkdir -p -- "$cluster_ssh_control_dir")
chmod 0700 "$cluster_ssh_control_dir"
}
load_cluster_env() {
[[ -f $cluster_env_file && ! -L $cluster_env_file ]] || {
@@ -91,20 +97,30 @@ load_cluster_env() {
cluster_ssh() {
local host=$1
shift
prepare_cluster_ssh_control_dir
ssh \
-i "$cluster_ssh_key" \
-o BatchMode=yes \
-o ConnectionAttempts=3 \
-o ConnectTimeout=10 \
-o ControlMaster=auto \
-o ControlPersist=120 \
-o "ControlPath=$cluster_ssh_control_dir/%C" \
-o ServerAliveInterval=15 \
-o ServerAliveCountMax=3 \
"$host" "$@"
}
cluster_scp() {
prepare_cluster_ssh_control_dir
scp \
-i "$cluster_ssh_key" \
-o BatchMode=yes \
-o ConnectionAttempts=3 \
-o ConnectTimeout=10 \
-o ControlMaster=auto \
-o ControlPersist=120 \
-o "ControlPath=$cluster_ssh_control_dir/%C" \
"$@"
}
+21 -3
View File
@@ -359,7 +359,11 @@ export_production_snapshot() {
--release-sha "$release_sha" \
--output "$output" >/dev/null
else
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$output" <<'REMOTE'
local attempt temporary_output
temporary_output=$output.remote
rm -f -- "$temporary_output"
for attempt in 1 2 3; do
if cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$temporary_output" <<'REMOTE'
set -euo pipefail
release_sha=$1
namespace=easyai
@@ -367,12 +371,14 @@ pod=$(k3s kubectl get pods -n "$namespace" \
-l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
--field-selector status.phase=Running \
-o 'jsonpath={.items[0].metadata.name}')
[[ $pod == easyai-api-ningbo-* ]]
[[ -n $pod && $pod == easyai-api-ningbo-* ]]
[[ $(k3s kubectl get pod "$pod" -n "$namespace" \
-o 'jsonpath={.status.containerStatuses[0].ready}') == true ]]
snapshot=/tmp/easyai-production-acceptance-snapshot-$release_sha.json
cleanup() {
k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
if [[ -n ${pod:-} ]]; then
k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
k3s kubectl exec -n "$namespace" "$pod" -- \
@@ -380,6 +386,18 @@ k3s kubectl exec -n "$namespace" "$pod" -- \
--release-sha "$release_sha" --output "$snapshot" >/dev/null
k3s kubectl exec -n "$namespace" "$pod" -- cat "$snapshot"
REMOTE
then
mv -- "$temporary_output" "$output"
break
fi
rm -f -- "$temporary_output"
if (( attempt == 3 )); then
echo 'production snapshot export failed after three read-only attempts' >&2
return 1
fi
echo "production snapshot export attempt $attempt failed; retrying read-only export" >&2
sleep "$attempt"
done
fi
chmod 0600 "$output"
"$acceptance_snapshot_binary" validate --input "$output" >/dev/null