fix(acceptance): 复用集群 SSH 控制连接
线上验收在宁波 SSH 建连超时时于负载前失败。为集群脚本增加受限 ControlMaster 复用和仅建连阶段的 ConnectionAttempts,避免重放远程命令。\n\n生产快照导出增加三次只读重试、原子落盘和空 Pod 清理防护。已通过 bash -n、ShellCheck、production acceptance 脚本测试、人工发布脚本测试,并验证宁波真实 SSH 控制连接可复用。
This commit is contained in:
@@ -5,6 +5,12 @@ set -euo pipefail
|
||||
cluster_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
cluster_env_file=${AI_GATEWAY_CLUSTER_ENV_FILE:-"$cluster_root/.env.local"}
|
||||
cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-"$HOME/.ssh/id_ed25519_easyai_gateway_cluster"}
|
||||
cluster_ssh_control_dir=${AI_GATEWAY_CLUSTER_SSH_CONTROL_DIR:-"/tmp/easyai-gateway-ssh-${UID:-$(id -u)}-$$"}
|
||||
|
||||
prepare_cluster_ssh_control_dir() {
|
||||
(umask 077 && mkdir -p -- "$cluster_ssh_control_dir")
|
||||
chmod 0700 "$cluster_ssh_control_dir"
|
||||
}
|
||||
|
||||
load_cluster_env() {
|
||||
[[ -f $cluster_env_file && ! -L $cluster_env_file ]] || {
|
||||
@@ -91,20 +97,30 @@ load_cluster_env() {
|
||||
cluster_ssh() {
|
||||
local host=$1
|
||||
shift
|
||||
prepare_cluster_ssh_control_dir
|
||||
ssh \
|
||||
-i "$cluster_ssh_key" \
|
||||
-o BatchMode=yes \
|
||||
-o ConnectionAttempts=3 \
|
||||
-o ConnectTimeout=10 \
|
||||
-o ControlMaster=auto \
|
||||
-o ControlPersist=120 \
|
||||
-o "ControlPath=$cluster_ssh_control_dir/%C" \
|
||||
-o ServerAliveInterval=15 \
|
||||
-o ServerAliveCountMax=3 \
|
||||
"$host" "$@"
|
||||
}
|
||||
|
||||
cluster_scp() {
|
||||
prepare_cluster_ssh_control_dir
|
||||
scp \
|
||||
-i "$cluster_ssh_key" \
|
||||
-o BatchMode=yes \
|
||||
-o ConnectionAttempts=3 \
|
||||
-o ConnectTimeout=10 \
|
||||
-o ControlMaster=auto \
|
||||
-o ControlPersist=120 \
|
||||
-o "ControlPath=$cluster_ssh_control_dir/%C" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
|
||||
@@ -359,7 +359,11 @@ export_production_snapshot() {
|
||||
--release-sha "$release_sha" \
|
||||
--output "$output" >/dev/null
|
||||
else
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$output" <<'REMOTE'
|
||||
local attempt temporary_output
|
||||
temporary_output=$output.remote
|
||||
rm -f -- "$temporary_output"
|
||||
for attempt in 1 2 3; do
|
||||
if cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$temporary_output" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
release_sha=$1
|
||||
namespace=easyai
|
||||
@@ -367,12 +371,14 @@ pod=$(k3s kubectl get pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
|
||||
--field-selector status.phase=Running \
|
||||
-o 'jsonpath={.items[0].metadata.name}')
|
||||
[[ $pod == easyai-api-ningbo-* ]]
|
||||
[[ -n $pod && $pod == easyai-api-ningbo-* ]]
|
||||
[[ $(k3s kubectl get pod "$pod" -n "$namespace" \
|
||||
-o 'jsonpath={.status.containerStatuses[0].ready}') == true ]]
|
||||
snapshot=/tmp/easyai-production-acceptance-snapshot-$release_sha.json
|
||||
cleanup() {
|
||||
k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
|
||||
if [[ -n ${pod:-} ]]; then
|
||||
k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
k3s kubectl exec -n "$namespace" "$pod" -- \
|
||||
@@ -380,6 +386,18 @@ k3s kubectl exec -n "$namespace" "$pod" -- \
|
||||
--release-sha "$release_sha" --output "$snapshot" >/dev/null
|
||||
k3s kubectl exec -n "$namespace" "$pod" -- cat "$snapshot"
|
||||
REMOTE
|
||||
then
|
||||
mv -- "$temporary_output" "$output"
|
||||
break
|
||||
fi
|
||||
rm -f -- "$temporary_output"
|
||||
if (( attempt == 3 )); then
|
||||
echo 'production snapshot export failed after three read-only attempts' >&2
|
||||
return 1
|
||||
fi
|
||||
echo "production snapshot export attempt $attempt failed; retrying read-only export" >&2
|
||||
sleep "$attempt"
|
||||
done
|
||||
fi
|
||||
chmod 0600 "$output"
|
||||
"$acceptance_snapshot_binary" validate --input "$output" >/dev/null
|
||||
|
||||
Reference in New Issue
Block a user