feat(deploy): 增加三节点 K3s 高可用迁移能力

新增 WireGuard 全互联、三 server embedded-etcd K3s、CloudNativePG 双实例、Barman OSS 备份、双 NGINX、Kubernetes Secret/RBAC 与本地旧文件按严格 24 小时清理。新增维护窗口数据迁移、digest 固定滚动发布、应用回滚、跨节点文件 E2E、节点和数据库故障演练、CNPG 恢复与 etcd 快照验收脚本；洛杉矶仅作为带 NoSchedule 污点的仲裁节点。所有生产 Secret 只在执行时从 0600 本地环境和旧生产容器导入，仓库不保存凭据；公网入口保持人工 DNS 故障切换边界。验证：bash -n、ShellCheck、kubectl kustomize、Node 语法检查、Secret 扫描、OSS put/head/delete 实测。
2026-07-28 04:37:31 +08:00
parent edf7e66941
commit 971540a2a4
38 changed files with 3386 additions and 3 deletions
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cluster_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+cluster_env_file=${AI_GATEWAY_CLUSTER_ENV_FILE:-"$cluster_root/.env.local"}
+cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-"$HOME/.ssh/id_ed25519_easyai_gateway_cluster"}
+
+load_cluster_env() {
+  [[ -f $cluster_env_file && ! -L $cluster_env_file ]] || {
+    echo "missing local cluster environment file: $cluster_env_file" >&2
+    return 1
+  }
+  set -a
+  # shellcheck source=/dev/null
+  source "$cluster_env_file"
+  set +a
+  cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-$cluster_ssh_key}
+
+  : "${AI_GATEWAY_DEPLOY_HOST:?}"
+  : "${AI_GATEWAY_DEPLOY_HOST_2:?}"
+  : "${AI_GATEWAY_DEPLOY_HOST_3:?}"
+  : "${ALI_KEY:?}"
+  : "${ALI_SECRET:?}"
+  : "${ALI_REGION:?}"
+  : "${ALI_BUCKET:?}"
+
+  CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST}
+  CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2}
+  CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3}
+  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST
+
+  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
+    [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || {
+      echo "cluster hosts must use root@host syntax" >&2
+      return 1
+    }
+  done
+  [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || {
+    echo 'invalid OSS region or bucket' >&2
+    return 1
+  }
+  [[ -f $cluster_ssh_key && ! -L $cluster_ssh_key ]] || {
+    echo "missing cluster SSH key: $cluster_ssh_key" >&2
+    return 1
+  }
+}
+
+cluster_ssh() {
+  local host=$1
+  shift
+  ssh \
+    -i "$cluster_ssh_key" \
+    -o BatchMode=yes \
+    -o ConnectTimeout=10 \
+    -o ServerAliveInterval=15 \
+    -o ServerAliveCountMax=3 \
+    "$host" "$@"
+}
+
+cluster_scp() {
+  scp \
+    -i "$cluster_ssh_key" \
+    -o BatchMode=yes \
+    -o ConnectTimeout=10 \
+    "$@"
+}
+
+require_commands() {
+  local command
+  for command in "$@"; do
+    command -v "$command" >/dev/null 2>&1 || {
+      echo "$command is required" >&2
+      return 1
+    }
+  done
+}
+
+assert_private_file() {
+  local path=$1
+  local mode
+  [[ -f $path && ! -L $path ]] || {
+    echo "missing private file: $path" >&2
+    return 1
+  }
+  if [[ $(uname -s) == Darwin ]]; then
+    mode=$(stat -f '%Lp' "$path")
+  else
+    mode=$(stat -c '%a' "$path")
+  fi
+  [[ $mode == 600 ]] || {
+    echo "private file must have mode 0600: $path" >&2
+    return 1
+  }
+}