feat(deploy): 增加三节点 K3s 高可用迁移能力

新增 WireGuard 全互联、三 server embedded-etcd K3s、CloudNativePG 双实例、Barman OSS 备份、双 NGINX、Kubernetes Secret/RBAC 与本地旧文件按严格 24 小时清理。新增维护窗口数据迁移、digest 固定滚动发布、应用回滚、跨节点文件 E2E、节点和数据库故障演练、CNPG 恢复与 etcd 快照验收脚本；洛杉矶仅作为带 NoSchedule 污点的仲裁节点。所有生产 Secret 只在执行时从 0600 本地环境和旧生产容器导入，仓库不保存凭据；公网入口保持人工 DNS 故障切换边界。验证：bash -n、ShellCheck、kubectl kustomize、Node 语法检查、Secret 扫描、OSS put/head/delete 实测。
2026-07-28 04:37:31 +08:00
parent edf7e66941
commit 971540a2a4
38 changed files with 3386 additions and 3 deletions
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+[[ ${1:-} == --execute ]] || {
+  echo 'usage: restore-backup-drill.sh --execute' >&2
+  exit 64
+}
+load_cluster_env
+
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+suffix=$(date -u '+%m%d%H%M%S')
+restore_cluster=easyai-restore-drill-"$suffix"
+cleanup() {
+  $kubectl delete cluster "$restore_cluster" -n easyai \
+    --ignore-not-found=true --wait=true --timeout=300s || true
+  $kubectl delete pvc -n easyai -l "cnpg.io/cluster=$restore_cluster" \
+    --ignore-not-found=true --wait=true --timeout=300s || true
+}
+trap cleanup EXIT HUP INT TERM
+
+source_primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
+acceptance_sql=$(mktemp)
+trap 'rm -f -- "$acceptance_sql"; cleanup' EXIT HUP INT TERM
+cat >"$acceptance_sql" <<'SQL'
+\pset tuples_only on
+\pset format unaligned
+SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
+ WHERE c.relkind='r' AND n.nspname='public';
+SELECT 'tasks=' || count(*) FROM gateway_tasks;
+SELECT 'attempts=' || count(*) FROM gateway_task_attempts;
+SELECT 'wallet=' || count(*) FROM gateway_wallet_transactions;
+SELECT 'users=' || count(*) FROM gateway_users;
+SELECT 'sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id),''))
+ FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) rows;
+SQL
+source_snapshot=$($kubectl exec -i -n easyai "$source_primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
+
+cat <<EOF | $kubectl apply -f - >/dev/null
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: $restore_cluster
+  namespace: easyai
+  labels:
+    easyai.io/acceptance-drill: backup-restore
+spec:
+  instances: 1
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c
+  bootstrap:
+    recovery:
+      source: origin
+      database: easyai_ai_gateway
+      owner: easyai
+      secret:
+        name: easyai-postgres-app
+  externalClusters:
+    - name: origin
+      plugin:
+        name: barman-cloud.cloudnative-pg.io
+        parameters:
+          barmanObjectName: easyai-postgres-oss
+          serverName: easyai-postgres
+  storage:
+    storageClass: local-path
+    size: 20Gi
+  walStorage:
+    storageClass: local-path
+    size: 5Gi
+  affinity:
+    nodeSelector:
+      easyai.io/site: ningbo
+  resources:
+    requests:
+      cpu: 250m
+      memory: 512Mi
+    limits:
+      cpu: "2"
+      memory: 3Gi
+EOF
+
+$kubectl wait --for=condition=Ready "cluster/$restore_cluster" -n easyai --timeout=1200s
+restore_primary=$($kubectl get cluster "$restore_cluster" -n easyai -o jsonpath='{.status.currentPrimary}')
+restored_snapshot=$($kubectl exec -i -n easyai "$restore_primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
+[[ $restored_snapshot == "$source_snapshot" ]] || {
+  diff -u <(printf '%s\n' "$source_snapshot") <(printf '%s\n' "$restored_snapshot")
+  exit 1
+}
+$kubectl exec -n easyai "$restore_primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
+  "SELECT current_user || ':' || current_database();" | grep -qx 'easyai:easyai_ai_gateway'
+cleanup
+trap - EXIT HUP INT TERM
+rm -f -- "$acceptance_sql"
+echo "backup_restore_drill=PASS cluster=$restore_cluster deleted=true"
+REMOTE