wangbo
971540a2a4
新增 WireGuard 全互联、三 server embedded-etcd K3s、CloudNativePG 双实例、Barman OSS 备份、双 NGINX、Kubernetes Secret/RBAC 与本地旧文件按严格 24 小时清理。 新增维护窗口数据迁移、digest 固定滚动发布、应用回滚、跨节点文件 E2E、节点和数据库故障演练、CNPG 恢复与 etcd 快照验收脚本；洛杉矶仅作为带 NoSchedule 污点的仲裁节点。 所有生产 Secret 只在执行时从 0600 本地环境和旧生产容器导入，仓库不保存凭据；公网入口保持人工 DNS 故障切换边界。 验证：bash -n、ShellCheck、kubectl kustomize、Node 语法检查、Secret 扫描、OSS put/head/delete 实测。
104 lines
3.5 KiB
Bash
Executable File
104 lines
3.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
# shellcheck source=scripts/cluster/common.sh
|
|
source "$script_dir/common.sh"
|
|
|
|
[[ ${1:-} == --execute ]] || {
|
|
echo 'usage: restore-backup-drill.sh --execute' >&2
|
|
exit 64
|
|
}
|
|
load_cluster_env
|
|
|
|
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
|
set -euo pipefail
|
|
kubectl='k3s kubectl'
|
|
suffix=$(date -u '+%m%d%H%M%S')
|
|
restore_cluster=easyai-restore-drill-"$suffix"
|
|
cleanup() {
|
|
$kubectl delete cluster "$restore_cluster" -n easyai \
|
|
--ignore-not-found=true --wait=true --timeout=300s || true
|
|
$kubectl delete pvc -n easyai -l "cnpg.io/cluster=$restore_cluster" \
|
|
--ignore-not-found=true --wait=true --timeout=300s || true
|
|
}
|
|
trap cleanup EXIT HUP INT TERM
|
|
|
|
source_primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
|
|
acceptance_sql=$(mktemp)
|
|
trap 'rm -f -- "$acceptance_sql"; cleanup' EXIT HUP INT TERM
|
|
cat >"$acceptance_sql" <<'SQL'
|
|
\pset tuples_only on
|
|
\pset format unaligned
|
|
SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
|
WHERE c.relkind='r' AND n.nspname='public';
|
|
SELECT 'tasks=' || count(*) FROM gateway_tasks;
|
|
SELECT 'attempts=' || count(*) FROM gateway_task_attempts;
|
|
SELECT 'wallet=' || count(*) FROM gateway_wallet_transactions;
|
|
SELECT 'users=' || count(*) FROM gateway_users;
|
|
SELECT 'sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id),''))
|
|
FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) rows;
|
|
SQL
|
|
source_snapshot=$($kubectl exec -i -n easyai "$source_primary" -c postgres -- \
|
|
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
|
|
|
|
cat <<EOF | $kubectl apply -f - >/dev/null
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: Cluster
|
|
metadata:
|
|
name: $restore_cluster
|
|
namespace: easyai
|
|
labels:
|
|
easyai.io/acceptance-drill: backup-restore
|
|
spec:
|
|
instances: 1
|
|
imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c
|
|
bootstrap:
|
|
recovery:
|
|
source: origin
|
|
database: easyai_ai_gateway
|
|
owner: easyai
|
|
secret:
|
|
name: easyai-postgres-app
|
|
externalClusters:
|
|
- name: origin
|
|
plugin:
|
|
name: barman-cloud.cloudnative-pg.io
|
|
parameters:
|
|
barmanObjectName: easyai-postgres-oss
|
|
serverName: easyai-postgres
|
|
storage:
|
|
storageClass: local-path
|
|
size: 20Gi
|
|
walStorage:
|
|
storageClass: local-path
|
|
size: 5Gi
|
|
affinity:
|
|
nodeSelector:
|
|
easyai.io/site: ningbo
|
|
resources:
|
|
requests:
|
|
cpu: 250m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: "2"
|
|
memory: 3Gi
|
|
EOF
|
|
|
|
$kubectl wait --for=condition=Ready "cluster/$restore_cluster" -n easyai --timeout=1200s
|
|
restore_primary=$($kubectl get cluster "$restore_cluster" -n easyai -o jsonpath='{.status.currentPrimary}')
|
|
restored_snapshot=$($kubectl exec -i -n easyai "$restore_primary" -c postgres -- \
|
|
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
|
|
[[ $restored_snapshot == "$source_snapshot" ]] || {
|
|
diff -u <(printf '%s\n' "$source_snapshot") <(printf '%s\n' "$restored_snapshot")
|
|
exit 1
|
|
}
|
|
$kubectl exec -n easyai "$restore_primary" -c postgres -- \
|
|
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
|
|
"SELECT current_user || ':' || current_database();" | grep -qx 'easyai:easyai_ai_gateway'
|
|
cleanup
|
|
trap - EXIT HUP INT TERM
|
|
rm -f -- "$acceptance_sql"
|
|
echo "backup_restore_drill=PASS cluster=$restore_cluster deleted=true"
|
|
REMOTE
|