fix(deploy): 补齐集群备份兼容与切换前门禁

阿里云 OSS 的 S3 兼容层要求 virtual-hosted addressing，且 boto3 需要使用 Signature V2。本提交为 Barman 的归档、备份、保留与恢复统一挂载 AWS 配置，并将实际 WAL 归档和近期全备设为切换硬门禁。

同时补充 CNPG/etcdutl 固定版本安装、显式主库切换、三节点逐台停机、OSS 快照下载校验与远端临时 Secret 清理。已通过 ShellCheck、Kustomize、服务端 dry-run、迁移测试、发布脚本测试、敏感信息扫描及生产 precutover 验收。

deploy/kubernetes/production/database.yaml

@@ -1,3 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: easyai-barman-aws-config
+  labels:
+    app.kubernetes.io/part-of: easyai-ai-gateway
+data:
+  config: |
+    [default]
+    region = oss-cn-shanghai
+    s3 =
+      addressing_style = virtual
+      signature_version = s3
+---
 apiVersion: barmancloud.cnpg.io/v1
 kind: ObjectStore
 metadata:
@@ -8,7 +22,7 @@ spec:
   retentionPolicy: 14d
   configuration:
     destinationPath: s3://wangbo0808/easyai-ai-gateway/production/postgresql
-    endpointURL: https://oss-cn-shanghai.aliyuncs.com
+    endpointURL: https://s3.oss-cn-shanghai.aliyuncs.com
     s3Credentials:
       accessKeyId:
         name: easyai-oss-backup
@@ -17,13 +31,23 @@ spec:
         name: easyai-oss-backup
         key: ACCESS_SECRET_KEY
     wal:
+      archiveAdditionalCommandArgs:
+        - --addressing-style=virtual
       compression: gzip
       maxParallel: 2
+      restoreAdditionalCommandArgs:
+        - --addressing-style=virtual
     data:
+      additionalCommandArgs:
+        - --addressing-style=virtual
       compression: gzip
       jobs: 2
+      restoreAdditionalCommandArgs:
+        - --addressing-style=virtual
   instanceSidecarConfiguration:
     env:
+      - name: AWS_CONFIG_FILE
+        value: /projected/barman-aws/config
       - name: AWS_DEFAULT_REGION
         value: oss-cn-shanghai
       - name: AWS_REQUEST_CHECKSUM_CALCULATION
@@ -88,6 +112,14 @@ spec:
       wal_keep_size: 1GB
   monitoring:
     enablePodMonitor: false
+  projectedVolumeTemplate:
+    defaultMode: 0444
+    sources:
+      - configMap:
+          name: easyai-barman-aws-config
+          items:
+            - key: config
+              path: barman-aws/config
   plugins:
     - name: barman-cloud.cloudnative-pg.io
       isWALArchiver: true

docs/operations/k3s-ha-runbook.md

@@ -26,7 +26,7 @@ pnpm install --frozen-lockfile
 pnpm lint
 pnpm test
 pnpm build
-shellcheck scripts/cluster/*.sh deploy/kubernetes/easyai-ai-gateway-cluster-release
+shellcheck -x -P . scripts/cluster/*.sh deploy/kubernetes/easyai-ai-gateway-cluster-release
 kubectl kustomize deploy/kubernetes/production >/dev/null
 ./tests/ci/migrations-test.sh
 node scripts/ci-validate-migrations.mjs <当前线上完整GitSHA>
@@ -45,6 +45,7 @@ node scripts/ci-validate-migrations.mjs <当前线上完整GitSHA>
 ./scripts/cluster/bootstrap-k3s.sh backup
 ./scripts/cluster/harden-node-network.sh
 ./scripts/cluster/bootstrap-k3s.sh install
+./scripts/cluster/verify-k3s-quorum.sh --execute
 ./scripts/cluster/verify-cluster.sh precutover
 ```
 
@@ -57,18 +58,34 @@ K3s containerd 同时配置 Docker Hub、GHCR 与 Quay 的镜像端点；已有
 ./scripts/cluster/configure-k3s-registries.sh
 ```
 
+`verify-k3s-quorum.sh` 会依次停止并恢复洛杉矶、香港、宁波 K3s，确认每次剩余两个
+server 都保持 etcd quorum。宁波停机可能令香港数据库副本晋升；首次切换前如需恢复宁波
+初始主库，显式执行 `promote-cnpg-primary.sh`，故障恢复流程不得自动回切。
+
 ### 3. 平台和入口预置
 
 ```bash
 ./scripts/cluster/bootstrap-platform.sh prepare dist/releases/<SHA>.json
+./scripts/cluster/install-cnpg-plugin.sh
+./scripts/cluster/install-etcdutl.sh
 ./scripts/cluster/install-nginx-edges.sh prepare
 ./scripts/cluster/install-certificate-sync.sh
 ./scripts/cluster/install-legacy-cleaner.sh
 ```
 
-`prepare` 将应用副本保持在 0，但启动 CNPG 双实例、WAL 归档和每日备份。SecretStore
+`prepare` 将应用副本保持在 0，但启动 CNPG 双实例、WAL 归档和每日备份。OSS 的 Barman
+链路使用 `s3.oss-cn-shanghai.aliyuncs.com`、virtual-hosted addressing 和 boto3
+Signature V2；`AWS_CONFIG_FILE` 通过 CNPG projected ConfigMap 同时挂载到主库、副本及恢复
+集群。预检会强制产生并归档一个新 WAL，并要求 48 小时内至少存在一份完成的全量备份。
+SecretStore
 的 4 个值从旧 volume 直接导入 Kubernetes Secret，脚本只校验数量与总字节数，不打印值。
 
+首次切换前如滚动安装令主库落在香港，可显式切回宁波。故障晋升后禁止自动执行该命令：
+
+```bash
+./scripts/cluster/promote-cnpg-primary.sh easyai-postgres-1
+```
+
 ### 4. 02:00–04:00 切换
 
 先执行只读预检：
@@ -86,7 +103,8 @@ K3s containerd 同时配置 Docker Hub、GHCR 与 Quay 的镜像端点；已有
 脚本依次启用维护页、停止旧 API、完整备份旧 PostgreSQL、上传 OSS、恢复 CNPG、比对数据库
 版本/扩展/catalog/表数/序列/关键行数/抽样 ID、运行 digest 固定 migrator、冻结 Worker 启动
 两地应用、做 NodePort 验收，再切换双 NGINX。公开流量提交后，先启用香港 Worker，再启用宁波
-Worker。旧 Docker PostgreSQL 和数据卷停止但保留，不自动删除。
+Worker。脚本等待切换后的 CNPG 全量备份状态变为 `completed` 后，才停止旧 Docker
+PostgreSQL；数据卷继续保留，不自动删除。
 
 在公共流量提交前，任一步失败或耗时超过 10 分钟都会恢复旧 Docker 与旧 NGINX。公共流量
 提交后禁止自动切回旧数据库，避免双写分叉。

scripts/cluster/bootstrap-platform.sh

@@ -135,6 +135,21 @@ set -euo pipefail
 source_sha=$1
 mode=$2
 kubectl='k3s kubectl'
+runtime_environment=
+cleanup_bootstrap_files() {
+  if [[ -n ${runtime_environment:-} && -f $runtime_environment &&
+        ! -L $runtime_environment ]]; then
+    shred -u "$runtime_environment"
+  fi
+  if [[ -f /root/bootstrap-secrets.yaml && ! -L /root/bootstrap-secrets.yaml ]]; then
+    shred -u /root/bootstrap-secrets.yaml
+  fi
+  for path in /root/cert-manager.yaml /root/cnpg.yaml /root/barman.yaml \
+    /root/easyai-production.yaml; do
+    [[ ! -e $path && ! -L $path ]] || unlink "$path"
+  done
+}
+trap cleanup_bootstrap_files EXIT HUP INT TERM
 
 $kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null
 $kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s
@@ -151,7 +166,6 @@ postgres_password=$($kubectl get secret easyai-postgres-app -n easyai \
   -o jsonpath='{.data.password}' | base64 -d)
 
 runtime_environment=$(mktemp)
-trap 'rm -f -- "$runtime_environment"' EXIT
 chmod 0600 "$runtime_environment"
 docker inspect easyai-ai-gateway-api-1 \
   --format '{{range .Config.Env}}{{println .}}{{end}}' |
@@ -206,6 +220,9 @@ $kubectl create secret generic easyai-registry -n easyai \
   --from-file=.dockerconfigjson="$registry_config" --dry-run=client -o yaml |
   $kubectl apply -f - >/dev/null
 
+$kubectl apply --server-side --dry-run=server --force-conflicts \
+  --field-manager=easyai-production-preflight \
+  -f /root/easyai-production.yaml >/dev/null
 $kubectl apply -f /root/easyai-production.yaml >/dev/null
 $kubectl annotate namespace easyai easyai.io/release="$source_sha" --overwrite >/dev/null
 if [[ $mode == activate ]]; then
@@ -215,8 +232,6 @@ if [[ $mode == activate ]]; then
   $kubectl rollout status deployment/easyai-web-ningbo -n easyai --timeout=300s
 fi
 
-rm -f -- /root/cert-manager.yaml /root/cnpg.yaml /root/barman.yaml \
-  /root/easyai-production.yaml /root/bootstrap-secrets.yaml
 REMOTE
 
 if [[ $mode == prepare ]]; then

scripts/cluster/failure-drill.sh

@@ -52,12 +52,12 @@ case $drill in
 set -euo pipefail
 primary=$1
 k3s kubectl exec -n easyai "$primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -c \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \
   "INSERT INTO system_settings(setting_key,value) VALUES('ha_acceptance_probe',jsonb_build_object('at',now())) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" >/dev/null
 for _ in $(seq 1 180); do
   if [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') == 2 ]]; then
     replication=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
-      psql -X -U easyai -d easyai_ai_gateway -At -c \
+      psql -X -U postgres -d easyai_ai_gateway -At -c \
       "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
     [[ $replication == 1 ]] && break
   fi
@@ -65,7 +65,7 @@ for _ in $(seq 1 180); do
 done
 [[ ${replication:-0} == 1 ]]
 k3s kubectl exec -n easyai "$primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -c \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \
   "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null
 REMOTE
     echo "database_failure_drill=PASS promotion_seconds=$elapsed old_primary=$old_primary new_primary=$new_primary"

scripts/cluster/install-cnpg-plugin.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+load_cluster_env
+require_commands curl shasum tar
+
+plugin_version=1.29.1
+plugin_archive=kubectl-cnpg_"$plugin_version"_linux_x86_64.tar.gz
+plugin_sha256=2a550d5ee2224e5bb66b95602f8052fc50da50485f59f97b817a0c3b44b58581
+working_directory=$(mktemp -d "$cluster_root/.local-secrets/cluster/cnpg-plugin.XXXXXX")
+trap '[[ $working_directory == "$cluster_root"/.local-secrets/cluster/cnpg-plugin.* ]] && rm -rf -- "$working_directory"' \
+  EXIT HUP INT TERM
+
+curl -fsSL \
+  "https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v${plugin_version}/${plugin_archive}" \
+  -o "$working_directory/$plugin_archive"
+actual_sha256=$(shasum -a 256 "$working_directory/$plugin_archive" | awk '{print $1}')
+[[ $actual_sha256 == "$plugin_sha256" ]] || {
+  echo 'kubectl-cnpg archive checksum mismatch' >&2
+  exit 1
+}
+tar -tzf "$working_directory/$plugin_archive" | grep -qx kubectl-cnpg
+tar -xzf "$working_directory/$plugin_archive" -C "$working_directory" kubectl-cnpg
+cluster_scp "$working_directory/kubectl-cnpg" "$CLUSTER_NINGBO_HOST:/root/kubectl-cnpg"
+cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$plugin_version" <<'REMOTE'
+set -euo pipefail
+plugin_version=$1
+install -m 0755 /root/kubectl-cnpg /usr/local/bin/kubectl-cnpg
+unlink /root/kubectl-cnpg
+k3s kubectl cnpg version | grep -q "Version:${plugin_version}"
+REMOTE
+
+echo "cnpg_plugin_install=PASS version=$plugin_version sha256=$plugin_sha256"

scripts/cluster/install-etcdutl.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+load_cluster_env
+require_commands curl shasum tar
+
+# K3s v1.36.2 embeds etcd 3.6.12. The closest upstream release utility is
+# 3.6.11 and uses the same v3 snapshot format.
+etcd_version=3.6.11
+etcd_archive=etcd-v"$etcd_version"-linux-amd64.tar.gz
+etcd_archive_sha256=8756f7a4eaf921668a83de0bf13c0f65cae9186a165696e3ae8396afe6f557ed
+working_directory=$(mktemp -d "$cluster_root/.local-secrets/cluster/etcdutl.XXXXXX")
+trap '[[ $working_directory == "$cluster_root"/.local-secrets/cluster/etcdutl.* ]] && rm -rf -- "$working_directory"' \
+  EXIT HUP INT TERM
+
+curl -fsSL \
+  "https://github.com/etcd-io/etcd/releases/download/v${etcd_version}/${etcd_archive}" \
+  -o "$working_directory/$etcd_archive"
+actual_sha256=$(shasum -a 256 "$working_directory/$etcd_archive" | awk '{print $1}')
+[[ $actual_sha256 == "$etcd_archive_sha256" ]] || {
+  echo 'etcd release archive checksum mismatch' >&2
+  exit 1
+}
+archive_member=etcd-v"$etcd_version"-linux-amd64/etcdutl
+tar -tzf "$working_directory/$etcd_archive" | grep -qx "$archive_member"
+tar -xzf "$working_directory/$etcd_archive" -C "$working_directory" "$archive_member"
+etcdutl_binary="$working_directory/$archive_member"
+
+for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
+  cluster_scp "$etcdutl_binary" "$host:/root/etcdutl"
+  cluster_ssh "$host" bash -s -- "$etcd_version" <<'REMOTE'
+set -euo pipefail
+etcd_version=$1
+install -m 0755 /root/etcdutl /usr/local/bin/etcdutl
+unlink /root/etcdutl
+etcdutl version | grep -q "etcdutl version: ${etcd_version}"
+REMOTE
+done
+
+echo "etcdutl_install=PASS version=$etcd_version nodes=3 sha256=$etcd_archive_sha256"

scripts/cluster/migrate-production.sh

@@ -37,6 +37,45 @@ docker exec easyai-ai-gateway-postgres-1 pg_isready >/dev/null
 k3s kubectl get --raw=/readyz >/dev/null
 [[ $(k3s kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
 [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
+primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
+[[ $primary =~ ^easyai-postgres-[0-9]+$ ]]
+for pod in $(k3s kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
+  -o jsonpath='{.items[*].metadata.name}'); do
+  k3s kubectl exec -n easyai "$pod" -c postgres -- \
+    test -r /projected/barman-aws/config
+done
+archived_before=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -U postgres -d postgres -At -c \
+  'SELECT archived_count FROM pg_stat_archiver;')
+archived_after=$(k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
+  bash -s -- "$archived_before" <<'POD'
+set -euo pipefail
+archived_before=$1
+psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres -At \
+  -c "SELECT pg_logical_emit_message(true, 'easyai-ha-preflight', clock_timestamp()::text);" \
+  -c 'SELECT pg_switch_wal();' >/dev/null
+for _ in $(seq 1 45); do
+  archived_after=$(psql -X -U postgres -d postgres -At -c \
+    'SELECT archived_count FROM pg_stat_archiver;')
+  if (( archived_after > archived_before )); then
+    printf '%s\n' "$archived_after"
+    exit 0
+  fi
+  sleep 2
+done
+exit 1
+POD
+)
+(( archived_after > archived_before ))
+[[ $(k3s kubectl get cluster easyai-postgres -n easyai \
+  -o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}') == True ]]
+latest_completed_backup=$(k3s kubectl get backups -n easyai -o json |
+  jq -r '[.items[] | select(
+    .spec.cluster.name == "easyai-postgres" and .status.phase == "completed"
+  ) | .status.stoppedAt] | sort | last // empty')
+[[ -n $latest_completed_backup ]]
+latest_completed_epoch=$(date -d "$latest_completed_backup" '+%s')
+(( $(date '+%s') - latest_completed_epoch < 172800 ))
 [[ $(df --output=avail -B1 /var/lib/docker | tail -1) -gt 2147483648 ]]
 [[ -f /etc/nginx/sites-available/ai.51easyai.com-maintenance ]]
 [[ -f /etc/nginx/sites-available/ai.51easyai.com-cluster ]]
@@ -196,7 +235,7 @@ k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
   pg_restore --exit-on-error --no-owner --role=easyai \
   -U postgres -d easyai_ai_gateway <"$backup_directory/database.dump"
 k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway \
   <"$snapshot_sql" >"$backup_directory/restored.snapshot"
 diff -u "$backup_directory/source.snapshot" "$backup_directory/restored.snapshot"
 
@@ -236,11 +275,11 @@ k3s kubectl apply -f "$backup_directory/migrator-job.yaml" >/dev/null
 k3s kubectl wait --for=condition=complete \
   "job/easyai-migrator-${source_sha:0:12}" -n easyai --timeout=300s
 k3s kubectl exec -n easyai "$primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
   "SELECT count(*) FROM schema_migrations WHERE version='0089_file_storage_request_asset_scene';" |
   grep -qx 1
 k3s kubectl exec -n easyai "$primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
   "SELECT count(*) FROM file_storage_channels WHERE deleted_at IS NULL AND provider='server_main_openapi' AND config->'scenes' ? 'request_asset';" |
   grep -Eq '^[1-9][0-9]*$'
 
@@ -316,7 +355,7 @@ $kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
 $kubectl set env deployment/easyai-api-ningbo -n easyai \
   AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true
 $kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
-cat <<'EOF' | $kubectl create -f - >/dev/null
+backup_resource=$(cat <<'EOF' | $kubectl create -f - -o name
 apiVersion: postgresql.cnpg.io/v1
 kind: Backup
 metadata:
@@ -329,10 +368,28 @@ spec:
   pluginConfiguration:
     name: barman-cloud.cloudnative-pg.io
 EOF
+)
+backup_name=${backup_resource#*/}
+backup_phase=
+for _ in $(seq 1 240); do
+  backup_phase=$($kubectl get backup "$backup_name" -n easyai \
+    -o jsonpath='{.status.phase}')
+  [[ $backup_phase == completed ]] && break
+  if [[ $backup_phase == failed ]]; then
+    echo "post-cutover CNPG backup failed: $backup_name" >&2
+    exit 1
+  fi
+  sleep 5
+done
+[[ $backup_phase == completed ]] || {
+  echo "post-cutover CNPG backup did not complete in time: $backup_name" >&2
+  exit 1
+}
 docker update --restart=no easyai-ai-gateway-api-1 \
   easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
 docker stop --time 30 easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
 rm -f -- /root/easyai-remote-cutover.sh /root/easyai-oss-object.mjs
+echo "post_cutover_backup=PASS name=$backup_name"
 REMOTE
 
 "$script_dir/install-release-helper.sh"

scripts/cluster/promote-cnpg-primary.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+target_primary=${1:-}
+[[ $target_primary =~ ^easyai-postgres-[12]$ ]] || {
+  echo 'usage: promote-cnpg-primary.sh easyai-postgres-{1|2}' >&2
+  exit 64
+}
+
+load_cluster_env
+cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$target_primary" <<'REMOTE'
+set -euo pipefail
+target_primary=$1
+kubectl='k3s kubectl'
+command -v kubectl-cnpg >/dev/null
+[[ $($kubectl get pod "$target_primary" -n easyai \
+  -o jsonpath='{.metadata.labels.cnpg\.io/cluster}') == easyai-postgres ]]
+current_primary=$($kubectl get cluster easyai-postgres -n easyai \
+  -o jsonpath='{.status.currentPrimary}')
+if [[ $current_primary != "$target_primary" ]]; then
+  $kubectl cnpg promote easyai-postgres "$target_primary" -n easyai
+fi
+for _ in $(seq 1 120); do
+  current_primary=$($kubectl get cluster easyai-postgres -n easyai \
+    -o jsonpath='{.status.currentPrimary}')
+  ready_instances=$($kubectl get cluster easyai-postgres -n easyai \
+    -o jsonpath='{.status.readyInstances}')
+  if [[ $current_primary == "$target_primary" && $ready_instances == 2 ]]; then
+    synchronous_replicas=$($kubectl exec -n easyai "$current_primary" -c postgres -- \
+      psql -X -U postgres -d postgres -At -c \
+      "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
+    if [[ $synchronous_replicas == 1 ]]; then
+      echo "cnpg_primary_promotion=PASS primary=$current_primary synchronous_replicas=1"
+      exit 0
+    fi
+  fi
+  sleep 3
+done
+echo "CNPG did not converge on target primary: $target_primary" >&2
+exit 1
+REMOTE

scripts/cluster/restore-backup-drill.sh

@@ -40,7 +40,7 @@ SELECT 'sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id),''))
  FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) rows;
 SQL
 source_snapshot=$($kubectl exec -i -n easyai "$source_primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway <"$acceptance_sql")
 
 cat <<EOF | $kubectl apply -f - >/dev/null
 apiVersion: postgresql.cnpg.io/v1
@@ -83,19 +83,27 @@ spec:
     limits:
       cpu: "2"
       memory: 3Gi
+  projectedVolumeTemplate:
+    defaultMode: 0444
+    sources:
+      - configMap:
+          name: easyai-barman-aws-config
+          items:
+            - key: config
+              path: barman-aws/config
 EOF
 
 $kubectl wait --for=condition=Ready "cluster/$restore_cluster" -n easyai --timeout=1200s
 restore_primary=$($kubectl get cluster "$restore_cluster" -n easyai -o jsonpath='{.status.currentPrimary}')
 restored_snapshot=$($kubectl exec -i -n easyai "$restore_primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway <"$acceptance_sql")
 [[ $restored_snapshot == "$source_snapshot" ]] || {
   diff -u <(printf '%s\n' "$source_snapshot") <(printf '%s\n' "$restored_snapshot")
   exit 1
 }
 $kubectl exec -n easyai "$restore_primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
-  "SELECT current_user || ':' || current_database();" | grep -qx 'easyai:easyai_ai_gateway'
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
+  "SELECT current_database();" | grep -qx 'easyai_ai_gateway'
 cleanup
 trap - EXIT HUP INT TERM
 rm -f -- "$acceptance_sql"

scripts/cluster/run-cross-node-file-e2e.sh

@@ -71,7 +71,7 @@ source_sha=$2
 primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
 query() {
   k3s kubectl exec -n easyai "$primary" -c postgres -- \
-    psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At "$@"
+    psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At "$@"
 }
 attempted_by=$(query -c \
   "SELECT attempted_by::text FROM river_job WHERE id=(SELECT river_job_id FROM gateway_tasks WHERE id='$task_id'::uuid);")

scripts/cluster/verify-cluster.sh

@@ -53,8 +53,7 @@ REMOTE
 [[ $cluster_state == nodes=3 ]]
 echo 'k3s_control_plane=PASS servers=3 witness_business_pods=0'
 
-if [[ $mode == postcutover ]]; then
-  database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
 set -euo pipefail
 kubectl='k3s kubectl'
 [[ $($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
@@ -62,9 +61,36 @@ primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.c
 [[ $($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-ningbo ||
    $($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-hongkong ]]
 replication=$($kubectl exec -n easyai "$primary" -c postgres -- \
-  psql -X -U easyai -d easyai_ai_gateway -At -c \
+  psql -X -U postgres -d easyai_ai_gateway -At -c \
   "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
 [[ $replication -eq 1 ]]
+archived_count=$($kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -U postgres -d postgres -At -c \
+  'SELECT archived_count FROM pg_stat_archiver;')
+(( archived_count > 0 ))
+[[ $($kubectl get cluster easyai-postgres -n easyai \
+  -o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}') == True ]]
+completed_backups=$($kubectl get backups -n easyai -o json |
+  jq '[.items[] | select(
+    .spec.cluster.name == "easyai-postgres" and .status.phase == "completed"
+  )] | length')
+(( completed_backups > 0 ))
+for pod in $($kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
+  -o jsonpath='{.items[*].metadata.name}'); do
+  $kubectl exec -n easyai "$pod" -c postgres -- \
+    test -r /projected/barman-aws/config
+done
+printf 'primary=%s archived=%s backups=%s\n' \
+  "$primary" "$archived_count" "$completed_backups"
+REMOTE
+)
+[[ $database_state == primary=easyai-postgres-* ]]
+echo "postgres_preflight=PASS $database_state synchronous_replicas=1"
+
+if [[ $mode == postcutover ]]; then
+  application_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
 [[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-api \
   --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
 [[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-web \
@@ -82,11 +108,11 @@ fi
   --as=system:serviceaccount:easyai:easyai-ai-gateway) == yes ]]
 [[ $($kubectl auth can-i update secret/easyai-ai-gateway-runtime -n easyai \
   --as=system:serviceaccount:easyai:easyai-ai-gateway) == no ]]
-printf 'primary=%s\n' "$primary"
+printf 'applications=ready\n'
 REMOTE
 )
-  [[ $database_state == primary=easyai-postgres-* ]]
-  echo "postgres_acceptance=PASS $database_state synchronous_replicas=1"
+  [[ $application_state == applications=ready ]]
+  echo 'application_acceptance=PASS api=2 web=2 digests=pinned'
 
   for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
     curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \

scripts/cluster/verify-etcd-backup.sh

@@ -6,29 +6,55 @@ script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 source "$script_dir/common.sh"
 load_cluster_env
 
+cluster_scp "$script_dir/oss-object.mjs" \
+  "$CLUSTER_NINGBO_HOST:/root/easyai-oss-object.mjs"
 cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
 set -euo pipefail
+working_directory=$(mktemp -d)
+cleanup() {
+  find "$working_directory" -type f -delete
+  find "$working_directory" -depth -type d -empty -delete
+  [[ ! -e /root/easyai-oss-object.mjs && ! -L /root/easyai-oss-object.mjs ]] ||
+    unlink /root/easyai-oss-object.mjs
+}
+trap cleanup EXIT HUP INT TERM
+command -v etcdutl >/dev/null
+command -v unzip >/dev/null
+command -v node >/dev/null
+
 snapshot_name=acceptance-$(date -u '+%Y%m%dT%H%M%SZ')
-k3s etcd-snapshot save --name "$snapshot_name" >/dev/null
+k3s etcd-snapshot save --name "$snapshot_name" >/dev/null 2>&1
 snapshot=$(find /var/lib/rancher/k3s/server/db/snapshots -maxdepth 1 -type f \
   -name "$snapshot_name-*" -printf '%T@ %p\n' | sort -rn | awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }')
 [[ -n $snapshot && -s $snapshot ]]
-etcdutl=$(find /var/lib/rancher/k3s/data -type f -path '*/bin/etcdutl' -print -quit)
-[[ -x $etcdutl ]]
-working_directory=$(mktemp -d)
-trap 'rm -rf -- "$working_directory"' EXIT
-case $snapshot in
-  *.zip)
-    if command -v unzip >/dev/null 2>&1 && unzip -t "$snapshot" >/dev/null 2>&1; then
-      unzip -p "$snapshot" >"$working_directory/snapshot.db"
-    else
-      gzip -dc "$snapshot" >"$working_directory/snapshot.db"
-    fi
-    ;;
-  *) cp "$snapshot" "$working_directory/snapshot.db" ;;
-esac
-"$etcdutl" snapshot status "$working_directory/snapshot.db" -w json |
+snapshot_file=$(basename "$snapshot")
+s3_location=$(k3s etcd-snapshot ls -o json 2>/dev/null |
+  jq -r --arg name "s3-$snapshot_file" \
+    '.items[] | select(.metadata.name == $name and .status.readyToUse == true) |
+      .spec.location')
+
+export ALI_KEY ALI_SECRET ALI_REGION ALI_BUCKET
+ALI_KEY=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \
+  -o jsonpath='{.data.etcd-s3-access-key}' | base64 -d)
+ALI_SECRET=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \
+  -o jsonpath='{.data.etcd-s3-secret-key}' | base64 -d)
+ALI_REGION=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \
+  -o jsonpath='{.data.etcd-s3-region}' | base64 -d)
+ALI_BUCKET=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \
+  -o jsonpath='{.data.etcd-s3-bucket}' | base64 -d)
+[[ $s3_location == \
+   "s3://$ALI_BUCKET/easyai-ai-gateway/production/k3s-etcd/$snapshot_file" ]]
+object_key=${s3_location#s3://"$ALI_BUCKET"/}
+node /root/easyai-oss-object.mjs get "$object_key" \
+  "$working_directory/s3-snapshot.zip" >/dev/null
+[[ $(sha256sum "$snapshot" | awk '{print $1}') == \
+   $(sha256sum "$working_directory/s3-snapshot.zip" | awk '{print $1}') ]]
+
+archive_entry=$(unzip -Z1 "$working_directory/s3-snapshot.zip")
+[[ -n $archive_entry && $archive_entry != *$'\n'* ]]
+unzip -p "$working_directory/s3-snapshot.zip" "$archive_entry" \
+  >"$working_directory/snapshot.db"
+etcdutl snapshot status "$working_directory/snapshot.db" -w json |
   jq -e '.hash != null and .totalKey > 0 and .totalSize > 0' >/dev/null
-k3s etcd-snapshot ls --s3 | grep -F "$snapshot_name" >/dev/null
-echo "etcd_backup_verification=PASS snapshot=$snapshot_name local=true oss=true"
+echo "etcd_backup_verification=PASS snapshot=$snapshot_file local=true oss_download=true status=true"
 REMOTE

scripts/cluster/verify-k3s-quorum.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+[[ ${1:-} == --execute ]] || {
+  echo 'usage: verify-k3s-quorum.sh --execute' >&2
+  exit 64
+}
+load_cluster_env
+
+active_host=
+restore_active_host() {
+  [[ -z ${active_host:-} ]] ||
+    cluster_ssh "$active_host" 'systemctl start k3s' >/dev/null 2>&1 || true
+}
+trap restore_active_host EXIT HUP INT TERM
+
+drill_server() {
+  local node_name=$1
+  local target_host=$2
+  local observer_host=$3
+  local started elapsed
+  active_host=$target_host
+  started=$(date +%s)
+  cluster_ssh "$target_host" 'systemctl stop k3s'
+  cluster_ssh "$target_host" '! systemctl is-active --quiet k3s'
+  cluster_ssh "$observer_host" bash -s <<'REMOTE'
+set -euo pipefail
+for _ in $(seq 1 60); do
+  if k3s kubectl get --raw='/readyz?verbose' 2>/dev/null |
+      grep -q '\[+\]etcd ok'; then
+    exit 0
+  fi
+  sleep 2
+done
+exit 1
+REMOTE
+  curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz |
+    grep -q '"ok":true'
+  cluster_ssh "$target_host" 'systemctl start k3s'
+  cluster_ssh "$observer_host" \
+    "k3s kubectl wait --for=condition=Ready node/'$node_name' --timeout=300s"
+  active_host=
+  elapsed=$(( $(date +%s) - started ))
+  echo "k3s_single_server_quorum=PASS node=$node_name recovery_seconds=$elapsed"
+}
+
+drill_server easyai-los-angeles "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_NINGBO_HOST"
+drill_server easyai-hongkong "$CLUSTER_HONGKONG_HOST" "$CLUSTER_NINGBO_HOST"
+drill_server easyai-ningbo "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"
+
+cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+for _ in $(seq 1 240); do
+  ready=$($kubectl get cluster easyai-postgres -n easyai \
+    -o jsonpath='{.status.readyInstances}')
+  if [[ $ready == 2 ]]; then
+    primary=$($kubectl get cluster easyai-postgres -n easyai \
+      -o jsonpath='{.status.currentPrimary}')
+    synchronous_replicas=$($kubectl exec -n easyai "$primary" -c postgres -- \
+      psql -X -U postgres -d postgres -At -c \
+      "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
+    if [[ $synchronous_replicas == 1 ]]; then
+      echo "k3s_quorum_drill=PASS servers=3 database_primary=$primary synchronous_replicas=1"
+      exit 0
+    fi
+  fi
+  sleep 3
+done
+exit 1
+REMOTE
+
+trap - EXIT HUP INT TERM