fix(deploy): 补齐集群备份兼容与切换前门禁

阿里云 OSS 的 S3 兼容层要求 virtual-hosted addressing，且 boto3 需要使用 Signature V2。本提交为 Barman 的归档、备份、保留与恢复统一挂载 AWS 配置，并将实际 WAL 归档和近期全备设为切换硬门禁。

同时补充 CNPG/etcdutl 固定版本安装、显式主库切换、三节点逐台停机、OSS 快照下载校验与远端临时 Secret 清理。已通过 ShellCheck、Kustomize、服务端 dry-run、迁移测试、发布脚本测试、敏感信息扫描及生产 precutover 验收。

scripts/cluster/migrate-production.sh

@@ -37,6 +37,45 @@ docker exec easyai-ai-gateway-postgres-1 pg_isready >/dev/null
 k3s kubectl get --raw=/readyz >/dev/null
 [[ $(k3s kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
 [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
+primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
+[[ $primary =~ ^easyai-postgres-[0-9]+$ ]]
+for pod in $(k3s kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
+  -o jsonpath='{.items[*].metadata.name}'); do
+  k3s kubectl exec -n easyai "$pod" -c postgres -- \
+    test -r /projected/barman-aws/config
+done
+archived_before=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -U postgres -d postgres -At -c \
+  'SELECT archived_count FROM pg_stat_archiver;')
+archived_after=$(k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
+  bash -s -- "$archived_before" <<'POD'
+set -euo pipefail
+archived_before=$1
+psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres -At \
+  -c "SELECT pg_logical_emit_message(true, 'easyai-ha-preflight', clock_timestamp()::text);" \
+  -c 'SELECT pg_switch_wal();' >/dev/null
+for _ in $(seq 1 45); do
+  archived_after=$(psql -X -U postgres -d postgres -At -c \
+    'SELECT archived_count FROM pg_stat_archiver;')
+  if (( archived_after > archived_before )); then
+    printf '%s\n' "$archived_after"
+    exit 0
+  fi
+  sleep 2
+done
+exit 1
+POD
+)
+(( archived_after > archived_before ))
+[[ $(k3s kubectl get cluster easyai-postgres -n easyai \
+  -o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}') == True ]]
+latest_completed_backup=$(k3s kubectl get backups -n easyai -o json |
+  jq -r '[.items[] | select(
+    .spec.cluster.name == "easyai-postgres" and .status.phase == "completed"
+  ) | .status.stoppedAt] | sort | last // empty')
+[[ -n $latest_completed_backup ]]
+latest_completed_epoch=$(date -d "$latest_completed_backup" '+%s')
+(( $(date '+%s') - latest_completed_epoch < 172800 ))
 [[ $(df --output=avail -B1 /var/lib/docker | tail -1) -gt 2147483648 ]]
 [[ -f /etc/nginx/sites-available/ai.51easyai.com-maintenance ]]
 [[ -f /etc/nginx/sites-available/ai.51easyai.com-cluster ]]
@@ -196,7 +235,7 @@ k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
   pg_restore --exit-on-error --no-owner --role=easyai \
   -U postgres -d easyai_ai_gateway <"$backup_directory/database.dump"
 k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway \
   <"$snapshot_sql" >"$backup_directory/restored.snapshot"
 diff -u "$backup_directory/source.snapshot" "$backup_directory/restored.snapshot"
 
@@ -236,11 +275,11 @@ k3s kubectl apply -f "$backup_directory/migrator-job.yaml" >/dev/null
 k3s kubectl wait --for=condition=complete \
   "job/easyai-migrator-${source_sha:0:12}" -n easyai --timeout=300s
 k3s kubectl exec -n easyai "$primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
   "SELECT count(*) FROM schema_migrations WHERE version='0089_file_storage_request_asset_scene';" |
   grep -qx 1
 k3s kubectl exec -n easyai "$primary" -c postgres -- \
-  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
   "SELECT count(*) FROM file_storage_channels WHERE deleted_at IS NULL AND provider='server_main_openapi' AND config->'scenes' ? 'request_asset';" |
   grep -Eq '^[1-9][0-9]*$'
 
@@ -316,7 +355,7 @@ $kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
 $kubectl set env deployment/easyai-api-ningbo -n easyai \
   AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true
 $kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
-cat <<'EOF' | $kubectl create -f - >/dev/null
+backup_resource=$(cat <<'EOF' | $kubectl create -f - -o name
 apiVersion: postgresql.cnpg.io/v1
 kind: Backup
 metadata:
@@ -329,10 +368,28 @@ spec:
   pluginConfiguration:
     name: barman-cloud.cloudnative-pg.io
 EOF
+)
+backup_name=${backup_resource#*/}
+backup_phase=
+for _ in $(seq 1 240); do
+  backup_phase=$($kubectl get backup "$backup_name" -n easyai \
+    -o jsonpath='{.status.phase}')
+  [[ $backup_phase == completed ]] && break
+  if [[ $backup_phase == failed ]]; then
+    echo "post-cutover CNPG backup failed: $backup_name" >&2
+    exit 1
+  fi
+  sleep 5
+done
+[[ $backup_phase == completed ]] || {
+  echo "post-cutover CNPG backup did not complete in time: $backup_name" >&2
+  exit 1
+}
 docker update --restart=no easyai-ai-gateway-api-1 \
   easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
 docker stop --time 30 easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
 rm -f -- /root/easyai-remote-cutover.sh /root/easyai-oss-object.mjs
+echo "post_cutover_backup=PASS name=$backup_name"
 REMOTE
 
 "$script_dir/install-release-helper.sh"