#!/usr/bin/env bash set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=scripts/cluster/common.sh source "$script_dir/common.sh" [[ ${1:-} == --execute ]] || { echo 'usage: restore-backup-drill.sh --execute' >&2 exit 64 } load_cluster_env cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' set -euo pipefail kubectl='k3s kubectl' suffix=$(date -u '+%m%d%H%M%S') restore_cluster=easyai-restore-drill-"$suffix" cleanup() { $kubectl delete cluster "$restore_cluster" -n easyai \ --ignore-not-found=true --wait=true --timeout=300s || true $kubectl delete pvc -n easyai -l "cnpg.io/cluster=$restore_cluster" \ --ignore-not-found=true --wait=true --timeout=300s || true } trap cleanup EXIT HUP INT TERM source_primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}') acceptance_sql=$(mktemp) trap 'rm -f -- "$acceptance_sql"; cleanup' EXIT HUP INT TERM cat >"$acceptance_sql" <<'SQL' \pset tuples_only on \pset format unaligned SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE c.relkind='r' AND n.nspname='public'; SELECT 'tasks=' || count(*) FROM gateway_tasks; SELECT 'attempts=' || count(*) FROM gateway_task_attempts; SELECT 'wallet=' || count(*) FROM gateway_wallet_transactions; SELECT 'users=' || count(*) FROM gateway_users; SELECT 'sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id),'')) FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) rows; SQL source_snapshot=$($kubectl exec -i -n easyai "$source_primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway <"$acceptance_sql") cat </dev/null apiVersion: postgresql.cnpg.io/v1 kind: Cluster metadata: name: $restore_cluster namespace: easyai labels: easyai.io/acceptance-drill: backup-restore spec: instances: 1 imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c bootstrap: recovery: source: origin database: easyai_ai_gateway owner: easyai secret: name: easyai-postgres-app externalClusters: - name: origin plugin: name: barman-cloud.cloudnative-pg.io parameters: barmanObjectName: easyai-postgres-oss serverName: easyai-postgres storage: storageClass: local-path size: 20Gi walStorage: storageClass: local-path size: 5Gi affinity: nodeSelector: easyai.io/site: ningbo resources: requests: cpu: 250m memory: 512Mi limits: cpu: "2" memory: 3Gi projectedVolumeTemplate: defaultMode: 0444 sources: - configMap: name: easyai-barman-aws-config items: - key: config path: barman-aws/config EOF $kubectl wait --for=condition=Ready "cluster/$restore_cluster" -n easyai --timeout=1200s restore_primary=$($kubectl get cluster "$restore_cluster" -n easyai -o jsonpath='{.status.currentPrimary}') restored_snapshot=$($kubectl exec -i -n easyai "$restore_primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway <"$acceptance_sql") [[ $restored_snapshot == "$source_snapshot" ]] || { diff -u <(printf '%s\n' "$source_snapshot") <(printf '%s\n' "$restored_snapshot") exit 1 } $kubectl exec -n easyai "$restore_primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \ "SELECT current_database();" | grep -qx 'easyai_ai_gateway' cleanup trap - EXIT HUP INT TERM rm -f -- "$acceptance_sql" echo "backup_restore_drill=PASS cluster=$restore_cluster deleted=true" REMOTE