#!/usr/bin/env bash set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=scripts/cluster/common.sh source "$script_dir/common.sh" load_cluster_env cluster_scp "$script_dir/oss-object.mjs" \ "$CLUSTER_NINGBO_HOST:/root/easyai-oss-object.mjs" cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' set -euo pipefail working_directory=$(mktemp -d) cleanup() { find "$working_directory" -type f -delete find "$working_directory" -depth -type d -empty -delete [[ ! -e /root/easyai-oss-object.mjs && ! -L /root/easyai-oss-object.mjs ]] || unlink /root/easyai-oss-object.mjs } trap cleanup EXIT HUP INT TERM command -v etcdutl >/dev/null command -v unzip >/dev/null command -v node >/dev/null snapshot_name=acceptance-$(date -u '+%Y%m%dT%H%M%SZ') k3s etcd-snapshot save --name "$snapshot_name" >/dev/null 2>&1 snapshot=$(find /var/lib/rancher/k3s/server/db/snapshots -maxdepth 1 -type f \ -name "$snapshot_name-*" -printf '%T@ %p\n' | sort -rn | awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }') [[ -n $snapshot && -s $snapshot ]] snapshot_file=$(basename "$snapshot") s3_location=$(k3s etcd-snapshot ls -o json 2>/dev/null | jq -r --arg name "s3-$snapshot_file" \ '.items[] | select(.metadata.name == $name and .status.readyToUse == true) | .spec.location') export ALI_KEY ALI_SECRET ALI_REGION ALI_BUCKET ALI_KEY=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \ -o jsonpath='{.data.etcd-s3-access-key}' | base64 -d) ALI_SECRET=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \ -o jsonpath='{.data.etcd-s3-secret-key}' | base64 -d) ALI_REGION=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \ -o jsonpath='{.data.etcd-s3-region}' | base64 -d) ALI_BUCKET=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \ -o jsonpath='{.data.etcd-s3-bucket}' | base64 -d) [[ $s3_location == \ "s3://$ALI_BUCKET/easyai-ai-gateway/production/k3s-etcd/$snapshot_file" ]] object_key=${s3_location#s3://"$ALI_BUCKET"/} node /root/easyai-oss-object.mjs get "$object_key" \ "$working_directory/s3-snapshot.zip" >/dev/null [[ $(sha256sum "$snapshot" | awk '{print $1}') == \ $(sha256sum "$working_directory/s3-snapshot.zip" | awk '{print $1}') ]] archive_entry=$(unzip -Z1 "$working_directory/s3-snapshot.zip") [[ -n $archive_entry && $archive_entry != *$'\n'* ]] unzip -p "$working_directory/s3-snapshot.zip" "$archive_entry" \ >"$working_directory/snapshot.db" etcdutl snapshot status "$working_directory/snapshot.db" -w json | jq -e '.hash != null and .totalKey > 0 and .totalSize > 0' >/dev/null echo "etcd_backup_verification=PASS snapshot=$snapshot_file local=true oss_download=true status=true" REMOTE