fix(deploy): 补齐集群备份兼容与切换前门禁
阿里云 OSS 的 S3 兼容层要求 virtual-hosted addressing，且 boto3 需要使用 Signature V2。本提交为 Barman 的归档、备份、保留与恢复统一挂载 AWS 配置，并将实际 WAL 归档和近期全备设为切换硬门禁。 同时补充 CNPG/etcdutl 固定版本安装、显式主库切换、三节点逐台停机、OSS 快照下载校验与远端临时 Secret 清理。已通过 ShellCheck、Kustomize、服务端 dry-run、迁移测试、发布脚本测试、敏感信息扫描及生产 precutover 验收。
This commit is contained in:
@@ -135,6 +135,21 @@ set -euo pipefail
|
||||
source_sha=$1
|
||||
mode=$2
|
||||
kubectl='k3s kubectl'
|
||||
runtime_environment=
|
||||
cleanup_bootstrap_files() {
|
||||
if [[ -n ${runtime_environment:-} && -f $runtime_environment &&
|
||||
! -L $runtime_environment ]]; then
|
||||
shred -u "$runtime_environment"
|
||||
fi
|
||||
if [[ -f /root/bootstrap-secrets.yaml && ! -L /root/bootstrap-secrets.yaml ]]; then
|
||||
shred -u /root/bootstrap-secrets.yaml
|
||||
fi
|
||||
for path in /root/cert-manager.yaml /root/cnpg.yaml /root/barman.yaml \
|
||||
/root/easyai-production.yaml; do
|
||||
[[ ! -e $path && ! -L $path ]] || unlink "$path"
|
||||
done
|
||||
}
|
||||
trap cleanup_bootstrap_files EXIT HUP INT TERM
|
||||
|
||||
$kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null
|
||||
$kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s
|
||||
@@ -151,7 +166,6 @@ postgres_password=$($kubectl get secret easyai-postgres-app -n easyai \
|
||||
-o jsonpath='{.data.password}' | base64 -d)
|
||||
|
||||
runtime_environment=$(mktemp)
|
||||
trap 'rm -f -- "$runtime_environment"' EXIT
|
||||
chmod 0600 "$runtime_environment"
|
||||
docker inspect easyai-ai-gateway-api-1 \
|
||||
--format '{{range .Config.Env}}{{println .}}{{end}}' |
|
||||
@@ -206,6 +220,9 @@ $kubectl create secret generic easyai-registry -n easyai \
|
||||
--from-file=.dockerconfigjson="$registry_config" --dry-run=client -o yaml |
|
||||
$kubectl apply -f - >/dev/null
|
||||
|
||||
$kubectl apply --server-side --dry-run=server --force-conflicts \
|
||||
--field-manager=easyai-production-preflight \
|
||||
-f /root/easyai-production.yaml >/dev/null
|
||||
$kubectl apply -f /root/easyai-production.yaml >/dev/null
|
||||
$kubectl annotate namespace easyai easyai.io/release="$source_sha" --overwrite >/dev/null
|
||||
if [[ $mode == activate ]]; then
|
||||
@@ -215,8 +232,6 @@ if [[ $mode == activate ]]; then
|
||||
$kubectl rollout status deployment/easyai-web-ningbo -n easyai --timeout=300s
|
||||
fi
|
||||
|
||||
rm -f -- /root/cert-manager.yaml /root/cnpg.yaml /root/barman.yaml \
|
||||
/root/easyai-production.yaml /root/bootstrap-secrets.yaml
|
||||
REMOTE
|
||||
|
||||
if [[ $mode == prepare ]]; then
|
||||
|
||||
@@ -52,12 +52,12 @@ case $drill in
|
||||
set -euo pipefail
|
||||
primary=$1
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -c \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \
|
||||
"INSERT INTO system_settings(setting_key,value) VALUES('ha_acceptance_probe',jsonb_build_object('at',now())) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" >/dev/null
|
||||
for _ in $(seq 1 180); do
|
||||
if [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') == 2 ]]; then
|
||||
replication=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -U easyai -d easyai_ai_gateway -At -c \
|
||||
psql -X -U postgres -d easyai_ai_gateway -At -c \
|
||||
"SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
|
||||
[[ $replication == 1 ]] && break
|
||||
fi
|
||||
@@ -65,7 +65,7 @@ for _ in $(seq 1 180); do
|
||||
done
|
||||
[[ ${replication:-0} == 1 ]]
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -c \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \
|
||||
"DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null
|
||||
REMOTE
|
||||
echo "database_failure_drill=PASS promotion_seconds=$elapsed old_primary=$old_primary new_primary=$new_primary"
|
||||
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=scripts/cluster/common.sh
|
||||
source "$script_dir/common.sh"
|
||||
|
||||
load_cluster_env
|
||||
require_commands curl shasum tar
|
||||
|
||||
plugin_version=1.29.1
|
||||
plugin_archive=kubectl-cnpg_"$plugin_version"_linux_x86_64.tar.gz
|
||||
plugin_sha256=2a550d5ee2224e5bb66b95602f8052fc50da50485f59f97b817a0c3b44b58581
|
||||
working_directory=$(mktemp -d "$cluster_root/.local-secrets/cluster/cnpg-plugin.XXXXXX")
|
||||
trap '[[ $working_directory == "$cluster_root"/.local-secrets/cluster/cnpg-plugin.* ]] && rm -rf -- "$working_directory"' \
|
||||
EXIT HUP INT TERM
|
||||
|
||||
curl -fsSL \
|
||||
"https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v${plugin_version}/${plugin_archive}" \
|
||||
-o "$working_directory/$plugin_archive"
|
||||
actual_sha256=$(shasum -a 256 "$working_directory/$plugin_archive" | awk '{print $1}')
|
||||
[[ $actual_sha256 == "$plugin_sha256" ]] || {
|
||||
echo 'kubectl-cnpg archive checksum mismatch' >&2
|
||||
exit 1
|
||||
}
|
||||
tar -tzf "$working_directory/$plugin_archive" | grep -qx kubectl-cnpg
|
||||
tar -xzf "$working_directory/$plugin_archive" -C "$working_directory" kubectl-cnpg
|
||||
cluster_scp "$working_directory/kubectl-cnpg" "$CLUSTER_NINGBO_HOST:/root/kubectl-cnpg"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$plugin_version" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
plugin_version=$1
|
||||
install -m 0755 /root/kubectl-cnpg /usr/local/bin/kubectl-cnpg
|
||||
unlink /root/kubectl-cnpg
|
||||
k3s kubectl cnpg version | grep -q "Version:${plugin_version}"
|
||||
REMOTE
|
||||
|
||||
echo "cnpg_plugin_install=PASS version=$plugin_version sha256=$plugin_sha256"
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=scripts/cluster/common.sh
|
||||
source "$script_dir/common.sh"
|
||||
|
||||
load_cluster_env
|
||||
require_commands curl shasum tar
|
||||
|
||||
# K3s v1.36.2 embeds etcd 3.6.12. The closest upstream release utility is
|
||||
# 3.6.11 and uses the same v3 snapshot format.
|
||||
etcd_version=3.6.11
|
||||
etcd_archive=etcd-v"$etcd_version"-linux-amd64.tar.gz
|
||||
etcd_archive_sha256=8756f7a4eaf921668a83de0bf13c0f65cae9186a165696e3ae8396afe6f557ed
|
||||
working_directory=$(mktemp -d "$cluster_root/.local-secrets/cluster/etcdutl.XXXXXX")
|
||||
trap '[[ $working_directory == "$cluster_root"/.local-secrets/cluster/etcdutl.* ]] && rm -rf -- "$working_directory"' \
|
||||
EXIT HUP INT TERM
|
||||
|
||||
curl -fsSL \
|
||||
"https://github.com/etcd-io/etcd/releases/download/v${etcd_version}/${etcd_archive}" \
|
||||
-o "$working_directory/$etcd_archive"
|
||||
actual_sha256=$(shasum -a 256 "$working_directory/$etcd_archive" | awk '{print $1}')
|
||||
[[ $actual_sha256 == "$etcd_archive_sha256" ]] || {
|
||||
echo 'etcd release archive checksum mismatch' >&2
|
||||
exit 1
|
||||
}
|
||||
archive_member=etcd-v"$etcd_version"-linux-amd64/etcdutl
|
||||
tar -tzf "$working_directory/$etcd_archive" | grep -qx "$archive_member"
|
||||
tar -xzf "$working_directory/$etcd_archive" -C "$working_directory" "$archive_member"
|
||||
etcdutl_binary="$working_directory/$archive_member"
|
||||
|
||||
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
|
||||
cluster_scp "$etcdutl_binary" "$host:/root/etcdutl"
|
||||
cluster_ssh "$host" bash -s -- "$etcd_version" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
etcd_version=$1
|
||||
install -m 0755 /root/etcdutl /usr/local/bin/etcdutl
|
||||
unlink /root/etcdutl
|
||||
etcdutl version | grep -q "etcdutl version: ${etcd_version}"
|
||||
REMOTE
|
||||
done
|
||||
|
||||
echo "etcdutl_install=PASS version=$etcd_version nodes=3 sha256=$etcd_archive_sha256"
|
||||
@@ -37,6 +37,45 @@ docker exec easyai-ai-gateway-postgres-1 pg_isready >/dev/null
|
||||
k3s kubectl get --raw=/readyz >/dev/null
|
||||
[[ $(k3s kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
|
||||
[[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
|
||||
primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
|
||||
[[ $primary =~ ^easyai-postgres-[0-9]+$ ]]
|
||||
for pod in $(k3s kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
|
||||
-o jsonpath='{.items[*].metadata.name}'); do
|
||||
k3s kubectl exec -n easyai "$pod" -c postgres -- \
|
||||
test -r /projected/barman-aws/config
|
||||
done
|
||||
archived_before=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -U postgres -d postgres -At -c \
|
||||
'SELECT archived_count FROM pg_stat_archiver;')
|
||||
archived_after=$(k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
|
||||
bash -s -- "$archived_before" <<'POD'
|
||||
set -euo pipefail
|
||||
archived_before=$1
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres -At \
|
||||
-c "SELECT pg_logical_emit_message(true, 'easyai-ha-preflight', clock_timestamp()::text);" \
|
||||
-c 'SELECT pg_switch_wal();' >/dev/null
|
||||
for _ in $(seq 1 45); do
|
||||
archived_after=$(psql -X -U postgres -d postgres -At -c \
|
||||
'SELECT archived_count FROM pg_stat_archiver;')
|
||||
if (( archived_after > archived_before )); then
|
||||
printf '%s\n' "$archived_after"
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
exit 1
|
||||
POD
|
||||
)
|
||||
(( archived_after > archived_before ))
|
||||
[[ $(k3s kubectl get cluster easyai-postgres -n easyai \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}') == True ]]
|
||||
latest_completed_backup=$(k3s kubectl get backups -n easyai -o json |
|
||||
jq -r '[.items[] | select(
|
||||
.spec.cluster.name == "easyai-postgres" and .status.phase == "completed"
|
||||
) | .status.stoppedAt] | sort | last // empty')
|
||||
[[ -n $latest_completed_backup ]]
|
||||
latest_completed_epoch=$(date -d "$latest_completed_backup" '+%s')
|
||||
(( $(date '+%s') - latest_completed_epoch < 172800 ))
|
||||
[[ $(df --output=avail -B1 /var/lib/docker | tail -1) -gt 2147483648 ]]
|
||||
[[ -f /etc/nginx/sites-available/ai.51easyai.com-maintenance ]]
|
||||
[[ -f /etc/nginx/sites-available/ai.51easyai.com-cluster ]]
|
||||
@@ -196,7 +235,7 @@ k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
|
||||
pg_restore --exit-on-error --no-owner --role=easyai \
|
||||
-U postgres -d easyai_ai_gateway <"$backup_directory/database.dump"
|
||||
k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway \
|
||||
<"$snapshot_sql" >"$backup_directory/restored.snapshot"
|
||||
diff -u "$backup_directory/source.snapshot" "$backup_directory/restored.snapshot"
|
||||
|
||||
@@ -236,11 +275,11 @@ k3s kubectl apply -f "$backup_directory/migrator-job.yaml" >/dev/null
|
||||
k3s kubectl wait --for=condition=complete \
|
||||
"job/easyai-migrator-${source_sha:0:12}" -n easyai --timeout=300s
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
|
||||
"SELECT count(*) FROM schema_migrations WHERE version='0089_file_storage_request_asset_scene';" |
|
||||
grep -qx 1
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
|
||||
"SELECT count(*) FROM file_storage_channels WHERE deleted_at IS NULL AND provider='server_main_openapi' AND config->'scenes' ? 'request_asset';" |
|
||||
grep -Eq '^[1-9][0-9]*$'
|
||||
|
||||
@@ -316,7 +355,7 @@ $kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
|
||||
$kubectl set env deployment/easyai-api-ningbo -n easyai \
|
||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true
|
||||
$kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
|
||||
cat <<'EOF' | $kubectl create -f - >/dev/null
|
||||
backup_resource=$(cat <<'EOF' | $kubectl create -f - -o name
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Backup
|
||||
metadata:
|
||||
@@ -329,10 +368,28 @@ spec:
|
||||
pluginConfiguration:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
EOF
|
||||
)
|
||||
backup_name=${backup_resource#*/}
|
||||
backup_phase=
|
||||
for _ in $(seq 1 240); do
|
||||
backup_phase=$($kubectl get backup "$backup_name" -n easyai \
|
||||
-o jsonpath='{.status.phase}')
|
||||
[[ $backup_phase == completed ]] && break
|
||||
if [[ $backup_phase == failed ]]; then
|
||||
echo "post-cutover CNPG backup failed: $backup_name" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
[[ $backup_phase == completed ]] || {
|
||||
echo "post-cutover CNPG backup did not complete in time: $backup_name" >&2
|
||||
exit 1
|
||||
}
|
||||
docker update --restart=no easyai-ai-gateway-api-1 \
|
||||
easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
|
||||
docker stop --time 30 easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
|
||||
rm -f -- /root/easyai-remote-cutover.sh /root/easyai-oss-object.mjs
|
||||
echo "post_cutover_backup=PASS name=$backup_name"
|
||||
REMOTE
|
||||
|
||||
"$script_dir/install-release-helper.sh"
|
||||
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=scripts/cluster/common.sh
|
||||
source "$script_dir/common.sh"
|
||||
|
||||
target_primary=${1:-}
|
||||
[[ $target_primary =~ ^easyai-postgres-[12]$ ]] || {
|
||||
echo 'usage: promote-cnpg-primary.sh easyai-postgres-{1|2}' >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
load_cluster_env
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$target_primary" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
target_primary=$1
|
||||
kubectl='k3s kubectl'
|
||||
command -v kubectl-cnpg >/dev/null
|
||||
[[ $($kubectl get pod "$target_primary" -n easyai \
|
||||
-o jsonpath='{.metadata.labels.cnpg\.io/cluster}') == easyai-postgres ]]
|
||||
current_primary=$($kubectl get cluster easyai-postgres -n easyai \
|
||||
-o jsonpath='{.status.currentPrimary}')
|
||||
if [[ $current_primary != "$target_primary" ]]; then
|
||||
$kubectl cnpg promote easyai-postgres "$target_primary" -n easyai
|
||||
fi
|
||||
for _ in $(seq 1 120); do
|
||||
current_primary=$($kubectl get cluster easyai-postgres -n easyai \
|
||||
-o jsonpath='{.status.currentPrimary}')
|
||||
ready_instances=$($kubectl get cluster easyai-postgres -n easyai \
|
||||
-o jsonpath='{.status.readyInstances}')
|
||||
if [[ $current_primary == "$target_primary" && $ready_instances == 2 ]]; then
|
||||
synchronous_replicas=$($kubectl exec -n easyai "$current_primary" -c postgres -- \
|
||||
psql -X -U postgres -d postgres -At -c \
|
||||
"SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
|
||||
if [[ $synchronous_replicas == 1 ]]; then
|
||||
echo "cnpg_primary_promotion=PASS primary=$current_primary synchronous_replicas=1"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
echo "CNPG did not converge on target primary: $target_primary" >&2
|
||||
exit 1
|
||||
REMOTE
|
||||
@@ -40,7 +40,7 @@ SELECT 'sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id),''))
|
||||
FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) rows;
|
||||
SQL
|
||||
source_snapshot=$($kubectl exec -i -n easyai "$source_primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway <"$acceptance_sql")
|
||||
|
||||
cat <<EOF | $kubectl apply -f - >/dev/null
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
@@ -83,19 +83,27 @@ spec:
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 3Gi
|
||||
projectedVolumeTemplate:
|
||||
defaultMode: 0444
|
||||
sources:
|
||||
- configMap:
|
||||
name: easyai-barman-aws-config
|
||||
items:
|
||||
- key: config
|
||||
path: barman-aws/config
|
||||
EOF
|
||||
|
||||
$kubectl wait --for=condition=Ready "cluster/$restore_cluster" -n easyai --timeout=1200s
|
||||
restore_primary=$($kubectl get cluster "$restore_cluster" -n easyai -o jsonpath='{.status.currentPrimary}')
|
||||
restored_snapshot=$($kubectl exec -i -n easyai "$restore_primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway <"$acceptance_sql")
|
||||
[[ $restored_snapshot == "$source_snapshot" ]] || {
|
||||
diff -u <(printf '%s\n' "$source_snapshot") <(printf '%s\n' "$restored_snapshot")
|
||||
exit 1
|
||||
}
|
||||
$kubectl exec -n easyai "$restore_primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
|
||||
"SELECT current_user || ':' || current_database();" | grep -qx 'easyai:easyai_ai_gateway'
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
|
||||
"SELECT current_database();" | grep -qx 'easyai_ai_gateway'
|
||||
cleanup
|
||||
trap - EXIT HUP INT TERM
|
||||
rm -f -- "$acceptance_sql"
|
||||
|
||||
@@ -71,7 +71,7 @@ source_sha=$2
|
||||
primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
|
||||
query() {
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At "$@"
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At "$@"
|
||||
}
|
||||
attempted_by=$(query -c \
|
||||
"SELECT attempted_by::text FROM river_job WHERE id=(SELECT river_job_id FROM gateway_tasks WHERE id='$task_id'::uuid);")
|
||||
|
||||
@@ -53,8 +53,7 @@ REMOTE
|
||||
[[ $cluster_state == nodes=3 ]]
|
||||
echo 'k3s_control_plane=PASS servers=3 witness_business_pods=0'
|
||||
|
||||
if [[ $mode == postcutover ]]; then
|
||||
database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
kubectl='k3s kubectl'
|
||||
[[ $($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
|
||||
@@ -62,9 +61,36 @@ primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.c
|
||||
[[ $($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-ningbo ||
|
||||
$($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-hongkong ]]
|
||||
replication=$($kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -U easyai -d easyai_ai_gateway -At -c \
|
||||
psql -X -U postgres -d easyai_ai_gateway -At -c \
|
||||
"SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
|
||||
[[ $replication -eq 1 ]]
|
||||
archived_count=$($kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -U postgres -d postgres -At -c \
|
||||
'SELECT archived_count FROM pg_stat_archiver;')
|
||||
(( archived_count > 0 ))
|
||||
[[ $($kubectl get cluster easyai-postgres -n easyai \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}') == True ]]
|
||||
completed_backups=$($kubectl get backups -n easyai -o json |
|
||||
jq '[.items[] | select(
|
||||
.spec.cluster.name == "easyai-postgres" and .status.phase == "completed"
|
||||
)] | length')
|
||||
(( completed_backups > 0 ))
|
||||
for pod in $($kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
|
||||
-o jsonpath='{.items[*].metadata.name}'); do
|
||||
$kubectl exec -n easyai "$pod" -c postgres -- \
|
||||
test -r /projected/barman-aws/config
|
||||
done
|
||||
printf 'primary=%s archived=%s backups=%s\n' \
|
||||
"$primary" "$archived_count" "$completed_backups"
|
||||
REMOTE
|
||||
)
|
||||
[[ $database_state == primary=easyai-postgres-* ]]
|
||||
echo "postgres_preflight=PASS $database_state synchronous_replicas=1"
|
||||
|
||||
if [[ $mode == postcutover ]]; then
|
||||
application_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
kubectl='k3s kubectl'
|
||||
[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-api \
|
||||
--field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
|
||||
[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-web \
|
||||
@@ -82,11 +108,11 @@ fi
|
||||
--as=system:serviceaccount:easyai:easyai-ai-gateway) == yes ]]
|
||||
[[ $($kubectl auth can-i update secret/easyai-ai-gateway-runtime -n easyai \
|
||||
--as=system:serviceaccount:easyai:easyai-ai-gateway) == no ]]
|
||||
printf 'primary=%s\n' "$primary"
|
||||
printf 'applications=ready\n'
|
||||
REMOTE
|
||||
)
|
||||
[[ $database_state == primary=easyai-postgres-* ]]
|
||||
echo "postgres_acceptance=PASS $database_state synchronous_replicas=1"
|
||||
[[ $application_state == applications=ready ]]
|
||||
echo 'application_acceptance=PASS api=2 web=2 digests=pinned'
|
||||
|
||||
for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
|
||||
curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
|
||||
@@ -6,29 +6,55 @@ script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
source "$script_dir/common.sh"
|
||||
load_cluster_env
|
||||
|
||||
cluster_scp "$script_dir/oss-object.mjs" \
|
||||
"$CLUSTER_NINGBO_HOST:/root/easyai-oss-object.mjs"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
working_directory=$(mktemp -d)
|
||||
cleanup() {
|
||||
find "$working_directory" -type f -delete
|
||||
find "$working_directory" -depth -type d -empty -delete
|
||||
[[ ! -e /root/easyai-oss-object.mjs && ! -L /root/easyai-oss-object.mjs ]] ||
|
||||
unlink /root/easyai-oss-object.mjs
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
command -v etcdutl >/dev/null
|
||||
command -v unzip >/dev/null
|
||||
command -v node >/dev/null
|
||||
|
||||
snapshot_name=acceptance-$(date -u '+%Y%m%dT%H%M%SZ')
|
||||
k3s etcd-snapshot save --name "$snapshot_name" >/dev/null
|
||||
k3s etcd-snapshot save --name "$snapshot_name" >/dev/null 2>&1
|
||||
snapshot=$(find /var/lib/rancher/k3s/server/db/snapshots -maxdepth 1 -type f \
|
||||
-name "$snapshot_name-*" -printf '%T@ %p\n' | sort -rn | awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }')
|
||||
[[ -n $snapshot && -s $snapshot ]]
|
||||
etcdutl=$(find /var/lib/rancher/k3s/data -type f -path '*/bin/etcdutl' -print -quit)
|
||||
[[ -x $etcdutl ]]
|
||||
working_directory=$(mktemp -d)
|
||||
trap 'rm -rf -- "$working_directory"' EXIT
|
||||
case $snapshot in
|
||||
*.zip)
|
||||
if command -v unzip >/dev/null 2>&1 && unzip -t "$snapshot" >/dev/null 2>&1; then
|
||||
unzip -p "$snapshot" >"$working_directory/snapshot.db"
|
||||
else
|
||||
gzip -dc "$snapshot" >"$working_directory/snapshot.db"
|
||||
fi
|
||||
;;
|
||||
*) cp "$snapshot" "$working_directory/snapshot.db" ;;
|
||||
esac
|
||||
"$etcdutl" snapshot status "$working_directory/snapshot.db" -w json |
|
||||
snapshot_file=$(basename "$snapshot")
|
||||
s3_location=$(k3s etcd-snapshot ls -o json 2>/dev/null |
|
||||
jq -r --arg name "s3-$snapshot_file" \
|
||||
'.items[] | select(.metadata.name == $name and .status.readyToUse == true) |
|
||||
.spec.location')
|
||||
|
||||
export ALI_KEY ALI_SECRET ALI_REGION ALI_BUCKET
|
||||
ALI_KEY=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \
|
||||
-o jsonpath='{.data.etcd-s3-access-key}' | base64 -d)
|
||||
ALI_SECRET=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \
|
||||
-o jsonpath='{.data.etcd-s3-secret-key}' | base64 -d)
|
||||
ALI_REGION=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \
|
||||
-o jsonpath='{.data.etcd-s3-region}' | base64 -d)
|
||||
ALI_BUCKET=$(k3s kubectl get secret easyai-etcd-s3 -n kube-system \
|
||||
-o jsonpath='{.data.etcd-s3-bucket}' | base64 -d)
|
||||
[[ $s3_location == \
|
||||
"s3://$ALI_BUCKET/easyai-ai-gateway/production/k3s-etcd/$snapshot_file" ]]
|
||||
object_key=${s3_location#s3://"$ALI_BUCKET"/}
|
||||
node /root/easyai-oss-object.mjs get "$object_key" \
|
||||
"$working_directory/s3-snapshot.zip" >/dev/null
|
||||
[[ $(sha256sum "$snapshot" | awk '{print $1}') == \
|
||||
$(sha256sum "$working_directory/s3-snapshot.zip" | awk '{print $1}') ]]
|
||||
|
||||
archive_entry=$(unzip -Z1 "$working_directory/s3-snapshot.zip")
|
||||
[[ -n $archive_entry && $archive_entry != *$'\n'* ]]
|
||||
unzip -p "$working_directory/s3-snapshot.zip" "$archive_entry" \
|
||||
>"$working_directory/snapshot.db"
|
||||
etcdutl snapshot status "$working_directory/snapshot.db" -w json |
|
||||
jq -e '.hash != null and .totalKey > 0 and .totalSize > 0' >/dev/null
|
||||
k3s etcd-snapshot ls --s3 | grep -F "$snapshot_name" >/dev/null
|
||||
echo "etcd_backup_verification=PASS snapshot=$snapshot_name local=true oss=true"
|
||||
echo "etcd_backup_verification=PASS snapshot=$snapshot_file local=true oss_download=true status=true"
|
||||
REMOTE
|
||||
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=scripts/cluster/common.sh
|
||||
source "$script_dir/common.sh"
|
||||
|
||||
[[ ${1:-} == --execute ]] || {
|
||||
echo 'usage: verify-k3s-quorum.sh --execute' >&2
|
||||
exit 64
|
||||
}
|
||||
load_cluster_env
|
||||
|
||||
active_host=
|
||||
restore_active_host() {
|
||||
[[ -z ${active_host:-} ]] ||
|
||||
cluster_ssh "$active_host" 'systemctl start k3s' >/dev/null 2>&1 || true
|
||||
}
|
||||
trap restore_active_host EXIT HUP INT TERM
|
||||
|
||||
drill_server() {
|
||||
local node_name=$1
|
||||
local target_host=$2
|
||||
local observer_host=$3
|
||||
local started elapsed
|
||||
active_host=$target_host
|
||||
started=$(date +%s)
|
||||
cluster_ssh "$target_host" 'systemctl stop k3s'
|
||||
cluster_ssh "$target_host" '! systemctl is-active --quiet k3s'
|
||||
cluster_ssh "$observer_host" bash -s <<'REMOTE'
|
||||
set -euo pipefail
|
||||
for _ in $(seq 1 60); do
|
||||
if k3s kubectl get --raw='/readyz?verbose' 2>/dev/null |
|
||||
grep -q '\[+\]etcd ok'; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
exit 1
|
||||
REMOTE
|
||||
curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz |
|
||||
grep -q '"ok":true'
|
||||
cluster_ssh "$target_host" 'systemctl start k3s'
|
||||
cluster_ssh "$observer_host" \
|
||||
"k3s kubectl wait --for=condition=Ready node/'$node_name' --timeout=300s"
|
||||
active_host=
|
||||
elapsed=$(( $(date +%s) - started ))
|
||||
echo "k3s_single_server_quorum=PASS node=$node_name recovery_seconds=$elapsed"
|
||||
}
|
||||
|
||||
drill_server easyai-los-angeles "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_NINGBO_HOST"
|
||||
drill_server easyai-hongkong "$CLUSTER_HONGKONG_HOST" "$CLUSTER_NINGBO_HOST"
|
||||
drill_server easyai-ningbo "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"
|
||||
|
||||
cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s <<'REMOTE'
|
||||
set -euo pipefail
|
||||
kubectl='k3s kubectl'
|
||||
for _ in $(seq 1 240); do
|
||||
ready=$($kubectl get cluster easyai-postgres -n easyai \
|
||||
-o jsonpath='{.status.readyInstances}')
|
||||
if [[ $ready == 2 ]]; then
|
||||
primary=$($kubectl get cluster easyai-postgres -n easyai \
|
||||
-o jsonpath='{.status.currentPrimary}')
|
||||
synchronous_replicas=$($kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -U postgres -d postgres -At -c \
|
||||
"SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
|
||||
if [[ $synchronous_replicas == 1 ]]; then
|
||||
echo "k3s_quorum_drill=PASS servers=3 database_primary=$primary synchronous_replicas=1"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
exit 1
|
||||
REMOTE
|
||||
|
||||
trap - EXIT HUP INT TERM
|
||||
Reference in New Issue
Block a user