wangbo
7c142f5960
阿里云 OSS 的 S3 兼容层要求 virtual-hosted addressing，且 boto3 需要使用 Signature V2。本提交为 Barman 的归档、备份、保留与恢复统一挂载 AWS 配置，并将实际 WAL 归档和近期全备设为切换硬门禁。 同时补充 CNPG/etcdutl 固定版本安装、显式主库切换、三节点逐台停机、OSS 快照下载校验与远端临时 Secret 清理。已通过 ShellCheck、Kustomize、服务端 dry-run、迁移测试、发布脚本测试、敏感信息扫描及生产 precutover 验收。
78 lines
2.4 KiB
Bash
Executable File
78 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
# shellcheck source=scripts/cluster/common.sh
|
|
source "$script_dir/common.sh"
|
|
|
|
[[ ${1:-} == --execute ]] || {
|
|
echo 'usage: verify-k3s-quorum.sh --execute' >&2
|
|
exit 64
|
|
}
|
|
load_cluster_env
|
|
|
|
active_host=
|
|
restore_active_host() {
|
|
[[ -z ${active_host:-} ]] ||
|
|
cluster_ssh "$active_host" 'systemctl start k3s' >/dev/null 2>&1 || true
|
|
}
|
|
trap restore_active_host EXIT HUP INT TERM
|
|
|
|
drill_server() {
|
|
local node_name=$1
|
|
local target_host=$2
|
|
local observer_host=$3
|
|
local started elapsed
|
|
active_host=$target_host
|
|
started=$(date +%s)
|
|
cluster_ssh "$target_host" 'systemctl stop k3s'
|
|
cluster_ssh "$target_host" '! systemctl is-active --quiet k3s'
|
|
cluster_ssh "$observer_host" bash -s <<'REMOTE'
|
|
set -euo pipefail
|
|
for _ in $(seq 1 60); do
|
|
if k3s kubectl get --raw='/readyz?verbose' 2>/dev/null |
|
|
grep -q '\[+\]etcd ok'; then
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
exit 1
|
|
REMOTE
|
|
curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz |
|
|
grep -q '"ok":true'
|
|
cluster_ssh "$target_host" 'systemctl start k3s'
|
|
cluster_ssh "$observer_host" \
|
|
"k3s kubectl wait --for=condition=Ready node/'$node_name' --timeout=300s"
|
|
active_host=
|
|
elapsed=$(( $(date +%s) - started ))
|
|
echo "k3s_single_server_quorum=PASS node=$node_name recovery_seconds=$elapsed"
|
|
}
|
|
|
|
drill_server easyai-los-angeles "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_NINGBO_HOST"
|
|
drill_server easyai-hongkong "$CLUSTER_HONGKONG_HOST" "$CLUSTER_NINGBO_HOST"
|
|
drill_server easyai-ningbo "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"
|
|
|
|
cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s <<'REMOTE'
|
|
set -euo pipefail
|
|
kubectl='k3s kubectl'
|
|
for _ in $(seq 1 240); do
|
|
ready=$($kubectl get cluster easyai-postgres -n easyai \
|
|
-o jsonpath='{.status.readyInstances}')
|
|
if [[ $ready == 2 ]]; then
|
|
primary=$($kubectl get cluster easyai-postgres -n easyai \
|
|
-o jsonpath='{.status.currentPrimary}')
|
|
synchronous_replicas=$($kubectl exec -n easyai "$primary" -c postgres -- \
|
|
psql -X -U postgres -d postgres -At -c \
|
|
"SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
|
|
if [[ $synchronous_replicas == 1 ]]; then
|
|
echo "k3s_quorum_drill=PASS servers=3 database_primary=$primary synchronous_replicas=1"
|
|
exit 0
|
|
fi
|
|
fi
|
|
sleep 3
|
|
done
|
|
exit 1
|
|
REMOTE
|
|
|
|
trap - EXIT HUP INT TERM
|