fix(cluster): 收敛宁波单实例数据库拓扑

将生产 CNPG 固定为宁波单实例,并显式关闭旧同步复制配置。发布流程在拓扑收敛前校验主库位置并创建 OSS 备份,应用后验证单实例、无复制连接和主库可用。同步更新生产监控、集群验收和发布脚本回归测试。
This commit is contained in:
2026-08-04 23:24:37 +08:00
parent 4b639df30a
commit 8ef9de0b06
6 changed files with 102 additions and 24 deletions
@@ -246,11 +246,11 @@ while (( $(date +%s) - started_epoch < AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS
if (value>max) max=value
} END {printf "%.0f",max}')
(( pod_max_memory < 1536 )) || failure_gate_id=${failure_gate_id:-pod_memory_hard}
[[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" \
-o 'jsonpath={.status.readyInstances}') == 2 ]] || failure_gate_id=${failure_gate_id:-postgres_ready}
sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
[[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] ||
failure_gate_id=${failure_gate_id:-postgres_replication}
postgres_state=$(remote_kubectl get cluster easyai-postgres -n "$namespace" -o json |
jq -r '[.spec.instances, (.status.readyInstances // 0)] | @tsv')
[[ $postgres_state == $'1\t1' ]] || failure_gate_id=${failure_gate_id:-postgres_ready}
replica_count=$(database_query "SELECT count(*) FROM pg_stat_replication;")
[[ $replica_count == 0 ]] || failure_gate_id=${failure_gate_id:-postgres_replication}
connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
(( connections < 150 && connections * 4 < max_connections * 3 )) ||
+11 -10
View File
@@ -96,22 +96,23 @@ echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witn
database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
set -euo pipefail
kubectl='k3s kubectl'
configured_instances=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.spec.instances}')
ready_instances=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}')
[[ $ready_instances -eq 2 ]] || {
echo "postgres ready instance gate failed: ready_instances=$ready_instances expected=2" >&2
[[ $configured_instances -eq 1 && $ready_instances -eq 1 ]] || {
echo "postgres single-instance gate failed: configured_instances=$configured_instances ready_instances=$ready_instances expected=1/1" >&2
exit 1
}
primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
primary_node=$($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}')
[[ $primary_node == easyai-ningbo || $primary_node == easyai-hongkong ]] || {
[[ $primary_node == easyai-ningbo ]] || {
echo "postgres primary placement gate failed: primary_node=$primary_node" >&2
exit 1
}
replication=$($kubectl exec -n easyai "$primary" -c postgres -- \
psql -X -U postgres -d easyai_ai_gateway -At -c \
"SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
[[ $replication -eq 1 ]] || {
echo "postgres synchronous replication gate failed: synchronous_replicas=$replication expected=1" >&2
"SELECT count(*) FROM pg_stat_replication;")
[[ $replication -eq 0 ]] || {
echo "postgres single-instance replication gate failed: replicas=$replication expected=0" >&2
exit 1
}
archived_count=$($kubectl exec -n easyai "$primary" -c postgres -- \
@@ -146,17 +147,17 @@ for pod in $($kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
exit 1
fi
done
[[ $readable_backup_configs -eq 2 ]] || {
echo "postgres projected backup config count failed: readable=$readable_backup_configs expected=2" >&2
[[ $readable_backup_configs -eq 1 ]] || {
echo "postgres projected backup config count failed: readable=$readable_backup_configs expected=1" >&2
exit 1
}
printf 'primary=%s archived=%s backups=%s ready=%s synchronous=%s continuous_archiving=%s backup_configs=%s\n' \
printf 'primary=%s archived=%s backups=%s ready=%s replicas=%s continuous_archiving=%s backup_configs=%s\n' \
"$primary" "$archived_count" "$completed_backups" "$ready_instances" \
"$replication" "$continuous_archiving" "$readable_backup_configs"
REMOTE
)
[[ $database_state == primary=easyai-postgres-* ]]
echo "postgres_preflight=PASS $database_state synchronous_replicas=1"
echo "postgres_preflight=PASS $database_state replicas=0 topology=ningbo-single-instance"
if [[ $mode == postcutover ]]; then
application_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'