BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(cluster): 收敛宁波单实例数据库拓扑

Browse Source 
将生产 CNPG 固定为宁波单实例，并显式关闭旧同步复制配置。发布流程在拓扑收敛前校验主库位置并创建 OSS 备份，应用后验证单实例、无复制连接和主库可用。同步更新生产监控、集群验收和发布脚本回归测试。
This commit is contained in:
wangbo
2026-08-04 23:24:37 +08:00
parent 4b639df30a
commit 8ef9de0b06
6 changed files with 102 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/kubernetes/easyai-ai-gateway-cluster-release
+68
View File
@@ -712,6 +712,65 @@ EOF
return 1
}
prepare_postgres_single_instance() {
local source_sha=$1
local current_instances current_primary primary_node synchronous_enabled
if ! grep -Eq '^ instances: 1$' "$DESIRED_STATE_DIR/database.yaml" ||
! grep -Eq '^ synchronous: null$' "$DESIRED_STATE_DIR/database.yaml"; then
echo 'production PostgreSQL desired state must declare one instance with synchronous replication disabled' >&2
return 1
fi
current_instances=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
-o jsonpath='{.spec.instances}')
synchronous_enabled=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" -o json |
jq -r '.spec.postgresql.synchronous != null')
if [[ $current_instances == 1 && $synchronous_enabled == false ]]; then
return 0
fi
current_primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
-o jsonpath='{.status.currentPrimary}')
[[ $current_primary =~ ^easyai-postgres-[0-9]+$ ]] || {
echo "production PostgreSQL primary is unavailable: primary=${current_primary:-unknown}" >&2
return 1
}
primary_node=$("${kubectl[@]}" get pod "$current_primary" -n "$NAMESPACE" \
-o jsonpath='{.spec.nodeName}')
[[ $primary_node == easyai-ningbo ]] || {
echo "refusing PostgreSQL downscale while primary is outside Ningbo: primary=$current_primary node=$primary_node" >&2
return 1
}
run_backup "$source_sha"
# A distinct preflight field manager cannot remove the existing synchronous
# object by omission. Remove it explicitly before applying instances=1 so
# CNPG never validates the old number=1 setting against the new topology.
"${kubectl[@]}" patch cluster easyai-postgres -n "$NAMESPACE" --type=merge \
-p '{"spec":{"postgresql":{"synchronous":null}}}' >/dev/null
}
verify_postgres_single_instance() {
local state primary primary_node replicas
for _ in $(seq 1 120); do
state=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" -o json |
jq -r '[.spec.instances, (.status.readyInstances // 0)] | @tsv')
if [[ $state == $'1\t1' ]]; then
primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
-o jsonpath='{.status.currentPrimary}')
primary_node=$("${kubectl[@]}" get pod "$primary" -n "$NAMESPACE" \
-o jsonpath='{.spec.nodeName}')
replicas=$("${kubectl[@]}" exec -n "$NAMESPACE" "$primary" -c postgres -- \
psql -X -U postgres -d easyai_ai_gateway -At -c \
'SELECT count(*) FROM pg_stat_replication;' 2>/dev/null || true)
if [[ $primary_node == easyai-ningbo && $replicas == 0 ]]; then
echo "production_postgres=PASS topology=ningbo-single-instance primary=$primary"
return 0
fi
fi
sleep 5
done
echo "production PostgreSQL did not converge to Ningbo single-instance topology: state=${state:-unknown}" >&2
return 1
}
run_migrator() {
local source_sha=$1
local api_image=$2
@@ -929,6 +988,9 @@ activate_manifest() {
deployment_snapshot=$(mktemp "$RELEASES_DIR/.deployments.XXXXXX")
snapshot_application_deployments "$deployment_snapshot"
if [[ $action == deploy ]]; then
prepare_postgres_single_instance "$source_sha"
fi
if [[ $action == deploy && $migrations_changed == true ]]; then
run_backup "$source_sha"
run_migrator "$source_sha" "$api_image"
@@ -943,6 +1005,12 @@ activate_manifest() {
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
if [[ $action == deploy ]] && ! verify_postgres_single_instance; then
echo '[cluster-release] PostgreSQL single-instance verification failed; restoring exact deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
if [[ $action == deploy && $api_changed == true ]] &&
! run_capacity_controller_preflight "$source_sha" "$api_image"; then
echo '[cluster-release] capacity controller preflight failed; restoring exact deployment snapshot' >&2
deploy/kubernetes/production/database.yaml
+10 -9
View File
@@ -63,7 +63,9 @@ metadata:
app.kubernetes.io/name: easyai-postgres
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
instances: 2
# Production currently uses one Ningbo primary. Cross-region replicas are
# intentionally disabled until a separately certified HA topology exists.
instances: 1
imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c
imagePullPolicy: IfNotPresent
env:
@@ -106,6 +108,7 @@ spec:
topologyKey: easyai.io/site
nodeSelector:
easyai.io/database: "true"
easyai.io/site: ningbo
resources:
requests:
cpu: 500m
@@ -114,10 +117,9 @@ spec:
cpu: "2"
memory: 3Gi
postgresql:
synchronous:
method: any
number: 1
dataDurability: preferred
# Keep this explicit null so server-side apply atomically removes the old
# two-instance synchronous setting during preflight field management.
synchronous: null
parameters:
# A 16 MiB WAL segment every 60 seconds requires at least 2.24 Mbit/s
# of continuous cross-region throughput even while the database is idle.
@@ -129,10 +131,9 @@ spec:
wal_compression: "on"
wal_keep_size: 1GB
# CNPG defaults these cross-region liveness timers to 5 seconds. A short
# control-plane or pod-network stall can otherwise tear down a healthy
# WAL stream and make preferred synchronous replication fall back to
# async. Keep failure detection bounded without amplifying a transient
# multi-second stall into repeated reconnects.
# control-plane or pod-network stall can otherwise tear down a future
# WAL stream. Keep failure detection bounded without amplifying a
# transient multi-second stall into repeated reconnects.
wal_sender_timeout: 30s
wal_receiver_timeout: 30s
monitoring:
scripts/cluster/monitor-production-release.sh
+5 -5
View File
@@ -246,11 +246,11 @@ while (( $(date +%s) - started_epoch < AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS
if (value>max) max=value
} END {printf "%.0f",max}')
(( pod_max_memory < 1536 )) || failure_gate_id=${failure_gate_id:-pod_memory_hard}
[[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" \
-o 'jsonpath={.status.readyInstances}') == 2 ]] || failure_gate_id=${failure_gate_id:-postgres_ready}
sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
[[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] ||
failure_gate_id=${failure_gate_id:-postgres_replication}
postgres_state=$(remote_kubectl get cluster easyai-postgres -n "$namespace" -o json |
jq -r '[.spec.instances, (.status.readyInstances // 0)] | @tsv')
[[ $postgres_state == $'1\t1' ]] || failure_gate_id=${failure_gate_id:-postgres_ready}
replica_count=$(database_query "SELECT count(*) FROM pg_stat_replication;")
[[ $replica_count == 0 ]] || failure_gate_id=${failure_gate_id:-postgres_replication}
connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
(( connections < 150 && connections * 4 < max_connections * 3 )) ||
scripts/cluster/verify-cluster.sh
+11 -10
View File
@@ -96,22 +96,23 @@ echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witn
database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
set -euo pipefail
kubectl='k3s kubectl'
configured_instances=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.spec.instances}')
ready_instances=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}')
[[ $ready_instances -eq 2 ]] || {
echo "postgres ready instance gate failed: ready_instances=$ready_instances expected=2" >&2
[[ $configured_instances -eq 1 && $ready_instances -eq 1 ]] || {
echo "postgres single-instance gate failed: configured_instances=$configured_instances ready_instances=$ready_instances expected=1/1" >&2
exit 1
}
primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
primary_node=$($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}')
[[ $primary_node == easyai-ningbo || $primary_node == easyai-hongkong ]] || {
[[ $primary_node == easyai-ningbo ]] || {
echo "postgres primary placement gate failed: primary_node=$primary_node" >&2
exit 1
}
replication=$($kubectl exec -n easyai "$primary" -c postgres -- \
psql -X -U postgres -d easyai_ai_gateway -At -c \
"SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
[[ $replication -eq 1 ]] || {
echo "postgres synchronous replication gate failed: synchronous_replicas=$replication expected=1" >&2
"SELECT count(*) FROM pg_stat_replication;")
[[ $replication -eq 0 ]] || {
echo "postgres single-instance replication gate failed: replicas=$replication expected=0" >&2
exit 1
}
archived_count=$($kubectl exec -n easyai "$primary" -c postgres -- \
@@ -146,17 +147,17 @@ for pod in $($kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
exit 1
fi
done
[[ $readable_backup_configs -eq 2 ]] || {
echo "postgres projected backup config count failed: readable=$readable_backup_configs expected=2" >&2
[[ $readable_backup_configs -eq 1 ]] || {
echo "postgres projected backup config count failed: readable=$readable_backup_configs expected=1" >&2
exit 1
}
printf 'primary=%s archived=%s backups=%s ready=%s synchronous=%s continuous_archiving=%s backup_configs=%s\n' \
printf 'primary=%s archived=%s backups=%s ready=%s replicas=%s continuous_archiving=%s backup_configs=%s\n' \
"$primary" "$archived_count" "$completed_backups" "$ready_instances" \
"$replication" "$continuous_archiving" "$readable_backup_configs"
REMOTE
)
[[ $database_state == primary=easyai-postgres-* ]]
echo "postgres_preflight=PASS $database_state synchronous_replicas=1"
echo "postgres_preflight=PASS $database_state replicas=0 topology=ningbo-single-instance"
if [[ $mode == postcutover ]]; then
application_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
tests/release/cluster-release-helper-test.sh
+5
View File
@@ -102,6 +102,11 @@ fi
grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1' \
"$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example"
grep -Fq "hongkong) replicas=\$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" "$helper"
grep -Fq "prepare_postgres_single_instance \"\$source_sha\"" "$helper"
grep -Fq "'production PostgreSQL desired state must declare one instance with synchronous replication disabled'" "$helper"
grep -Fq 'refusing PostgreSQL downscale while primary is outside Ningbo' "$helper"
grep -Fq 'production_postgres=PASS topology=ningbo-single-instance' "$helper"
grep -Fq -- "-p '{\"spec\":{\"postgresql\":{\"synchronous\":null}}}'" "$helper"
grep -Fq "\"\${kubectl[@]}\" replace -f -" "$helper"
if grep -Fq "\"\${kubectl[@]}\" apply -f \"\$snapshot\"" "$helper"; then
echo 'deployment rollback still uses merge semantics instead of exact restore' >&2
tests/release/manual-release-test.sh
+3
View File
@@ -314,6 +314,9 @@ if grep -Fq 'archive_timeout: 60s' "$root/deploy/kubernetes/production/database.
echo 'production archive timeout would exceed the certified cross-region bandwidth floor' >&2
exit 1
fi
grep -Fq ' instances: 1' "$root/deploy/kubernetes/production/database.yaml"
grep -Fq ' easyai.io/site: ningbo' "$root/deploy/kubernetes/production/database.yaml"
grep -Fq ' synchronous: null' "$root/deploy/kubernetes/production/database.yaml"
grep -Fq 'wal_sender_timeout: 30s' "$root/deploy/kubernetes/production/database.yaml"
grep -Fq 'wal_receiver_timeout: 30s' "$root/deploy/kubernetes/production/database.yaml"
bash -n "$root/scripts/cluster/configure-control-plane-io.sh"