fix(cluster): 收敛宁波单实例数据库拓扑
将生产 CNPG 固定为宁波单实例,并显式关闭旧同步复制配置。发布流程在拓扑收敛前校验主库位置并创建 OSS 备份,应用后验证单实例、无复制连接和主库可用。同步更新生产监控、集群验收和发布脚本回归测试。
This commit is contained in:
@@ -712,6 +712,65 @@ EOF
|
||||
return 1
|
||||
}
|
||||
|
||||
prepare_postgres_single_instance() {
|
||||
local source_sha=$1
|
||||
local current_instances current_primary primary_node synchronous_enabled
|
||||
if ! grep -Eq '^ instances: 1$' "$DESIRED_STATE_DIR/database.yaml" ||
|
||||
! grep -Eq '^ synchronous: null$' "$DESIRED_STATE_DIR/database.yaml"; then
|
||||
echo 'production PostgreSQL desired state must declare one instance with synchronous replication disabled' >&2
|
||||
return 1
|
||||
fi
|
||||
current_instances=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.instances}')
|
||||
synchronous_enabled=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" -o json |
|
||||
jq -r '.spec.postgresql.synchronous != null')
|
||||
if [[ $current_instances == 1 && $synchronous_enabled == false ]]; then
|
||||
return 0
|
||||
fi
|
||||
current_primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
|
||||
-o jsonpath='{.status.currentPrimary}')
|
||||
[[ $current_primary =~ ^easyai-postgres-[0-9]+$ ]] || {
|
||||
echo "production PostgreSQL primary is unavailable: primary=${current_primary:-unknown}" >&2
|
||||
return 1
|
||||
}
|
||||
primary_node=$("${kubectl[@]}" get pod "$current_primary" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.nodeName}')
|
||||
[[ $primary_node == easyai-ningbo ]] || {
|
||||
echo "refusing PostgreSQL downscale while primary is outside Ningbo: primary=$current_primary node=$primary_node" >&2
|
||||
return 1
|
||||
}
|
||||
run_backup "$source_sha"
|
||||
# A distinct preflight field manager cannot remove the existing synchronous
|
||||
# object by omission. Remove it explicitly before applying instances=1 so
|
||||
# CNPG never validates the old number=1 setting against the new topology.
|
||||
"${kubectl[@]}" patch cluster easyai-postgres -n "$NAMESPACE" --type=merge \
|
||||
-p '{"spec":{"postgresql":{"synchronous":null}}}' >/dev/null
|
||||
}
|
||||
|
||||
verify_postgres_single_instance() {
|
||||
local state primary primary_node replicas
|
||||
for _ in $(seq 1 120); do
|
||||
state=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" -o json |
|
||||
jq -r '[.spec.instances, (.status.readyInstances // 0)] | @tsv')
|
||||
if [[ $state == $'1\t1' ]]; then
|
||||
primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
|
||||
-o jsonpath='{.status.currentPrimary}')
|
||||
primary_node=$("${kubectl[@]}" get pod "$primary" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.nodeName}')
|
||||
replicas=$("${kubectl[@]}" exec -n "$NAMESPACE" "$primary" -c postgres -- \
|
||||
psql -X -U postgres -d easyai_ai_gateway -At -c \
|
||||
'SELECT count(*) FROM pg_stat_replication;' 2>/dev/null || true)
|
||||
if [[ $primary_node == easyai-ningbo && $replicas == 0 ]]; then
|
||||
echo "production_postgres=PASS topology=ningbo-single-instance primary=$primary"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "production PostgreSQL did not converge to Ningbo single-instance topology: state=${state:-unknown}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
run_migrator() {
|
||||
local source_sha=$1
|
||||
local api_image=$2
|
||||
@@ -929,6 +988,9 @@ activate_manifest() {
|
||||
deployment_snapshot=$(mktemp "$RELEASES_DIR/.deployments.XXXXXX")
|
||||
snapshot_application_deployments "$deployment_snapshot"
|
||||
|
||||
if [[ $action == deploy ]]; then
|
||||
prepare_postgres_single_instance "$source_sha"
|
||||
fi
|
||||
if [[ $action == deploy && $migrations_changed == true ]]; then
|
||||
run_backup "$source_sha"
|
||||
run_migrator "$source_sha" "$api_image"
|
||||
@@ -943,6 +1005,12 @@ activate_manifest() {
|
||||
rm -f -- "$deployment_snapshot" "$current_file"
|
||||
return 1
|
||||
fi
|
||||
if [[ $action == deploy ]] && ! verify_postgres_single_instance; then
|
||||
echo '[cluster-release] PostgreSQL single-instance verification failed; restoring exact deployment snapshot' >&2
|
||||
restore_application_deployments "$deployment_snapshot" || true
|
||||
rm -f -- "$deployment_snapshot" "$current_file"
|
||||
return 1
|
||||
fi
|
||||
if [[ $action == deploy && $api_changed == true ]] &&
|
||||
! run_capacity_controller_preflight "$source_sha" "$api_image"; then
|
||||
echo '[cluster-release] capacity controller preflight failed; restoring exact deployment snapshot' >&2
|
||||
|
||||
@@ -63,7 +63,9 @@ metadata:
|
||||
app.kubernetes.io/name: easyai-postgres
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
spec:
|
||||
instances: 2
|
||||
# Production currently uses one Ningbo primary. Cross-region replicas are
|
||||
# intentionally disabled until a separately certified HA topology exists.
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
@@ -106,6 +108,7 @@ spec:
|
||||
topologyKey: easyai.io/site
|
||||
nodeSelector:
|
||||
easyai.io/database: "true"
|
||||
easyai.io/site: ningbo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
@@ -114,10 +117,9 @@ spec:
|
||||
cpu: "2"
|
||||
memory: 3Gi
|
||||
postgresql:
|
||||
synchronous:
|
||||
method: any
|
||||
number: 1
|
||||
dataDurability: preferred
|
||||
# Keep this explicit null so server-side apply atomically removes the old
|
||||
# two-instance synchronous setting during preflight field management.
|
||||
synchronous: null
|
||||
parameters:
|
||||
# A 16 MiB WAL segment every 60 seconds requires at least 2.24 Mbit/s
|
||||
# of continuous cross-region throughput even while the database is idle.
|
||||
@@ -129,10 +131,9 @@ spec:
|
||||
wal_compression: "on"
|
||||
wal_keep_size: 1GB
|
||||
# CNPG defaults these cross-region liveness timers to 5 seconds. A short
|
||||
# control-plane or pod-network stall can otherwise tear down a healthy
|
||||
# WAL stream and make preferred synchronous replication fall back to
|
||||
# async. Keep failure detection bounded without amplifying a transient
|
||||
# multi-second stall into repeated reconnects.
|
||||
# control-plane or pod-network stall can otherwise tear down a future
|
||||
# WAL stream. Keep failure detection bounded without amplifying a
|
||||
# transient multi-second stall into repeated reconnects.
|
||||
wal_sender_timeout: 30s
|
||||
wal_receiver_timeout: 30s
|
||||
monitoring:
|
||||
|
||||
Reference in New Issue
Block a user