diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index f2b9bf6..a2c115a 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -712,6 +712,65 @@ EOF return 1 } +prepare_postgres_single_instance() { + local source_sha=$1 + local current_instances current_primary primary_node synchronous_enabled + if ! grep -Eq '^ instances: 1$' "$DESIRED_STATE_DIR/database.yaml" || + ! grep -Eq '^ synchronous: null$' "$DESIRED_STATE_DIR/database.yaml"; then + echo 'production PostgreSQL desired state must declare one instance with synchronous replication disabled' >&2 + return 1 + fi + current_instances=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \ + -o jsonpath='{.spec.instances}') + synchronous_enabled=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" -o json | + jq -r '.spec.postgresql.synchronous != null') + if [[ $current_instances == 1 && $synchronous_enabled == false ]]; then + return 0 + fi + current_primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \ + -o jsonpath='{.status.currentPrimary}') + [[ $current_primary =~ ^easyai-postgres-[0-9]+$ ]] || { + echo "production PostgreSQL primary is unavailable: primary=${current_primary:-unknown}" >&2 + return 1 + } + primary_node=$("${kubectl[@]}" get pod "$current_primary" -n "$NAMESPACE" \ + -o jsonpath='{.spec.nodeName}') + [[ $primary_node == easyai-ningbo ]] || { + echo "refusing PostgreSQL downscale while primary is outside Ningbo: primary=$current_primary node=$primary_node" >&2 + return 1 + } + run_backup "$source_sha" + # A distinct preflight field manager cannot remove the existing synchronous + # object by omission. Remove it explicitly before applying instances=1 so + # CNPG never validates the old number=1 setting against the new topology. + "${kubectl[@]}" patch cluster easyai-postgres -n "$NAMESPACE" --type=merge \ + -p '{"spec":{"postgresql":{"synchronous":null}}}' >/dev/null +} + +verify_postgres_single_instance() { + local state primary primary_node replicas + for _ in $(seq 1 120); do + state=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" -o json | + jq -r '[.spec.instances, (.status.readyInstances // 0)] | @tsv') + if [[ $state == $'1\t1' ]]; then + primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \ + -o jsonpath='{.status.currentPrimary}') + primary_node=$("${kubectl[@]}" get pod "$primary" -n "$NAMESPACE" \ + -o jsonpath='{.spec.nodeName}') + replicas=$("${kubectl[@]}" exec -n "$NAMESPACE" "$primary" -c postgres -- \ + psql -X -U postgres -d easyai_ai_gateway -At -c \ + 'SELECT count(*) FROM pg_stat_replication;' 2>/dev/null || true) + if [[ $primary_node == easyai-ningbo && $replicas == 0 ]]; then + echo "production_postgres=PASS topology=ningbo-single-instance primary=$primary" + return 0 + fi + fi + sleep 5 + done + echo "production PostgreSQL did not converge to Ningbo single-instance topology: state=${state:-unknown}" >&2 + return 1 +} + run_migrator() { local source_sha=$1 local api_image=$2 @@ -929,6 +988,9 @@ activate_manifest() { deployment_snapshot=$(mktemp "$RELEASES_DIR/.deployments.XXXXXX") snapshot_application_deployments "$deployment_snapshot" + if [[ $action == deploy ]]; then + prepare_postgres_single_instance "$source_sha" + fi if [[ $action == deploy && $migrations_changed == true ]]; then run_backup "$source_sha" run_migrator "$source_sha" "$api_image" @@ -943,6 +1005,12 @@ activate_manifest() { rm -f -- "$deployment_snapshot" "$current_file" return 1 fi + if [[ $action == deploy ]] && ! verify_postgres_single_instance; then + echo '[cluster-release] PostgreSQL single-instance verification failed; restoring exact deployment snapshot' >&2 + restore_application_deployments "$deployment_snapshot" || true + rm -f -- "$deployment_snapshot" "$current_file" + return 1 + fi if [[ $action == deploy && $api_changed == true ]] && ! run_capacity_controller_preflight "$source_sha" "$api_image"; then echo '[cluster-release] capacity controller preflight failed; restoring exact deployment snapshot' >&2 diff --git a/deploy/kubernetes/production/database.yaml b/deploy/kubernetes/production/database.yaml index 8dd6c38..d50d7e4 100644 --- a/deploy/kubernetes/production/database.yaml +++ b/deploy/kubernetes/production/database.yaml @@ -63,7 +63,9 @@ metadata: app.kubernetes.io/name: easyai-postgres app.kubernetes.io/part-of: easyai-ai-gateway spec: - instances: 2 + # Production currently uses one Ningbo primary. Cross-region replicas are + # intentionally disabled until a separately certified HA topology exists. + instances: 1 imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c imagePullPolicy: IfNotPresent env: @@ -106,6 +108,7 @@ spec: topologyKey: easyai.io/site nodeSelector: easyai.io/database: "true" + easyai.io/site: ningbo resources: requests: cpu: 500m @@ -114,10 +117,9 @@ spec: cpu: "2" memory: 3Gi postgresql: - synchronous: - method: any - number: 1 - dataDurability: preferred + # Keep this explicit null so server-side apply atomically removes the old + # two-instance synchronous setting during preflight field management. + synchronous: null parameters: # A 16 MiB WAL segment every 60 seconds requires at least 2.24 Mbit/s # of continuous cross-region throughput even while the database is idle. @@ -129,10 +131,9 @@ spec: wal_compression: "on" wal_keep_size: 1GB # CNPG defaults these cross-region liveness timers to 5 seconds. A short - # control-plane or pod-network stall can otherwise tear down a healthy - # WAL stream and make preferred synchronous replication fall back to - # async. Keep failure detection bounded without amplifying a transient - # multi-second stall into repeated reconnects. + # control-plane or pod-network stall can otherwise tear down a future + # WAL stream. Keep failure detection bounded without amplifying a + # transient multi-second stall into repeated reconnects. wal_sender_timeout: 30s wal_receiver_timeout: 30s monitoring: diff --git a/scripts/cluster/monitor-production-release.sh b/scripts/cluster/monitor-production-release.sh index 11e5af4..48e0004 100755 --- a/scripts/cluster/monitor-production-release.sh +++ b/scripts/cluster/monitor-production-release.sh @@ -246,11 +246,11 @@ while (( $(date +%s) - started_epoch < AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS if (value>max) max=value } END {printf "%.0f",max}') (( pod_max_memory < 1536 )) || failure_gate_id=${failure_gate_id:-pod_memory_hard} - [[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" \ - -o 'jsonpath={.status.readyInstances}') == 2 ]] || failure_gate_id=${failure_gate_id:-postgres_ready} - sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;") - [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] || - failure_gate_id=${failure_gate_id:-postgres_replication} + postgres_state=$(remote_kubectl get cluster easyai-postgres -n "$namespace" -o json | + jq -r '[.spec.instances, (.status.readyInstances // 0)] | @tsv') + [[ $postgres_state == $'1\t1' ]] || failure_gate_id=${failure_gate_id:-postgres_ready} + replica_count=$(database_query "SELECT count(*) FROM pg_stat_replication;") + [[ $replica_count == 0 ]] || failure_gate_id=${failure_gate_id:-postgres_replication} connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';") max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';") (( connections < 150 && connections * 4 < max_connections * 3 )) || diff --git a/scripts/cluster/verify-cluster.sh b/scripts/cluster/verify-cluster.sh index 1f15518..720e0a3 100755 --- a/scripts/cluster/verify-cluster.sh +++ b/scripts/cluster/verify-cluster.sh @@ -96,22 +96,23 @@ echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witn database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' set -euo pipefail kubectl='k3s kubectl' +configured_instances=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.spec.instances}') ready_instances=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -[[ $ready_instances -eq 2 ]] || { - echo "postgres ready instance gate failed: ready_instances=$ready_instances expected=2" >&2 +[[ $configured_instances -eq 1 && $ready_instances -eq 1 ]] || { + echo "postgres single-instance gate failed: configured_instances=$configured_instances ready_instances=$ready_instances expected=1/1" >&2 exit 1 } primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}') primary_node=$($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') -[[ $primary_node == easyai-ningbo || $primary_node == easyai-hongkong ]] || { +[[ $primary_node == easyai-ningbo ]] || { echo "postgres primary placement gate failed: primary_node=$primary_node" >&2 exit 1 } replication=$($kubectl exec -n easyai "$primary" -c postgres -- \ psql -X -U postgres -d easyai_ai_gateway -At -c \ - "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');") -[[ $replication -eq 1 ]] || { - echo "postgres synchronous replication gate failed: synchronous_replicas=$replication expected=1" >&2 + "SELECT count(*) FROM pg_stat_replication;") +[[ $replication -eq 0 ]] || { + echo "postgres single-instance replication gate failed: replicas=$replication expected=0" >&2 exit 1 } archived_count=$($kubectl exec -n easyai "$primary" -c postgres -- \ @@ -146,17 +147,17 @@ for pod in $($kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \ exit 1 fi done -[[ $readable_backup_configs -eq 2 ]] || { - echo "postgres projected backup config count failed: readable=$readable_backup_configs expected=2" >&2 +[[ $readable_backup_configs -eq 1 ]] || { + echo "postgres projected backup config count failed: readable=$readable_backup_configs expected=1" >&2 exit 1 } -printf 'primary=%s archived=%s backups=%s ready=%s synchronous=%s continuous_archiving=%s backup_configs=%s\n' \ +printf 'primary=%s archived=%s backups=%s ready=%s replicas=%s continuous_archiving=%s backup_configs=%s\n' \ "$primary" "$archived_count" "$completed_backups" "$ready_instances" \ "$replication" "$continuous_archiving" "$readable_backup_configs" REMOTE ) [[ $database_state == primary=easyai-postgres-* ]] -echo "postgres_preflight=PASS $database_state synchronous_replicas=1" +echo "postgres_preflight=PASS $database_state replicas=0 topology=ningbo-single-instance" if [[ $mode == postcutover ]]; then application_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' diff --git a/tests/release/cluster-release-helper-test.sh b/tests/release/cluster-release-helper-test.sh index a862bbf..3289c9a 100755 --- a/tests/release/cluster-release-helper-test.sh +++ b/tests/release/cluster-release-helper-test.sh @@ -102,6 +102,11 @@ fi grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1' \ "$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example" grep -Fq "hongkong) replicas=\$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" "$helper" +grep -Fq "prepare_postgres_single_instance \"\$source_sha\"" "$helper" +grep -Fq "'production PostgreSQL desired state must declare one instance with synchronous replication disabled'" "$helper" +grep -Fq 'refusing PostgreSQL downscale while primary is outside Ningbo' "$helper" +grep -Fq 'production_postgres=PASS topology=ningbo-single-instance' "$helper" +grep -Fq -- "-p '{\"spec\":{\"postgresql\":{\"synchronous\":null}}}'" "$helper" grep -Fq "\"\${kubectl[@]}\" replace -f -" "$helper" if grep -Fq "\"\${kubectl[@]}\" apply -f \"\$snapshot\"" "$helper"; then echo 'deployment rollback still uses merge semantics instead of exact restore' >&2 diff --git a/tests/release/manual-release-test.sh b/tests/release/manual-release-test.sh index 829da40..f4bfd57 100755 --- a/tests/release/manual-release-test.sh +++ b/tests/release/manual-release-test.sh @@ -314,6 +314,9 @@ if grep -Fq 'archive_timeout: 60s' "$root/deploy/kubernetes/production/database. echo 'production archive timeout would exceed the certified cross-region bandwidth floor' >&2 exit 1 fi +grep -Fq ' instances: 1' "$root/deploy/kubernetes/production/database.yaml" +grep -Fq ' easyai.io/site: ningbo' "$root/deploy/kubernetes/production/database.yaml" +grep -Fq ' synchronous: null' "$root/deploy/kubernetes/production/database.yaml" grep -Fq 'wal_sender_timeout: 30s' "$root/deploy/kubernetes/production/database.yaml" grep -Fq 'wal_receiver_timeout: 30s' "$root/deploy/kubernetes/production/database.yaml" bash -n "$root/scripts/cluster/configure-control-plane-io.sh"