feat(cluster): 接入深圳专用 Worker 节点

新增深圳 K3s agent 与四节点 WireGuard 全互联接入脚本，将深圳归入香港逻辑 Worker 池并用污点限制为 Worker 专用。\n\n生产验收支持宁波 0、香港逻辑池 2 的双物理节点基线，补充副本拓扑分散、容量档位、链路巡检和零副本站点校验。\n\n已通过 Go 全量测试、go vet、govulncheck、迁移安全检查、ShellCheck、发布脚本测试、前后端 lint/test/build、Compose 与 Kubernetes 渲染校验。

scripts/cluster/add-shenzhen-worker-node.sh

@@ -0,0 +1,271 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+action=${1:-all}
+[[ $action == preflight || $action == network || $action == install ||
+   $action == verify || $action == all ]] || {
+  echo 'usage: add-shenzhen-worker-node.sh [preflight|network|install|verify|all]' >&2
+  exit 64
+}
+
+load_cluster_env
+require_commands ssh scp shasum
+: "${CLUSTER_SHENZHEN_HOST:?AI_GATEWAY_SHENZHEN_HOST is required}"
+
+node_name=easyai-shenzhen
+node_ip=10.77.0.4
+k3s_version=v1.36.2+k3s1
+k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de
+secret_root="$cluster_root/.local-secrets/cluster"
+cluster_environment="$secret_root/cluster.env"
+k3s_binary="$secret_root/k3s-${k3s_version}-amd64"
+registries_file="$cluster_root/deploy/kubernetes/registries.yaml"
+
+assert_join_material() {
+  assert_private_file "$cluster_environment"
+  [[ -f $k3s_binary && ! -L $k3s_binary && -f $registries_file && ! -L $registries_file ]] || {
+    echo 'pinned K3s binary or registry configuration is missing' >&2
+    return 1
+  }
+  [[ $(shasum -a 256 "$k3s_binary" | awk '{print $1}') == "$k3s_sha256" ]] || {
+    echo 'pinned K3s binary checksum mismatch' >&2
+    return 1
+  }
+}
+
+preflight_node() {
+  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s <<'REMOTE'
+set -euo pipefail
+[[ $(id -u) -eq 0 && $(uname -m) == x86_64 ]]
+. /etc/os-release
+[[ $ID == debian && ${VERSION_ID%%.*} -ge 12 ]]
+(( $(nproc) >= 4 ))
+available_kib=$(awk '/MemAvailable:/ {print $2}' /proc/meminfo)
+(( available_kib >= 6 * 1024 * 1024 ))
+available_bytes=$(df -B1 --output=avail / | tail -n 1 | tr -d ' ')
+(( available_bytes >= 20 * 1024 * 1024 * 1024 ))
+timedatectl show -p NTPSynchronized --value | grep -Fxq yes
+[[ ! -e /etc/rancher/k3s/config.yaml || -f /etc/rancher/k3s/config.yaml ]]
+printf 'shenzhen_worker_preflight=PASS cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \
+  "$(nproc)" "$((available_kib / 1024))" "$((available_bytes / 1024 / 1024 / 1024))"
+REMOTE
+}
+
+configure_wireguard() {
+  local -a server_hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
+  local -a server_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
+  local -a server_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
+  local -a server_keys=()
+  local shenzhen_key host key
+
+  shenzhen_key=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" 'bash -s' <<'REMOTE' | tail -n 1
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+if ! command -v wg >/dev/null 2>&1; then
+  apt-get update -qq
+  apt-get install -y -qq wireguard-tools
+fi
+install -d -m 0700 /etc/wireguard
+if [[ ! -s /etc/wireguard/privatekey ]]; then
+  umask 077
+  wg genkey >/etc/wireguard/privatekey
+fi
+chmod 0600 /etc/wireguard/privatekey
+wg pubkey </etc/wireguard/privatekey >/etc/wireguard/publickey
+cat /etc/wireguard/publickey
+REMOTE
+)
+  [[ $shenzhen_key =~ ^[A-Za-z0-9+/]{43}=$ ]]
+
+  for host in "${server_hosts[@]}"; do
+    key=$(cluster_ssh "$host" 'wg show wg0 public-key')
+    [[ $key =~ ^[A-Za-z0-9+/]{43}=$ ]]
+    server_keys+=("$key")
+  done
+
+  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \
+    "${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \
+    "${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \
+    "${server_keys[2]}" "${server_endpoints[2]}" "${server_ips[2]}" <<'REMOTE'
+set -euo pipefail
+private_key=$(cat /etc/wireguard/privatekey)
+umask 077
+temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
+cat >"$temporary_config" <<EOF
+[Interface]
+Address = 10.77.0.4/24
+ListenPort = 51820
+PrivateKey = ${private_key}
+
+[Peer]
+PublicKey = $1
+Endpoint = $2:51820
+AllowedIPs = $3/32
+PersistentKeepalive = 25
+
+[Peer]
+PublicKey = $4
+Endpoint = $5:51820
+AllowedIPs = $6/32
+PersistentKeepalive = 25
+
+[Peer]
+PublicKey = $7
+Endpoint = $8:51820
+AllowedIPs = $9/32
+PersistentKeepalive = 25
+EOF
+chmod 0600 "$temporary_config"
+mv "$temporary_config" /etc/wireguard/wg0.conf
+for peer in "$2" "$5" "$8"; do
+  ufw allow from "$peer" to any port 51820 proto udp comment easyai-wireguard >/dev/null
+done
+systemctl enable wg-quick@wg0 >/dev/null
+systemctl restart wg-quick@wg0
+REMOTE
+
+  for host in "${server_hosts[@]}"; do
+    cluster_ssh "$host" bash -s -- "$shenzhen_key" "${CLUSTER_SHENZHEN_HOST#*@}" "$node_ip" <<'REMOTE'
+set -euo pipefail
+wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:51820" persistent-keepalive 25
+wg-quick save wg0
+REMOTE
+  done
+  verify_wireguard
+}
+
+verify_path() {
+  local source_host=$1 target_ip=$2 max_rtt=$3 label=$4 output loss average
+  output=$(cluster_ssh "$source_host" "ping -q -i 0.2 -c 10 -W 2 $target_ip")
+  loss=$(awk -F',' '/packet loss/ {gsub(/[^0-9.]/, "", $3); print $3}' <<<"$output")
+  average=$(awk -F'=' '/min\/avg\/max/ {split($2, values, "/"); gsub(/ /, "", values[2]); print values[2]}' <<<"$output")
+  [[ -n $loss && -n $average ]]
+  awk -v loss="$loss" -v average="$average" -v max="$max_rtt" \
+    'BEGIN {exit !(loss < 1 && average < max)}'
+  echo "wireguard_path=$label loss_percent=$loss average_rtt_ms=$average"
+}
+
+verify_wireguard() {
+  verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_shenzhen
+  verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 shenzhen_to_ningbo
+  verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_shenzhen
+  verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 shenzhen_to_hongkong
+  verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_shenzhen
+  verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 shenzhen_to_losangeles
+  local host recent
+  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_SHENZHEN_HOST"; do
+    recent=$(cluster_ssh "$host" \
+      "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
+    [[ $recent == 3 ]]
+  done
+  echo 'shenzhen_wireguard=PASS peers=4 recent_handshakes=12'
+}
+
+install_agent() {
+  assert_join_material
+  # shellcheck source=/dev/null
+  source "$cluster_environment"
+  : "${K3S_TOKEN:?}"
+  [[ $K3S_TOKEN =~ ^[0-9a-f]{64}$ ]]
+
+  cluster_ssh "$CLUSTER_SHENZHEN_HOST" 'install -d -m 0700 /etc/rancher/k3s'
+  printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_SHENZHEN_HOST" \
+    'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token'
+  cluster_scp "$k3s_binary" "$CLUSTER_SHENZHEN_HOST:/usr/local/bin/k3s"
+  cluster_scp "$registries_file" "$CLUSTER_SHENZHEN_HOST:/etc/rancher/k3s/registries.yaml"
+  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- "$k3s_version" "$k3s_sha256" <<'REMOTE'
+set -euo pipefail
+version=$1
+expected_sha256=$2
+chmod 0755 /usr/local/bin/k3s
+echo "$expected_sha256  /usr/local/bin/k3s" | sha256sum -c - >/dev/null
+umask 077
+cat >/etc/rancher/k3s/config.yaml <<'EOF'
+server: https://10.77.0.1:6443
+token-file: /etc/rancher/k3s/cluster-token
+node-name: easyai-shenzhen
+node-ip: 10.77.0.4
+flannel-iface: wg0
+node-label:
+  - easyai.io/site=hongkong
+  - easyai.io/workload=true
+  - easyai.io/database=false
+  - easyai.io/worker=true
+node-taint:
+  - easyai.io/worker-only=true:NoSchedule
+EOF
+chmod 0600 /etc/rancher/k3s/config.yaml /etc/rancher/k3s/registries.yaml
+export DEBIAN_FRONTEND=noninteractive
+if ! command -v netfilter-persistent >/dev/null 2>&1; then
+  apt-get update -qq
+  apt-get install -y -qq iptables-persistent
+fi
+ensure_rule() {
+  if ! iptables -C INPUT "$@" 2>/dev/null; then iptables -I INPUT 1 "$@"; fi
+}
+public_interface=$(ip -o route get 1.1.1.1 | awk '{for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}')
+ensure_rule -i wg0 -s 10.77.0.0/24 -p tcp -m multiport --dports 6443,10250 \
+  -m comment --comment easyai-k3s-private -j ACCEPT
+ensure_rule -i wg0 -s 10.77.0.0/24 -p udp --dport 8472 \
+  -m comment --comment easyai-flannel-private -j ACCEPT
+ensure_rule -i "$public_interface" -p tcp -m multiport --dports 6443,10250 \
+  -m comment --comment easyai-k3s-public-drop -j DROP
+ensure_rule -i "$public_interface" -p udp --dport 8472 \
+  -m comment --comment easyai-flannel-public-drop -j DROP
+ensure_rule -i "$public_interface" -p tcp --dport 30000:31057 \
+  -m comment --comment easyai-nodeport-public-drop-low -j DROP
+ensure_rule -i "$public_interface" -p tcp --dport 31059:32767 \
+  -m comment --comment easyai-nodeport-public-drop-high -j DROP
+netfilter-persistent save >/dev/null
+install_script=$(mktemp)
+trap 'rm -f -- "$install_script"' EXIT
+curl -fsSL https://get.k3s.io -o "$install_script"
+chmod 0700 "$install_script"
+INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION="$version" \
+  INSTALL_K3S_EXEC=agent "$install_script"
+systemctl is-active --quiet k3s-agent
+systemctl is-enabled --quiet k3s-agent
+[[ ! -e /var/lib/rancher/k3s/server/db/etcd ]]
+REMOTE
+
+  local ready=false
+  for _ in $(seq 1 90); do
+    if cluster_ssh "$CLUSTER_NINGBO_HOST" \
+      "k3s kubectl get node $node_name --no-headers 2>/dev/null | awk '\$2==\"Ready\" {found=1} END {exit !found}'"; then
+      ready=true
+      break
+    fi
+    sleep 2
+  done
+  [[ $ready == true ]]
+  verify_agent
+}
+
+verify_agent() {
+  verify_wireguard
+  cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$node_name" <<'REMOTE'
+set -euo pipefail
+node=$1
+kubectl='k3s kubectl'
+[[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/workload}') == true ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]]
+[[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]
+[[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |
+  jq '[.items[] | select(.metadata.namespace!="kube-system")] | length') -eq 0 ]]
+printf 'shenzhen_k3s_agent=PASS node=%s role=worker-only etcd_member=false\n' "$node"
+REMOTE
+}
+
+case $action in
+  preflight) preflight_node ;;
+  network) preflight_node; configure_wireguard ;;
+  install) preflight_node; install_agent ;;
+  verify) preflight_node; verify_agent ;;
+  all) preflight_node; configure_wireguard; install_agent ;;
+esac

scripts/cluster/bootstrap-wireguard.sh

@@ -11,6 +11,11 @@ require_commands ssh scp
 hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
 public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
 wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
+if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+  hosts+=("$CLUSTER_SHENZHEN_HOST")
+  public_endpoints+=("${CLUSTER_SHENZHEN_HOST#*@}")
+  wireguard_ips+=(10.77.0.4)
+fi
 public_keys=()
 
 for host in "${hosts[@]}"; do
@@ -39,26 +44,24 @@ REMOTE
   public_keys+=("$public_key")
 done
 
-for local_index in 0 1 2; do
-  peer_indexes=()
-  for candidate in 0 1 2; do
-    [[ $candidate -eq $local_index ]] || peer_indexes+=("$candidate")
+for local_index in "${!hosts[@]}"; do
+  peer_arguments=()
+  for candidate in "${!hosts[@]}"; do
+    if [[ $candidate -ne $local_index ]]; then
+      peer_arguments+=(
+        "${public_keys[$candidate]}"
+        "${public_endpoints[$candidate]}"
+        "${wireguard_ips[$candidate]}"
+      )
+    fi
   done
-  first=${peer_indexes[0]}
-  second=${peer_indexes[1]}
   echo "[wireguard] configuring ${hosts[$local_index]} as ${wireguard_ips[$local_index]}"
   cluster_ssh "${hosts[$local_index]}" bash -s -- \
     "${wireguard_ips[$local_index]}" \
-    "${public_keys[$first]}" "${public_endpoints[$first]}" "${wireguard_ips[$first]}" \
-    "${public_keys[$second]}" "${public_endpoints[$second]}" "${wireguard_ips[$second]}" <<'REMOTE'
+    "${peer_arguments[@]}" <<'REMOTE'
 set -euo pipefail
 local_ip=$1
-peer_one_key=$2
-peer_one_endpoint=$3
-peer_one_ip=$4
-peer_two_key=$5
-peer_two_endpoint=$6
-peer_two_ip=$7
+shift
 private_key=$(cat /etc/wireguard/privatekey)
 umask 077
 temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
@@ -67,19 +70,19 @@ cat >"$temporary_config" <<EOF
 Address = ${local_ip}/24
 ListenPort = 51820
 PrivateKey = ${private_key}
+EOF
+while (( $# >= 3 )); do
+  cat >>"$temporary_config" <<EOF
 
 [Peer]
-PublicKey = ${peer_one_key}
-Endpoint = ${peer_one_endpoint}:51820
-AllowedIPs = ${peer_one_ip}/32
-PersistentKeepalive = 25
-
-[Peer]
-PublicKey = ${peer_two_key}
-Endpoint = ${peer_two_endpoint}:51820
-AllowedIPs = ${peer_two_ip}/32
+PublicKey = $1
+Endpoint = $2:51820
+AllowedIPs = $3/32
 PersistentKeepalive = 25
 EOF
+  shift 3
+done
+(( $# == 0 ))
 chmod 0600 "$temporary_config"
 mv "$temporary_config" /etc/wireguard/wg0.conf
 systemctl enable wg-quick@wg0 >/dev/null
@@ -87,8 +90,8 @@ systemctl restart wg-quick@wg0
 REMOTE
 done
 
-for source_index in 0 1 2; do
-  for target_index in 0 1 2; do
+for source_index in "${!hosts[@]}"; do
+  for target_index in "${!hosts[@]}"; do
     [[ $source_index -eq $target_index ]] && continue
     cluster_ssh "${hosts[$source_index]}" \
       "ping -c 3 -W 2 ${wireguard_ips[$target_index]} >/dev/null"
@@ -98,10 +101,10 @@ done
 for host in "${hosts[@]}"; do
   latest_handshake=$(cluster_ssh "$host" \
     "wg show wg0 latest-handshakes | awk '\$2 > 0 { count++ } END { print count + 0 }'")
-  [[ $latest_handshake == 2 ]] || {
+  [[ $latest_handshake == $((${#hosts[@]} - 1)) ]] || {
     echo "WireGuard handshake verification failed on $host" >&2
     exit 1
   }
 done
 
-echo 'wireguard_bootstrap=PASS peers=3'
+echo "wireguard_bootstrap=PASS peers=${#hosts[@]}"

scripts/cluster/common.sh

@@ -28,7 +28,8 @@ load_cluster_env() {
   CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST}
   CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2}
   CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3}
-  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST
+  CLUSTER_SHENZHEN_HOST=${AI_GATEWAY_SHENZHEN_HOST:-}
+  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST CLUSTER_SHENZHEN_HOST
 
   for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
     [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || {
@@ -36,6 +37,10 @@ load_cluster_env() {
       return 1
     }
   done
+  if [[ -n $CLUSTER_SHENZHEN_HOST && ! $CLUSTER_SHENZHEN_HOST =~ ^root@[A-Za-z0-9.-]+$ ]]; then
+    echo "Shenzhen Worker host must use root@host syntax" >&2
+    return 1
+  fi
   [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || {
     echo 'invalid OSS region or bucket' >&2
     return 1

scripts/cluster/monitor-production-release.sh

@@ -140,12 +140,17 @@ pool_saturated_pods() {
 check_wireguard() {
   local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
   local -a ips=(10.77.0.1 10.77.0.2 10.77.0.3)
-  local source target output loss average handshakes
-  for source in 0 1 2; do
+  [[ -z $CLUSTER_SHENZHEN_HOST ]] || {
+    hosts+=("$CLUSTER_SHENZHEN_HOST")
+    ips+=(10.77.0.4)
+  }
+  local source target output loss average handshakes peer_count
+  peer_count=$((${#hosts[@]} - 1))
+  for source in "${!hosts[@]}"; do
     handshakes=$(cluster_ssh "${hosts[$source]}" \
       "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
-    (( handshakes == 2 )) || return 1
-    for target in 0 1 2; do
+    (( handshakes == peer_count )) || return 1
+    for target in "${!hosts[@]}"; do
       (( source == target )) && continue
       output=$(cluster_ssh "${hosts[$source]}" "ping -q -c 10 -W 2 ${ips[$target]}") || return 1
       loss=$(awk -F',' '/packet loss/ {gsub(/[^0-9.]/,"",$3); print $3}' <<<"$output")

scripts/cluster/run-production-acceptance.sh

@@ -99,6 +99,10 @@ require_commands curl git go jq node openssl sed shasum
 : "${AI_GATEWAY_ACCEPTANCE_ETCD_WAL_FSYNC_MAX_MS:=15}"
 : "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}"
 : "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}"
+: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO:=1}"
+: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG:=1}"
+: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO}"
+: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG}"
 : "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
 : "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
 : "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
@@ -167,11 +171,20 @@ fi
   echo 'etcd latency gates must be positive integer milliseconds' >&2
   exit 1
 }
-[[ $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[1-9][0-9]*$ &&
-   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[1-9][0-9]*$ &&
+[[ $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO =~ ^[0-9]+$ &&
+   $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
+   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO =~ ^[0-9]+$ &&
+   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
+   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[0-9]+$ &&
+   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
+   $((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG)) -eq 2 &&
+   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
+   $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
+   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
+   $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG &&
    $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
    $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG -le 16 ]] || {
-  echo 'acceptance autoscaling replica caps must be between 1 and 16' >&2
+  echo 'acceptance requires exactly two baseline Workers and ordered 0..16 per-site autoscaling bounds' >&2
   exit 1
 }
 
@@ -263,8 +276,8 @@ image_stable_throughput=
 image_admitted_throughput=
 video_stable_throughput=
 video_admitted_throughput=
-certified_max_replicas_ningbo=1
-certified_max_replicas_hongkong=1
+certified_max_replicas_ningbo=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
+certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
 runtime_observation_started_at=
 
 cleanup() {
@@ -636,21 +649,45 @@ verify_release_cas() {
       -o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
   done
   [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
-    -o 'jsonpath={.spec.replicas}') == 1 ]]
+    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
   [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
-    -o 'jsonpath={.spec.replicas}') == 1 ]]
+    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
+  if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
+        AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) &&
+     [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+    local worker_node pod_count
+    for worker_node in easyai-hongkong easyai-shenzhen; do
+      pod_count=$(remote_kubectl get pods -n "$namespace" \
+        -l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \
+        --field-selector "spec.nodeName=$worker_node" -o json | jq \
+        '[.items[] | select(.status.phase=="Running")] | length')
+      [[ $pod_count == 1 ]] || {
+        echo "Hong Kong logical Worker pool is not spread across both physical nodes: node=$worker_node running_pods=$pod_count" >&2
+        return 1
+      }
+    done
+  fi
 }
 
 verify_resource_preconditions() {
   local report=$temporary_root/resource-preconditions.csv
-  local ningbo_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
-  ningbo_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
-    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
+  local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
+  worker_nodes=
+  if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then
+    worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
+      -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
+    worker_nodes+=$'\n'
+  fi
+  if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then
+    worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong' \
+      -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
+  fi
+  worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u)
   los_angeles_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=los-angeles' \
     -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
-  [[ -n $ningbo_nodes && -n $los_angeles_nodes ]]
+  [[ -n $worker_nodes && -n $los_angeles_nodes ]]
   invalid_mixed_workloads=$(remote_kubectl get pods -A -o json |
-    jq --argjson nodes "$(jq -Rn --arg value "$ningbo_nodes" '$value | split("\n") | map(select(length>0))')" '
+    jq --argjson nodes "$(jq -Rn --arg value "$worker_nodes" '$value | split("\n") | map(select(length>0))')" '
       [.items[]
         | select(.spec.nodeName as $name | $nodes | index($name))
         | select(.metadata.name | test("staging|runner|(^|-)ci(-|$)"; "i"))
@@ -661,7 +698,7 @@ verify_resource_preconditions() {
             (.resources.limits.cpu // "") == ""))]
       | length')
   [[ $invalid_mixed_workloads == 0 ]] || {
-    echo 'Ningbo staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2
+    echo 'Worker-site staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2
     return 1
   }
   witness_workloads=$(remote_kubectl get pods -A -o json |
@@ -687,12 +724,12 @@ verify_resource_preconditions() {
         awk '{value=$5; sub(/%$/, "", value); print value}')
       [[ $memory_percent =~ ^[0-9]+$ &&
          $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
-        echo "Ningbo memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
+        echo "Worker node memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
         return 1
       }
       printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
         "$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report"
-    done <<<"$ningbo_nodes"
+    done <<<"$worker_nodes"
     (( sample == 90 )) || sleep 10
   done
   cp "$report" "$report_root/resource-preconditions.csv"
@@ -1440,19 +1477,27 @@ fail_acceptance_gate() {
 
 apply_capacity_profile() {
   local profile=$1
-  local slots pool media global
+  local slots pool media global baseline_replicas
   local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
   local min_idle=4
   local max_idle_seconds=30
   case $profile in
-    P24) slots=24; pool=32; media=24; global=48 ;;
-    P28) slots=28; pool=36; media=28; global=56 ;;
-    P32) slots=32; pool=40; media=32; global=64 ;;
+    P24) slots=24; pool=32; media=24 ;;
+    P28) slots=28; pool=36; media=28 ;;
+    P32) slots=32; pool=40; media=32 ;;
     *) return 1 ;;
   esac
+  baseline_replicas=$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG))
+  global=$((slots * baseline_replicas))
   local command_text capacity_output
   printf -v command_text \
-    'AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
+    'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
+    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
+    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
+    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
+    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
+    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
+    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
     "$slots" "$global" "$global" "$pool" "$api_pool" \
     "$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" "$AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS" \
     "$min_idle" "$max_idle_seconds" \
@@ -1605,8 +1650,9 @@ apply_autoscaling_profile() {
   local max_replicas_ningbo=${2:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO}
   local max_replicas_hongkong=${3:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG}
   local slots pool media global
-  [[ $max_replicas_ningbo =~ ^[1-9][0-9]*$ &&
-     $max_replicas_hongkong =~ ^[1-9][0-9]*$ ]] || return 1
+  [[ $max_replicas_ningbo =~ ^[0-9]+$ &&
+     $max_replicas_hongkong =~ ^[0-9]+$ &&
+     $((max_replicas_ningbo + max_replicas_hongkong)) -ge 1 ]] || return 1
   (( max_replicas_ningbo <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
      max_replicas_hongkong <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG )) || return 1
   case $profile in
@@ -1619,7 +1665,11 @@ apply_autoscaling_profile() {
   global=$((slots * (max_replicas_ningbo + max_replicas_hongkong)))
   local command_text capacity_output
   printf -v command_text \
-    'AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
+    'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
+    "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
+    "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
+    "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
+    "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
     "$max_replicas_ningbo" \
     "$max_replicas_hongkong" \
     "$slots" "$global" "$global" "$pool" \
@@ -1697,14 +1747,15 @@ run_autoscaling_topology_phase() {
       "$(jq -r '.plan.rawDesired // 0' <<<"${status:-{}}")" \
       "$(jq -r '.plan.desiredTotal // 0' <<<"${status:-{}}")" \
       "$(jq -r '.plan.frozenReason // ""' <<<"${status:-{}}")" "$resource_max" >>"$observations"
-    if (( ningbo_replicas == 1 && hongkong_replicas == 1 )); then
+    if (( ningbo_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO &&
+          hongkong_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG )); then
       drained=true
       break
     fi
     sleep 10
   done
   [[ $drained == true ]] || {
-    echo "capacity controller did not safely drain $phase to topology 1+1" >&2
+    echo "capacity controller did not safely drain $phase to configured minimum topology" >&2
     return 1
   }
   local unsafe_residue
@@ -1734,15 +1785,18 @@ run_autoscaling_acceptance() {
   local report=$report_root/autoscaling.json
   local observations=$report_root/autoscaling-observations.csv
   local status deadline resource_max_hongkong resource_max_ningbo
-  local tested_1_plus_2=false tested_2_plus_2=false
+  local tested_south_pool_scale_out=false tested_1_plus_2=false tested_2_plus_2=false
   local requested_hongkong_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG
   local requested_ningbo_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO
   (( requested_hongkong_max > 2 )) && requested_hongkong_max=2
   (( requested_ningbo_max > 2 )) && requested_ningbo_max=2
-  autoscaling_phase_observed_max=2
+  autoscaling_phase_observed_max=$((
+    AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO +
+    AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG
+  ))
   echo 'timestamp,phase,ningbo_replicas,hongkong_replicas,raw_desired,desired_total,frozen_reason,resource_max_total' >"$observations"
 
-  apply_autoscaling_profile "$stable_profile" 1 "$requested_hongkong_max"
+  apply_autoscaling_profile "$stable_profile" "$requested_ningbo_max" "$requested_hongkong_max"
   deadline=$((SECONDS + 180))
   while (( SECONDS < deadline )); do
     status=$(capacity_controller_status || true)
@@ -1755,7 +1809,17 @@ run_autoscaling_acceptance() {
   }
   resource_max_hongkong=$(jq -r \
     '[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
-  if (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then
+  if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
+        AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG < AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
+        requested_hongkong_max >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
+        resource_max_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG )); then
+    run_autoscaling_topology_phase south-pool-scale-out \
+      "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
+      "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" "$observations"
+    tested_south_pool_scale_out=true
+    certified_max_replicas_ningbo=0
+    certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
+  elif (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then
     run_autoscaling_topology_phase 1-plus-2 1 2 "$observations"
     tested_1_plus_2=true
     certified_max_replicas_hongkong=2
@@ -1785,6 +1849,9 @@ run_autoscaling_acceptance() {
 
   jq -n \
     --arg profile "$stable_profile" \
+    --argjson baselineNingbo "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
+    --argjson baselineHongkong "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
+    --argjson testedSouthPoolScaleOut "$tested_south_pool_scale_out" \
     --argjson testedOnePlusTwo "$tested_1_plus_2" \
     --argjson testedTwoPlusTwo "$tested_2_plus_2" \
     --argjson observedMaxReplicas "$autoscaling_phase_observed_max" \
@@ -1792,8 +1859,10 @@ run_autoscaling_acceptance() {
       passed:true,
       drained:true,
       capacityProfile:$profile,
+      baseline:{ningbo:$baselineNingbo,hongkong:$baselineHongkong},
       topologies:{
-        "1+1":true,
+        "1+1":($baselineNingbo == 1 and $baselineHongkong == 1),
+        "0+1_to_0+2":$testedSouthPoolScaleOut,
         "1+2":$testedOnePlusTwo,
         "2+2":$testedTwoPlusTwo
       },
@@ -2002,11 +2071,17 @@ WHERE acceptance_run_id IS NULL
 }
 
 activate_certified_autoscaling() {
-  local deadline status resource_max
+  local deadline status resource_max test_min_ningbo test_min_hongkong
+  test_min_ningbo=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO
+  test_min_hongkong=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG
+  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
+  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
   apply_autoscaling_profile \
     "$stable_profile" \
     "$certified_max_replicas_ningbo" \
     "$certified_max_replicas_hongkong"
+  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$test_min_ningbo
+  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$test_min_hongkong
   deadline=$((SECONDS + 180))
   while (( SECONDS < deadline )); do
     status=$(capacity_controller_status || true)
@@ -2023,7 +2098,9 @@ activate_certified_autoscaling() {
     '[.plan.sites[] | select(.site == "ningbo") | .resourceMax][0] // 0' <<<"$status")
   certified_max_replicas_hongkong=$(jq -r \
     '[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
-  (( certified_max_replicas_ningbo >= 1 && certified_max_replicas_hongkong >= 1 ))
+  (( certified_max_replicas_ningbo >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
+     certified_max_replicas_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
+     certified_max_replicas_ningbo + certified_max_replicas_hongkong >= 2 ))
   jq -n \
     --arg profile "$stable_profile" \
     --argjson ningbo "$certified_max_replicas_ningbo" \
@@ -2035,9 +2112,9 @@ activate_certified_autoscaling() {
       resourceMaxTotal:$total
     }' >"$report_root/certified-site-capacity.json"
   [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
-    -o 'jsonpath={.spec.replicas}') == 1 ]]
+    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
   [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
-    -o 'jsonpath={.spec.replicas}') == 1 ]]
+    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
   stage_capacity_profiles
   verify_runtime_gates
 }
@@ -2332,6 +2409,14 @@ verify_cluster_links() {
   verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
   verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
   verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
+  if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+    verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_shenzhen'
+    verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 'shenzhen_to_ningbo'
+    verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_shenzhen'
+    verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 'shenzhen_to_hongkong'
+    verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_shenzhen'
+    verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 'shenzhen_to_los_angeles'
+  fi
 }
 
 verify_control_plane_and_recent_logs() {
@@ -2349,6 +2434,11 @@ verify_control_plane_and_recent_logs() {
       [[ $error_count == 0 ]] || return 1
     fi
   done
+  if [[ -n $CLUSTER_SHENZHEN_HOST && -n $runtime_observation_started_at ]]; then
+    error_count=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" \
+      "journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
+    [[ $error_count == 0 ]] || return 1
+  fi
   for workload in api worker; do
     while read -r pod; do
       error_count=$(remote_kubectl logs "$pod" -n "$namespace" "$log_window" |

scripts/cluster/verify-cluster.sh

@@ -15,14 +15,20 @@ load_cluster_env
 hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
 wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
 site_labels=(ningbo hongkong losangeles)
-for source_index in 0 1 2; do
+if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+  hosts+=("$CLUSTER_SHENZHEN_HOST")
+  wireguard_ips+=(10.77.0.4)
+  site_labels+=(shenzhen)
+fi
+peer_count=$((${#hosts[@]} - 1))
+for source_index in "${!hosts[@]}"; do
   handshakes=$(cluster_ssh "${hosts[$source_index]}" \
     "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime() - \$2 < 180 { count++ } END { print count + 0 }'")
-  [[ $handshakes -eq 2 ]] || {
+  [[ $handshakes -eq $peer_count ]] || {
     echo "stale WireGuard handshake on ${site_labels[$source_index]}: recent_peers=$handshakes" >&2
     exit 1
   }
-  for target_index in 0 1 2; do
+  for target_index in "${!hosts[@]}"; do
     [[ $source_index -eq $target_index ]] && continue
     ping_output=$(cluster_ssh "${hosts[$source_index]}" \
       "ping -i 0.2 -c 10 -W 2 ${wireguard_ips[$target_index]}")
@@ -46,12 +52,15 @@ for source_index in 0 1 2; do
     fi
   done
 done
-echo 'wireguard_acceptance=PASS handshakes=6 packet_loss_lt_1=true'
+echo "wireguard_acceptance=PASS peers=${#hosts[@]} handshakes=$((${#hosts[@]} * peer_count)) packet_loss_lt_1=true"
 
-cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+expected_nodes=3
+[[ -z $CLUSTER_SHENZHEN_HOST ]] || expected_nodes=4
+cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$expected_nodes" <<'REMOTE'
 set -euo pipefail
+expected_nodes=$1
 kubectl='k3s kubectl'
-[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
+[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq $expected_nodes ]]
 ready_output=$($kubectl get --raw='/readyz?verbose')
 grep -Fq '[+]etcd ok' <<<"$ready_output"
 [[ $($kubectl get node easyai-los-angeles -o json |
@@ -62,8 +71,24 @@ grep -Fq '[+]etcd ok' <<<"$ready_output"
 printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)"
 REMOTE
 )
-[[ $cluster_state == nodes=3 ]]
-echo 'k3s_control_plane=PASS servers=3 witness_business_pods=0'
+[[ $cluster_state == nodes="$expected_nodes" ]]
+if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+  worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+node=easyai-shenzhen
+[[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]]
+[[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]
+[[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |
+  jq '[.items[] | select(.metadata.namespace!="kube-system" and .metadata.labels["app.kubernetes.io/name"]!="easyai-worker")] | length') -eq 0 ]]
+printf 'worker_node=ready\n'
+REMOTE
+)
+  [[ $worker_state == worker_node=ready ]]
+fi
+echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witness_business_pods=0"
 
 database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
 set -euo pipefail
@@ -143,8 +168,11 @@ kubectl='k3s kubectl'
 for site in ningbo hongkong; do
   [[ $($kubectl get deployment "easyai-api-$site" -n easyai \
     -o jsonpath='{.status.readyReplicas}') -eq 1 ]]
-  [[ $($kubectl get deployment "easyai-worker-$site" -n easyai \
-    -o jsonpath='{.status.readyReplicas}') -eq 1 ]]
+  worker_replicas=$($kubectl get deployment "easyai-worker-$site" -n easyai \
+    -o json | jq -r '.spec.replicas')
+  worker_ready=$($kubectl get deployment "easyai-worker-$site" -n easyai \
+    -o json | jq -r '.status.readyReplicas // 0')
+  [[ $worker_ready -eq $worker_replicas ]]
   [[ $($kubectl get deployment "easyai-api-$site" -n easyai \
     -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]]
   [[ $($kubectl get deployment "easyai-worker-$site" -n easyai \

scripts/release-manifest.mjs

@@ -149,9 +149,11 @@ function validate(manifest) {
         !Number.isInteger(manifest.certification.workerInstanceSlots) ||
         manifest.certification.workerInstanceSlots < 1 ||
         !Number.isInteger(manifest.certification.workerMaxReplicasNingbo) ||
-        manifest.certification.workerMaxReplicasNingbo < 1 ||
+        manifest.certification.workerMaxReplicasNingbo < 0 ||
         !Number.isInteger(manifest.certification.workerMaxReplicasHongkong) ||
-        manifest.certification.workerMaxReplicasHongkong < 1) {
+        manifest.certification.workerMaxReplicasHongkong < 0 ||
+        manifest.certification.workerMaxReplicasNingbo +
+          manifest.certification.workerMaxReplicasHongkong < 1) {
       fail('certification record is invalid')
     }
   }