fix(cluster): 以宁波专用节点替换深圳 Worker

移除深圳节点及中继拓扑，新增第二台宁波 K3s agent 的全互联 WireGuard 接入和严格 UFW 门禁。\n\nWorker Deployment 与容量控制器仅选择 easyai.io/worker=true 节点，使原宁波混部节点退出 Worker 资源预算，生产基线恢复为宁波专用节点与香港节点各一实例。\n\n已通过 Go 全量测试、go vet、gofmt、迁移安全检查、bash -n、ShellCheck、发布脚本测试和 Kubernetes 清单渲染。

apps/api/internal/capacitycontroller/kubernetes.go

@@ -172,7 +172,9 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
 			} `json:"status"`
 		} `json:"items"`
 	}
-	nodePath := "/api/v1/nodes?labelSelector=" + url.QueryEscape("easyai.io/site="+site)
+	nodePath := "/api/v1/nodes?labelSelector=" + url.QueryEscape(
+		"easyai.io/site="+site+",easyai.io/worker=true",
+	)
 	if err := client.getJSON(ctx, nodePath, &nodes); err != nil {
 		return KubernetesSiteState{}, err
 	}

apps/api/internal/capacitycontroller/kubernetes_test.go

@@ -27,6 +27,10 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) {
 				}]}}}
 			}`))
 		case request.Method == http.MethodGet && request.URL.Path == "/api/v1/nodes":
+			if request.URL.Query().Get("labelSelector") != "easyai.io/site=hongkong,easyai.io/worker=true" {
+				http.Error(w, "unexpected Worker node selector", http.StatusBadRequest)
+				return
+			}
 			_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"easyai-hongkong"},"status":{
 				"allocatable":{"memory":"8Gi","cpu":"4"},
 				"conditions":[{"type":"MemoryPressure","status":"False"}]

deploy/kubernetes/production/application.yaml

@@ -458,6 +458,7 @@ spec:
       nodeSelector:
         easyai.io/site: ningbo
         easyai.io/workload: "true"
+        easyai.io/worker: "true"
       tolerations:
         - key: easyai.io/worker-only
           operator: Equal
@@ -622,6 +623,7 @@ spec:
       nodeSelector:
         easyai.io/site: hongkong
         easyai.io/workload: "true"
+        easyai.io/worker: "true"
       tolerations:
         - key: easyai.io/worker-only
           operator: Equal

docs/operations/k3s-ha-runbook.md

@@ -6,22 +6,23 @@
 CloudNativePG；洛杉矶只作为 K3s server/etcd 仲裁节点，并带
 `easyai.io/witness=true:NoSchedule` 污点。
 
-深圳扩展节点是纯 K3s agent，不加入 etcd 或 CloudNativePG。它使用节点名
+第二台宁波服务器是纯 K3s agent，不加入 etcd 或 CloudNativePG。它使用节点名
-`easyai-shenzhen`、WireGuard 地址 `10.77.0.4`，并归入逻辑 `hongkong` Worker 池。
+`easyai-ningbo-worker-2`、WireGuard 地址 `10.77.0.4`，并归入逻辑 `ningbo` Worker 池。
 `easyai.io/worker-only=true:NoSchedule` 污点确保除 Worker 外的业务 Pod 不会调度到该机；
-香港池副本通过 hostname topology spread 分散到香港、深圳。接入使用：
+`easyai.io/worker=true` 标签确保容量控制器只把专用 Worker 节点纳入资源预算。接入使用：
 
 ```bash
-./scripts/cluster/add-shenzhen-worker-node.sh all
+./scripts/cluster/add-ningbo-worker-node.sh all
 ```
 
-香港—深圳公网直连实测无法满足生产链路门禁，因此两地 `10.77.0.2/10.77.0.4` 数据前缀
+新节点与宁波、香港、洛杉矶使用 WireGuard 全互联直连，不经过中继。公网 K3s 直连虽然可以
-固定经宁波 WireGuard 中继；香港—深圳直接 peer 只保留加密心跳。巡检仍以两端实际数据路径
+单独保护 API TLS，但当前 Flannel、Pod 和数据库网络依赖 `10.77.0.0/24`，禁止改成未加密的
-的 10 包丢包率低于 1%、平均 RTT 低于 80 ms 为硬门禁，禁止自动回退到劣化的公网直连。
+公网 VXLAN。巡检以 10 包丢包率低于 1%为硬门禁；宁波和香港路径 RTT 低于 80 ms，洛杉矶
+路径低于 300 ms。
 
-本地 `.env.local` 只保存 `AI_GATEWAY_SHENZHEN_HOST` 和 SSH 接入密码，必须保持 `0600` 且
+本地 `.env.local` 只保存 `AI_GATEWAY_NINGBO_WORKER_HOST` 和 SSH 接入密码，必须保持 `0600`
-不得进入 Git。深圳已有宿主机服务时，接入脚本保留其公开端口，只封锁 K3s API、kubelet、
+且不得进入 Git。新节点只保留原有 SSH 入口，封锁 K3s API、kubelet、VXLAN 和全部 NodePort
-VXLAN 和除既有 `31058` 外的 NodePort 公网入口。深圳防火墙继续由原有 UFW 管理，接入脚本
+公网入口。节点防火墙继续由原有 UFW 管理，接入脚本
 禁止安装与 UFW 冲突的 `iptables-persistent`；UFW 未安装或未启用时必须在预检阶段停止。
 
 生产上线前的双节点高媒体负载、Worker 强杀和 P24/P28/P32 容量搜索见

docs/operations/production-acceptance.md

@@ -89,21 +89,22 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG
 各一个 2 GiB Worker，再按实时内存、CPU 和 PostgreSQL 预算验证 `1+2`、条件允许时的
 `2+2`，以及安全 drain 回到 `1+1`：
 
-宁波因正式混部不具备 Worker 余量时，验收可改用 `0+2` 基线：
+生产基线使用两个独立物理 Worker 节点：第二台宁波服务器承载 `easyai-worker-ningbo`，
+香港服务器承载 `easyai-worker-hongkong`。两个 Deployment 都要求节点带
+`easyai.io/worker=true`，原宁波混部节点不再承载 Worker。默认验收配置为：
 
 ```bash
-AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO=0
+AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO=1
-AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG=2
+AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG=1
-AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=0
+AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=1
 AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=1
-AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO=0
+AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO=2
 AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG=2
 ```
 
-此时香港逻辑池的两个 Worker 必须通过 hostname topology spread 分布在香港和深圳物理节点；
+资源前置门禁和容量控制器只采样 `easyai.io/worker=true` 节点，避免把原宁波混部节点的
-固定容量仍是两个实例，自动伸缩阶段验证 `0+1 → 0+2 → 0+1`，最终 certified profile
+可分配资源错误计入弹性上限。每站点能否升到两个副本仍由实测 RSS、CPU 和 PostgreSQL
-恢复 `0+2` 最小副本以保留双物理节点容灾。资源前置门禁采样实际启用的 Worker 站点节点，
+连接预算决定，不因新增节点而预先承诺。
-宁波仍受运行时 80% 目标和 85% 硬门禁约束。
 
 | 档位 | 单实例执行槽 | 数据库池 | 媒体并发 | 全局执行槽 |
 |---|---:|---:|---:|---:|

scripts/cluster/add-ningbo-worker-node.sh

@@ -8,15 +8,15 @@ source "$script_dir/common.sh"
 action=${1:-all}
 [[ $action == preflight || $action == network || $action == install ||
    $action == verify || $action == all ]] || {
-  echo 'usage: add-shenzhen-worker-node.sh [preflight|network|install|verify|all]' >&2
+  echo 'usage: add-ningbo-worker-node.sh [preflight|network|install|verify|all]' >&2
   exit 64
 }
 
 load_cluster_env
 require_commands ssh scp shasum
-: "${CLUSTER_SHENZHEN_HOST:?AI_GATEWAY_SHENZHEN_HOST is required}"
+: "${CLUSTER_NINGBO_WORKER_HOST:?AI_GATEWAY_NINGBO_WORKER_HOST is required}"
 
-node_name=easyai-shenzhen
+node_name=easyai-ningbo-worker-2
 node_ip=10.77.0.4
 k3s_version=v1.36.2+k3s1
 k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de
@@ -38,11 +38,12 @@ assert_join_material() {
 }
 
 preflight_node() {
-  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s <<'REMOTE'
+  cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s <<'REMOTE'
 set -euo pipefail
 [[ $(id -u) -eq 0 && $(uname -m) == x86_64 ]]
 . /etc/os-release
-[[ $ID == debian && ${VERSION_ID%%.*} -ge 12 ]]
+[[ ($ID == debian && ${VERSION_ID%%.*} -ge 12) ||
+   ($ID == ubuntu && ${VERSION_ID%%.*} -ge 24) ]]
 (( $(nproc) >= 4 ))
 available_kib=$(awk '/MemAvailable:/ {print $2}' /proc/meminfo)
 (( available_kib >= 6 * 1024 * 1024 ))
@@ -52,7 +53,7 @@ timedatectl show -p NTPSynchronized --value | grep -Fxq yes
 [[ ! -e /etc/rancher/k3s/config.yaml || -f /etc/rancher/k3s/config.yaml ]]
 command -v ufw >/dev/null
 ufw status | grep -Fqx 'Status: active'
-printf 'shenzhen_worker_preflight=PASS cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \
+printf 'ningbo_worker_preflight=PASS cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \
   "$(nproc)" "$((available_kib / 1024))" "$((available_bytes / 1024 / 1024 / 1024))"
 REMOTE
 }
@@ -62,9 +63,9 @@ configure_wireguard() {
   local -a server_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
   local -a server_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
   local -a server_keys=()
-  local shenzhen_key host key
+  local worker_key host key
 
-  shenzhen_key=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" 'bash -s' <<'REMOTE' | tail -n 1
+  worker_key=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" 'bash -s' <<'REMOTE' | tail -n 1
 set -euo pipefail
 export DEBIAN_FRONTEND=noninteractive
 if ! command -v wg >/dev/null 2>&1; then
@@ -81,7 +82,7 @@ wg pubkey </etc/wireguard/privatekey >/etc/wireguard/publickey
 cat /etc/wireguard/publickey
 REMOTE
 )
-  [[ $shenzhen_key =~ ^[A-Za-z0-9+/]{43}=$ ]]
+  [[ $worker_key =~ ^[A-Za-z0-9+/]{43}=$ ]]
 
   for host in "${server_hosts[@]}"; do
     key=$(cluster_ssh "$host" 'wg show wg0 public-key')
@@ -89,7 +90,7 @@ REMOTE
     server_keys+=("$key")
   done
 
-  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \
+  cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- \
     "${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \
     "${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \
     "${server_keys[2]}" "${server_endpoints[2]}" "${server_ips[2]}" <<'REMOTE'
@@ -131,43 +132,13 @@ systemctl restart wg-quick@wg0
 REMOTE
 
   for host in "${server_hosts[@]}"; do
-    cluster_ssh "$host" bash -s -- "$shenzhen_key" "${CLUSTER_SHENZHEN_HOST#*@}" "$node_ip" <<'REMOTE'
+    cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" "$node_ip" <<'REMOTE'
 set -euo pipefail
 wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:51820" persistent-keepalive 25
 wg-quick save wg0
 REMOTE
   done
 
-  # The Hong Kong-Shenzhen public route is unstable. Keep the direct peers for
-  # encrypted liveness, but route their data prefixes through Ningbo's healthy
-  # WireGuard paths. AllowedIPs remain unique, so WireGuard selects the relay
-  # deterministically instead of oscillating between direct and relayed paths.
-  cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- \
-    "${server_keys[0]}" "$shenzhen_key" <<'REMOTE'
-set -euo pipefail
-wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.4/32
-wg set wg0 peer "$2" allowed-ips 0.0.0.0/32
-wg-quick save wg0
-REMOTE
-  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \
-    "${server_keys[0]}" "${server_keys[1]}" <<'REMOTE'
-set -euo pipefail
-wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.2/32
-wg set wg0 peer "$2" allowed-ips 0.0.0.0/32
-wg-quick save wg0
-REMOTE
-  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
-set -euo pipefail
-sysctl -qw net.ipv4.ip_forward=1
-ensure_forward_rule() {
-  if ! iptables -C FORWARD "$@" 2>/dev/null; then iptables -I FORWARD 1 "$@"; fi
-}
-ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.2/32 -d 10.77.0.4/32 \
-  -m comment --comment easyai-wireguard-hongkong-shenzhen -j ACCEPT
-ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.4/32 -d 10.77.0.2/32 \
-  -m comment --comment easyai-wireguard-shenzhen-hongkong -j ACCEPT
-netfilter-persistent save >/dev/null
-REMOTE
   verify_wireguard
 }
 
@@ -183,19 +154,19 @@ verify_path() {
 }
 
 verify_wireguard() {
-  verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_shenzhen
+  verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_worker_2
-  verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 shenzhen_to_ningbo
+  verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 worker_2_to_ningbo
-  verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_shenzhen
+  verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_worker_2
-  verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 shenzhen_to_hongkong
+  verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 worker_2_to_hongkong
-  verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_shenzhen
+  verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_worker_2
-  verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 shenzhen_to_losangeles
+  verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 worker_2_to_losangeles
   local host recent
-  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_SHENZHEN_HOST"; do
+  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_NINGBO_WORKER_HOST"; do
     recent=$(cluster_ssh "$host" \
       "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
     [[ $recent == 3 ]]
   done
-  echo 'shenzhen_wireguard=PASS peers=4 recent_handshakes=12 hongkong_shenzhen_via=ningbo'
+  echo 'ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct'
 }
 
 install_agent() {
@@ -205,12 +176,12 @@ install_agent() {
   : "${K3S_TOKEN:?}"
   [[ $K3S_TOKEN =~ ^[0-9a-f]{64}$ ]]
 
-  cluster_ssh "$CLUSTER_SHENZHEN_HOST" 'install -d -m 0700 /etc/rancher/k3s'
+  cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" 'install -d -m 0700 /etc/rancher/k3s'
-  printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_SHENZHEN_HOST" \
+  printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \
     'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token'
-  cluster_scp "$k3s_binary" "$CLUSTER_SHENZHEN_HOST:/usr/local/bin/k3s"
+  cluster_scp "$k3s_binary" "$CLUSTER_NINGBO_WORKER_HOST:/usr/local/bin/k3s"
-  cluster_scp "$registries_file" "$CLUSTER_SHENZHEN_HOST:/etc/rancher/k3s/registries.yaml"
+  cluster_scp "$registries_file" "$CLUSTER_NINGBO_WORKER_HOST:/etc/rancher/k3s/registries.yaml"
-  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- "$k3s_version" "$k3s_sha256" <<'REMOTE'
+  cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- "$k3s_version" "$k3s_sha256" <<'REMOTE'
 set -euo pipefail
 version=$1
 expected_sha256=$2
@@ -220,11 +191,11 @@ umask 077
 cat >/etc/rancher/k3s/config.yaml <<'EOF'
 server: https://10.77.0.1:6443
 token-file: /etc/rancher/k3s/cluster-token
-node-name: easyai-shenzhen
+node-name: easyai-ningbo-worker-2
 node-ip: 10.77.0.4
 flannel-iface: wg0
 node-label:
-  - easyai.io/site=hongkong
+  - easyai.io/site=ningbo
   - easyai.io/workload=true
   - easyai.io/database=false
   - easyai.io/worker=true
@@ -241,10 +212,8 @@ ufw deny in on "$public_interface" to any port 6443 proto tcp comment easyai-k3s
 ufw deny in on "$public_interface" to any port 10250 proto tcp comment easyai-kubelet-public-deny >/dev/null
 ufw deny in on "$public_interface" to any port 8472 proto udp comment easyai-flannel-public-deny >/dev/null
 for protocol in tcp udp; do
-  ufw deny in on "$public_interface" to any port 30000:31057 proto "$protocol" \
+  ufw deny in on "$public_interface" to any port 30000:32767 proto "$protocol" \
-    comment easyai-nodeport-public-deny-low >/dev/null
+    comment easyai-nodeport-public-deny >/dev/null
-  ufw deny in on "$public_interface" to any port 31059:32767 proto "$protocol" \
-    comment easyai-nodeport-public-deny-high >/dev/null
 done
 install_script=$(mktemp)
 trap 'rm -f -- "$install_script"' EXIT
@@ -267,6 +236,13 @@ REMOTE
     sleep 2
   done
   [[ $ready == true ]]
+  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+$kubectl label node easyai-ningbo easyai.io/worker=false --overwrite >/dev/null
+$kubectl label node easyai-hongkong easyai.io/worker=true --overwrite >/dev/null
+$kubectl label node easyai-los-angeles easyai.io/worker=false --overwrite >/dev/null
+REMOTE
   verify_agent
 }
 
@@ -277,13 +253,15 @@ set -euo pipefail
 node=$1
 kubectl='k3s kubectl'
 [[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]]
-[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == ningbo ]]
 [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/workload}') == true ]]
 [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]]
+[[ $($kubectl get node easyai-hongkong -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]]
+[[ $($kubectl get node easyai-ningbo -o jsonpath='{.metadata.labels.easyai\.io/worker}') == false ]]
 [[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]
 [[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |
   jq '[.items[] | select(.metadata.namespace!="kube-system")] | length') -eq 0 ]]
-printf 'shenzhen_k3s_agent=PASS node=%s role=worker-only etcd_member=false\n' "$node"
+printf 'ningbo_worker_k3s_agent=PASS node=%s role=worker-only etcd_member=false\n' "$node"
 REMOTE
 }
 

scripts/cluster/bootstrap-k3s.sh

@@ -256,9 +256,9 @@ REMOTE
   cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
 set -euo pipefail
 kubectl='k3s kubectl'
-$kubectl label node easyai-ningbo easyai.io/site=ningbo easyai.io/workload=true easyai.io/database=true --overwrite
+$kubectl label node easyai-ningbo easyai.io/site=ningbo easyai.io/workload=true easyai.io/database=true easyai.io/worker=false --overwrite
-$kubectl label node easyai-hongkong easyai.io/site=hongkong easyai.io/workload=true easyai.io/database=true --overwrite
+$kubectl label node easyai-hongkong easyai.io/site=hongkong easyai.io/workload=true easyai.io/database=true easyai.io/worker=true --overwrite
-$kubectl label node easyai-los-angeles easyai.io/site=los-angeles easyai.io/workload=false easyai.io/database=false --overwrite
+$kubectl label node easyai-los-angeles easyai.io/site=los-angeles easyai.io/workload=false easyai.io/database=false easyai.io/worker=false --overwrite
 $kubectl taint node easyai-los-angeles easyai.io/witness=true:NoSchedule --overwrite
 k3s secrets-encrypt status
 k3s etcd-snapshot save --name post-bootstrap

scripts/cluster/bootstrap-wireguard.sh

@@ -11,9 +11,9 @@ require_commands ssh scp
 hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
 public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
 wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
-if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
-  hosts+=("$CLUSTER_SHENZHEN_HOST")
+  hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
-  public_endpoints+=("${CLUSTER_SHENZHEN_HOST#*@}")
+  public_endpoints+=("${CLUSTER_NINGBO_WORKER_HOST#*@}")
   wireguard_ips+=(10.77.0.4)
 fi
 public_keys=()
@@ -90,35 +90,6 @@ systemctl restart wg-quick@wg0
 REMOTE
 done
 
-if (( ${#hosts[@]} == 4 )); then
-  cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- \
-    "${public_keys[0]}" "${public_keys[3]}" <<'REMOTE'
-set -euo pipefail
-wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.4/32
-wg set wg0 peer "$2" allowed-ips 0.0.0.0/32
-wg-quick save wg0
-REMOTE
-  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \
-    "${public_keys[0]}" "${public_keys[1]}" <<'REMOTE'
-set -euo pipefail
-wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.2/32
-wg set wg0 peer "$2" allowed-ips 0.0.0.0/32
-wg-quick save wg0
-REMOTE
-  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
-set -euo pipefail
-sysctl -qw net.ipv4.ip_forward=1
-ensure_forward_rule() {
-  if ! iptables -C FORWARD "$@" 2>/dev/null; then iptables -I FORWARD 1 "$@"; fi
-}
-ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.2/32 -d 10.77.0.4/32 \
-  -m comment --comment easyai-wireguard-hongkong-shenzhen -j ACCEPT
-ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.4/32 -d 10.77.0.2/32 \
-  -m comment --comment easyai-wireguard-shenzhen-hongkong -j ACCEPT
-netfilter-persistent save >/dev/null
-REMOTE
-fi
-
 for source_index in "${!hosts[@]}"; do
   for target_index in "${!hosts[@]}"; do
     [[ $source_index -eq $target_index ]] && continue
@@ -136,8 +107,4 @@ for host in "${hosts[@]}"; do
   }
 done
 
-if (( ${#hosts[@]} == 4 )); then
+echo "wireguard_bootstrap=PASS peers=${#hosts[@]} routing=direct"
-  echo 'wireguard_bootstrap=PASS peers=4 hongkong_shenzhen_via=ningbo'
-else
-  echo "wireguard_bootstrap=PASS peers=${#hosts[@]}"
-fi

scripts/cluster/common.sh

@@ -28,8 +28,8 @@ load_cluster_env() {
   CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST}
   CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2}
   CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3}
-  CLUSTER_SHENZHEN_HOST=${AI_GATEWAY_SHENZHEN_HOST:-}
+  CLUSTER_NINGBO_WORKER_HOST=${AI_GATEWAY_NINGBO_WORKER_HOST:-}
-  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST CLUSTER_SHENZHEN_HOST
+  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST CLUSTER_NINGBO_WORKER_HOST
 
   for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
     [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || {
@@ -37,8 +37,8 @@ load_cluster_env() {
       return 1
     }
   done
-  if [[ -n $CLUSTER_SHENZHEN_HOST && ! $CLUSTER_SHENZHEN_HOST =~ ^root@[A-Za-z0-9.-]+$ ]]; then
+  if [[ -n $CLUSTER_NINGBO_WORKER_HOST && ! $CLUSTER_NINGBO_WORKER_HOST =~ ^root@[A-Za-z0-9.-]+$ ]]; then
-    echo "Shenzhen Worker host must use root@host syntax" >&2
+    echo "dedicated Ningbo Worker host must use root@host syntax" >&2
     return 1
   fi
   [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || {

scripts/cluster/monitor-production-release.sh

@@ -140,8 +140,8 @@ pool_saturated_pods() {
 check_wireguard() {
   local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
   local -a ips=(10.77.0.1 10.77.0.2 10.77.0.3)
-  [[ -z $CLUSTER_SHENZHEN_HOST ]] || {
+  [[ -z $CLUSTER_NINGBO_WORKER_HOST ]] || {
-    hosts+=("$CLUSTER_SHENZHEN_HOST")
+    hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
     ips+=(10.77.0.4)
   }
   local source target output loss average handshakes peer_count

scripts/cluster/run-production-acceptance.sh

@@ -652,17 +652,25 @@ verify_release_cas() {
     -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
   [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
     -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
-  if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
+  if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
-        AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) &&
+    local expected_node expected_replicas site pod_count
-     [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+    for site in ningbo hongkong; do
-    local worker_node pod_count
+      case $site in
-    for worker_node in easyai-hongkong easyai-shenzhen; do
+        ningbo)
+          expected_node=easyai-ningbo-worker-2
+          expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
+          ;;
+        hongkong)
+          expected_node=easyai-hongkong
+          expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
+          ;;
+      esac
       pod_count=$(remote_kubectl get pods -n "$namespace" \
-        -l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \
+        -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" \
-        --field-selector "spec.nodeName=$worker_node" -o json | jq \
+        --field-selector "spec.nodeName=$expected_node" -o json | jq \
         '[.items[] | select(.status.phase=="Running")] | length')
-      [[ $pod_count == 1 ]] || {
+      [[ $pod_count == "$expected_replicas" ]] || {
-        echo "Hong Kong logical Worker pool is not spread across both physical nodes: node=$worker_node running_pods=$pod_count" >&2
+        echo "Worker placement does not match the dedicated node: site=$site node=$expected_node running_pods=$pod_count expected=$expected_replicas" >&2
         return 1
       }
     done
@@ -674,12 +682,12 @@ verify_resource_preconditions() {
   local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
   worker_nodes=
   if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then
-    worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
+    worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo,easyai.io/worker=true' \
       -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
     worker_nodes+=$'\n'
   fi
   if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then
-    worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong' \
+    worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong,easyai.io/worker=true' \
       -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
   fi
   worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u)
@@ -2409,13 +2417,13 @@ verify_cluster_links() {
   verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
   verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
   verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
-  if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+  if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
-    verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_shenzhen'
+    verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_2'
-    verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 'shenzhen_to_ningbo'
+    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 'worker_2_to_ningbo'
-    verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_shenzhen'
+    verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_2'
-    verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 'shenzhen_to_hongkong'
+    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 'worker_2_to_hongkong'
-    verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_shenzhen'
+    verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_2'
-    verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 'shenzhen_to_los_angeles'
+    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 'worker_2_to_los_angeles'
   fi
 }
 
@@ -2434,8 +2442,8 @@ verify_control_plane_and_recent_logs() {
       [[ $error_count == 0 ]] || return 1
     fi
   done
-  if [[ -n $CLUSTER_SHENZHEN_HOST && -n $runtime_observation_started_at ]]; then
+  if [[ -n $CLUSTER_NINGBO_WORKER_HOST && -n $runtime_observation_started_at ]]; then
-    error_count=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" \
+    error_count=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \
       "journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
     [[ $error_count == 0 ]] || return 1
   fi

scripts/cluster/verify-cluster.sh

@@ -15,10 +15,10 @@ load_cluster_env
 hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
 wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
 site_labels=(ningbo hongkong losangeles)
-if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
-  hosts+=("$CLUSTER_SHENZHEN_HOST")
+  hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
   wireguard_ips+=(10.77.0.4)
-  site_labels+=(shenzhen)
+  site_labels+=(ningbo-worker-2)
 fi
 peer_count=$((${#hosts[@]} - 1))
 for source_index in "${!hosts[@]}"; do
@@ -55,7 +55,7 @@ done
 echo "wireguard_acceptance=PASS peers=${#hosts[@]} handshakes=$((${#hosts[@]} * peer_count)) packet_loss_lt_1=true"
 
 expected_nodes=3
-[[ -z $CLUSTER_SHENZHEN_HOST ]] || expected_nodes=4
+[[ -z $CLUSTER_NINGBO_WORKER_HOST ]] || expected_nodes=4
 cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$expected_nodes" <<'REMOTE'
 set -euo pipefail
 expected_nodes=$1
@@ -72,13 +72,14 @@ printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)"
 REMOTE
 )
 [[ $cluster_state == nodes="$expected_nodes" ]]
-if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
+if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
   worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
 set -euo pipefail
 kubectl='k3s kubectl'
-node=easyai-shenzhen
+node=easyai-ningbo-worker-2
 [[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]]
-[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == ningbo ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]]
 [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]]
 [[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]
 [[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |