refactor(cluster): 通用化第四个 Worker 节点接入

将原宁波专用接入脚本改为由环境变量控制节点名称、站点和 WireGuard 端口，支持香港双物理节点 Worker 池。增加候选节点启用开关和服务商网络许可门禁，未明确批准时禁止任何 WireGuard 或 K3s 写操作。同步调整生产验收、巡检、拓扑校验和运行手册。\n\n验证：\n- scripts/cluster 全部 Shell 脚本 bash -n\n- scripts/cluster 全部 ShellCheck\n- git diff --check\n- 本地凭据值扫描\n- 当前三节点 WireGuard/K3s/PostgreSQL 只读基线检查

docs/operations/k3s-ha-runbook.md

@@ -2,29 +2,40 @@
 
 ## 范围与目标
 
-本手册覆盖宁波、香港、洛杉矶三节点迁移。宁波和香港承载 API、Web、River Worker 与
+本手册覆盖宁波、香港、洛杉矶三个 K3s server 和一个可选专用 Worker agent。宁波、香港
-CloudNativePG；洛杉矶只作为 K3s server/etcd 仲裁节点，并带
+承载 API、Web 与 CloudNativePG；Worker 只调度到带 `easyai.io/worker=true` 标签的节点。
+洛杉矶只作为 K3s server/etcd 仲裁节点，并带
 `easyai.io/witness=true:NoSchedule` 污点。
 
-第二台宁波服务器是纯 K3s agent，不加入 etcd 或 CloudNativePG。它使用节点名
+第 4 台服务器是站点可配置的纯 K3s agent，不加入 etcd 或 CloudNativePG。节点名和逻辑站点
-`easyai-ningbo-worker-2`、WireGuard 地址 `10.77.0.4`，并归入逻辑 `ningbo` Worker 池。
+分别由 `AI_GATEWAY_WORKER_NODE_4_NAME`、`AI_GATEWAY_WORKER_NODE_4_SITE` 指定，固定使用
+WireGuard 地址 `10.77.0.4`。当前计划节点为 `easyai-hongkong-worker-2`，归入逻辑
+`hongkong` Worker 池；稳定基线为宁波 0 副本、香港 2 副本，并跨香港两台物理机分布。
 `easyai.io/worker-only=true:NoSchedule` 污点确保除 Worker 外的业务 Pod 不会调度到该机；
 `easyai.io/worker=true` 标签确保容量控制器只把专用 Worker 节点纳入资源预算。接入使用：
 
 ```bash
-./scripts/cluster/add-ningbo-worker-node.sh all
+./scripts/cluster/add-worker-node.sh preflight
+./scripts/cluster/add-worker-node.sh all
 ```
 
 新节点与宁波、香港、洛杉矶使用 WireGuard 全互联直连，不经过中继。公网 K3s 直连虽然可以
 单独保护 API TLS，但当前 Flannel、Pod 和数据库网络依赖 `10.77.0.0/24`，禁止改成未加密的
 公网 VXLAN。巡检以 10 包丢包率低于 1%为硬门禁；宁波和香港路径 RTT 低于 80 ms，洛杉矶
-路径低于 300 ms。新节点监听 UDP/443；实测 UDP/51820 在香港方向存在 QoS 抖动，禁止在
+路径低于 300 ms。新节点默认监听标准 UDP/51820，可通过
-未重新完成三轮六向链路验收前改回默认端口。
+`AI_GATEWAY_WORKER_NODE_4_WIREGUARD_PORT` 配置；不得为了绕过服务商风控擅自伪装端口。
 
-本地 `.env.local` 只保存 `AI_GATEWAY_NINGBO_WORKER_HOST` 和 SSH 接入密码，必须保持 `0600`
+服务商必须先通过工单书面确认“仅用于 Kubernetes 私有集群互联，不提供 VPN、代理或流量
-且不得进入 Git。新节点只保留原有 SSH 入口，封锁 K3s API、kubelet、VXLAN 和全部 NodePort
+转发服务”符合其政策；确认后才可在 `.env.local` 设置
-公网入口。节点防火墙继续由原有 UFW 管理，接入脚本
+`AI_GATEWAY_WORKER_NODE_4_NETWORK_APPROVED=true`。未设置时，脚本只允许执行只读
+`preflight`，所有 WireGuard 与 K3s 写操作都会停止。
+
+本地 `.env.local` 保存 `AI_GATEWAY_WORKER_NODE_4_HOST/NAME/SITE` 和 SSH 接入密码，必须保持
+`0600` 且不得进入 Git。新节点只保留原有 SSH 入口，封锁 K3s API、kubelet、VXLAN和全部
+NodePort 公网入口。节点防火墙继续由原有 UFW 管理，接入脚本
 禁止安装与 UFW 冲突的 `iptables-persistent`；UFW 未安装或未启用时必须在预检阶段停止。
+节点完成加入和验收后才设置 `AI_GATEWAY_WORKER_NODE_4_ENABLED=true`；候选服务器仅写入本地
+配置但尚未加入时必须保持 `false`，避免日常巡检误把候选机计入生产拓扑。
 
 生产上线前的双节点高媒体负载、Worker 强杀和 P24/P28/P32 容量搜索见
 [生产同构验收模式与高媒体压力测试](production-acceptance.md)。

scripts/cluster/add-worker-node.sh

@@ -8,17 +8,18 @@ source "$script_dir/common.sh"
 action=${1:-all}
 [[ $action == preflight || $action == network || $action == install ||
    $action == verify || $action == all ]] || {
-  echo 'usage: add-ningbo-worker-node.sh [preflight|network|install|verify|all]' >&2
+  echo 'usage: add-worker-node.sh [preflight|network|install|verify|all]' >&2
   exit 64
 }
 
 load_cluster_env
 require_commands ssh scp shasum
-: "${CLUSTER_NINGBO_WORKER_HOST:?AI_GATEWAY_NINGBO_WORKER_HOST is required}"
+: "${CLUSTER_WORKER_NODE_4_HOST:?AI_GATEWAY_WORKER_NODE_4_HOST is required}"
 
-node_name=easyai-ningbo-worker-2
+node_name=$CLUSTER_WORKER_NODE_4_NAME
+node_site=$CLUSTER_WORKER_NODE_4_SITE
 node_ip=10.77.0.4
-worker_wireguard_port=443
+worker_wireguard_port=$CLUSTER_WORKER_NODE_4_WIREGUARD_PORT
 k3s_version=v1.36.2+k3s1
 k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de
 secret_root="$cluster_root/.local-secrets/cluster"
@@ -39,8 +40,10 @@ assert_join_material() {
 }
 
 preflight_node() {
-  cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s <<'REMOTE'
+  cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" bash -s -- "$node_name" "$node_site" <<'REMOTE'
 set -euo pipefail
+node_name=$1
+node_site=$2
 [[ $(id -u) -eq 0 && $(uname -m) == x86_64 ]]
 . /etc/os-release
 [[ ($ID == debian && ${VERSION_ID%%.*} -ge 12) ||
@@ -54,11 +57,19 @@ timedatectl show -p NTPSynchronized --value | grep -Fxq yes
 [[ ! -e /etc/rancher/k3s/config.yaml || -f /etc/rancher/k3s/config.yaml ]]
 command -v ufw >/dev/null
 ufw status | grep -Fqx 'Status: active'
-printf 'ningbo_worker_preflight=PASS cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \
+printf 'worker_node_preflight=PASS node=%s site=%s cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \
-  "$(nproc)" "$((available_kib / 1024))" "$((available_bytes / 1024 / 1024 / 1024))"
+  "$node_name" "$node_site" "$(nproc)" "$((available_kib / 1024))" \
+  "$((available_bytes / 1024 / 1024 / 1024))"
 REMOTE
 }
 
+assert_network_approved() {
+  [[ $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == true ]] || {
+    echo 'dedicated Worker private-network setup is blocked until the provider approves this non-proxy Kubernetes use in writing' >&2
+    return 1
+  }
+}
+
 configure_wireguard() {
   local -a server_hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
   local -a server_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
@@ -66,7 +77,7 @@ configure_wireguard() {
   local -a server_keys=()
   local worker_key host key
 
-  worker_key=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" 'bash -s' <<'REMOTE' | tail -n 1
+  worker_key=$(cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" 'bash -s' <<'REMOTE' | tail -n 1
 set -euo pipefail
 export DEBIAN_FRONTEND=noninteractive
 if ! command -v wg >/dev/null 2>&1; then
@@ -91,7 +102,7 @@ REMOTE
     server_keys+=("$key")
   done
 
-  cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- \
+  cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" bash -s -- \
     "$worker_wireguard_port" \
     "${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \
     "${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \
@@ -136,7 +147,7 @@ systemctl restart wg-quick@wg0
 REMOTE
 
   for host in "${server_hosts[@]}"; do
-    cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" \
+    cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_WORKER_NODE_4_HOST#*@}" \
       "$node_ip" "$worker_wireguard_port" <<'REMOTE'
 set -euo pipefail
 wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:$4" persistent-keepalive 25
@@ -159,19 +170,19 @@ verify_path() {
 }
 
 verify_wireguard() {
-  verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_worker_2
+  verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_worker_node_4
-  verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 worker_2_to_ningbo
+  verify_path "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.1 80 worker_node_4_to_ningbo
-  verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_worker_2
+  verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_worker_node_4
-  verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 worker_2_to_hongkong
+  verify_path "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.2 80 worker_node_4_to_hongkong
-  verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_worker_2
+  verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_worker_node_4
-  verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 worker_2_to_losangeles
+  verify_path "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.3 300 worker_node_4_to_losangeles
   local host recent
-  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_NINGBO_WORKER_HOST"; do
+  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_WORKER_NODE_4_HOST"; do
     recent=$(cluster_ssh "$host" \
       "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
     [[ $recent == 3 ]]
   done
-  echo "ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct listen_port=$worker_wireguard_port"
+  echo "worker_node_wireguard=PASS node=$node_name site=$node_site peers=4 recent_handshakes=12 routing=direct listen_port=$worker_wireguard_port"
 }
 
 install_agent() {
@@ -181,26 +192,29 @@ install_agent() {
   : "${K3S_TOKEN:?}"
   [[ $K3S_TOKEN =~ ^[0-9a-f]{64}$ ]]
 
-  cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" 'install -d -m 0700 /etc/rancher/k3s'
+  cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" 'install -d -m 0700 /etc/rancher/k3s'
-  printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \
+  printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" \
     'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token'
-  cluster_scp "$k3s_binary" "$CLUSTER_NINGBO_WORKER_HOST:/usr/local/bin/k3s"
+  cluster_scp "$k3s_binary" "$CLUSTER_WORKER_NODE_4_HOST:/usr/local/bin/k3s"
-  cluster_scp "$registries_file" "$CLUSTER_NINGBO_WORKER_HOST:/etc/rancher/k3s/registries.yaml"
+  cluster_scp "$registries_file" "$CLUSTER_WORKER_NODE_4_HOST:/etc/rancher/k3s/registries.yaml"
-  cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- "$k3s_version" "$k3s_sha256" <<'REMOTE'
+  cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" bash -s -- \
+    "$k3s_version" "$k3s_sha256" "$node_name" "$node_site" <<'REMOTE'
 set -euo pipefail
 version=$1
 expected_sha256=$2
+node_name=$3
+node_site=$4
 chmod 0755 /usr/local/bin/k3s
 echo "$expected_sha256  /usr/local/bin/k3s" | sha256sum -c - >/dev/null
 umask 077
-cat >/etc/rancher/k3s/config.yaml <<'EOF'
+cat >/etc/rancher/k3s/config.yaml <<EOF
 server: https://10.77.0.1:6443
 token-file: /etc/rancher/k3s/cluster-token
-node-name: easyai-ningbo-worker-2
+node-name: ${node_name}
 node-ip: 10.77.0.4
 flannel-iface: wg0
 node-label:
-  - easyai.io/site=ningbo
+  - easyai.io/site=${node_site}
   - easyai.io/workload=true
   - easyai.io/database=false
   - easyai.io/worker=true
@@ -253,12 +267,13 @@ REMOTE
 
 verify_agent() {
   verify_wireguard
-  cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$node_name" <<'REMOTE'
+  cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$node_name" "$node_site" <<'REMOTE'
 set -euo pipefail
 node=$1
+site=$2
 kubectl='k3s kubectl'
 [[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]]
-[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == ningbo ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == "$site" ]]
 [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/workload}') == true ]]
 [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]]
 [[ $($kubectl get node easyai-hongkong -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]]
@@ -266,14 +281,14 @@ kubectl='k3s kubectl'
 [[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]
 [[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |
   jq '[.items[] | select(.metadata.namespace!="kube-system")] | length') -eq 0 ]]
-printf 'ningbo_worker_k3s_agent=PASS node=%s role=worker-only etcd_member=false\n' "$node"
+printf 'worker_node_k3s_agent=PASS node=%s site=%s role=worker-only etcd_member=false\n' "$node" "$site"
 REMOTE
 }
 
 case $action in
   preflight) preflight_node ;;
-  network) preflight_node; configure_wireguard ;;
+  network) assert_network_approved; preflight_node; configure_wireguard ;;
-  install) preflight_node; install_agent ;;
+  install) assert_network_approved; preflight_node; install_agent ;;
   verify) preflight_node; verify_agent ;;
-  all) preflight_node; configure_wireguard; install_agent ;;
+  all) assert_network_approved; preflight_node; configure_wireguard; install_agent ;;
 esac

scripts/cluster/bootstrap-wireguard.sh

@@ -12,11 +12,15 @@ hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOS
 public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
 wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
 wireguard_ports=(51820 51820 51820)
-if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
+if [[ -n $CLUSTER_WORKER_NODE_4_HOST ]]; then
-  hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
+  [[ $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == true ]] || {
-  public_endpoints+=("${CLUSTER_NINGBO_WORKER_HOST#*@}")
+    echo 'dedicated Worker private-network setup is blocked until the provider approves this non-proxy Kubernetes use in writing' >&2
+    exit 1
+  }
+  hosts+=("$CLUSTER_WORKER_NODE_4_HOST")
+  public_endpoints+=("${CLUSTER_WORKER_NODE_4_HOST#*@}")
   wireguard_ips+=(10.77.0.4)
-  wireguard_ports+=(443)
+  wireguard_ports+=("$CLUSTER_WORKER_NODE_4_WIREGUARD_PORT")
 fi
 public_keys=()
 

scripts/cluster/common.sh

@@ -28,8 +28,16 @@ load_cluster_env() {
   CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST}
   CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2}
   CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3}
-  CLUSTER_NINGBO_WORKER_HOST=${AI_GATEWAY_NINGBO_WORKER_HOST:-}
+  CLUSTER_WORKER_NODE_4_HOST=${AI_GATEWAY_WORKER_NODE_4_HOST:-}
-  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST CLUSTER_NINGBO_WORKER_HOST
+  CLUSTER_WORKER_NODE_4_NAME=${AI_GATEWAY_WORKER_NODE_4_NAME:-}
+  CLUSTER_WORKER_NODE_4_SITE=${AI_GATEWAY_WORKER_NODE_4_SITE:-}
+  CLUSTER_WORKER_NODE_4_WIREGUARD_PORT=${AI_GATEWAY_WORKER_NODE_4_WIREGUARD_PORT:-51820}
+  CLUSTER_WORKER_NODE_4_NETWORK_APPROVED=${AI_GATEWAY_WORKER_NODE_4_NETWORK_APPROVED:-false}
+  CLUSTER_WORKER_NODE_4_ENABLED=${AI_GATEWAY_WORKER_NODE_4_ENABLED:-false}
+  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST
+  export CLUSTER_WORKER_NODE_4_HOST CLUSTER_WORKER_NODE_4_NAME CLUSTER_WORKER_NODE_4_SITE
+  export CLUSTER_WORKER_NODE_4_WIREGUARD_PORT CLUSTER_WORKER_NODE_4_NETWORK_APPROVED
+  export CLUSTER_WORKER_NODE_4_ENABLED
 
   for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
     [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || {
@@ -37,8 +45,37 @@ load_cluster_env() {
       return 1
     }
   done
-  if [[ -n $CLUSTER_NINGBO_WORKER_HOST && ! $CLUSTER_NINGBO_WORKER_HOST =~ ^root@[A-Za-z0-9.-]+$ ]]; then
+  if [[ -n $CLUSTER_WORKER_NODE_4_HOST ]]; then
-    echo "dedicated Ningbo Worker host must use root@host syntax" >&2
+    [[ $CLUSTER_WORKER_NODE_4_HOST =~ ^root@[A-Za-z0-9.-]+$ ]] || {
+      echo "dedicated Worker node 4 host must use root@host syntax" >&2
+      return 1
+    }
+    [[ ${#CLUSTER_WORKER_NODE_4_NAME} -le 63 &&
+       $CLUSTER_WORKER_NODE_4_NAME =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] || {
+      echo "dedicated Worker node 4 name must be a Kubernetes DNS label" >&2
+      return 1
+    }
+    [[ $CLUSTER_WORKER_NODE_4_SITE == ningbo || $CLUSTER_WORKER_NODE_4_SITE == hongkong ]] || {
+      echo "dedicated Worker node 4 site must be ningbo or hongkong" >&2
+      return 1
+    }
+    [[ $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT =~ ^[0-9]+$ &&
+       $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT -ge 1 &&
+       $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT -le 65535 ]] || {
+      echo "dedicated Worker node 4 WireGuard port is invalid" >&2
+      return 1
+    }
+    [[ $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == true ||
+       $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == false ]] || {
+      echo "dedicated Worker node 4 network approval must be true or false" >&2
+      return 1
+    }
+    [[ $CLUSTER_WORKER_NODE_4_ENABLED == true || $CLUSTER_WORKER_NODE_4_ENABLED == false ]] || {
+      echo "dedicated Worker node 4 enabled flag must be true or false" >&2
+      return 1
+    }
+  elif [[ -n $CLUSTER_WORKER_NODE_4_NAME || -n $CLUSTER_WORKER_NODE_4_SITE ]]; then
+    echo "dedicated Worker node 4 metadata requires AI_GATEWAY_WORKER_NODE_4_HOST" >&2
     return 1
   fi
   [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || {

scripts/cluster/monitor-production-release.sh

@@ -140,8 +140,8 @@ pool_saturated_pods() {
 check_wireguard() {
   local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
   local -a ips=(10.77.0.1 10.77.0.2 10.77.0.3)
-  [[ -z $CLUSTER_NINGBO_WORKER_HOST ]] || {
+  [[ $CLUSTER_WORKER_NODE_4_ENABLED != true ]] || {
-    hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
+    hosts+=("$CLUSTER_WORKER_NODE_4_HOST")
     ips+=(10.77.0.4)
   }
   local source target output loss average handshakes peer_count

scripts/cluster/run-production-acceptance.sh

@@ -652,28 +652,41 @@ verify_release_cas() {
     -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
   [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
     -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
-  if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
+  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
-    local expected_node expected_replicas site pod_count
+    local expected_replicas eligible_nodes eligible_count pod_report
-    for site in ningbo hongkong; do
+    local running_count invalid_count node4_count used_nodes
-      case $site in
+    case $CLUSTER_WORKER_NODE_4_SITE in
-        ningbo)
+      ningbo) expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO ;;
-          expected_node=easyai-ningbo-worker-2
+      hongkong) expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG ;;
-          expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
+    esac
-          ;;
+    eligible_nodes=$(remote_kubectl get nodes \
-        hongkong)
+      -l "easyai.io/site=$CLUSTER_WORKER_NODE_4_SITE,easyai.io/worker=true" \
-          expected_node=easyai-hongkong
+      -o json | jq -c '[.items[].metadata.name]')
-          expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
+    eligible_count=$(jq 'length' <<<"$eligible_nodes")
-          ;;
+    jq -e --arg node "$CLUSTER_WORKER_NODE_4_NAME" 'index($node) != null' \
-      esac
+      <<<"$eligible_nodes" >/dev/null
-      pod_count=$(remote_kubectl get pods -n "$namespace" \
+    pod_report=$(remote_kubectl get pods -n "$namespace" \
-        -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" \
+      -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$CLUSTER_WORKER_NODE_4_SITE" -o json)
-        --field-selector "spec.nodeName=$expected_node" -o json | jq \
+    running_count=$(jq '[.items[] | select(.status.phase=="Running")] | length' <<<"$pod_report")
-        '[.items[] | select(.status.phase=="Running")] | length')
+    invalid_count=$(jq --argjson nodes "$eligible_nodes" \
-      [[ $pod_count == "$expected_replicas" ]] || {
+      '[.items[] | select(.status.phase=="Running") | select(.spec.nodeName as $node | $nodes | index($node) | not)] | length' \
-        echo "Worker placement does not match the dedicated node: site=$site node=$expected_node running_pods=$pod_count expected=$expected_replicas" >&2
+      <<<"$pod_report")
-        return 1
+    node4_count=$(jq --arg node "$CLUSTER_WORKER_NODE_4_NAME" \
-      }
+      '[.items[] | select(.status.phase=="Running" and .spec.nodeName==$node)] | length' <<<"$pod_report")
-    done
+    used_nodes=$(jq '[.items[] | select(.status.phase=="Running") | .spec.nodeName] | unique | length' \
+      <<<"$pod_report")
+    [[ $running_count == "$expected_replicas" && $invalid_count == 0 ]] || {
+      echo "Worker placement is outside eligible nodes: site=$CLUSTER_WORKER_NODE_4_SITE running_pods=$running_count expected=$expected_replicas invalid=$invalid_count" >&2
+      return 1
+    }
+    if (( expected_replicas > 0 && node4_count == 0 )); then
+      echo "dedicated Worker node has no baseline pod: site=$CLUSTER_WORKER_NODE_4_SITE node=$CLUSTER_WORKER_NODE_4_NAME" >&2
+      return 1
+    fi
+    if (( expected_replicas >= 2 && eligible_count >= 2 && used_nodes < 2 )); then
+      echo "Worker baseline is not distributed across physical nodes: site=$CLUSTER_WORKER_NODE_4_SITE used_nodes=$used_nodes" >&2
+      return 1
+    fi
   fi
 }
 
@@ -2417,13 +2430,13 @@ verify_cluster_links() {
   verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
   verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
   verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
-  if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
+  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
-    verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_2'
+    verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_node_4'
-    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 'worker_2_to_ningbo'
+    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.1 80 'worker_node_4_to_ningbo'
-    verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_2'
+    verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_node_4'
-    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 'worker_2_to_hongkong'
+    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.2 80 'worker_node_4_to_hongkong'
-    verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_2'
+    verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_node_4'
-    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 'worker_2_to_los_angeles'
+    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.3 300 'worker_node_4_to_losangeles'
   fi
 }
 
@@ -2442,8 +2455,8 @@ verify_control_plane_and_recent_logs() {
       [[ $error_count == 0 ]] || return 1
     fi
   done
-  if [[ -n $CLUSTER_NINGBO_WORKER_HOST && -n $runtime_observation_started_at ]]; then
+  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true && -n $runtime_observation_started_at ]]; then
-    error_count=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \
+    error_count=$(cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" \
       "journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
     [[ $error_count == 0 ]] || return 1
   fi

scripts/cluster/verify-cluster.sh

@@ -15,10 +15,10 @@ load_cluster_env
 hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
 wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
 site_labels=(ningbo hongkong losangeles)
-if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
+if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
-  hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
+  hosts+=("$CLUSTER_WORKER_NODE_4_HOST")
   wireguard_ips+=(10.77.0.4)
-  site_labels+=(ningbo-worker-2)
+  site_labels+=("$CLUSTER_WORKER_NODE_4_NAME")
 fi
 peer_count=$((${#hosts[@]} - 1))
 for source_index in "${!hosts[@]}"; do
@@ -55,7 +55,7 @@ done
 echo "wireguard_acceptance=PASS peers=${#hosts[@]} handshakes=$((${#hosts[@]} * peer_count)) packet_loss_lt_1=true"
 
 expected_nodes=3
-[[ -z $CLUSTER_NINGBO_WORKER_HOST ]] || expected_nodes=4
+[[ $CLUSTER_WORKER_NODE_4_ENABLED != true ]] || expected_nodes=4
 cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$expected_nodes" <<'REMOTE'
 set -euo pipefail
 expected_nodes=$1
@@ -72,22 +72,24 @@ printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)"
 REMOTE
 )
 [[ $cluster_state == nodes="$expected_nodes" ]]
-if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
+if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
-  worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+  worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- \
+    "$CLUSTER_WORKER_NODE_4_NAME" "$CLUSTER_WORKER_NODE_4_SITE" <<'REMOTE'
 set -euo pipefail
 kubectl='k3s kubectl'
-node=easyai-ningbo-worker-2
+node=$1
+site=$2
 [[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]]
-[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == ningbo ]]
+[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == "$site" ]]
 [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]]
 [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]]
 [[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]
 [[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |
   jq '[.items[] | select(.metadata.namespace!="kube-system" and .metadata.labels["app.kubernetes.io/name"]!="easyai-worker")] | length') -eq 0 ]]
-printf 'worker_node=ready\n'
+printf 'worker_node=ready node=%s site=%s\n' "$node" "$site"
 REMOTE
 )
-  [[ $worker_state == worker_node=ready ]]
+  [[ $worker_state == "worker_node=ready node=$CLUSTER_WORKER_NODE_4_NAME site=$CLUSTER_WORKER_NODE_4_SITE" ]]
 fi
 echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witness_business_pods=0"