From e77bd177547e2fad9e18117e95116533c02d4f34 Mon Sep 17 00:00:00 2001
From: wangbo <wangbolhr@163.com>
Date: Sat, 1 Aug 2026 10:54:30 +0800
Subject: [PATCH] =?UTF-8?q?fix(cluster):=20=E5=9B=BA=E5=8C=96=E5=AE=81?=
 =?UTF-8?q?=E6=B3=A2=20Worker=20=E9=9A=A7=E9=81=93=E7=AB=AF=E5=8F=A3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

新宁波节点改用 UDP/443 监听，其他三台节点保留 UDP/51820，并在全量 WireGuard 引导中按节点配置 endpoint 端口。\n\n三轮六向实测均为零丢包，香港 RTT 从 73-77ms 降至 48-53ms；已通过 bash -n、ShellCheck 和差异检查。
---
 docs/operations/k3s-ha-runbook.md         |  3 ++-
 scripts/cluster/add-ningbo-worker-node.sh | 15 ++++++++++-----
 scripts/cluster/bootstrap-wireguard.sh    | 17 +++++++++++------
 3 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md
index 63a31d7..0100c7a 100644
--- a/docs/operations/k3s-ha-runbook.md
+++ b/docs/operations/k3s-ha-runbook.md
@@ -18,7 +18,8 @@ CloudNativePG；洛杉矶只作为 K3s server/etcd 仲裁节点，并带
 新节点与宁波、香港、洛杉矶使用 WireGuard 全互联直连，不经过中继。公网 K3s 直连虽然可以
 单独保护 API TLS，但当前 Flannel、Pod 和数据库网络依赖 `10.77.0.0/24`，禁止改成未加密的
 公网 VXLAN。巡检以 10 包丢包率低于 1%为硬门禁；宁波和香港路径 RTT 低于 80 ms，洛杉矶
-路径低于 300 ms。
+路径低于 300 ms。新节点监听 UDP/443；实测 UDP/51820 在香港方向存在 QoS 抖动，禁止在
+未重新完成三轮六向链路验收前改回默认端口。
 
 本地 `.env.local` 只保存 `AI_GATEWAY_NINGBO_WORKER_HOST` 和 SSH 接入密码，必须保持 `0600`
 且不得进入 Git。新节点只保留原有 SSH 入口，封锁 K3s API、kubelet、VXLAN 和全部 NodePort
diff --git a/scripts/cluster/add-ningbo-worker-node.sh b/scripts/cluster/add-ningbo-worker-node.sh
index 59ffee1..1ac0a2e 100755
--- a/scripts/cluster/add-ningbo-worker-node.sh
+++ b/scripts/cluster/add-ningbo-worker-node.sh
@@ -18,6 +18,7 @@ require_commands ssh scp shasum
 
 node_name=easyai-ningbo-worker-2
 node_ip=10.77.0.4
+worker_wireguard_port=443
 k3s_version=v1.36.2+k3s1
 k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de
 secret_root="$cluster_root/.local-secrets/cluster"
@@ -91,17 +92,20 @@ REMOTE
   done
 
   cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- \
+    "$worker_wireguard_port" \
     "${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \
     "${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \
     "${server_keys[2]}" "${server_endpoints[2]}" "${server_ips[2]}" <<'REMOTE'
 set -euo pipefail
+listen_port=$1
+shift
 private_key=$(cat /etc/wireguard/privatekey)
 umask 077
 temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
 cat >"$temporary_config" <<EOF
 [Interface]
 Address = 10.77.0.4/24
-ListenPort = 51820
+ListenPort = ${listen_port}
 PrivateKey = ${private_key}
 
 [Peer]
@@ -125,16 +129,17 @@ EOF
 chmod 0600 "$temporary_config"
 mv "$temporary_config" /etc/wireguard/wg0.conf
 for peer in "$2" "$5" "$8"; do
-  ufw allow from "$peer" to any port 51820 proto udp comment easyai-wireguard >/dev/null
+  ufw allow from "$peer" to any port "$listen_port" proto udp comment easyai-wireguard >/dev/null
 done
 systemctl enable wg-quick@wg0 >/dev/null
 systemctl restart wg-quick@wg0
 REMOTE
 
   for host in "${server_hosts[@]}"; do
-    cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" "$node_ip" <<'REMOTE'
+    cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" \
+      "$node_ip" "$worker_wireguard_port" <<'REMOTE'
 set -euo pipefail
-wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:51820" persistent-keepalive 25
+wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:$4" persistent-keepalive 25
 wg-quick save wg0
 REMOTE
   done
@@ -166,7 +171,7 @@ verify_wireguard() {
       "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
     [[ $recent == 3 ]]
   done
-  echo 'ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct'
+  echo "ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct listen_port=$worker_wireguard_port"
 }
 
 install_agent() {
diff --git a/scripts/cluster/bootstrap-wireguard.sh b/scripts/cluster/bootstrap-wireguard.sh
index f71edfd..6b6bd28 100755
--- a/scripts/cluster/bootstrap-wireguard.sh
+++ b/scripts/cluster/bootstrap-wireguard.sh
@@ -11,10 +11,12 @@ require_commands ssh scp
 hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
 public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
 wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
+wireguard_ports=(51820 51820 51820)
 if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
   hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
   public_endpoints+=("${CLUSTER_NINGBO_WORKER_HOST#*@}")
   wireguard_ips+=(10.77.0.4)
+  wireguard_ports+=(443)
 fi
 public_keys=()
 
@@ -51,6 +53,7 @@ for local_index in "${!hosts[@]}"; do
       peer_arguments+=(
         "${public_keys[$candidate]}"
         "${public_endpoints[$candidate]}"
+        "${wireguard_ports[$candidate]}"
         "${wireguard_ips[$candidate]}"
       )
     fi
@@ -58,29 +61,31 @@ for local_index in "${!hosts[@]}"; do
   echo "[wireguard] configuring ${hosts[$local_index]} as ${wireguard_ips[$local_index]}"
   cluster_ssh "${hosts[$local_index]}" bash -s -- \
     "${wireguard_ips[$local_index]}" \
+    "${wireguard_ports[$local_index]}" \
     "${peer_arguments[@]}" <<'REMOTE'
 set -euo pipefail
 local_ip=$1
-shift
+local_port=$2
+shift 2
 private_key=$(cat /etc/wireguard/privatekey)
 umask 077
 temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
 cat >"$temporary_config" <<EOF
 [Interface]
 Address = ${local_ip}/24
-ListenPort = 51820
+ListenPort = ${local_port}
 PrivateKey = ${private_key}
 EOF
-while (( $# >= 3 )); do
+while (( $# >= 4 )); do
   cat >>"$temporary_config" <<EOF
 
 [Peer]
 PublicKey = $1
-Endpoint = $2:51820
-AllowedIPs = $3/32
+Endpoint = $2:$3
+AllowedIPs = $4/32
 PersistentKeepalive = 25
 EOF
-  shift 3
+  shift 4
 done
 (( $# == 0 ))
 chmod 0600 "$temporary_config"