From c63709d785a431626690dae803c11ae5f3b3d565 Mon Sep 17 00:00:00 2001
From: wangbo <wangbolhr@163.com>
Date: Sat, 1 Aug 2026 09:42:59 +0800
Subject: [PATCH] =?UTF-8?q?fix(cluster):=20=E7=BB=8F=E5=AE=81=E6=B3=A2?=
 =?UTF-8?q?=E4=B8=AD=E7=BB=A7=E6=B7=B1=E5=9C=B3=E9=A6=99=E6=B8=AF=E9=93=BE?=
 =?UTF-8?q?=E8=B7=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

香港与深圳公网直连出现高丢包和高时延时，固定将两地 WireGuard 数据前缀经宁波转发，同时保留直接 peer 心跳。\n\n中继规则和转发状态持久化，接入与全量 WireGuard 引导脚本保持一致；链路验收仍使用端到端丢包和 RTT 硬门禁。\n\n已通过 bash -n、ShellCheck、深圳资源预检和差异检查。
---
 docs/operations/k3s-ha-runbook.md           |  4 +++
 scripts/cluster/add-shenzhen-worker-node.sh | 33 ++++++++++++++++++-
 scripts/cluster/bootstrap-wireguard.sh      | 35 ++++++++++++++++++++-
 3 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md
index 500dcf4..60a17fd 100644
--- a/docs/operations/k3s-ha-runbook.md
+++ b/docs/operations/k3s-ha-runbook.md
@@ -15,6 +15,10 @@ CloudNativePG；洛杉矶只作为 K3s server/etcd 仲裁节点，并带
 ./scripts/cluster/add-shenzhen-worker-node.sh all
 ```
 
+香港—深圳公网直连实测无法满足生产链路门禁，因此两地 `10.77.0.2/10.77.0.4` 数据前缀
+固定经宁波 WireGuard 中继；香港—深圳直接 peer 只保留加密心跳。巡检仍以两端实际数据路径
+的 10 包丢包率低于 1%、平均 RTT 低于 80 ms 为硬门禁，禁止自动回退到劣化的公网直连。
+
 本地 `.env.local` 只保存 `AI_GATEWAY_SHENZHEN_HOST` 和 SSH 接入密码，必须保持 `0600` 且
 不得进入 Git。深圳已有宿主机服务时，接入脚本保留其公开端口，只封锁 K3s API、kubelet、
 VXLAN 和除既有 `31058` 外的 NodePort 公网入口。
diff --git a/scripts/cluster/add-shenzhen-worker-node.sh b/scripts/cluster/add-shenzhen-worker-node.sh
index 2c61e26..e6d86f5 100755
--- a/scripts/cluster/add-shenzhen-worker-node.sh
+++ b/scripts/cluster/add-shenzhen-worker-node.sh
@@ -135,6 +135,37 @@ wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:51820" persistent-keepaliv
 wg-quick save wg0
 REMOTE
   done
+
+  # The Hong Kong-Shenzhen public route is unstable. Keep the direct peers for
+  # encrypted liveness, but route their data prefixes through Ningbo's healthy
+  # WireGuard paths. AllowedIPs remain unique, so WireGuard selects the relay
+  # deterministically instead of oscillating between direct and relayed paths.
+  cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- \
+    "${server_keys[0]}" "$shenzhen_key" <<'REMOTE'
+set -euo pipefail
+wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.4/32
+wg set wg0 peer "$2" allowed-ips 0.0.0.0/32
+wg-quick save wg0
+REMOTE
+  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \
+    "${server_keys[0]}" "${server_keys[1]}" <<'REMOTE'
+set -euo pipefail
+wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.2/32
+wg set wg0 peer "$2" allowed-ips 0.0.0.0/32
+wg-quick save wg0
+REMOTE
+  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+sysctl -qw net.ipv4.ip_forward=1
+ensure_forward_rule() {
+  if ! iptables -C FORWARD "$@" 2>/dev/null; then iptables -I FORWARD 1 "$@"; fi
+}
+ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.2/32 -d 10.77.0.4/32 \
+  -m comment --comment easyai-wireguard-hongkong-shenzhen -j ACCEPT
+ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.4/32 -d 10.77.0.2/32 \
+  -m comment --comment easyai-wireguard-shenzhen-hongkong -j ACCEPT
+netfilter-persistent save >/dev/null
+REMOTE
   verify_wireguard
 }
 
@@ -162,7 +193,7 @@ verify_wireguard() {
       "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
     [[ $recent == 3 ]]
   done
-  echo 'shenzhen_wireguard=PASS peers=4 recent_handshakes=12'
+  echo 'shenzhen_wireguard=PASS peers=4 recent_handshakes=12 hongkong_shenzhen_via=ningbo'
 }
 
 install_agent() {
diff --git a/scripts/cluster/bootstrap-wireguard.sh b/scripts/cluster/bootstrap-wireguard.sh
index 14f8922..b5146b0 100755
--- a/scripts/cluster/bootstrap-wireguard.sh
+++ b/scripts/cluster/bootstrap-wireguard.sh
@@ -90,6 +90,35 @@ systemctl restart wg-quick@wg0
 REMOTE
 done
 
+if (( ${#hosts[@]} == 4 )); then
+  cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- \
+    "${public_keys[0]}" "${public_keys[3]}" <<'REMOTE'
+set -euo pipefail
+wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.4/32
+wg set wg0 peer "$2" allowed-ips 0.0.0.0/32
+wg-quick save wg0
+REMOTE
+  cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \
+    "${public_keys[0]}" "${public_keys[1]}" <<'REMOTE'
+set -euo pipefail
+wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.2/32
+wg set wg0 peer "$2" allowed-ips 0.0.0.0/32
+wg-quick save wg0
+REMOTE
+  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+sysctl -qw net.ipv4.ip_forward=1
+ensure_forward_rule() {
+  if ! iptables -C FORWARD "$@" 2>/dev/null; then iptables -I FORWARD 1 "$@"; fi
+}
+ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.2/32 -d 10.77.0.4/32 \
+  -m comment --comment easyai-wireguard-hongkong-shenzhen -j ACCEPT
+ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.4/32 -d 10.77.0.2/32 \
+  -m comment --comment easyai-wireguard-shenzhen-hongkong -j ACCEPT
+netfilter-persistent save >/dev/null
+REMOTE
+fi
+
 for source_index in "${!hosts[@]}"; do
   for target_index in "${!hosts[@]}"; do
     [[ $source_index -eq $target_index ]] && continue
@@ -107,4 +136,8 @@ for host in "${hosts[@]}"; do
   }
 done
 
-echo "wireguard_bootstrap=PASS peers=${#hosts[@]}"
+if (( ${#hosts[@]} == 4 )); then
+  echo 'wireguard_bootstrap=PASS peers=4 hongkong_shenzhen_via=ningbo'
+else
+  echo "wireguard_bootstrap=PASS peers=${#hosts[@]}"
+fi