fix(cluster): 固化宁波 Worker 隧道端口
新宁波节点改用 UDP/443 监听,其他三台节点保留 UDP/51820,并在全量 WireGuard 引导中按节点配置 endpoint 端口。\n\n三轮六向实测均为零丢包,香港 RTT 从 73-77ms 降至 48-53ms;已通过 bash -n、ShellCheck 和差异检查。
This commit is contained in:
@@ -18,7 +18,8 @@ CloudNativePG;洛杉矶只作为 K3s server/etcd 仲裁节点,并带
|
||||
新节点与宁波、香港、洛杉矶使用 WireGuard 全互联直连,不经过中继。公网 K3s 直连虽然可以
|
||||
单独保护 API TLS,但当前 Flannel、Pod 和数据库网络依赖 `10.77.0.0/24`,禁止改成未加密的
|
||||
公网 VXLAN。巡检以 10 包丢包率低于 1%为硬门禁;宁波和香港路径 RTT 低于 80 ms,洛杉矶
|
||||
路径低于 300 ms。
|
||||
路径低于 300 ms。新节点监听 UDP/443;实测 UDP/51820 在香港方向存在 QoS 抖动,禁止在
|
||||
未重新完成三轮六向链路验收前改回默认端口。
|
||||
|
||||
本地 `.env.local` 只保存 `AI_GATEWAY_NINGBO_WORKER_HOST` 和 SSH 接入密码,必须保持 `0600`
|
||||
且不得进入 Git。新节点只保留原有 SSH 入口,封锁 K3s API、kubelet、VXLAN 和全部 NodePort
|
||||
|
||||
@@ -18,6 +18,7 @@ require_commands ssh scp shasum
|
||||
|
||||
node_name=easyai-ningbo-worker-2
|
||||
node_ip=10.77.0.4
|
||||
worker_wireguard_port=443
|
||||
k3s_version=v1.36.2+k3s1
|
||||
k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de
|
||||
secret_root="$cluster_root/.local-secrets/cluster"
|
||||
@@ -91,17 +92,20 @@ REMOTE
|
||||
done
|
||||
|
||||
cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- \
|
||||
"$worker_wireguard_port" \
|
||||
"${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \
|
||||
"${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \
|
||||
"${server_keys[2]}" "${server_endpoints[2]}" "${server_ips[2]}" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
listen_port=$1
|
||||
shift
|
||||
private_key=$(cat /etc/wireguard/privatekey)
|
||||
umask 077
|
||||
temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
|
||||
cat >"$temporary_config" <<EOF
|
||||
[Interface]
|
||||
Address = 10.77.0.4/24
|
||||
ListenPort = 51820
|
||||
ListenPort = ${listen_port}
|
||||
PrivateKey = ${private_key}
|
||||
|
||||
[Peer]
|
||||
@@ -125,16 +129,17 @@ EOF
|
||||
chmod 0600 "$temporary_config"
|
||||
mv "$temporary_config" /etc/wireguard/wg0.conf
|
||||
for peer in "$2" "$5" "$8"; do
|
||||
ufw allow from "$peer" to any port 51820 proto udp comment easyai-wireguard >/dev/null
|
||||
ufw allow from "$peer" to any port "$listen_port" proto udp comment easyai-wireguard >/dev/null
|
||||
done
|
||||
systemctl enable wg-quick@wg0 >/dev/null
|
||||
systemctl restart wg-quick@wg0
|
||||
REMOTE
|
||||
|
||||
for host in "${server_hosts[@]}"; do
|
||||
cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" "$node_ip" <<'REMOTE'
|
||||
cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" \
|
||||
"$node_ip" "$worker_wireguard_port" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:51820" persistent-keepalive 25
|
||||
wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:$4" persistent-keepalive 25
|
||||
wg-quick save wg0
|
||||
REMOTE
|
||||
done
|
||||
@@ -166,7 +171,7 @@ verify_wireguard() {
|
||||
"wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
|
||||
[[ $recent == 3 ]]
|
||||
done
|
||||
echo 'ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct'
|
||||
echo "ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct listen_port=$worker_wireguard_port"
|
||||
}
|
||||
|
||||
install_agent() {
|
||||
|
||||
@@ -11,10 +11,12 @@ require_commands ssh scp
|
||||
hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
|
||||
public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
|
||||
wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
|
||||
wireguard_ports=(51820 51820 51820)
|
||||
if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
|
||||
hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
|
||||
public_endpoints+=("${CLUSTER_NINGBO_WORKER_HOST#*@}")
|
||||
wireguard_ips+=(10.77.0.4)
|
||||
wireguard_ports+=(443)
|
||||
fi
|
||||
public_keys=()
|
||||
|
||||
@@ -51,6 +53,7 @@ for local_index in "${!hosts[@]}"; do
|
||||
peer_arguments+=(
|
||||
"${public_keys[$candidate]}"
|
||||
"${public_endpoints[$candidate]}"
|
||||
"${wireguard_ports[$candidate]}"
|
||||
"${wireguard_ips[$candidate]}"
|
||||
)
|
||||
fi
|
||||
@@ -58,29 +61,31 @@ for local_index in "${!hosts[@]}"; do
|
||||
echo "[wireguard] configuring ${hosts[$local_index]} as ${wireguard_ips[$local_index]}"
|
||||
cluster_ssh "${hosts[$local_index]}" bash -s -- \
|
||||
"${wireguard_ips[$local_index]}" \
|
||||
"${wireguard_ports[$local_index]}" \
|
||||
"${peer_arguments[@]}" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
local_ip=$1
|
||||
shift
|
||||
local_port=$2
|
||||
shift 2
|
||||
private_key=$(cat /etc/wireguard/privatekey)
|
||||
umask 077
|
||||
temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
|
||||
cat >"$temporary_config" <<EOF
|
||||
[Interface]
|
||||
Address = ${local_ip}/24
|
||||
ListenPort = 51820
|
||||
ListenPort = ${local_port}
|
||||
PrivateKey = ${private_key}
|
||||
EOF
|
||||
while (( $# >= 3 )); do
|
||||
while (( $# >= 4 )); do
|
||||
cat >>"$temporary_config" <<EOF
|
||||
|
||||
[Peer]
|
||||
PublicKey = $1
|
||||
Endpoint = $2:51820
|
||||
AllowedIPs = $3/32
|
||||
Endpoint = $2:$3
|
||||
AllowedIPs = $4/32
|
||||
PersistentKeepalive = 25
|
||||
EOF
|
||||
shift 3
|
||||
shift 4
|
||||
done
|
||||
(( $# == 0 ))
|
||||
chmod 0600 "$temporary_config"
|
||||
|
||||
Reference in New Issue
Block a user