fix(cluster): 固化宁波 Worker 隧道端口
新宁波节点改用 UDP/443 监听,其他三台节点保留 UDP/51820,并在全量 WireGuard 引导中按节点配置 endpoint 端口。\n\n三轮六向实测均为零丢包,香港 RTT 从 73-77ms 降至 48-53ms;已通过 bash -n、ShellCheck 和差异检查。
This commit is contained in:
@@ -18,7 +18,8 @@ CloudNativePG;洛杉矶只作为 K3s server/etcd 仲裁节点,并带
|
|||||||
新节点与宁波、香港、洛杉矶使用 WireGuard 全互联直连,不经过中继。公网 K3s 直连虽然可以
|
新节点与宁波、香港、洛杉矶使用 WireGuard 全互联直连,不经过中继。公网 K3s 直连虽然可以
|
||||||
单独保护 API TLS,但当前 Flannel、Pod 和数据库网络依赖 `10.77.0.0/24`,禁止改成未加密的
|
单独保护 API TLS,但当前 Flannel、Pod 和数据库网络依赖 `10.77.0.0/24`,禁止改成未加密的
|
||||||
公网 VXLAN。巡检以 10 包丢包率低于 1%为硬门禁;宁波和香港路径 RTT 低于 80 ms,洛杉矶
|
公网 VXLAN。巡检以 10 包丢包率低于 1%为硬门禁;宁波和香港路径 RTT 低于 80 ms,洛杉矶
|
||||||
路径低于 300 ms。
|
路径低于 300 ms。新节点监听 UDP/443;实测 UDP/51820 在香港方向存在 QoS 抖动,禁止在
|
||||||
|
未重新完成三轮六向链路验收前改回默认端口。
|
||||||
|
|
||||||
本地 `.env.local` 只保存 `AI_GATEWAY_NINGBO_WORKER_HOST` 和 SSH 接入密码,必须保持 `0600`
|
本地 `.env.local` 只保存 `AI_GATEWAY_NINGBO_WORKER_HOST` 和 SSH 接入密码,必须保持 `0600`
|
||||||
且不得进入 Git。新节点只保留原有 SSH 入口,封锁 K3s API、kubelet、VXLAN 和全部 NodePort
|
且不得进入 Git。新节点只保留原有 SSH 入口,封锁 K3s API、kubelet、VXLAN 和全部 NodePort
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ require_commands ssh scp shasum
|
|||||||
|
|
||||||
node_name=easyai-ningbo-worker-2
|
node_name=easyai-ningbo-worker-2
|
||||||
node_ip=10.77.0.4
|
node_ip=10.77.0.4
|
||||||
|
worker_wireguard_port=443
|
||||||
k3s_version=v1.36.2+k3s1
|
k3s_version=v1.36.2+k3s1
|
||||||
k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de
|
k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de
|
||||||
secret_root="$cluster_root/.local-secrets/cluster"
|
secret_root="$cluster_root/.local-secrets/cluster"
|
||||||
@@ -91,17 +92,20 @@ REMOTE
|
|||||||
done
|
done
|
||||||
|
|
||||||
cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- \
|
cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- \
|
||||||
|
"$worker_wireguard_port" \
|
||||||
"${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \
|
"${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \
|
||||||
"${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \
|
"${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \
|
||||||
"${server_keys[2]}" "${server_endpoints[2]}" "${server_ips[2]}" <<'REMOTE'
|
"${server_keys[2]}" "${server_endpoints[2]}" "${server_ips[2]}" <<'REMOTE'
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
listen_port=$1
|
||||||
|
shift
|
||||||
private_key=$(cat /etc/wireguard/privatekey)
|
private_key=$(cat /etc/wireguard/privatekey)
|
||||||
umask 077
|
umask 077
|
||||||
temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
|
temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
|
||||||
cat >"$temporary_config" <<EOF
|
cat >"$temporary_config" <<EOF
|
||||||
[Interface]
|
[Interface]
|
||||||
Address = 10.77.0.4/24
|
Address = 10.77.0.4/24
|
||||||
ListenPort = 51820
|
ListenPort = ${listen_port}
|
||||||
PrivateKey = ${private_key}
|
PrivateKey = ${private_key}
|
||||||
|
|
||||||
[Peer]
|
[Peer]
|
||||||
@@ -125,16 +129,17 @@ EOF
|
|||||||
chmod 0600 "$temporary_config"
|
chmod 0600 "$temporary_config"
|
||||||
mv "$temporary_config" /etc/wireguard/wg0.conf
|
mv "$temporary_config" /etc/wireguard/wg0.conf
|
||||||
for peer in "$2" "$5" "$8"; do
|
for peer in "$2" "$5" "$8"; do
|
||||||
ufw allow from "$peer" to any port 51820 proto udp comment easyai-wireguard >/dev/null
|
ufw allow from "$peer" to any port "$listen_port" proto udp comment easyai-wireguard >/dev/null
|
||||||
done
|
done
|
||||||
systemctl enable wg-quick@wg0 >/dev/null
|
systemctl enable wg-quick@wg0 >/dev/null
|
||||||
systemctl restart wg-quick@wg0
|
systemctl restart wg-quick@wg0
|
||||||
REMOTE
|
REMOTE
|
||||||
|
|
||||||
for host in "${server_hosts[@]}"; do
|
for host in "${server_hosts[@]}"; do
|
||||||
cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" "$node_ip" <<'REMOTE'
|
cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" \
|
||||||
|
"$node_ip" "$worker_wireguard_port" <<'REMOTE'
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:51820" persistent-keepalive 25
|
wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:$4" persistent-keepalive 25
|
||||||
wg-quick save wg0
|
wg-quick save wg0
|
||||||
REMOTE
|
REMOTE
|
||||||
done
|
done
|
||||||
@@ -166,7 +171,7 @@ verify_wireguard() {
|
|||||||
"wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
|
"wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
|
||||||
[[ $recent == 3 ]]
|
[[ $recent == 3 ]]
|
||||||
done
|
done
|
||||||
echo 'ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct'
|
echo "ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct listen_port=$worker_wireguard_port"
|
||||||
}
|
}
|
||||||
|
|
||||||
install_agent() {
|
install_agent() {
|
||||||
|
|||||||
@@ -11,10 +11,12 @@ require_commands ssh scp
|
|||||||
hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
|
hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
|
||||||
public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
|
public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
|
||||||
wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
|
wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
|
||||||
|
wireguard_ports=(51820 51820 51820)
|
||||||
if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
|
if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
|
||||||
hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
|
hosts+=("$CLUSTER_NINGBO_WORKER_HOST")
|
||||||
public_endpoints+=("${CLUSTER_NINGBO_WORKER_HOST#*@}")
|
public_endpoints+=("${CLUSTER_NINGBO_WORKER_HOST#*@}")
|
||||||
wireguard_ips+=(10.77.0.4)
|
wireguard_ips+=(10.77.0.4)
|
||||||
|
wireguard_ports+=(443)
|
||||||
fi
|
fi
|
||||||
public_keys=()
|
public_keys=()
|
||||||
|
|
||||||
@@ -51,6 +53,7 @@ for local_index in "${!hosts[@]}"; do
|
|||||||
peer_arguments+=(
|
peer_arguments+=(
|
||||||
"${public_keys[$candidate]}"
|
"${public_keys[$candidate]}"
|
||||||
"${public_endpoints[$candidate]}"
|
"${public_endpoints[$candidate]}"
|
||||||
|
"${wireguard_ports[$candidate]}"
|
||||||
"${wireguard_ips[$candidate]}"
|
"${wireguard_ips[$candidate]}"
|
||||||
)
|
)
|
||||||
fi
|
fi
|
||||||
@@ -58,29 +61,31 @@ for local_index in "${!hosts[@]}"; do
|
|||||||
echo "[wireguard] configuring ${hosts[$local_index]} as ${wireguard_ips[$local_index]}"
|
echo "[wireguard] configuring ${hosts[$local_index]} as ${wireguard_ips[$local_index]}"
|
||||||
cluster_ssh "${hosts[$local_index]}" bash -s -- \
|
cluster_ssh "${hosts[$local_index]}" bash -s -- \
|
||||||
"${wireguard_ips[$local_index]}" \
|
"${wireguard_ips[$local_index]}" \
|
||||||
|
"${wireguard_ports[$local_index]}" \
|
||||||
"${peer_arguments[@]}" <<'REMOTE'
|
"${peer_arguments[@]}" <<'REMOTE'
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
local_ip=$1
|
local_ip=$1
|
||||||
shift
|
local_port=$2
|
||||||
|
shift 2
|
||||||
private_key=$(cat /etc/wireguard/privatekey)
|
private_key=$(cat /etc/wireguard/privatekey)
|
||||||
umask 077
|
umask 077
|
||||||
temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
|
temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
|
||||||
cat >"$temporary_config" <<EOF
|
cat >"$temporary_config" <<EOF
|
||||||
[Interface]
|
[Interface]
|
||||||
Address = ${local_ip}/24
|
Address = ${local_ip}/24
|
||||||
ListenPort = 51820
|
ListenPort = ${local_port}
|
||||||
PrivateKey = ${private_key}
|
PrivateKey = ${private_key}
|
||||||
EOF
|
EOF
|
||||||
while (( $# >= 3 )); do
|
while (( $# >= 4 )); do
|
||||||
cat >>"$temporary_config" <<EOF
|
cat >>"$temporary_config" <<EOF
|
||||||
|
|
||||||
[Peer]
|
[Peer]
|
||||||
PublicKey = $1
|
PublicKey = $1
|
||||||
Endpoint = $2:51820
|
Endpoint = $2:$3
|
||||||
AllowedIPs = $3/32
|
AllowedIPs = $4/32
|
||||||
PersistentKeepalive = 25
|
PersistentKeepalive = 25
|
||||||
EOF
|
EOF
|
||||||
shift 3
|
shift 4
|
||||||
done
|
done
|
||||||
(( $# == 0 ))
|
(( $# == 0 ))
|
||||||
chmod 0600 "$temporary_config"
|
chmod 0600 "$temporary_config"
|
||||||
|
|||||||
Reference in New Issue
Block a user