fix(cluster): 以宁波专用节点替换深圳 Worker

移除深圳节点及中继拓扑,新增第二台宁波 K3s agent 的全互联 WireGuard 接入和严格 UFW 门禁。\n\nWorker Deployment 与容量控制器仅选择 easyai.io/worker=true 节点,使原宁波混部节点退出 Worker 资源预算,生产基线恢复为宁波专用节点与香港节点各一实例。\n\n已通过 Go 全量测试、go vet、gofmt、迁移安全检查、bash -n、ShellCheck、发布脚本测试和 Kubernetes 清单渲染。
This commit is contained in:
2026-08-01 10:47:01 +08:00
parent d3b36cf63d
commit 53589fe3ef
12 changed files with 119 additions and 155 deletions
+28 -20
View File
@@ -652,17 +652,25 @@ verify_release_cas() {
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
[[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) &&
[[ -n $CLUSTER_SHENZHEN_HOST ]]; then
local worker_node pod_count
for worker_node in easyai-hongkong easyai-shenzhen; do
if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
local expected_node expected_replicas site pod_count
for site in ningbo hongkong; do
case $site in
ningbo)
expected_node=easyai-ningbo-worker-2
expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
;;
hongkong)
expected_node=easyai-hongkong
expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
;;
esac
pod_count=$(remote_kubectl get pods -n "$namespace" \
-l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \
--field-selector "spec.nodeName=$worker_node" -o json | jq \
-l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" \
--field-selector "spec.nodeName=$expected_node" -o json | jq \
'[.items[] | select(.status.phase=="Running")] | length')
[[ $pod_count == 1 ]] || {
echo "Hong Kong logical Worker pool is not spread across both physical nodes: node=$worker_node running_pods=$pod_count" >&2
[[ $pod_count == "$expected_replicas" ]] || {
echo "Worker placement does not match the dedicated node: site=$site node=$expected_node running_pods=$pod_count expected=$expected_replicas" >&2
return 1
}
done
@@ -674,12 +682,12 @@ verify_resource_preconditions() {
local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
worker_nodes=
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo,easyai.io/worker=true' \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
worker_nodes+=$'\n'
fi
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong' \
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong,easyai.io/worker=true' \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
fi
worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u)
@@ -2409,13 +2417,13 @@ verify_cluster_links() {
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_shenzhen'
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 'shenzhen_to_ningbo'
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_shenzhen'
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 'shenzhen_to_hongkong'
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_shenzhen'
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 'shenzhen_to_los_angeles'
if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_2'
verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 'worker_2_to_ningbo'
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_2'
verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 'worker_2_to_hongkong'
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_2'
verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 'worker_2_to_los_angeles'
fi
}
@@ -2434,8 +2442,8 @@ verify_control_plane_and_recent_logs() {
[[ $error_count == 0 ]] || return 1
fi
done
if [[ -n $CLUSTER_SHENZHEN_HOST && -n $runtime_observation_started_at ]]; then
error_count=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" \
if [[ -n $CLUSTER_NINGBO_WORKER_HOST && -n $runtime_observation_started_at ]]; then
error_count=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \
"journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
[[ $error_count == 0 ]] || return 1
fi