fix(cluster): 以宁波专用节点替换深圳 Worker
移除深圳节点及中继拓扑,新增第二台宁波 K3s agent 的全互联 WireGuard 接入和严格 UFW 门禁。\n\nWorker Deployment 与容量控制器仅选择 easyai.io/worker=true 节点,使原宁波混部节点退出 Worker 资源预算,生产基线恢复为宁波专用节点与香港节点各一实例。\n\n已通过 Go 全量测试、go vet、gofmt、迁移安全检查、bash -n、ShellCheck、发布脚本测试和 Kubernetes 清单渲染。
This commit is contained in:
@@ -652,17 +652,25 @@ verify_release_cas() {
|
||||
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
|
||||
[[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
|
||||
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
|
||||
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
|
||||
AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) &&
|
||||
[[ -n $CLUSTER_SHENZHEN_HOST ]]; then
|
||||
local worker_node pod_count
|
||||
for worker_node in easyai-hongkong easyai-shenzhen; do
|
||||
if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
|
||||
local expected_node expected_replicas site pod_count
|
||||
for site in ningbo hongkong; do
|
||||
case $site in
|
||||
ningbo)
|
||||
expected_node=easyai-ningbo-worker-2
|
||||
expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
|
||||
;;
|
||||
hongkong)
|
||||
expected_node=easyai-hongkong
|
||||
expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
|
||||
;;
|
||||
esac
|
||||
pod_count=$(remote_kubectl get pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \
|
||||
--field-selector "spec.nodeName=$worker_node" -o json | jq \
|
||||
-l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" \
|
||||
--field-selector "spec.nodeName=$expected_node" -o json | jq \
|
||||
'[.items[] | select(.status.phase=="Running")] | length')
|
||||
[[ $pod_count == 1 ]] || {
|
||||
echo "Hong Kong logical Worker pool is not spread across both physical nodes: node=$worker_node running_pods=$pod_count" >&2
|
||||
[[ $pod_count == "$expected_replicas" ]] || {
|
||||
echo "Worker placement does not match the dedicated node: site=$site node=$expected_node running_pods=$pod_count expected=$expected_replicas" >&2
|
||||
return 1
|
||||
}
|
||||
done
|
||||
@@ -674,12 +682,12 @@ verify_resource_preconditions() {
|
||||
local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
|
||||
worker_nodes=
|
||||
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then
|
||||
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
|
||||
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo,easyai.io/worker=true' \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
worker_nodes+=$'\n'
|
||||
fi
|
||||
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then
|
||||
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong' \
|
||||
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong,easyai.io/worker=true' \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
fi
|
||||
worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u)
|
||||
@@ -2409,13 +2417,13 @@ verify_cluster_links() {
|
||||
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
|
||||
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
|
||||
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
|
||||
if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
|
||||
verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_shenzhen'
|
||||
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 'shenzhen_to_ningbo'
|
||||
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_shenzhen'
|
||||
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 'shenzhen_to_hongkong'
|
||||
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_shenzhen'
|
||||
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 'shenzhen_to_los_angeles'
|
||||
if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
|
||||
verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_2'
|
||||
verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 'worker_2_to_ningbo'
|
||||
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_2'
|
||||
verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 'worker_2_to_hongkong'
|
||||
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_2'
|
||||
verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 'worker_2_to_los_angeles'
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -2434,8 +2442,8 @@ verify_control_plane_and_recent_logs() {
|
||||
[[ $error_count == 0 ]] || return 1
|
||||
fi
|
||||
done
|
||||
if [[ -n $CLUSTER_SHENZHEN_HOST && -n $runtime_observation_started_at ]]; then
|
||||
error_count=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" \
|
||||
if [[ -n $CLUSTER_NINGBO_WORKER_HOST && -n $runtime_observation_started_at ]]; then
|
||||
error_count=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \
|
||||
"journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
|
||||
[[ $error_count == 0 ]] || return 1
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user