refactor(cluster): 通用化第四个 Worker 节点接入

将原宁波专用接入脚本改为由环境变量控制节点名称、站点和 WireGuard 端口，支持香港双物理节点 Worker 池。增加候选节点启用开关和服务商网络许可门禁，未明确批准时禁止任何 WireGuard 或 K3s 写操作。同步调整生产验收、巡检、拓扑校验和运行手册。\n\n验证：\n- scripts/cluster 全部 Shell 脚本 bash -n\n- scripts/cluster 全部 ShellCheck\n- git diff --check\n- 本地凭据值扫描\n- 当前三节点 WireGuard/K3s/PostgreSQL 只读基线检查
2026-08-01 11:23:19 +08:00
parent e77bd17754
commit ed3b970178
7 changed files with 175 additions and 93 deletions
@@ -652,28 +652,41 @@ verify_release_cas() {
    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
  [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
-  if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
-    local expected_node expected_replicas site pod_count
-    for site in ningbo hongkong; do
-      case $site in
-        ningbo)
-          expected_node=easyai-ningbo-worker-2
-          expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
-          ;;
-        hongkong)
-          expected_node=easyai-hongkong
-          expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
-          ;;
-      esac
-      pod_count=$(remote_kubectl get pods -n "$namespace" \
-        -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" \
-        --field-selector "spec.nodeName=$expected_node" -o json | jq \
-        '[.items[] | select(.status.phase=="Running")] | length')
-      [[ $pod_count == "$expected_replicas" ]] || {
-        echo "Worker placement does not match the dedicated node: site=$site node=$expected_node running_pods=$pod_count expected=$expected_replicas" >&2
-        return 1
-      }
-    done
+  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
+    local expected_replicas eligible_nodes eligible_count pod_report
+    local running_count invalid_count node4_count used_nodes
+    case $CLUSTER_WORKER_NODE_4_SITE in
+      ningbo) expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO ;;
+      hongkong) expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG ;;
+    esac
+    eligible_nodes=$(remote_kubectl get nodes \
+      -l "easyai.io/site=$CLUSTER_WORKER_NODE_4_SITE,easyai.io/worker=true" \
+      -o json | jq -c '[.items[].metadata.name]')
+    eligible_count=$(jq 'length' <<<"$eligible_nodes")
+    jq -e --arg node "$CLUSTER_WORKER_NODE_4_NAME" 'index($node) != null' \
+      <<<"$eligible_nodes" >/dev/null
+    pod_report=$(remote_kubectl get pods -n "$namespace" \
+      -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$CLUSTER_WORKER_NODE_4_SITE" -o json)
+    running_count=$(jq '[.items[] | select(.status.phase=="Running")] | length' <<<"$pod_report")
+    invalid_count=$(jq --argjson nodes "$eligible_nodes" \
+      '[.items[] | select(.status.phase=="Running") | select(.spec.nodeName as $node | $nodes | index($node) | not)] | length' \
+      <<<"$pod_report")
+    node4_count=$(jq --arg node "$CLUSTER_WORKER_NODE_4_NAME" \
+      '[.items[] | select(.status.phase=="Running" and .spec.nodeName==$node)] | length' <<<"$pod_report")
+    used_nodes=$(jq '[.items[] | select(.status.phase=="Running") | .spec.nodeName] | unique | length' \
+      <<<"$pod_report")
+    [[ $running_count == "$expected_replicas" && $invalid_count == 0 ]] || {
+      echo "Worker placement is outside eligible nodes: site=$CLUSTER_WORKER_NODE_4_SITE running_pods=$running_count expected=$expected_replicas invalid=$invalid_count" >&2
+      return 1
+    }
+    if (( expected_replicas > 0 && node4_count == 0 )); then
+      echo "dedicated Worker node has no baseline pod: site=$CLUSTER_WORKER_NODE_4_SITE node=$CLUSTER_WORKER_NODE_4_NAME" >&2
+      return 1
+    fi
+    if (( expected_replicas >= 2 && eligible_count >= 2 && used_nodes < 2 )); then
+      echo "Worker baseline is not distributed across physical nodes: site=$CLUSTER_WORKER_NODE_4_SITE used_nodes=$used_nodes" >&2
+      return 1
+    fi
  fi
 }

@@ -2417,13 +2430,13 @@ verify_cluster_links() {
  verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
  verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
  verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
-  if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then
-    verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_2'
-    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 'worker_2_to_ningbo'
-    verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_2'
-    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 'worker_2_to_hongkong'
-    verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_2'
-    verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 'worker_2_to_los_angeles'
+  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
+    verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_node_4'
+    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.1 80 'worker_node_4_to_ningbo'
+    verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_node_4'
+    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.2 80 'worker_node_4_to_hongkong'
+    verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_node_4'
+    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.3 300 'worker_node_4_to_losangeles'
  fi
 }

@@ -2442,8 +2455,8 @@ verify_control_plane_and_recent_logs() {
      [[ $error_count == 0 ]] || return 1
    fi
  done
-  if [[ -n $CLUSTER_NINGBO_WORKER_HOST && -n $runtime_observation_started_at ]]; then
-    error_count=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \
+  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true && -n $runtime_observation_started_at ]]; then
+    error_count=$(cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" \
      "journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
    [[ $error_count == 0 ]] || return 1
  fi