From ed3b970178eaa28cf0c4aefc18763bfd683cb444 Mon Sep 17 00:00:00 2001 From: wangbo Date: Sat, 1 Aug 2026 11:23:19 +0800 Subject: [PATCH] =?UTF-8?q?refactor(cluster):=20=E9=80=9A=E7=94=A8?= =?UTF-8?q?=E5=8C=96=E7=AC=AC=E5=9B=9B=E4=B8=AA=20Worker=20=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E6=8E=A5=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将原宁波专用接入脚本改为由环境变量控制节点名称、站点和 WireGuard 端口，支持香港双物理节点 Worker 池。增加候选节点启用开关和服务商网络许可门禁，未明确批准时禁止任何 WireGuard 或 K3s 写操作。同步调整生产验收、巡检、拓扑校验和运行手册。\n\n验证：\n- scripts/cluster 全部 Shell 脚本 bash -n\n- scripts/cluster 全部 ShellCheck\n- git diff --check\n- 本地凭据值扫描\n- 当前三节点 WireGuard/K3s/PostgreSQL 只读基线检查 --- docs/operations/k3s-ha-runbook.md | 31 +++++--- ...ngbo-worker-node.sh => add-worker-node.sh} | 79 +++++++++++-------- scripts/cluster/bootstrap-wireguard.sh | 12 ++- scripts/cluster/common.sh | 45 ++++++++++- scripts/cluster/monitor-production-release.sh | 4 +- scripts/cluster/run-production-acceptance.sh | 75 ++++++++++-------- scripts/cluster/verify-cluster.sh | 22 +++--- 7 files changed, 175 insertions(+), 93 deletions(-) rename scripts/cluster/{add-ningbo-worker-node.sh => add-worker-node.sh} (77%) diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md index 0100c7a..aed8df7 100644 --- a/docs/operations/k3s-ha-runbook.md +++ b/docs/operations/k3s-ha-runbook.md @@ -2,29 +2,40 @@ ## 范围与目标 -本手册覆盖宁波、香港、洛杉矶三节点迁移。宁波和香港承载 API、Web、River Worker 与 -CloudNativePG；洛杉矶只作为 K3s server/etcd 仲裁节点，并带 +本手册覆盖宁波、香港、洛杉矶三个 K3s server 和一个可选专用 Worker agent。宁波、香港 +承载 API、Web 与 CloudNativePG；Worker 只调度到带 `easyai.io/worker=true` 标签的节点。 +洛杉矶只作为 K3s server/etcd 仲裁节点，并带 `easyai.io/witness=true:NoSchedule` 污点。 -第二台宁波服务器是纯 K3s agent，不加入 etcd 或 CloudNativePG。它使用节点名 -`easyai-ningbo-worker-2`、WireGuard 地址 `10.77.0.4`，并归入逻辑 `ningbo` Worker 池。 +第 4 台服务器是站点可配置的纯 K3s agent，不加入 etcd 或 CloudNativePG。节点名和逻辑站点 +分别由 `AI_GATEWAY_WORKER_NODE_4_NAME`、`AI_GATEWAY_WORKER_NODE_4_SITE` 指定，固定使用 +WireGuard 地址 `10.77.0.4`。当前计划节点为 `easyai-hongkong-worker-2`，归入逻辑 +`hongkong` Worker 池；稳定基线为宁波 0 副本、香港 2 副本，并跨香港两台物理机分布。 `easyai.io/worker-only=true:NoSchedule` 污点确保除 Worker 外的业务 Pod 不会调度到该机； `easyai.io/worker=true` 标签确保容量控制器只把专用 Worker 节点纳入资源预算。接入使用： ```bash -./scripts/cluster/add-ningbo-worker-node.sh all +./scripts/cluster/add-worker-node.sh preflight +./scripts/cluster/add-worker-node.sh all ``` 新节点与宁波、香港、洛杉矶使用 WireGuard 全互联直连，不经过中继。公网 K3s 直连虽然可以 单独保护 API TLS，但当前 Flannel、Pod 和数据库网络依赖 `10.77.0.0/24`，禁止改成未加密的 公网 VXLAN。巡检以 10 包丢包率低于 1%为硬门禁；宁波和香港路径 RTT 低于 80 ms，洛杉矶 -路径低于 300 ms。新节点监听 UDP/443；实测 UDP/51820 在香港方向存在 QoS 抖动，禁止在 -未重新完成三轮六向链路验收前改回默认端口。 +路径低于 300 ms。新节点默认监听标准 UDP/51820，可通过 +`AI_GATEWAY_WORKER_NODE_4_WIREGUARD_PORT` 配置；不得为了绕过服务商风控擅自伪装端口。 -本地 `.env.local` 只保存 `AI_GATEWAY_NINGBO_WORKER_HOST` 和 SSH 接入密码，必须保持 `0600` -且不得进入 Git。新节点只保留原有 SSH 入口，封锁 K3s API、kubelet、VXLAN 和全部 NodePort -公网入口。节点防火墙继续由原有 UFW 管理，接入脚本 +服务商必须先通过工单书面确认“仅用于 Kubernetes 私有集群互联，不提供 VPN、代理或流量 +转发服务”符合其政策；确认后才可在 `.env.local` 设置 +`AI_GATEWAY_WORKER_NODE_4_NETWORK_APPROVED=true`。未设置时，脚本只允许执行只读 +`preflight`，所有 WireGuard 与 K3s 写操作都会停止。 + +本地 `.env.local` 保存 `AI_GATEWAY_WORKER_NODE_4_HOST/NAME/SITE` 和 SSH 接入密码，必须保持 +`0600` 且不得进入 Git。新节点只保留原有 SSH 入口，封锁 K3s API、kubelet、VXLAN和全部 +NodePort 公网入口。节点防火墙继续由原有 UFW 管理，接入脚本 禁止安装与 UFW 冲突的 `iptables-persistent`；UFW 未安装或未启用时必须在预检阶段停止。 +节点完成加入和验收后才设置 `AI_GATEWAY_WORKER_NODE_4_ENABLED=true`；候选服务器仅写入本地 +配置但尚未加入时必须保持 `false`，避免日常巡检误把候选机计入生产拓扑。 生产上线前的双节点高媒体负载、Worker 强杀和 P24/P28/P32 容量搜索见 [生产同构验收模式与高媒体压力测试](production-acceptance.md)。 diff --git a/scripts/cluster/add-ningbo-worker-node.sh b/scripts/cluster/add-worker-node.sh similarity index 77% rename from scripts/cluster/add-ningbo-worker-node.sh rename to scripts/cluster/add-worker-node.sh index 1ac0a2e..89b7cdf 100755 --- a/scripts/cluster/add-ningbo-worker-node.sh +++ b/scripts/cluster/add-worker-node.sh @@ -8,17 +8,18 @@ source "$script_dir/common.sh" action=${1:-all} [[ $action == preflight || $action == network || $action == install || $action == verify || $action == all ]] || { - echo 'usage: add-ningbo-worker-node.sh [preflight|network|install|verify|all]' >&2 + echo 'usage: add-worker-node.sh [preflight|network|install|verify|all]' >&2 exit 64 } load_cluster_env require_commands ssh scp shasum -: "${CLUSTER_NINGBO_WORKER_HOST:?AI_GATEWAY_NINGBO_WORKER_HOST is required}" +: "${CLUSTER_WORKER_NODE_4_HOST:?AI_GATEWAY_WORKER_NODE_4_HOST is required}" -node_name=easyai-ningbo-worker-2 +node_name=$CLUSTER_WORKER_NODE_4_NAME +node_site=$CLUSTER_WORKER_NODE_4_SITE node_ip=10.77.0.4 -worker_wireguard_port=443 +worker_wireguard_port=$CLUSTER_WORKER_NODE_4_WIREGUARD_PORT k3s_version=v1.36.2+k3s1 k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de secret_root="$cluster_root/.local-secrets/cluster" @@ -39,8 +40,10 @@ assert_join_material() { } preflight_node() { - cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s <<'REMOTE' + cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" bash -s -- "$node_name" "$node_site" <<'REMOTE' set -euo pipefail +node_name=$1 +node_site=$2 [[ $(id -u) -eq 0 && $(uname -m) == x86_64 ]] . /etc/os-release [[ ($ID == debian && ${VERSION_ID%%.*} -ge 12) || @@ -54,11 +57,19 @@ timedatectl show -p NTPSynchronized --value | grep -Fxq yes [[ ! -e /etc/rancher/k3s/config.yaml || -f /etc/rancher/k3s/config.yaml ]] command -v ufw >/dev/null ufw status | grep -Fqx 'Status: active' -printf 'ningbo_worker_preflight=PASS cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \ - "$(nproc)" "$((available_kib / 1024))" "$((available_bytes / 1024 / 1024 / 1024))" +printf 'worker_node_preflight=PASS node=%s site=%s cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \ + "$node_name" "$node_site" "$(nproc)" "$((available_kib / 1024))" \ + "$((available_bytes / 1024 / 1024 / 1024))" REMOTE } +assert_network_approved() { + [[ $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == true ]] || { + echo 'dedicated Worker private-network setup is blocked until the provider approves this non-proxy Kubernetes use in writing' >&2 + return 1 + } +} + configure_wireguard() { local -a server_hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") local -a server_ips=(10.77.0.1 10.77.0.2 10.77.0.3) @@ -66,7 +77,7 @@ configure_wireguard() { local -a server_keys=() local worker_key host key - worker_key=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" 'bash -s' <<'REMOTE' | tail -n 1 + worker_key=$(cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" 'bash -s' <<'REMOTE' | tail -n 1 set -euo pipefail export DEBIAN_FRONTEND=noninteractive if ! command -v wg >/dev/null 2>&1; then @@ -91,7 +102,7 @@ REMOTE server_keys+=("$key") done - cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- \ + cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" bash -s -- \ "$worker_wireguard_port" \ "${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \ "${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \ @@ -136,7 +147,7 @@ systemctl restart wg-quick@wg0 REMOTE for host in "${server_hosts[@]}"; do - cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" \ + cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_WORKER_NODE_4_HOST#*@}" \ "$node_ip" "$worker_wireguard_port" <<'REMOTE' set -euo pipefail wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:$4" persistent-keepalive 25 @@ -159,19 +170,19 @@ verify_path() { } verify_wireguard() { - verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_worker_2 - verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 worker_2_to_ningbo - verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_worker_2 - verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 worker_2_to_hongkong - verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_worker_2 - verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 worker_2_to_losangeles + verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_worker_node_4 + verify_path "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.1 80 worker_node_4_to_ningbo + verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_worker_node_4 + verify_path "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.2 80 worker_node_4_to_hongkong + verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_worker_node_4 + verify_path "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.3 300 worker_node_4_to_losangeles local host recent - for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_NINGBO_WORKER_HOST"; do + for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_WORKER_NODE_4_HOST"; do recent=$(cluster_ssh "$host" \ "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'") [[ $recent == 3 ]] done - echo "ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct listen_port=$worker_wireguard_port" + echo "worker_node_wireguard=PASS node=$node_name site=$node_site peers=4 recent_handshakes=12 routing=direct listen_port=$worker_wireguard_port" } install_agent() { @@ -181,26 +192,29 @@ install_agent() { : "${K3S_TOKEN:?}" [[ $K3S_TOKEN =~ ^[0-9a-f]{64}$ ]] - cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" 'install -d -m 0700 /etc/rancher/k3s' - printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \ + cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" 'install -d -m 0700 /etc/rancher/k3s' + printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" \ 'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token' - cluster_scp "$k3s_binary" "$CLUSTER_NINGBO_WORKER_HOST:/usr/local/bin/k3s" - cluster_scp "$registries_file" "$CLUSTER_NINGBO_WORKER_HOST:/etc/rancher/k3s/registries.yaml" - cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- "$k3s_version" "$k3s_sha256" <<'REMOTE' + cluster_scp "$k3s_binary" "$CLUSTER_WORKER_NODE_4_HOST:/usr/local/bin/k3s" + cluster_scp "$registries_file" "$CLUSTER_WORKER_NODE_4_HOST:/etc/rancher/k3s/registries.yaml" + cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" bash -s -- \ + "$k3s_version" "$k3s_sha256" "$node_name" "$node_site" <<'REMOTE' set -euo pipefail version=$1 expected_sha256=$2 +node_name=$3 +node_site=$4 chmod 0755 /usr/local/bin/k3s echo "$expected_sha256 /usr/local/bin/k3s" | sha256sum -c - >/dev/null umask 077 -cat >/etc/rancher/k3s/config.yaml <<'EOF' +cat >/etc/rancher/k3s/config.yaml <&2 + exit 1 + } + hosts+=("$CLUSTER_WORKER_NODE_4_HOST") + public_endpoints+=("${CLUSTER_WORKER_NODE_4_HOST#*@}") wireguard_ips+=(10.77.0.4) - wireguard_ports+=(443) + wireguard_ports+=("$CLUSTER_WORKER_NODE_4_WIREGUARD_PORT") fi public_keys=() diff --git a/scripts/cluster/common.sh b/scripts/cluster/common.sh index fab6cab..b203648 100755 --- a/scripts/cluster/common.sh +++ b/scripts/cluster/common.sh @@ -28,8 +28,16 @@ load_cluster_env() { CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST} CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2} CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3} - CLUSTER_NINGBO_WORKER_HOST=${AI_GATEWAY_NINGBO_WORKER_HOST:-} - export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST CLUSTER_NINGBO_WORKER_HOST + CLUSTER_WORKER_NODE_4_HOST=${AI_GATEWAY_WORKER_NODE_4_HOST:-} + CLUSTER_WORKER_NODE_4_NAME=${AI_GATEWAY_WORKER_NODE_4_NAME:-} + CLUSTER_WORKER_NODE_4_SITE=${AI_GATEWAY_WORKER_NODE_4_SITE:-} + CLUSTER_WORKER_NODE_4_WIREGUARD_PORT=${AI_GATEWAY_WORKER_NODE_4_WIREGUARD_PORT:-51820} + CLUSTER_WORKER_NODE_4_NETWORK_APPROVED=${AI_GATEWAY_WORKER_NODE_4_NETWORK_APPROVED:-false} + CLUSTER_WORKER_NODE_4_ENABLED=${AI_GATEWAY_WORKER_NODE_4_ENABLED:-false} + export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST + export CLUSTER_WORKER_NODE_4_HOST CLUSTER_WORKER_NODE_4_NAME CLUSTER_WORKER_NODE_4_SITE + export CLUSTER_WORKER_NODE_4_WIREGUARD_PORT CLUSTER_WORKER_NODE_4_NETWORK_APPROVED + export CLUSTER_WORKER_NODE_4_ENABLED for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || { @@ -37,8 +45,37 @@ load_cluster_env() { return 1 } done - if [[ -n $CLUSTER_NINGBO_WORKER_HOST && ! $CLUSTER_NINGBO_WORKER_HOST =~ ^root@[A-Za-z0-9.-]+$ ]]; then - echo "dedicated Ningbo Worker host must use root@host syntax" >&2 + if [[ -n $CLUSTER_WORKER_NODE_4_HOST ]]; then + [[ $CLUSTER_WORKER_NODE_4_HOST =~ ^root@[A-Za-z0-9.-]+$ ]] || { + echo "dedicated Worker node 4 host must use root@host syntax" >&2 + return 1 + } + [[ ${#CLUSTER_WORKER_NODE_4_NAME} -le 63 && + $CLUSTER_WORKER_NODE_4_NAME =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] || { + echo "dedicated Worker node 4 name must be a Kubernetes DNS label" >&2 + return 1 + } + [[ $CLUSTER_WORKER_NODE_4_SITE == ningbo || $CLUSTER_WORKER_NODE_4_SITE == hongkong ]] || { + echo "dedicated Worker node 4 site must be ningbo or hongkong" >&2 + return 1 + } + [[ $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT =~ ^[0-9]+$ && + $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT -ge 1 && + $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT -le 65535 ]] || { + echo "dedicated Worker node 4 WireGuard port is invalid" >&2 + return 1 + } + [[ $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == true || + $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == false ]] || { + echo "dedicated Worker node 4 network approval must be true or false" >&2 + return 1 + } + [[ $CLUSTER_WORKER_NODE_4_ENABLED == true || $CLUSTER_WORKER_NODE_4_ENABLED == false ]] || { + echo "dedicated Worker node 4 enabled flag must be true or false" >&2 + return 1 + } + elif [[ -n $CLUSTER_WORKER_NODE_4_NAME || -n $CLUSTER_WORKER_NODE_4_SITE ]]; then + echo "dedicated Worker node 4 metadata requires AI_GATEWAY_WORKER_NODE_4_HOST" >&2 return 1 fi [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || { diff --git a/scripts/cluster/monitor-production-release.sh b/scripts/cluster/monitor-production-release.sh index fcf6693..11e5af4 100755 --- a/scripts/cluster/monitor-production-release.sh +++ b/scripts/cluster/monitor-production-release.sh @@ -140,8 +140,8 @@ pool_saturated_pods() { check_wireguard() { local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") local -a ips=(10.77.0.1 10.77.0.2 10.77.0.3) - [[ -z $CLUSTER_NINGBO_WORKER_HOST ]] || { - hosts+=("$CLUSTER_NINGBO_WORKER_HOST") + [[ $CLUSTER_WORKER_NODE_4_ENABLED != true ]] || { + hosts+=("$CLUSTER_WORKER_NODE_4_HOST") ips+=(10.77.0.4) } local source target output loss average handshakes peer_count diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 0bdf1ef..b46acd5 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -652,28 +652,41 @@ verify_release_cas() { -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]] [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \ -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]] - if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then - local expected_node expected_replicas site pod_count - for site in ningbo hongkong; do - case $site in - ningbo) - expected_node=easyai-ningbo-worker-2 - expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO - ;; - hongkong) - expected_node=easyai-hongkong - expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG - ;; - esac - pod_count=$(remote_kubectl get pods -n "$namespace" \ - -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" \ - --field-selector "spec.nodeName=$expected_node" -o json | jq \ - '[.items[] | select(.status.phase=="Running")] | length') - [[ $pod_count == "$expected_replicas" ]] || { - echo "Worker placement does not match the dedicated node: site=$site node=$expected_node running_pods=$pod_count expected=$expected_replicas" >&2 - return 1 - } - done + if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then + local expected_replicas eligible_nodes eligible_count pod_report + local running_count invalid_count node4_count used_nodes + case $CLUSTER_WORKER_NODE_4_SITE in + ningbo) expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO ;; + hongkong) expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG ;; + esac + eligible_nodes=$(remote_kubectl get nodes \ + -l "easyai.io/site=$CLUSTER_WORKER_NODE_4_SITE,easyai.io/worker=true" \ + -o json | jq -c '[.items[].metadata.name]') + eligible_count=$(jq 'length' <<<"$eligible_nodes") + jq -e --arg node "$CLUSTER_WORKER_NODE_4_NAME" 'index($node) != null' \ + <<<"$eligible_nodes" >/dev/null + pod_report=$(remote_kubectl get pods -n "$namespace" \ + -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$CLUSTER_WORKER_NODE_4_SITE" -o json) + running_count=$(jq '[.items[] | select(.status.phase=="Running")] | length' <<<"$pod_report") + invalid_count=$(jq --argjson nodes "$eligible_nodes" \ + '[.items[] | select(.status.phase=="Running") | select(.spec.nodeName as $node | $nodes | index($node) | not)] | length' \ + <<<"$pod_report") + node4_count=$(jq --arg node "$CLUSTER_WORKER_NODE_4_NAME" \ + '[.items[] | select(.status.phase=="Running" and .spec.nodeName==$node)] | length' <<<"$pod_report") + used_nodes=$(jq '[.items[] | select(.status.phase=="Running") | .spec.nodeName] | unique | length' \ + <<<"$pod_report") + [[ $running_count == "$expected_replicas" && $invalid_count == 0 ]] || { + echo "Worker placement is outside eligible nodes: site=$CLUSTER_WORKER_NODE_4_SITE running_pods=$running_count expected=$expected_replicas invalid=$invalid_count" >&2 + return 1 + } + if (( expected_replicas > 0 && node4_count == 0 )); then + echo "dedicated Worker node has no baseline pod: site=$CLUSTER_WORKER_NODE_4_SITE node=$CLUSTER_WORKER_NODE_4_NAME" >&2 + return 1 + fi + if (( expected_replicas >= 2 && eligible_count >= 2 && used_nodes < 2 )); then + echo "Worker baseline is not distributed across physical nodes: site=$CLUSTER_WORKER_NODE_4_SITE used_nodes=$used_nodes" >&2 + return 1 + fi fi } @@ -2417,13 +2430,13 @@ verify_cluster_links() { verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo' verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles' verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong' - if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then - verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_2' - verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 'worker_2_to_ningbo' - verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_2' - verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 'worker_2_to_hongkong' - verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_2' - verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 'worker_2_to_los_angeles' + if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then + verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_node_4' + verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.1 80 'worker_node_4_to_ningbo' + verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_node_4' + verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.2 80 'worker_node_4_to_hongkong' + verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_node_4' + verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.3 300 'worker_node_4_to_losangeles' fi } @@ -2442,8 +2455,8 @@ verify_control_plane_and_recent_logs() { [[ $error_count == 0 ]] || return 1 fi done - if [[ -n $CLUSTER_NINGBO_WORKER_HOST && -n $runtime_observation_started_at ]]; then - error_count=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \ + if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true && -n $runtime_observation_started_at ]]; then + error_count=$(cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" \ "journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'") [[ $error_count == 0 ]] || return 1 fi diff --git a/scripts/cluster/verify-cluster.sh b/scripts/cluster/verify-cluster.sh index 738fdd9..1f15518 100755 --- a/scripts/cluster/verify-cluster.sh +++ b/scripts/cluster/verify-cluster.sh @@ -15,10 +15,10 @@ load_cluster_env hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3) site_labels=(ningbo hongkong losangeles) -if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then - hosts+=("$CLUSTER_NINGBO_WORKER_HOST") +if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then + hosts+=("$CLUSTER_WORKER_NODE_4_HOST") wireguard_ips+=(10.77.0.4) - site_labels+=(ningbo-worker-2) + site_labels+=("$CLUSTER_WORKER_NODE_4_NAME") fi peer_count=$((${#hosts[@]} - 1)) for source_index in "${!hosts[@]}"; do @@ -55,7 +55,7 @@ done echo "wireguard_acceptance=PASS peers=${#hosts[@]} handshakes=$((${#hosts[@]} * peer_count)) packet_loss_lt_1=true" expected_nodes=3 -[[ -z $CLUSTER_NINGBO_WORKER_HOST ]] || expected_nodes=4 +[[ $CLUSTER_WORKER_NODE_4_ENABLED != true ]] || expected_nodes=4 cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$expected_nodes" <<'REMOTE' set -euo pipefail expected_nodes=$1 @@ -72,22 +72,24 @@ printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)" REMOTE ) [[ $cluster_state == nodes="$expected_nodes" ]] -if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then - worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then + worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- \ + "$CLUSTER_WORKER_NODE_4_NAME" "$CLUSTER_WORKER_NODE_4_SITE" <<'REMOTE' set -euo pipefail kubectl='k3s kubectl' -node=easyai-ningbo-worker-2 +node=$1 +site=$2 [[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]] -[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == ningbo ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == "$site" ]] [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]] [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]] [[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]] [[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json | jq '[.items[] | select(.metadata.namespace!="kube-system" and .metadata.labels["app.kubernetes.io/name"]!="easyai-worker")] | length') -eq 0 ]] -printf 'worker_node=ready\n' +printf 'worker_node=ready node=%s site=%s\n' "$node" "$site" REMOTE ) - [[ $worker_state == worker_node=ready ]] + [[ $worker_state == "worker_node=ready node=$CLUSTER_WORKER_NODE_4_NAME site=$CLUSTER_WORKER_NODE_4_SITE" ]] fi echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witness_business_pods=0"