diff --git a/apps/api/internal/capacitycontroller/kubernetes.go b/apps/api/internal/capacitycontroller/kubernetes.go index c4459cb..b1d4e56 100644 --- a/apps/api/internal/capacitycontroller/kubernetes.go +++ b/apps/api/internal/capacitycontroller/kubernetes.go @@ -172,7 +172,9 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub } `json:"status"` } `json:"items"` } - nodePath := "/api/v1/nodes?labelSelector=" + url.QueryEscape("easyai.io/site="+site) + nodePath := "/api/v1/nodes?labelSelector=" + url.QueryEscape( + "easyai.io/site="+site+",easyai.io/worker=true", + ) if err := client.getJSON(ctx, nodePath, &nodes); err != nil { return KubernetesSiteState{}, err } diff --git a/apps/api/internal/capacitycontroller/kubernetes_test.go b/apps/api/internal/capacitycontroller/kubernetes_test.go index 90d79ed..33a0597 100644 --- a/apps/api/internal/capacitycontroller/kubernetes_test.go +++ b/apps/api/internal/capacitycontroller/kubernetes_test.go @@ -27,6 +27,10 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) { }]}}} }`)) case request.Method == http.MethodGet && request.URL.Path == "/api/v1/nodes": + if request.URL.Query().Get("labelSelector") != "easyai.io/site=hongkong,easyai.io/worker=true" { + http.Error(w, "unexpected Worker node selector", http.StatusBadRequest) + return + } _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"easyai-hongkong"},"status":{ "allocatable":{"memory":"8Gi","cpu":"4"}, "conditions":[{"type":"MemoryPressure","status":"False"}] diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index d2a4459..04b8bb2 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -458,6 +458,7 @@ spec: nodeSelector: easyai.io/site: ningbo easyai.io/workload: "true" + easyai.io/worker: "true" tolerations: - key: easyai.io/worker-only operator: Equal @@ -622,6 +623,7 @@ spec: nodeSelector: easyai.io/site: hongkong easyai.io/workload: "true" + easyai.io/worker: "true" tolerations: - key: easyai.io/worker-only operator: Equal diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md index 4bb4016..63a31d7 100644 --- a/docs/operations/k3s-ha-runbook.md +++ b/docs/operations/k3s-ha-runbook.md @@ -6,22 +6,23 @@ CloudNativePG;洛杉矶只作为 K3s server/etcd 仲裁节点,并带 `easyai.io/witness=true:NoSchedule` 污点。 -深圳扩展节点是纯 K3s agent,不加入 etcd 或 CloudNativePG。它使用节点名 -`easyai-shenzhen`、WireGuard 地址 `10.77.0.4`,并归入逻辑 `hongkong` Worker 池。 +第二台宁波服务器是纯 K3s agent,不加入 etcd 或 CloudNativePG。它使用节点名 +`easyai-ningbo-worker-2`、WireGuard 地址 `10.77.0.4`,并归入逻辑 `ningbo` Worker 池。 `easyai.io/worker-only=true:NoSchedule` 污点确保除 Worker 外的业务 Pod 不会调度到该机; -香港池副本通过 hostname topology spread 分散到香港、深圳。接入使用: +`easyai.io/worker=true` 标签确保容量控制器只把专用 Worker 节点纳入资源预算。接入使用: ```bash -./scripts/cluster/add-shenzhen-worker-node.sh all +./scripts/cluster/add-ningbo-worker-node.sh all ``` -香港—深圳公网直连实测无法满足生产链路门禁,因此两地 `10.77.0.2/10.77.0.4` 数据前缀 -固定经宁波 WireGuard 中继;香港—深圳直接 peer 只保留加密心跳。巡检仍以两端实际数据路径 -的 10 包丢包率低于 1%、平均 RTT 低于 80 ms 为硬门禁,禁止自动回退到劣化的公网直连。 +新节点与宁波、香港、洛杉矶使用 WireGuard 全互联直连,不经过中继。公网 K3s 直连虽然可以 +单独保护 API TLS,但当前 Flannel、Pod 和数据库网络依赖 `10.77.0.0/24`,禁止改成未加密的 +公网 VXLAN。巡检以 10 包丢包率低于 1%为硬门禁;宁波和香港路径 RTT 低于 80 ms,洛杉矶 +路径低于 300 ms。 -本地 `.env.local` 只保存 `AI_GATEWAY_SHENZHEN_HOST` 和 SSH 接入密码,必须保持 `0600` 且 -不得进入 Git。深圳已有宿主机服务时,接入脚本保留其公开端口,只封锁 K3s API、kubelet、 -VXLAN 和除既有 `31058` 外的 NodePort 公网入口。深圳防火墙继续由原有 UFW 管理,接入脚本 +本地 `.env.local` 只保存 `AI_GATEWAY_NINGBO_WORKER_HOST` 和 SSH 接入密码,必须保持 `0600` +且不得进入 Git。新节点只保留原有 SSH 入口,封锁 K3s API、kubelet、VXLAN 和全部 NodePort +公网入口。节点防火墙继续由原有 UFW 管理,接入脚本 禁止安装与 UFW 冲突的 `iptables-persistent`;UFW 未安装或未启用时必须在预检阶段停止。 生产上线前的双节点高媒体负载、Worker 强杀和 P24/P28/P32 容量搜索见 diff --git a/docs/operations/production-acceptance.md b/docs/operations/production-acceptance.md index 904cff5..bd79eba 100644 --- a/docs/operations/production-acceptance.md +++ b/docs/operations/production-acceptance.md @@ -89,21 +89,22 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG 各一个 2 GiB Worker,再按实时内存、CPU 和 PostgreSQL 预算验证 `1+2`、条件允许时的 `2+2`,以及安全 drain 回到 `1+1`: -宁波因正式混部不具备 Worker 余量时,验收可改用 `0+2` 基线: +生产基线使用两个独立物理 Worker 节点:第二台宁波服务器承载 `easyai-worker-ningbo`, +香港服务器承载 `easyai-worker-hongkong`。两个 Deployment 都要求节点带 +`easyai.io/worker=true`,原宁波混部节点不再承载 Worker。默认验收配置为: ```bash -AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO=0 -AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG=2 -AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=0 +AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO=1 +AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG=1 +AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=1 -AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO=0 +AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO=2 AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG=2 ``` -此时香港逻辑池的两个 Worker 必须通过 hostname topology spread 分布在香港和深圳物理节点; -固定容量仍是两个实例,自动伸缩阶段验证 `0+1 → 0+2 → 0+1`,最终 certified profile -恢复 `0+2` 最小副本以保留双物理节点容灾。资源前置门禁采样实际启用的 Worker 站点节点, -宁波仍受运行时 80% 目标和 85% 硬门禁约束。 +资源前置门禁和容量控制器只采样 `easyai.io/worker=true` 节点,避免把原宁波混部节点的 +可分配资源错误计入弹性上限。每站点能否升到两个副本仍由实测 RSS、CPU 和 PostgreSQL +连接预算决定,不因新增节点而预先承诺。 | 档位 | 单实例执行槽 | 数据库池 | 媒体并发 | 全局执行槽 | |---|---:|---:|---:|---:| diff --git a/scripts/cluster/add-shenzhen-worker-node.sh b/scripts/cluster/add-ningbo-worker-node.sh similarity index 71% rename from scripts/cluster/add-shenzhen-worker-node.sh rename to scripts/cluster/add-ningbo-worker-node.sh index 94460b1..59ffee1 100755 --- a/scripts/cluster/add-shenzhen-worker-node.sh +++ b/scripts/cluster/add-ningbo-worker-node.sh @@ -8,15 +8,15 @@ source "$script_dir/common.sh" action=${1:-all} [[ $action == preflight || $action == network || $action == install || $action == verify || $action == all ]] || { - echo 'usage: add-shenzhen-worker-node.sh [preflight|network|install|verify|all]' >&2 + echo 'usage: add-ningbo-worker-node.sh [preflight|network|install|verify|all]' >&2 exit 64 } load_cluster_env require_commands ssh scp shasum -: "${CLUSTER_SHENZHEN_HOST:?AI_GATEWAY_SHENZHEN_HOST is required}" +: "${CLUSTER_NINGBO_WORKER_HOST:?AI_GATEWAY_NINGBO_WORKER_HOST is required}" -node_name=easyai-shenzhen +node_name=easyai-ningbo-worker-2 node_ip=10.77.0.4 k3s_version=v1.36.2+k3s1 k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de @@ -38,11 +38,12 @@ assert_join_material() { } preflight_node() { - cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s <<'REMOTE' + cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s <<'REMOTE' set -euo pipefail [[ $(id -u) -eq 0 && $(uname -m) == x86_64 ]] . /etc/os-release -[[ $ID == debian && ${VERSION_ID%%.*} -ge 12 ]] +[[ ($ID == debian && ${VERSION_ID%%.*} -ge 12) || + ($ID == ubuntu && ${VERSION_ID%%.*} -ge 24) ]] (( $(nproc) >= 4 )) available_kib=$(awk '/MemAvailable:/ {print $2}' /proc/meminfo) (( available_kib >= 6 * 1024 * 1024 )) @@ -52,7 +53,7 @@ timedatectl show -p NTPSynchronized --value | grep -Fxq yes [[ ! -e /etc/rancher/k3s/config.yaml || -f /etc/rancher/k3s/config.yaml ]] command -v ufw >/dev/null ufw status | grep -Fqx 'Status: active' -printf 'shenzhen_worker_preflight=PASS cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \ +printf 'ningbo_worker_preflight=PASS cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \ "$(nproc)" "$((available_kib / 1024))" "$((available_bytes / 1024 / 1024 / 1024))" REMOTE } @@ -62,9 +63,9 @@ configure_wireguard() { local -a server_ips=(10.77.0.1 10.77.0.2 10.77.0.3) local -a server_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}") local -a server_keys=() - local shenzhen_key host key + local worker_key host key - shenzhen_key=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" 'bash -s' <<'REMOTE' | tail -n 1 + worker_key=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" 'bash -s' <<'REMOTE' | tail -n 1 set -euo pipefail export DEBIAN_FRONTEND=noninteractive if ! command -v wg >/dev/null 2>&1; then @@ -81,7 +82,7 @@ wg pubkey /etc/wireguard/publickey cat /etc/wireguard/publickey REMOTE ) - [[ $shenzhen_key =~ ^[A-Za-z0-9+/]{43}=$ ]] + [[ $worker_key =~ ^[A-Za-z0-9+/]{43}=$ ]] for host in "${server_hosts[@]}"; do key=$(cluster_ssh "$host" 'wg show wg0 public-key') @@ -89,7 +90,7 @@ REMOTE server_keys+=("$key") done - cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \ + cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- \ "${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \ "${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \ "${server_keys[2]}" "${server_endpoints[2]}" "${server_ips[2]}" <<'REMOTE' @@ -131,43 +132,13 @@ systemctl restart wg-quick@wg0 REMOTE for host in "${server_hosts[@]}"; do - cluster_ssh "$host" bash -s -- "$shenzhen_key" "${CLUSTER_SHENZHEN_HOST#*@}" "$node_ip" <<'REMOTE' + cluster_ssh "$host" bash -s -- "$worker_key" "${CLUSTER_NINGBO_WORKER_HOST#*@}" "$node_ip" <<'REMOTE' set -euo pipefail wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:51820" persistent-keepalive 25 wg-quick save wg0 REMOTE done - # The Hong Kong-Shenzhen public route is unstable. Keep the direct peers for - # encrypted liveness, but route their data prefixes through Ningbo's healthy - # WireGuard paths. AllowedIPs remain unique, so WireGuard selects the relay - # deterministically instead of oscillating between direct and relayed paths. - cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- \ - "${server_keys[0]}" "$shenzhen_key" <<'REMOTE' -set -euo pipefail -wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.4/32 -wg set wg0 peer "$2" allowed-ips 0.0.0.0/32 -wg-quick save wg0 -REMOTE - cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \ - "${server_keys[0]}" "${server_keys[1]}" <<'REMOTE' -set -euo pipefail -wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.2/32 -wg set wg0 peer "$2" allowed-ips 0.0.0.0/32 -wg-quick save wg0 -REMOTE - cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' -set -euo pipefail -sysctl -qw net.ipv4.ip_forward=1 -ensure_forward_rule() { - if ! iptables -C FORWARD "$@" 2>/dev/null; then iptables -I FORWARD 1 "$@"; fi -} -ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.2/32 -d 10.77.0.4/32 \ - -m comment --comment easyai-wireguard-hongkong-shenzhen -j ACCEPT -ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.4/32 -d 10.77.0.2/32 \ - -m comment --comment easyai-wireguard-shenzhen-hongkong -j ACCEPT -netfilter-persistent save >/dev/null -REMOTE verify_wireguard } @@ -183,19 +154,19 @@ verify_path() { } verify_wireguard() { - verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_shenzhen - verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 shenzhen_to_ningbo - verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_shenzhen - verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 shenzhen_to_hongkong - verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_shenzhen - verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 shenzhen_to_losangeles + verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_worker_2 + verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 worker_2_to_ningbo + verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_worker_2 + verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 worker_2_to_hongkong + verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_worker_2 + verify_path "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 worker_2_to_losangeles local host recent - for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_SHENZHEN_HOST"; do + for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_NINGBO_WORKER_HOST"; do recent=$(cluster_ssh "$host" \ "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'") [[ $recent == 3 ]] done - echo 'shenzhen_wireguard=PASS peers=4 recent_handshakes=12 hongkong_shenzhen_via=ningbo' + echo 'ningbo_worker_wireguard=PASS peers=4 recent_handshakes=12 routing=direct' } install_agent() { @@ -205,12 +176,12 @@ install_agent() { : "${K3S_TOKEN:?}" [[ $K3S_TOKEN =~ ^[0-9a-f]{64}$ ]] - cluster_ssh "$CLUSTER_SHENZHEN_HOST" 'install -d -m 0700 /etc/rancher/k3s' - printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_SHENZHEN_HOST" \ + cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" 'install -d -m 0700 /etc/rancher/k3s' + printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \ 'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token' - cluster_scp "$k3s_binary" "$CLUSTER_SHENZHEN_HOST:/usr/local/bin/k3s" - cluster_scp "$registries_file" "$CLUSTER_SHENZHEN_HOST:/etc/rancher/k3s/registries.yaml" - cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- "$k3s_version" "$k3s_sha256" <<'REMOTE' + cluster_scp "$k3s_binary" "$CLUSTER_NINGBO_WORKER_HOST:/usr/local/bin/k3s" + cluster_scp "$registries_file" "$CLUSTER_NINGBO_WORKER_HOST:/etc/rancher/k3s/registries.yaml" + cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" bash -s -- "$k3s_version" "$k3s_sha256" <<'REMOTE' set -euo pipefail version=$1 expected_sha256=$2 @@ -220,11 +191,11 @@ umask 077 cat >/etc/rancher/k3s/config.yaml <<'EOF' server: https://10.77.0.1:6443 token-file: /etc/rancher/k3s/cluster-token -node-name: easyai-shenzhen +node-name: easyai-ningbo-worker-2 node-ip: 10.77.0.4 flannel-iface: wg0 node-label: - - easyai.io/site=hongkong + - easyai.io/site=ningbo - easyai.io/workload=true - easyai.io/database=false - easyai.io/worker=true @@ -241,10 +212,8 @@ ufw deny in on "$public_interface" to any port 6443 proto tcp comment easyai-k3s ufw deny in on "$public_interface" to any port 10250 proto tcp comment easyai-kubelet-public-deny >/dev/null ufw deny in on "$public_interface" to any port 8472 proto udp comment easyai-flannel-public-deny >/dev/null for protocol in tcp udp; do - ufw deny in on "$public_interface" to any port 30000:31057 proto "$protocol" \ - comment easyai-nodeport-public-deny-low >/dev/null - ufw deny in on "$public_interface" to any port 31059:32767 proto "$protocol" \ - comment easyai-nodeport-public-deny-high >/dev/null + ufw deny in on "$public_interface" to any port 30000:32767 proto "$protocol" \ + comment easyai-nodeport-public-deny >/dev/null done install_script=$(mktemp) trap 'rm -f -- "$install_script"' EXIT @@ -267,6 +236,13 @@ REMOTE sleep 2 done [[ $ready == true ]] + cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +kubectl='k3s kubectl' +$kubectl label node easyai-ningbo easyai.io/worker=false --overwrite >/dev/null +$kubectl label node easyai-hongkong easyai.io/worker=true --overwrite >/dev/null +$kubectl label node easyai-los-angeles easyai.io/worker=false --overwrite >/dev/null +REMOTE verify_agent } @@ -277,13 +253,15 @@ set -euo pipefail node=$1 kubectl='k3s kubectl' [[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]] -[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == ningbo ]] [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/workload}') == true ]] [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]] +[[ $($kubectl get node easyai-hongkong -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]] +[[ $($kubectl get node easyai-ningbo -o jsonpath='{.metadata.labels.easyai\.io/worker}') == false ]] [[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]] [[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json | jq '[.items[] | select(.metadata.namespace!="kube-system")] | length') -eq 0 ]] -printf 'shenzhen_k3s_agent=PASS node=%s role=worker-only etcd_member=false\n' "$node" +printf 'ningbo_worker_k3s_agent=PASS node=%s role=worker-only etcd_member=false\n' "$node" REMOTE } diff --git a/scripts/cluster/bootstrap-k3s.sh b/scripts/cluster/bootstrap-k3s.sh index 84fe19f..3f20f9b 100755 --- a/scripts/cluster/bootstrap-k3s.sh +++ b/scripts/cluster/bootstrap-k3s.sh @@ -256,9 +256,9 @@ REMOTE cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' set -euo pipefail kubectl='k3s kubectl' -$kubectl label node easyai-ningbo easyai.io/site=ningbo easyai.io/workload=true easyai.io/database=true --overwrite -$kubectl label node easyai-hongkong easyai.io/site=hongkong easyai.io/workload=true easyai.io/database=true --overwrite -$kubectl label node easyai-los-angeles easyai.io/site=los-angeles easyai.io/workload=false easyai.io/database=false --overwrite +$kubectl label node easyai-ningbo easyai.io/site=ningbo easyai.io/workload=true easyai.io/database=true easyai.io/worker=false --overwrite +$kubectl label node easyai-hongkong easyai.io/site=hongkong easyai.io/workload=true easyai.io/database=true easyai.io/worker=true --overwrite +$kubectl label node easyai-los-angeles easyai.io/site=los-angeles easyai.io/workload=false easyai.io/database=false easyai.io/worker=false --overwrite $kubectl taint node easyai-los-angeles easyai.io/witness=true:NoSchedule --overwrite k3s secrets-encrypt status k3s etcd-snapshot save --name post-bootstrap diff --git a/scripts/cluster/bootstrap-wireguard.sh b/scripts/cluster/bootstrap-wireguard.sh index b5146b0..f71edfd 100755 --- a/scripts/cluster/bootstrap-wireguard.sh +++ b/scripts/cluster/bootstrap-wireguard.sh @@ -11,9 +11,9 @@ require_commands ssh scp hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}") wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3) -if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then - hosts+=("$CLUSTER_SHENZHEN_HOST") - public_endpoints+=("${CLUSTER_SHENZHEN_HOST#*@}") +if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then + hosts+=("$CLUSTER_NINGBO_WORKER_HOST") + public_endpoints+=("${CLUSTER_NINGBO_WORKER_HOST#*@}") wireguard_ips+=(10.77.0.4) fi public_keys=() @@ -90,35 +90,6 @@ systemctl restart wg-quick@wg0 REMOTE done -if (( ${#hosts[@]} == 4 )); then - cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- \ - "${public_keys[0]}" "${public_keys[3]}" <<'REMOTE' -set -euo pipefail -wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.4/32 -wg set wg0 peer "$2" allowed-ips 0.0.0.0/32 -wg-quick save wg0 -REMOTE - cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \ - "${public_keys[0]}" "${public_keys[1]}" <<'REMOTE' -set -euo pipefail -wg set wg0 peer "$1" allowed-ips 10.77.0.1/32,10.77.0.2/32 -wg set wg0 peer "$2" allowed-ips 0.0.0.0/32 -wg-quick save wg0 -REMOTE - cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' -set -euo pipefail -sysctl -qw net.ipv4.ip_forward=1 -ensure_forward_rule() { - if ! iptables -C FORWARD "$@" 2>/dev/null; then iptables -I FORWARD 1 "$@"; fi -} -ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.2/32 -d 10.77.0.4/32 \ - -m comment --comment easyai-wireguard-hongkong-shenzhen -j ACCEPT -ensure_forward_rule -i wg0 -o wg0 -s 10.77.0.4/32 -d 10.77.0.2/32 \ - -m comment --comment easyai-wireguard-shenzhen-hongkong -j ACCEPT -netfilter-persistent save >/dev/null -REMOTE -fi - for source_index in "${!hosts[@]}"; do for target_index in "${!hosts[@]}"; do [[ $source_index -eq $target_index ]] && continue @@ -136,8 +107,4 @@ for host in "${hosts[@]}"; do } done -if (( ${#hosts[@]} == 4 )); then - echo 'wireguard_bootstrap=PASS peers=4 hongkong_shenzhen_via=ningbo' -else - echo "wireguard_bootstrap=PASS peers=${#hosts[@]}" -fi +echo "wireguard_bootstrap=PASS peers=${#hosts[@]} routing=direct" diff --git a/scripts/cluster/common.sh b/scripts/cluster/common.sh index e3c89a8..fab6cab 100755 --- a/scripts/cluster/common.sh +++ b/scripts/cluster/common.sh @@ -28,8 +28,8 @@ load_cluster_env() { CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST} CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2} CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3} - CLUSTER_SHENZHEN_HOST=${AI_GATEWAY_SHENZHEN_HOST:-} - export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST CLUSTER_SHENZHEN_HOST + CLUSTER_NINGBO_WORKER_HOST=${AI_GATEWAY_NINGBO_WORKER_HOST:-} + export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST CLUSTER_NINGBO_WORKER_HOST for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || { @@ -37,8 +37,8 @@ load_cluster_env() { return 1 } done - if [[ -n $CLUSTER_SHENZHEN_HOST && ! $CLUSTER_SHENZHEN_HOST =~ ^root@[A-Za-z0-9.-]+$ ]]; then - echo "Shenzhen Worker host must use root@host syntax" >&2 + if [[ -n $CLUSTER_NINGBO_WORKER_HOST && ! $CLUSTER_NINGBO_WORKER_HOST =~ ^root@[A-Za-z0-9.-]+$ ]]; then + echo "dedicated Ningbo Worker host must use root@host syntax" >&2 return 1 fi [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || { diff --git a/scripts/cluster/monitor-production-release.sh b/scripts/cluster/monitor-production-release.sh index 9d84dbf..fcf6693 100755 --- a/scripts/cluster/monitor-production-release.sh +++ b/scripts/cluster/monitor-production-release.sh @@ -140,8 +140,8 @@ pool_saturated_pods() { check_wireguard() { local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") local -a ips=(10.77.0.1 10.77.0.2 10.77.0.3) - [[ -z $CLUSTER_SHENZHEN_HOST ]] || { - hosts+=("$CLUSTER_SHENZHEN_HOST") + [[ -z $CLUSTER_NINGBO_WORKER_HOST ]] || { + hosts+=("$CLUSTER_NINGBO_WORKER_HOST") ips+=(10.77.0.4) } local source target output loss average handshakes peer_count diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index f5ab501..0bdf1ef 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -652,17 +652,25 @@ verify_release_cas() { -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]] [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \ -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]] - if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 && - AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) && - [[ -n $CLUSTER_SHENZHEN_HOST ]]; then - local worker_node pod_count - for worker_node in easyai-hongkong easyai-shenzhen; do + if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then + local expected_node expected_replicas site pod_count + for site in ningbo hongkong; do + case $site in + ningbo) + expected_node=easyai-ningbo-worker-2 + expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + ;; + hongkong) + expected_node=easyai-hongkong + expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG + ;; + esac pod_count=$(remote_kubectl get pods -n "$namespace" \ - -l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \ - --field-selector "spec.nodeName=$worker_node" -o json | jq \ + -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" \ + --field-selector "spec.nodeName=$expected_node" -o json | jq \ '[.items[] | select(.status.phase=="Running")] | length') - [[ $pod_count == 1 ]] || { - echo "Hong Kong logical Worker pool is not spread across both physical nodes: node=$worker_node running_pods=$pod_count" >&2 + [[ $pod_count == "$expected_replicas" ]] || { + echo "Worker placement does not match the dedicated node: site=$site node=$expected_node running_pods=$pod_count expected=$expected_replicas" >&2 return 1 } done @@ -674,12 +682,12 @@ verify_resource_preconditions() { local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads worker_nodes= if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then - worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \ + worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo,easyai.io/worker=true' \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') worker_nodes+=$'\n' fi if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then - worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong' \ + worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong,easyai.io/worker=true' \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') fi worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u) @@ -2409,13 +2417,13 @@ verify_cluster_links() { verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo' verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles' verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong' - if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then - verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_shenzhen' - verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 'shenzhen_to_ningbo' - verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_shenzhen' - verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 'shenzhen_to_hongkong' - verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_shenzhen' - verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 'shenzhen_to_los_angeles' + if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then + verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_2' + verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.1 80 'worker_2_to_ningbo' + verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_2' + verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.2 80 'worker_2_to_hongkong' + verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_2' + verify_wireguard "$CLUSTER_NINGBO_WORKER_HOST" 10.77.0.3 300 'worker_2_to_los_angeles' fi } @@ -2434,8 +2442,8 @@ verify_control_plane_and_recent_logs() { [[ $error_count == 0 ]] || return 1 fi done - if [[ -n $CLUSTER_SHENZHEN_HOST && -n $runtime_observation_started_at ]]; then - error_count=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" \ + if [[ -n $CLUSTER_NINGBO_WORKER_HOST && -n $runtime_observation_started_at ]]; then + error_count=$(cluster_ssh "$CLUSTER_NINGBO_WORKER_HOST" \ "journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'") [[ $error_count == 0 ]] || return 1 fi diff --git a/scripts/cluster/verify-cluster.sh b/scripts/cluster/verify-cluster.sh index 2fb4702..738fdd9 100755 --- a/scripts/cluster/verify-cluster.sh +++ b/scripts/cluster/verify-cluster.sh @@ -15,10 +15,10 @@ load_cluster_env hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3) site_labels=(ningbo hongkong losangeles) -if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then - hosts+=("$CLUSTER_SHENZHEN_HOST") +if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then + hosts+=("$CLUSTER_NINGBO_WORKER_HOST") wireguard_ips+=(10.77.0.4) - site_labels+=(shenzhen) + site_labels+=(ningbo-worker-2) fi peer_count=$((${#hosts[@]} - 1)) for source_index in "${!hosts[@]}"; do @@ -55,7 +55,7 @@ done echo "wireguard_acceptance=PASS peers=${#hosts[@]} handshakes=$((${#hosts[@]} * peer_count)) packet_loss_lt_1=true" expected_nodes=3 -[[ -z $CLUSTER_SHENZHEN_HOST ]] || expected_nodes=4 +[[ -z $CLUSTER_NINGBO_WORKER_HOST ]] || expected_nodes=4 cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$expected_nodes" <<'REMOTE' set -euo pipefail expected_nodes=$1 @@ -72,13 +72,14 @@ printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)" REMOTE ) [[ $cluster_state == nodes="$expected_nodes" ]] -if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then +if [[ -n $CLUSTER_NINGBO_WORKER_HOST ]]; then worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' set -euo pipefail kubectl='k3s kubectl' -node=easyai-shenzhen +node=easyai-ningbo-worker-2 [[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]] -[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == ningbo ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]] [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]] [[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]] [[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |