diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index 3c9f9eb..a4762ff 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -55,9 +55,19 @@ source "$config_file" echo 'production desired-state bundle is unavailable' >&2 exit 1 } -for capacity_value in \ +for replica_value in \ "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ + "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ + "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG"; do + [[ $replica_value =~ ^[0-9]+$ ]] || { + echo 'Worker replica configuration must contain non-negative integers' >&2 + exit 1 + } +done +for capacity_value in \ "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \ @@ -71,10 +81,6 @@ for capacity_value in \ "$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \ "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \ "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \ - "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ - "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \ "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \ "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \ @@ -103,6 +109,7 @@ done } (( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 && AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 && + AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG >= 1 && AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 && AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 && AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 && @@ -332,7 +339,7 @@ restore_application_deployments() { rollout_worker_site() { local site=$1 local api_image=$2 - local replicas + local replicas ready_replicas case $site in ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;; hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;; @@ -371,8 +378,9 @@ rollout_worker_site() { "${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \ -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]] - [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \ - -o jsonpath='{.status.readyReplicas}') == "$replicas" ]] + ready_replicas=$("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \ + -o json | jq -r '.status.readyReplicas // 0') + [[ $ready_replicas == "$replicas" ]] } rollout_capacity_controller() { diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index 72a12e2..d2a4459 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -458,6 +458,11 @@ spec: nodeSelector: easyai.io/site: ningbo easyai.io/workload: "true" + tolerations: + - key: easyai.io/worker-only + operator: Equal + value: "true" + effect: NoSchedule securityContext: runAsNonRoot: true runAsUser: 10001 @@ -617,6 +622,11 @@ spec: nodeSelector: easyai.io/site: hongkong easyai.io/workload: "true" + tolerations: + - key: easyai.io/worker-only + operator: Equal + value: "true" + effect: NoSchedule securityContext: runAsNonRoot: true runAsUser: 10001 diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md index 614c199..500dcf4 100644 --- a/docs/operations/k3s-ha-runbook.md +++ b/docs/operations/k3s-ha-runbook.md @@ -6,6 +6,19 @@ CloudNativePG;洛杉矶只作为 K3s server/etcd 仲裁节点,并带 `easyai.io/witness=true:NoSchedule` 污点。 +深圳扩展节点是纯 K3s agent,不加入 etcd 或 CloudNativePG。它使用节点名 +`easyai-shenzhen`、WireGuard 地址 `10.77.0.4`,并归入逻辑 `hongkong` Worker 池。 +`easyai.io/worker-only=true:NoSchedule` 污点确保除 Worker 外的业务 Pod 不会调度到该机; +香港池副本通过 hostname topology spread 分散到香港、深圳。接入使用: + +```bash +./scripts/cluster/add-shenzhen-worker-node.sh all +``` + +本地 `.env.local` 只保存 `AI_GATEWAY_SHENZHEN_HOST` 和 SSH 接入密码,必须保持 `0600` 且 +不得进入 Git。深圳已有宿主机服务时,接入脚本保留其公开端口,只封锁 K3s API、kubelet、 +VXLAN 和除既有 `31058` 外的 NodePort 公网入口。 + 生产上线前的双节点高媒体负载、Worker 强杀和 P24/P28/P32 容量搜索见 [生产同构验收模式与高媒体压力测试](production-acceptance.md)。 diff --git a/docs/operations/production-acceptance.md b/docs/operations/production-acceptance.md index 52f63aa..904cff5 100644 --- a/docs/operations/production-acceptance.md +++ b/docs/operations/production-acceptance.md @@ -89,6 +89,22 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG 各一个 2 GiB Worker,再按实时内存、CPU 和 PostgreSQL 预算验证 `1+2`、条件允许时的 `2+2`,以及安全 drain 回到 `1+1`: +宁波因正式混部不具备 Worker 余量时,验收可改用 `0+2` 基线: + +```bash +AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO=0 +AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG=2 +AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=0 +AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=1 +AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO=0 +AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG=2 +``` + +此时香港逻辑池的两个 Worker 必须通过 hostname topology spread 分布在香港和深圳物理节点; +固定容量仍是两个实例,自动伸缩阶段验证 `0+1 → 0+2 → 0+1`,最终 certified profile +恢复 `0+2` 最小副本以保留双物理节点容灾。资源前置门禁采样实际启用的 Worker 站点节点, +宁波仍受运行时 80% 目标和 85% 硬门禁约束。 + | 档位 | 单实例执行槽 | 数据库池 | 媒体并发 | 全局执行槽 | |---|---:|---:|---:|---:| | P24 | 24 | 32 | 24 | 48 | diff --git a/scripts/cluster/add-shenzhen-worker-node.sh b/scripts/cluster/add-shenzhen-worker-node.sh new file mode 100755 index 0000000..2c61e26 --- /dev/null +++ b/scripts/cluster/add-shenzhen-worker-node.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +action=${1:-all} +[[ $action == preflight || $action == network || $action == install || + $action == verify || $action == all ]] || { + echo 'usage: add-shenzhen-worker-node.sh [preflight|network|install|verify|all]' >&2 + exit 64 +} + +load_cluster_env +require_commands ssh scp shasum +: "${CLUSTER_SHENZHEN_HOST:?AI_GATEWAY_SHENZHEN_HOST is required}" + +node_name=easyai-shenzhen +node_ip=10.77.0.4 +k3s_version=v1.36.2+k3s1 +k3s_sha256=65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de +secret_root="$cluster_root/.local-secrets/cluster" +cluster_environment="$secret_root/cluster.env" +k3s_binary="$secret_root/k3s-${k3s_version}-amd64" +registries_file="$cluster_root/deploy/kubernetes/registries.yaml" + +assert_join_material() { + assert_private_file "$cluster_environment" + [[ -f $k3s_binary && ! -L $k3s_binary && -f $registries_file && ! -L $registries_file ]] || { + echo 'pinned K3s binary or registry configuration is missing' >&2 + return 1 + } + [[ $(shasum -a 256 "$k3s_binary" | awk '{print $1}') == "$k3s_sha256" ]] || { + echo 'pinned K3s binary checksum mismatch' >&2 + return 1 + } +} + +preflight_node() { + cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s <<'REMOTE' +set -euo pipefail +[[ $(id -u) -eq 0 && $(uname -m) == x86_64 ]] +. /etc/os-release +[[ $ID == debian && ${VERSION_ID%%.*} -ge 12 ]] +(( $(nproc) >= 4 )) +available_kib=$(awk '/MemAvailable:/ {print $2}' /proc/meminfo) +(( available_kib >= 6 * 1024 * 1024 )) +available_bytes=$(df -B1 --output=avail / | tail -n 1 | tr -d ' ') +(( available_bytes >= 20 * 1024 * 1024 * 1024 )) +timedatectl show -p NTPSynchronized --value | grep -Fxq yes +[[ ! -e /etc/rancher/k3s/config.yaml || -f /etc/rancher/k3s/config.yaml ]] +printf 'shenzhen_worker_preflight=PASS cpu=%s memory_available_mib=%s disk_available_gib=%s\n' \ + "$(nproc)" "$((available_kib / 1024))" "$((available_bytes / 1024 / 1024 / 1024))" +REMOTE +} + +configure_wireguard() { + local -a server_hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") + local -a server_ips=(10.77.0.1 10.77.0.2 10.77.0.3) + local -a server_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}") + local -a server_keys=() + local shenzhen_key host key + + shenzhen_key=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" 'bash -s' <<'REMOTE' | tail -n 1 +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +if ! command -v wg >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq wireguard-tools +fi +install -d -m 0700 /etc/wireguard +if [[ ! -s /etc/wireguard/privatekey ]]; then + umask 077 + wg genkey >/etc/wireguard/privatekey +fi +chmod 0600 /etc/wireguard/privatekey +wg pubkey /etc/wireguard/publickey +cat /etc/wireguard/publickey +REMOTE +) + [[ $shenzhen_key =~ ^[A-Za-z0-9+/]{43}=$ ]] + + for host in "${server_hosts[@]}"; do + key=$(cluster_ssh "$host" 'wg show wg0 public-key') + [[ $key =~ ^[A-Za-z0-9+/]{43}=$ ]] + server_keys+=("$key") + done + + cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- \ + "${server_keys[0]}" "${server_endpoints[0]}" "${server_ips[0]}" \ + "${server_keys[1]}" "${server_endpoints[1]}" "${server_ips[1]}" \ + "${server_keys[2]}" "${server_endpoints[2]}" "${server_ips[2]}" <<'REMOTE' +set -euo pipefail +private_key=$(cat /etc/wireguard/privatekey) +umask 077 +temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX) +cat >"$temporary_config" </dev/null +done +systemctl enable wg-quick@wg0 >/dev/null +systemctl restart wg-quick@wg0 +REMOTE + + for host in "${server_hosts[@]}"; do + cluster_ssh "$host" bash -s -- "$shenzhen_key" "${CLUSTER_SHENZHEN_HOST#*@}" "$node_ip" <<'REMOTE' +set -euo pipefail +wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:51820" persistent-keepalive 25 +wg-quick save wg0 +REMOTE + done + verify_wireguard +} + +verify_path() { + local source_host=$1 target_ip=$2 max_rtt=$3 label=$4 output loss average + output=$(cluster_ssh "$source_host" "ping -q -i 0.2 -c 10 -W 2 $target_ip") + loss=$(awk -F',' '/packet loss/ {gsub(/[^0-9.]/, "", $3); print $3}' <<<"$output") + average=$(awk -F'=' '/min\/avg\/max/ {split($2, values, "/"); gsub(/ /, "", values[2]); print values[2]}' <<<"$output") + [[ -n $loss && -n $average ]] + awk -v loss="$loss" -v average="$average" -v max="$max_rtt" \ + 'BEGIN {exit !(loss < 1 && average < max)}' + echo "wireguard_path=$label loss_percent=$loss average_rtt_ms=$average" +} + +verify_wireguard() { + verify_path "$CLUSTER_NINGBO_HOST" "$node_ip" 80 ningbo_to_shenzhen + verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 shenzhen_to_ningbo + verify_path "$CLUSTER_HONGKONG_HOST" "$node_ip" 80 hongkong_to_shenzhen + verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 shenzhen_to_hongkong + verify_path "$CLUSTER_LOS_ANGELES_HOST" "$node_ip" 300 losangeles_to_shenzhen + verify_path "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 shenzhen_to_losangeles + local host recent + for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_SHENZHEN_HOST"; do + recent=$(cluster_ssh "$host" \ + "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'") + [[ $recent == 3 ]] + done + echo 'shenzhen_wireguard=PASS peers=4 recent_handshakes=12' +} + +install_agent() { + assert_join_material + # shellcheck source=/dev/null + source "$cluster_environment" + : "${K3S_TOKEN:?}" + [[ $K3S_TOKEN =~ ^[0-9a-f]{64}$ ]] + + cluster_ssh "$CLUSTER_SHENZHEN_HOST" 'install -d -m 0700 /etc/rancher/k3s' + printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$CLUSTER_SHENZHEN_HOST" \ + 'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token' + cluster_scp "$k3s_binary" "$CLUSTER_SHENZHEN_HOST:/usr/local/bin/k3s" + cluster_scp "$registries_file" "$CLUSTER_SHENZHEN_HOST:/etc/rancher/k3s/registries.yaml" + cluster_ssh "$CLUSTER_SHENZHEN_HOST" bash -s -- "$k3s_version" "$k3s_sha256" <<'REMOTE' +set -euo pipefail +version=$1 +expected_sha256=$2 +chmod 0755 /usr/local/bin/k3s +echo "$expected_sha256 /usr/local/bin/k3s" | sha256sum -c - >/dev/null +umask 077 +cat >/etc/rancher/k3s/config.yaml <<'EOF' +server: https://10.77.0.1:6443 +token-file: /etc/rancher/k3s/cluster-token +node-name: easyai-shenzhen +node-ip: 10.77.0.4 +flannel-iface: wg0 +node-label: + - easyai.io/site=hongkong + - easyai.io/workload=true + - easyai.io/database=false + - easyai.io/worker=true +node-taint: + - easyai.io/worker-only=true:NoSchedule +EOF +chmod 0600 /etc/rancher/k3s/config.yaml /etc/rancher/k3s/registries.yaml +export DEBIAN_FRONTEND=noninteractive +if ! command -v netfilter-persistent >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq iptables-persistent +fi +ensure_rule() { + if ! iptables -C INPUT "$@" 2>/dev/null; then iptables -I INPUT 1 "$@"; fi +} +public_interface=$(ip -o route get 1.1.1.1 | awk '{for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}') +ensure_rule -i wg0 -s 10.77.0.0/24 -p tcp -m multiport --dports 6443,10250 \ + -m comment --comment easyai-k3s-private -j ACCEPT +ensure_rule -i wg0 -s 10.77.0.0/24 -p udp --dport 8472 \ + -m comment --comment easyai-flannel-private -j ACCEPT +ensure_rule -i "$public_interface" -p tcp -m multiport --dports 6443,10250 \ + -m comment --comment easyai-k3s-public-drop -j DROP +ensure_rule -i "$public_interface" -p udp --dport 8472 \ + -m comment --comment easyai-flannel-public-drop -j DROP +ensure_rule -i "$public_interface" -p tcp --dport 30000:31057 \ + -m comment --comment easyai-nodeport-public-drop-low -j DROP +ensure_rule -i "$public_interface" -p tcp --dport 31059:32767 \ + -m comment --comment easyai-nodeport-public-drop-high -j DROP +netfilter-persistent save >/dev/null +install_script=$(mktemp) +trap 'rm -f -- "$install_script"' EXIT +curl -fsSL https://get.k3s.io -o "$install_script" +chmod 0700 "$install_script" +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION="$version" \ + INSTALL_K3S_EXEC=agent "$install_script" +systemctl is-active --quiet k3s-agent +systemctl is-enabled --quiet k3s-agent +[[ ! -e /var/lib/rancher/k3s/server/db/etcd ]] +REMOTE + + local ready=false + for _ in $(seq 1 90); do + if cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "k3s kubectl get node $node_name --no-headers 2>/dev/null | awk '\$2==\"Ready\" {found=1} END {exit !found}'"; then + ready=true + break + fi + sleep 2 + done + [[ $ready == true ]] + verify_agent +} + +verify_agent() { + verify_wireguard + cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$node_name" <<'REMOTE' +set -euo pipefail +node=$1 +kubectl='k3s kubectl' +[[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/workload}') == true ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]] +[[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]] +[[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json | + jq '[.items[] | select(.metadata.namespace!="kube-system")] | length') -eq 0 ]] +printf 'shenzhen_k3s_agent=PASS node=%s role=worker-only etcd_member=false\n' "$node" +REMOTE +} + +case $action in + preflight) preflight_node ;; + network) preflight_node; configure_wireguard ;; + install) preflight_node; install_agent ;; + verify) preflight_node; verify_agent ;; + all) preflight_node; configure_wireguard; install_agent ;; +esac diff --git a/scripts/cluster/bootstrap-wireguard.sh b/scripts/cluster/bootstrap-wireguard.sh index 9827784..14f8922 100755 --- a/scripts/cluster/bootstrap-wireguard.sh +++ b/scripts/cluster/bootstrap-wireguard.sh @@ -11,6 +11,11 @@ require_commands ssh scp hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}") wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3) +if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then + hosts+=("$CLUSTER_SHENZHEN_HOST") + public_endpoints+=("${CLUSTER_SHENZHEN_HOST#*@}") + wireguard_ips+=(10.77.0.4) +fi public_keys=() for host in "${hosts[@]}"; do @@ -39,26 +44,24 @@ REMOTE public_keys+=("$public_key") done -for local_index in 0 1 2; do - peer_indexes=() - for candidate in 0 1 2; do - [[ $candidate -eq $local_index ]] || peer_indexes+=("$candidate") +for local_index in "${!hosts[@]}"; do + peer_arguments=() + for candidate in "${!hosts[@]}"; do + if [[ $candidate -ne $local_index ]]; then + peer_arguments+=( + "${public_keys[$candidate]}" + "${public_endpoints[$candidate]}" + "${wireguard_ips[$candidate]}" + ) + fi done - first=${peer_indexes[0]} - second=${peer_indexes[1]} echo "[wireguard] configuring ${hosts[$local_index]} as ${wireguard_ips[$local_index]}" cluster_ssh "${hosts[$local_index]}" bash -s -- \ "${wireguard_ips[$local_index]}" \ - "${public_keys[$first]}" "${public_endpoints[$first]}" "${wireguard_ips[$first]}" \ - "${public_keys[$second]}" "${public_endpoints[$second]}" "${wireguard_ips[$second]}" <<'REMOTE' + "${peer_arguments[@]}" <<'REMOTE' set -euo pipefail local_ip=$1 -peer_one_key=$2 -peer_one_endpoint=$3 -peer_one_ip=$4 -peer_two_key=$5 -peer_two_endpoint=$6 -peer_two_ip=$7 +shift private_key=$(cat /etc/wireguard/privatekey) umask 077 temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX) @@ -67,19 +70,19 @@ cat >"$temporary_config" <= 3 )); do + cat >>"$temporary_config" </dev/null @@ -87,8 +90,8 @@ systemctl restart wg-quick@wg0 REMOTE done -for source_index in 0 1 2; do - for target_index in 0 1 2; do +for source_index in "${!hosts[@]}"; do + for target_index in "${!hosts[@]}"; do [[ $source_index -eq $target_index ]] && continue cluster_ssh "${hosts[$source_index]}" \ "ping -c 3 -W 2 ${wireguard_ips[$target_index]} >/dev/null" @@ -98,10 +101,10 @@ done for host in "${hosts[@]}"; do latest_handshake=$(cluster_ssh "$host" \ "wg show wg0 latest-handshakes | awk '\$2 > 0 { count++ } END { print count + 0 }'") - [[ $latest_handshake == 2 ]] || { + [[ $latest_handshake == $((${#hosts[@]} - 1)) ]] || { echo "WireGuard handshake verification failed on $host" >&2 exit 1 } done -echo 'wireguard_bootstrap=PASS peers=3' +echo "wireguard_bootstrap=PASS peers=${#hosts[@]}" diff --git a/scripts/cluster/common.sh b/scripts/cluster/common.sh index 9098619..e3c89a8 100755 --- a/scripts/cluster/common.sh +++ b/scripts/cluster/common.sh @@ -28,7 +28,8 @@ load_cluster_env() { CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST} CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2} CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3} - export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST + CLUSTER_SHENZHEN_HOST=${AI_GATEWAY_SHENZHEN_HOST:-} + export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST CLUSTER_SHENZHEN_HOST for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || { @@ -36,6 +37,10 @@ load_cluster_env() { return 1 } done + if [[ -n $CLUSTER_SHENZHEN_HOST && ! $CLUSTER_SHENZHEN_HOST =~ ^root@[A-Za-z0-9.-]+$ ]]; then + echo "Shenzhen Worker host must use root@host syntax" >&2 + return 1 + fi [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || { echo 'invalid OSS region or bucket' >&2 return 1 diff --git a/scripts/cluster/monitor-production-release.sh b/scripts/cluster/monitor-production-release.sh index d64213b..9d84dbf 100755 --- a/scripts/cluster/monitor-production-release.sh +++ b/scripts/cluster/monitor-production-release.sh @@ -140,12 +140,17 @@ pool_saturated_pods() { check_wireguard() { local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") local -a ips=(10.77.0.1 10.77.0.2 10.77.0.3) - local source target output loss average handshakes - for source in 0 1 2; do + [[ -z $CLUSTER_SHENZHEN_HOST ]] || { + hosts+=("$CLUSTER_SHENZHEN_HOST") + ips+=(10.77.0.4) + } + local source target output loss average handshakes peer_count + peer_count=$((${#hosts[@]} - 1)) + for source in "${!hosts[@]}"; do handshakes=$(cluster_ssh "${hosts[$source]}" \ "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'") - (( handshakes == 2 )) || return 1 - for target in 0 1 2; do + (( handshakes == peer_count )) || return 1 + for target in "${!hosts[@]}"; do (( source == target )) && continue output=$(cluster_ssh "${hosts[$source]}" "ping -q -c 10 -W 2 ${ips[$target]}") || return 1 loss=$(awk -F',' '/packet loss/ {gsub(/[^0-9.]/,"",$3); print $3}' <<<"$output") diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 118dc84..f5ab501 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -99,6 +99,10 @@ require_commands curl git go jq node openssl sed shasum : "${AI_GATEWAY_ACCEPTANCE_ETCD_WAL_FSYNC_MAX_MS:=15}" : "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}" : "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}" +: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO:=1}" +: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG:=1}" +: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO}" +: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG}" : "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}" : "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}" : "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}" @@ -167,11 +171,20 @@ fi echo 'etcd latency gates must be positive integer milliseconds' >&2 exit 1 } -[[ $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[1-9][0-9]*$ && - $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[1-9][0-9]*$ && +[[ $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO =~ ^[0-9]+$ && + $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG =~ ^[0-9]+$ && + $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO =~ ^[0-9]+$ && + $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG =~ ^[0-9]+$ && + $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[0-9]+$ && + $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[0-9]+$ && + $((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG)) -eq 2 && + $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO && + $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO && + $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG && + $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG && $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 && $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG -le 16 ]] || { - echo 'acceptance autoscaling replica caps must be between 1 and 16' >&2 + echo 'acceptance requires exactly two baseline Workers and ordered 0..16 per-site autoscaling bounds' >&2 exit 1 } @@ -263,8 +276,8 @@ image_stable_throughput= image_admitted_throughput= video_stable_throughput= video_admitted_throughput= -certified_max_replicas_ningbo=1 -certified_max_replicas_hongkong=1 +certified_max_replicas_ningbo=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO +certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG runtime_observation_started_at= cleanup() { @@ -636,21 +649,45 @@ verify_release_cas() { -o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]] done [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \ - -o 'jsonpath={.spec.replicas}') == 1 ]] + -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]] [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \ - -o 'jsonpath={.spec.replicas}') == 1 ]] + -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]] + if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 && + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) && + [[ -n $CLUSTER_SHENZHEN_HOST ]]; then + local worker_node pod_count + for worker_node in easyai-hongkong easyai-shenzhen; do + pod_count=$(remote_kubectl get pods -n "$namespace" \ + -l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \ + --field-selector "spec.nodeName=$worker_node" -o json | jq \ + '[.items[] | select(.status.phase=="Running")] | length') + [[ $pod_count == 1 ]] || { + echo "Hong Kong logical Worker pool is not spread across both physical nodes: node=$worker_node running_pods=$pod_count" >&2 + return 1 + } + done + fi } verify_resource_preconditions() { local report=$temporary_root/resource-preconditions.csv - local ningbo_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads - ningbo_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads + worker_nodes= + if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then + worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + worker_nodes+=$'\n' + fi + if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then + worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong' \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + fi + worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u) los_angeles_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=los-angeles' \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') - [[ -n $ningbo_nodes && -n $los_angeles_nodes ]] + [[ -n $worker_nodes && -n $los_angeles_nodes ]] invalid_mixed_workloads=$(remote_kubectl get pods -A -o json | - jq --argjson nodes "$(jq -Rn --arg value "$ningbo_nodes" '$value | split("\n") | map(select(length>0))')" ' + jq --argjson nodes "$(jq -Rn --arg value "$worker_nodes" '$value | split("\n") | map(select(length>0))')" ' [.items[] | select(.spec.nodeName as $name | $nodes | index($name)) | select(.metadata.name | test("staging|runner|(^|-)ci(-|$)"; "i")) @@ -661,7 +698,7 @@ verify_resource_preconditions() { (.resources.limits.cpu // "") == ""))] | length') [[ $invalid_mixed_workloads == 0 ]] || { - echo 'Ningbo staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2 + echo 'Worker-site staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2 return 1 } witness_workloads=$(remote_kubectl get pods -A -o json | @@ -687,12 +724,12 @@ verify_resource_preconditions() { awk '{value=$5; sub(/%$/, "", value); print value}') [[ $memory_percent =~ ^[0-9]+$ && $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || { - echo "Ningbo memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2 + echo "Worker node memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2 return 1 } printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \ "$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report" - done <<<"$ningbo_nodes" + done <<<"$worker_nodes" (( sample == 90 )) || sleep 10 done cp "$report" "$report_root/resource-preconditions.csv" @@ -1440,19 +1477,27 @@ fail_acceptance_gate() { apply_capacity_profile() { local profile=$1 - local slots pool media global + local slots pool media global baseline_replicas local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS local min_idle=4 local max_idle_seconds=30 case $profile in - P24) slots=24; pool=32; media=24; global=48 ;; - P28) slots=28; pool=36; media=28; global=56 ;; - P32) slots=32; pool=40; media=32; global=64 ;; + P24) slots=24; pool=32; media=24 ;; + P28) slots=28; pool=36; media=28 ;; + P32) slots=32; pool=40; media=32 ;; *) return 1 ;; esac + baseline_replicas=$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG)) + global=$((slots * baseline_replicas)) local command_text capacity_output printf -v command_text \ - 'AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \ + 'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \ + "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \ + "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \ + "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \ + "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \ "$slots" "$global" "$global" "$pool" "$api_pool" \ "$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" "$AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS" \ "$min_idle" "$max_idle_seconds" \ @@ -1605,8 +1650,9 @@ apply_autoscaling_profile() { local max_replicas_ningbo=${2:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO} local max_replicas_hongkong=${3:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG} local slots pool media global - [[ $max_replicas_ningbo =~ ^[1-9][0-9]*$ && - $max_replicas_hongkong =~ ^[1-9][0-9]*$ ]] || return 1 + [[ $max_replicas_ningbo =~ ^[0-9]+$ && + $max_replicas_hongkong =~ ^[0-9]+$ && + $((max_replicas_ningbo + max_replicas_hongkong)) -ge 1 ]] || return 1 (( max_replicas_ningbo <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO && max_replicas_hongkong <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG )) || return 1 case $profile in @@ -1619,7 +1665,11 @@ apply_autoscaling_profile() { global=$((slots * (max_replicas_ningbo + max_replicas_hongkong))) local command_text capacity_output printf -v command_text \ - 'AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \ + 'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \ + "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \ + "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \ + "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \ "$max_replicas_ningbo" \ "$max_replicas_hongkong" \ "$slots" "$global" "$global" "$pool" \ @@ -1697,14 +1747,15 @@ run_autoscaling_topology_phase() { "$(jq -r '.plan.rawDesired // 0' <<<"${status:-{}}")" \ "$(jq -r '.plan.desiredTotal // 0' <<<"${status:-{}}")" \ "$(jq -r '.plan.frozenReason // ""' <<<"${status:-{}}")" "$resource_max" >>"$observations" - if (( ningbo_replicas == 1 && hongkong_replicas == 1 )); then + if (( ningbo_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO && + hongkong_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG )); then drained=true break fi sleep 10 done [[ $drained == true ]] || { - echo "capacity controller did not safely drain $phase to topology 1+1" >&2 + echo "capacity controller did not safely drain $phase to configured minimum topology" >&2 return 1 } local unsafe_residue @@ -1734,15 +1785,18 @@ run_autoscaling_acceptance() { local report=$report_root/autoscaling.json local observations=$report_root/autoscaling-observations.csv local status deadline resource_max_hongkong resource_max_ningbo - local tested_1_plus_2=false tested_2_plus_2=false + local tested_south_pool_scale_out=false tested_1_plus_2=false tested_2_plus_2=false local requested_hongkong_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG local requested_ningbo_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO (( requested_hongkong_max > 2 )) && requested_hongkong_max=2 (( requested_ningbo_max > 2 )) && requested_ningbo_max=2 - autoscaling_phase_observed_max=2 + autoscaling_phase_observed_max=$(( + AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO + + AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG + )) echo 'timestamp,phase,ningbo_replicas,hongkong_replicas,raw_desired,desired_total,frozen_reason,resource_max_total' >"$observations" - apply_autoscaling_profile "$stable_profile" 1 "$requested_hongkong_max" + apply_autoscaling_profile "$stable_profile" "$requested_ningbo_max" "$requested_hongkong_max" deadline=$((SECONDS + 180)) while (( SECONDS < deadline )); do status=$(capacity_controller_status || true) @@ -1755,7 +1809,17 @@ run_autoscaling_acceptance() { } resource_max_hongkong=$(jq -r \ '[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status") - if (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then + if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 && + AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG < AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG && + requested_hongkong_max >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG && + resource_max_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG )); then + run_autoscaling_topology_phase south-pool-scale-out \ + "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \ + "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" "$observations" + tested_south_pool_scale_out=true + certified_max_replicas_ningbo=0 + certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG + elif (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then run_autoscaling_topology_phase 1-plus-2 1 2 "$observations" tested_1_plus_2=true certified_max_replicas_hongkong=2 @@ -1785,6 +1849,9 @@ run_autoscaling_acceptance() { jq -n \ --arg profile "$stable_profile" \ + --argjson baselineNingbo "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \ + --argjson baselineHongkong "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \ + --argjson testedSouthPoolScaleOut "$tested_south_pool_scale_out" \ --argjson testedOnePlusTwo "$tested_1_plus_2" \ --argjson testedTwoPlusTwo "$tested_2_plus_2" \ --argjson observedMaxReplicas "$autoscaling_phase_observed_max" \ @@ -1792,8 +1859,10 @@ run_autoscaling_acceptance() { passed:true, drained:true, capacityProfile:$profile, + baseline:{ningbo:$baselineNingbo,hongkong:$baselineHongkong}, topologies:{ - "1+1":true, + "1+1":($baselineNingbo == 1 and $baselineHongkong == 1), + "0+1_to_0+2":$testedSouthPoolScaleOut, "1+2":$testedOnePlusTwo, "2+2":$testedTwoPlusTwo }, @@ -2002,11 +2071,17 @@ WHERE acceptance_run_id IS NULL } activate_certified_autoscaling() { - local deadline status resource_max + local deadline status resource_max test_min_ningbo test_min_hongkong + test_min_ningbo=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO + test_min_hongkong=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG + AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG apply_autoscaling_profile \ "$stable_profile" \ "$certified_max_replicas_ningbo" \ "$certified_max_replicas_hongkong" + AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$test_min_ningbo + AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$test_min_hongkong deadline=$((SECONDS + 180)) while (( SECONDS < deadline )); do status=$(capacity_controller_status || true) @@ -2023,7 +2098,9 @@ activate_certified_autoscaling() { '[.plan.sites[] | select(.site == "ningbo") | .resourceMax][0] // 0' <<<"$status") certified_max_replicas_hongkong=$(jq -r \ '[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status") - (( certified_max_replicas_ningbo >= 1 && certified_max_replicas_hongkong >= 1 )) + (( certified_max_replicas_ningbo >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO && + certified_max_replicas_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG && + certified_max_replicas_ningbo + certified_max_replicas_hongkong >= 2 )) jq -n \ --arg profile "$stable_profile" \ --argjson ningbo "$certified_max_replicas_ningbo" \ @@ -2035,9 +2112,9 @@ activate_certified_autoscaling() { resourceMaxTotal:$total }' >"$report_root/certified-site-capacity.json" [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \ - -o 'jsonpath={.spec.replicas}') == 1 ]] + -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]] [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \ - -o 'jsonpath={.spec.replicas}') == 1 ]] + -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]] stage_capacity_profiles verify_runtime_gates } @@ -2332,6 +2409,14 @@ verify_cluster_links() { verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo' verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles' verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong' + if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then + verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_shenzhen' + verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 'shenzhen_to_ningbo' + verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_shenzhen' + verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 'shenzhen_to_hongkong' + verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_shenzhen' + verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 'shenzhen_to_los_angeles' + fi } verify_control_plane_and_recent_logs() { @@ -2349,6 +2434,11 @@ verify_control_plane_and_recent_logs() { [[ $error_count == 0 ]] || return 1 fi done + if [[ -n $CLUSTER_SHENZHEN_HOST && -n $runtime_observation_started_at ]]; then + error_count=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" \ + "journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'") + [[ $error_count == 0 ]] || return 1 + fi for workload in api worker; do while read -r pod; do error_count=$(remote_kubectl logs "$pod" -n "$namespace" "$log_window" | diff --git a/scripts/cluster/verify-cluster.sh b/scripts/cluster/verify-cluster.sh index 9a74e91..2fb4702 100755 --- a/scripts/cluster/verify-cluster.sh +++ b/scripts/cluster/verify-cluster.sh @@ -15,14 +15,20 @@ load_cluster_env hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3) site_labels=(ningbo hongkong losangeles) -for source_index in 0 1 2; do +if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then + hosts+=("$CLUSTER_SHENZHEN_HOST") + wireguard_ips+=(10.77.0.4) + site_labels+=(shenzhen) +fi +peer_count=$((${#hosts[@]} - 1)) +for source_index in "${!hosts[@]}"; do handshakes=$(cluster_ssh "${hosts[$source_index]}" \ "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime() - \$2 < 180 { count++ } END { print count + 0 }'") - [[ $handshakes -eq 2 ]] || { + [[ $handshakes -eq $peer_count ]] || { echo "stale WireGuard handshake on ${site_labels[$source_index]}: recent_peers=$handshakes" >&2 exit 1 } - for target_index in 0 1 2; do + for target_index in "${!hosts[@]}"; do [[ $source_index -eq $target_index ]] && continue ping_output=$(cluster_ssh "${hosts[$source_index]}" \ "ping -i 0.2 -c 10 -W 2 ${wireguard_ips[$target_index]}") @@ -46,12 +52,15 @@ for source_index in 0 1 2; do fi done done -echo 'wireguard_acceptance=PASS handshakes=6 packet_loss_lt_1=true' +echo "wireguard_acceptance=PASS peers=${#hosts[@]} handshakes=$((${#hosts[@]} * peer_count)) packet_loss_lt_1=true" -cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +expected_nodes=3 +[[ -z $CLUSTER_SHENZHEN_HOST ]] || expected_nodes=4 +cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$expected_nodes" <<'REMOTE' set -euo pipefail +expected_nodes=$1 kubectl='k3s kubectl' -[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]] +[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq $expected_nodes ]] ready_output=$($kubectl get --raw='/readyz?verbose') grep -Fq '[+]etcd ok' <<<"$ready_output" [[ $($kubectl get node easyai-los-angeles -o json | @@ -62,8 +71,24 @@ grep -Fq '[+]etcd ok' <<<"$ready_output" printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)" REMOTE ) -[[ $cluster_state == nodes=3 ]] -echo 'k3s_control_plane=PASS servers=3 witness_business_pods=0' +[[ $cluster_state == nodes="$expected_nodes" ]] +if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then + worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +kubectl='k3s kubectl' +node=easyai-shenzhen +[[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]] +[[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]] +[[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json | + jq '[.items[] | select(.metadata.namespace!="kube-system" and .metadata.labels["app.kubernetes.io/name"]!="easyai-worker")] | length') -eq 0 ]] +printf 'worker_node=ready\n' +REMOTE +) + [[ $worker_state == worker_node=ready ]] +fi +echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witness_business_pods=0" database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' set -euo pipefail @@ -143,8 +168,11 @@ kubectl='k3s kubectl' for site in ningbo hongkong; do [[ $($kubectl get deployment "easyai-api-$site" -n easyai \ -o jsonpath='{.status.readyReplicas}') -eq 1 ]] - [[ $($kubectl get deployment "easyai-worker-$site" -n easyai \ - -o jsonpath='{.status.readyReplicas}') -eq 1 ]] + worker_replicas=$($kubectl get deployment "easyai-worker-$site" -n easyai \ + -o json | jq -r '.spec.replicas') + worker_ready=$($kubectl get deployment "easyai-worker-$site" -n easyai \ + -o json | jq -r '.status.readyReplicas // 0') + [[ $worker_ready -eq $worker_replicas ]] [[ $($kubectl get deployment "easyai-api-$site" -n easyai \ -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]] [[ $($kubectl get deployment "easyai-worker-$site" -n easyai \ diff --git a/scripts/release-manifest.mjs b/scripts/release-manifest.mjs index eeb4f58..9afcf8a 100755 --- a/scripts/release-manifest.mjs +++ b/scripts/release-manifest.mjs @@ -149,9 +149,11 @@ function validate(manifest) { !Number.isInteger(manifest.certification.workerInstanceSlots) || manifest.certification.workerInstanceSlots < 1 || !Number.isInteger(manifest.certification.workerMaxReplicasNingbo) || - manifest.certification.workerMaxReplicasNingbo < 1 || + manifest.certification.workerMaxReplicasNingbo < 0 || !Number.isInteger(manifest.certification.workerMaxReplicasHongkong) || - manifest.certification.workerMaxReplicasHongkong < 1) { + manifest.certification.workerMaxReplicasHongkong < 0 || + manifest.certification.workerMaxReplicasNingbo + + manifest.certification.workerMaxReplicasHongkong < 1) { fail('certification record is invalid') } } diff --git a/tests/release/manual-release-test.sh b/tests/release/manual-release-test.sh index ef337a0..4b80b67 100755 --- a/tests/release/manual-release-test.sh +++ b/tests/release/manual-release-test.sh @@ -151,11 +151,12 @@ RELEASE_ACCEPTANCE_RUN_ID=11111111-1111-4111-8111-111111111111 \ RELEASE_CONFIG_HASH=$(printf '3%.0s' {1..64}) \ RELEASE_CAPACITY_PROFILE=P24 \ RELEASE_WORKER_INSTANCE_SLOTS=24 \ -RELEASE_WORKER_MAX_REPLICAS_NINGBO=1 \ +RELEASE_WORKER_MAX_REPLICAS_NINGBO=0 \ RELEASE_WORKER_MAX_REPLICAS_HONGKONG=2 \ node "$root/scripts/release-manifest.mjs" record-certification "$recorded" "$certified" >/dev/null node "$root/scripts/release-manifest.mjs" validate "$certified" >/dev/null [[ $(node "$root/scripts/release-manifest.mjs" get "$certified" certification.capacityProfile) == P24 ]] +[[ $(node "$root/scripts/release-manifest.mjs" get "$certified" certification.workerMaxReplicasNingbo) == 0 ]] sensitive=$tmp/sensitive.json node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync(process.argv[1])); value.accessToken="forbidden"; fs.writeFileSync(process.argv[2], JSON.stringify(value))' "$manifest" "$sensitive"