diff --git a/scripts/cluster/add-worker-node.sh b/scripts/cluster/add-worker-node.sh index 89b7cdf..b41f559 100755 --- a/scripts/cluster/add-worker-node.sh +++ b/scripts/cluster/add-worker-node.sh @@ -151,10 +151,36 @@ REMOTE "$node_ip" "$worker_wireguard_port" <<'REMOTE' set -euo pipefail wg set wg0 peer "$1" allowed-ips "$3/32" endpoint "$2:$4" persistent-keepalive 25 -wg-quick save wg0 +wg-quick save wg0 >/dev/null REMOTE done + local handshakes_ready=false recent + for _ in $(seq 1 30); do + handshakes_ready=true + for host in "${server_hosts[@]}" "$CLUSTER_WORKER_NODE_4_HOST"; do + recent=$(cluster_ssh "$host" \ + "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'") + if [[ $recent != 3 ]]; then + handshakes_ready=false + break + fi + done + [[ $handshakes_ready == true ]] && break + sleep 2 + done + [[ $handshakes_ready == true ]] || { + echo 'Worker node WireGuard peers did not complete their initial handshakes within 60 seconds' >&2 + return 1 + } + # Prime both directions after the new peer endpoint is installed. These packets are not + # part of the measured 10-packet gate and only establish routes/NAT state. + cluster_ssh "$CLUSTER_NINGBO_HOST" "ping -q -c 3 -W 2 $node_ip >/dev/null" || true + cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" 'ping -q -c 3 -W 2 10.77.0.1 >/dev/null' || true + cluster_ssh "$CLUSTER_HONGKONG_HOST" "ping -q -c 3 -W 2 $node_ip >/dev/null" || true + cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" 'ping -q -c 3 -W 2 10.77.0.2 >/dev/null' || true + cluster_ssh "$CLUSTER_LOS_ANGELES_HOST" "ping -q -c 3 -W 2 $node_ip >/dev/null" || true + cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" 'ping -q -c 3 -W 2 10.77.0.3 >/dev/null' || true verify_wireguard } @@ -240,6 +266,8 @@ curl -fsSL https://get.k3s.io -o "$install_script" chmod 0700 "$install_script" INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION="$version" \ INSTALL_K3S_EXEC=agent "$install_script" +systemctl enable k3s-agent >/dev/null +systemctl is-active --quiet k3s-agent || systemctl start k3s-agent systemctl is-active --quiet k3s-agent systemctl is-enabled --quiet k3s-agent [[ ! -e /var/lib/rancher/k3s/server/db/etcd ]] @@ -276,6 +304,7 @@ kubectl='k3s kubectl' [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == "$site" ]] [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/workload}') == true ]] [[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]] +[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]] [[ $($kubectl get node easyai-hongkong -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]] [[ $($kubectl get node easyai-ningbo -o jsonpath='{.metadata.labels.easyai\.io/worker}') == false ]] [[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]