feat(cluster): 接入深圳专用 Worker 节点
新增深圳 K3s agent 与四节点 WireGuard 全互联接入脚本,将深圳归入香港逻辑 Worker 池并用污点限制为 Worker 专用。\n\n生产验收支持宁波 0、香港逻辑池 2 的双物理节点基线,补充副本拓扑分散、容量档位、链路巡检和零副本站点校验。\n\n已通过 Go 全量测试、go vet、govulncheck、迁移安全检查、ShellCheck、发布脚本测试、前后端 lint/test/build、Compose 与 Kubernetes 渲染校验。
This commit is contained in:
@@ -15,14 +15,20 @@ load_cluster_env
|
||||
hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
|
||||
wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
|
||||
site_labels=(ningbo hongkong losangeles)
|
||||
for source_index in 0 1 2; do
|
||||
if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
|
||||
hosts+=("$CLUSTER_SHENZHEN_HOST")
|
||||
wireguard_ips+=(10.77.0.4)
|
||||
site_labels+=(shenzhen)
|
||||
fi
|
||||
peer_count=$((${#hosts[@]} - 1))
|
||||
for source_index in "${!hosts[@]}"; do
|
||||
handshakes=$(cluster_ssh "${hosts[$source_index]}" \
|
||||
"wg show wg0 latest-handshakes | awk '\$2 > 0 && systime() - \$2 < 180 { count++ } END { print count + 0 }'")
|
||||
[[ $handshakes -eq 2 ]] || {
|
||||
[[ $handshakes -eq $peer_count ]] || {
|
||||
echo "stale WireGuard handshake on ${site_labels[$source_index]}: recent_peers=$handshakes" >&2
|
||||
exit 1
|
||||
}
|
||||
for target_index in 0 1 2; do
|
||||
for target_index in "${!hosts[@]}"; do
|
||||
[[ $source_index -eq $target_index ]] && continue
|
||||
ping_output=$(cluster_ssh "${hosts[$source_index]}" \
|
||||
"ping -i 0.2 -c 10 -W 2 ${wireguard_ips[$target_index]}")
|
||||
@@ -46,12 +52,15 @@ for source_index in 0 1 2; do
|
||||
fi
|
||||
done
|
||||
done
|
||||
echo 'wireguard_acceptance=PASS handshakes=6 packet_loss_lt_1=true'
|
||||
echo "wireguard_acceptance=PASS peers=${#hosts[@]} handshakes=$((${#hosts[@]} * peer_count)) packet_loss_lt_1=true"
|
||||
|
||||
cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
expected_nodes=3
|
||||
[[ -z $CLUSTER_SHENZHEN_HOST ]] || expected_nodes=4
|
||||
cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$expected_nodes" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
expected_nodes=$1
|
||||
kubectl='k3s kubectl'
|
||||
[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
|
||||
[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq $expected_nodes ]]
|
||||
ready_output=$($kubectl get --raw='/readyz?verbose')
|
||||
grep -Fq '[+]etcd ok' <<<"$ready_output"
|
||||
[[ $($kubectl get node easyai-los-angeles -o json |
|
||||
@@ -62,8 +71,24 @@ grep -Fq '[+]etcd ok' <<<"$ready_output"
|
||||
printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)"
|
||||
REMOTE
|
||||
)
|
||||
[[ $cluster_state == nodes=3 ]]
|
||||
echo 'k3s_control_plane=PASS servers=3 witness_business_pods=0'
|
||||
[[ $cluster_state == nodes="$expected_nodes" ]]
|
||||
if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
|
||||
worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
kubectl='k3s kubectl'
|
||||
node=easyai-shenzhen
|
||||
[[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]]
|
||||
[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == hongkong ]]
|
||||
[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]]
|
||||
[[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]
|
||||
[[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |
|
||||
jq '[.items[] | select(.metadata.namespace!="kube-system" and .metadata.labels["app.kubernetes.io/name"]!="easyai-worker")] | length') -eq 0 ]]
|
||||
printf 'worker_node=ready\n'
|
||||
REMOTE
|
||||
)
|
||||
[[ $worker_state == worker_node=ready ]]
|
||||
fi
|
||||
echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witness_business_pods=0"
|
||||
|
||||
database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
@@ -143,8 +168,11 @@ kubectl='k3s kubectl'
|
||||
for site in ningbo hongkong; do
|
||||
[[ $($kubectl get deployment "easyai-api-$site" -n easyai \
|
||||
-o jsonpath='{.status.readyReplicas}') -eq 1 ]]
|
||||
[[ $($kubectl get deployment "easyai-worker-$site" -n easyai \
|
||||
-o jsonpath='{.status.readyReplicas}') -eq 1 ]]
|
||||
worker_replicas=$($kubectl get deployment "easyai-worker-$site" -n easyai \
|
||||
-o json | jq -r '.spec.replicas')
|
||||
worker_ready=$($kubectl get deployment "easyai-worker-$site" -n easyai \
|
||||
-o json | jq -r '.status.readyReplicas // 0')
|
||||
[[ $worker_ready -eq $worker_replicas ]]
|
||||
[[ $($kubectl get deployment "easyai-api-$site" -n easyai \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]]
|
||||
[[ $($kubectl get deployment "easyai-worker-$site" -n easyai \
|
||||
|
||||
Reference in New Issue
Block a user