easyai-ai-gateway/scripts/cluster/verify-cluster.sh

#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=scripts/cluster/common.sh
source "$script_dir/common.sh"

mode=${1:-precutover}
[[ $mode == precutover || $mode == postcutover ]] || {
  echo 'usage: verify-cluster.sh [precutover|postcutover]' >&2
  exit 64
}
load_cluster_env

hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
site_labels=(ningbo hongkong losangeles)
if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
  hosts+=("$CLUSTER_WORKER_NODE_4_HOST")
  wireguard_ips+=(10.77.0.4)
  site_labels+=("$CLUSTER_WORKER_NODE_4_NAME")
fi
peer_count=$((${#hosts[@]} - 1))
for source_index in "${!hosts[@]}"; do
  handshakes=$(cluster_ssh "${hosts[$source_index]}" \
    "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime() - \$2 < 180 { count++ } END { print count + 0 }'")
  [[ $handshakes -eq $peer_count ]] || {
    echo "stale WireGuard handshake on ${site_labels[$source_index]}: recent_peers=$handshakes" >&2
    exit 1
  }
  for target_index in "${!hosts[@]}"; do
    [[ $source_index -eq $target_index ]] && continue
    ping_output=$(cluster_ssh "${hosts[$source_index]}" \
      "ping -i 0.2 -c 10 -W 2 ${wireguard_ips[$target_index]}")
    packet_loss=$(sed -nE 's/.* ([0-9.]+)% packet loss.*/\1/p' <<<"$ping_output")
    average_rtt=$(sed -nE 's#.* = [0-9.]+/([0-9.]+)/.*#\1#p' <<<"$ping_output")
    echo "wireguard_path=${site_labels[$source_index]}_to_${site_labels[$target_index]} loss_percent=$packet_loss average_rtt_ms=$average_rtt"
    awk -v loss="$packet_loss" 'BEGIN { exit !(loss < 1) }' || {
      echo "WireGuard packet loss gate failed on ${site_labels[$source_index]} to ${site_labels[$target_index]}: loss_percent=$packet_loss" >&2
      exit 1
    }
    if [[ $source_index -eq 2 || $target_index -eq 2 ]]; then
      awk -v rtt="$average_rtt" 'BEGIN { exit !(rtt < 300) }' || {
        echo "WireGuard RTT gate failed on ${site_labels[$source_index]} to ${site_labels[$target_index]}: average_rtt_ms=$average_rtt limit_ms=300" >&2
        exit 1
      }
    else
      awk -v rtt="$average_rtt" 'BEGIN { exit !(rtt < 80) }' || {
        echo "WireGuard RTT gate failed on ${site_labels[$source_index]} to ${site_labels[$target_index]}: average_rtt_ms=$average_rtt limit_ms=80" >&2
        exit 1
      }
    fi
  done
done
echo "wireguard_acceptance=PASS peers=${#hosts[@]} handshakes=$((${#hosts[@]} * peer_count)) packet_loss_lt_1=true"

expected_nodes=3
[[ $CLUSTER_WORKER_NODE_4_ENABLED != true ]] || expected_nodes=4
cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$expected_nodes" <<'REMOTE'
set -euo pipefail
expected_nodes=$1
kubectl='k3s kubectl'
[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq $expected_nodes ]]
ready_output=$($kubectl get --raw='/readyz?verbose')
grep -Fq '[+]etcd ok' <<<"$ready_output"
[[ $($kubectl get node easyai-los-angeles -o json |
  jq '[.spec.taints[]? | select(.key=="easyai.io/witness" and .effect=="NoSchedule")] | length') -eq 1 ]]
[[ $($kubectl get pods -A -o json |
  jq '[.items[] | select(.spec.nodeName=="easyai-los-angeles") |
    select(.metadata.namespace != "kube-system" and .metadata.namespace != "cnpg-system" and .metadata.namespace != "cert-manager")] | length') -eq 0 ]]
printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)"
REMOTE
)
[[ $cluster_state == nodes="$expected_nodes" ]]
if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
  worker_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- \
    "$CLUSTER_WORKER_NODE_4_NAME" "$CLUSTER_WORKER_NODE_4_SITE" <<'REMOTE'
set -euo pipefail
kubectl='k3s kubectl'
node=$1
site=$2
[[ $($kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') == True ]]
[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/site}') == "$site" ]]
[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/worker}') == true ]]
[[ $($kubectl get node "$node" -o jsonpath='{.metadata.labels.easyai\.io/database}') == false ]]
[[ $($kubectl get node "$node" -o json | jq '[.spec.taints[]? | select(.key=="easyai.io/worker-only" and .effect=="NoSchedule")] | length') -eq 1 ]]
[[ $($kubectl get pods -A --field-selector "spec.nodeName=$node" -o json |
  jq '[.items[] | select(.metadata.namespace!="kube-system" and .metadata.labels["app.kubernetes.io/name"]!="easyai-worker")] | length') -eq 0 ]]
printf 'worker_node=ready node=%s site=%s\n' "$node" "$site"
REMOTE
)
  [[ $worker_state == "worker_node=ready node=$CLUSTER_WORKER_NODE_4_NAME site=$CLUSTER_WORKER_NODE_4_SITE" ]]
fi
echo "k3s_control_plane=PASS servers=3 worker_nodes=$((expected_nodes - 3)) witness_business_pods=0"

database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
set -euo pipefail
kubectl='k3s kubectl'
ready_instances=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}')
[[ $ready_instances -eq 2 ]] || {
  echo "postgres ready instance gate failed: ready_instances=$ready_instances expected=2" >&2
  exit 1
}
primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
primary_node=$($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}')
[[ $primary_node == easyai-ningbo || $primary_node == easyai-hongkong ]] || {
  echo "postgres primary placement gate failed: primary_node=$primary_node" >&2
  exit 1
}
replication=$($kubectl exec -n easyai "$primary" -c postgres -- \
  psql -X -U postgres -d easyai_ai_gateway -At -c \
  "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
[[ $replication -eq 1 ]] || {
  echo "postgres synchronous replication gate failed: synchronous_replicas=$replication expected=1" >&2
  exit 1
}
archived_count=$($kubectl exec -n easyai "$primary" -c postgres -- \
  psql -X -U postgres -d postgres -At -c \
  'SELECT archived_count FROM pg_stat_archiver;')
(( archived_count > 0 )) || {
  echo "postgres archive gate failed: archived_count=$archived_count" >&2
  exit 1
}
continuous_archiving=$($kubectl get cluster easyai-postgres -n easyai \
  -o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}')
[[ $continuous_archiving == True ]] || {
  echo "postgres continuous archiving gate failed: status=$continuous_archiving" >&2
  exit 1
}
completed_backups=$($kubectl get backups -n easyai -o json |
  jq '[.items[] | select(
    .spec.cluster.name == "easyai-postgres" and .status.phase == "completed"
  )] | length')
(( completed_backups > 0 )) || {
  echo "postgres backup gate failed: completed_backups=$completed_backups" >&2
  exit 1
}
readable_backup_configs=0
for pod in $($kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
  -o jsonpath='{.items[*].metadata.name}'); do
  if $kubectl exec -n easyai "$pod" -c postgres -- \
    test -r /projected/barman-aws/config; then
    readable_backup_configs=$((readable_backup_configs + 1))
  else
    echo "postgres projected backup config gate failed: pod=$pod" >&2
    exit 1
  fi
done
[[ $readable_backup_configs -eq 2 ]] || {
  echo "postgres projected backup config count failed: readable=$readable_backup_configs expected=2" >&2
  exit 1
}
printf 'primary=%s archived=%s backups=%s ready=%s synchronous=%s continuous_archiving=%s backup_configs=%s\n' \
  "$primary" "$archived_count" "$completed_backups" "$ready_instances" \
  "$replication" "$continuous_archiving" "$readable_backup_configs"
REMOTE
)
[[ $database_state == primary=easyai-postgres-* ]]
echo "postgres_preflight=PASS $database_state synchronous_replicas=1"

if [[ $mode == postcutover ]]; then
  application_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
set -euo pipefail
kubectl='k3s kubectl'
[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-api \
  --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-web \
  --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-worker \
  --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
for site in ningbo hongkong; do
  [[ $($kubectl get deployment "easyai-api-$site" -n easyai \
    -o jsonpath='{.status.readyReplicas}') -eq 1 ]]
  worker_replicas=$($kubectl get deployment "easyai-worker-$site" -n easyai \
    -o json | jq -r '.spec.replicas')
  worker_ready=$($kubectl get deployment "easyai-worker-$site" -n easyai \
    -o json | jq -r '.status.readyReplicas // 0')
  [[ $worker_ready -eq $worker_replicas ]]
  [[ $($kubectl get deployment "easyai-api-$site" -n easyai \
    -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]]
  [[ $($kubectl get deployment "easyai-worker-$site" -n easyai \
    -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
done
images=$($kubectl get deployments -n easyai -o json |
  jq -r '.items[] | select(.metadata.name | startswith("easyai-")) |
    .spec.template.spec.containers[].image')
if grep -Ev '@sha256:[0-9a-f]{64}$' <<<"$images" | grep -q .; then
  echo 'one or more application images are not digest-pinned' >&2
  exit 1
fi
[[ $($kubectl get secret easyai-gateway-security-events -n easyai \
  -o json | jq '.data | length') -eq 4 ]]
[[ $($kubectl auth can-i update secret/easyai-gateway-security-events -n easyai \
  --as=system:serviceaccount:easyai:easyai-ai-gateway) == yes ]]
[[ $($kubectl auth can-i update secret/easyai-ai-gateway-runtime -n easyai \
  --as=system:serviceaccount:easyai:easyai-ai-gateway) == no ]]
printf 'applications=ready\n'
REMOTE
)
  [[ $application_state == applications=ready ]]
  echo 'application_acceptance=PASS api=2 worker=2 web=2 roles=isolated digests=pinned'

  for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
    health_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
      https://ai.51easyai.com/api/v1/healthz)
    grep -q easyai-ai-gateway <<<"$health_body"
    ready_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
      https://ai.51easyai.com/api/v1/readyz)
    grep -q '"ok":true' <<<"$ready_body"
    openapi_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
      https://ai.51easyai.com/api/v1/openapi.json)
    grep -q '"/api/v1/chat/completions"' <<<"$openapi_body"
    web_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
      https://ai.51easyai.com/)
    grep -q 'EasyAI AI Gateway' <<<"$web_body"
  done
  echo 'nginx_edges=PASS ningbo=true hongkong=true dns_changed=false'
fi

echo "cluster_verification=PASS mode=$mode"