#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=scripts/cluster/common.sh
source "$script_dir/common.sh"

[[ ${1:-} == --execute ]] || {
  echo 'usage: verify-k3s-quorum.sh --execute' >&2
  exit 64
}
load_cluster_env

active_host=
restore_active_host() {
  [[ -z ${active_host:-} ]] ||
    cluster_ssh "$active_host" 'systemctl start k3s' >/dev/null 2>&1 || true
}
trap restore_active_host EXIT HUP INT TERM

drill_server() {
  local node_name=$1
  local target_host=$2
  local observer_host=$3
  local started elapsed
  active_host=$target_host
  started=$(date +%s)
  cluster_ssh "$target_host" 'systemctl stop k3s'
  cluster_ssh "$target_host" '! systemctl is-active --quiet k3s'
  cluster_ssh "$observer_host" bash -s <<'REMOTE'
set -euo pipefail
for _ in $(seq 1 60); do
  if k3s kubectl get --raw='/readyz?verbose' 2>/dev/null |
      grep -q '\[+\]etcd ok'; then
    exit 0
  fi
  sleep 2
done
exit 1
REMOTE
  curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz |
    grep -q '"ok":true'
  cluster_ssh "$target_host" 'systemctl start k3s'
  cluster_ssh "$observer_host" \
    "k3s kubectl wait --for=condition=Ready node/'$node_name' --timeout=300s"
  active_host=
  elapsed=$(( $(date +%s) - started ))
  echo "k3s_single_server_quorum=PASS node=$node_name recovery_seconds=$elapsed"
}

drill_server easyai-los-angeles "$CLUSTER_LOS_ANGELES_HOST" "$CLUSTER_NINGBO_HOST"
drill_server easyai-hongkong "$CLUSTER_HONGKONG_HOST" "$CLUSTER_NINGBO_HOST"
drill_server easyai-ningbo "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"

cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s <<'REMOTE'
set -euo pipefail
kubectl='k3s kubectl'
for _ in $(seq 1 240); do
  ready=$($kubectl get cluster easyai-postgres -n easyai \
    -o jsonpath='{.status.readyInstances}')
  if [[ $ready == 2 ]]; then
    primary=$($kubectl get cluster easyai-postgres -n easyai \
      -o jsonpath='{.status.currentPrimary}')
    synchronous_replicas=$($kubectl exec -n easyai "$primary" -c postgres -- \
      psql -X -U postgres -d postgres -At -c \
      "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
    if [[ $synchronous_replicas == 1 ]]; then
      echo "k3s_quorum_drill=PASS servers=3 database_primary=$primary synchronous_replicas=1"
      exit 0
    fi
  fi
  sleep 3
done
exit 1
REMOTE

trap - EXIT HUP INT TERM