easyai-ai-gateway/scripts/cluster/common.sh

#!/usr/bin/env bash

set -euo pipefail

cluster_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
cluster_env_file=${AI_GATEWAY_CLUSTER_ENV_FILE:-"$cluster_root/.env.local"}
cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-"$HOME/.ssh/id_ed25519_easyai_gateway_cluster"}
cluster_ssh_control_dir=${AI_GATEWAY_CLUSTER_SSH_CONTROL_DIR:-"/tmp/easyai-gateway-ssh-${UID:-$(id -u)}-$$"}

prepare_cluster_ssh_control_dir() {
  (umask 077 && mkdir -p -- "$cluster_ssh_control_dir")
  chmod 0700 "$cluster_ssh_control_dir"
}

load_cluster_env() {
  [[ -f $cluster_env_file && ! -L $cluster_env_file ]] || {
    echo "missing local cluster environment file: $cluster_env_file" >&2
    return 1
  }
  set -a
  # shellcheck source=/dev/null
  source "$cluster_env_file"
  set +a
  cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-$cluster_ssh_key}

  : "${AI_GATEWAY_DEPLOY_HOST:?}"
  : "${AI_GATEWAY_DEPLOY_HOST_2:?}"
  : "${AI_GATEWAY_DEPLOY_HOST_3:?}"
  : "${ALI_KEY:?}"
  : "${ALI_SECRET:?}"
  : "${ALI_REGION:?}"
  : "${ALI_BUCKET:?}"

  CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST}
  CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2}
  CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3}
  CLUSTER_WORKER_NODE_4_HOST=${AI_GATEWAY_WORKER_NODE_4_HOST:-}
  CLUSTER_WORKER_NODE_4_NAME=${AI_GATEWAY_WORKER_NODE_4_NAME:-}
  CLUSTER_WORKER_NODE_4_SITE=${AI_GATEWAY_WORKER_NODE_4_SITE:-}
  CLUSTER_WORKER_NODE_4_WIREGUARD_PORT=${AI_GATEWAY_WORKER_NODE_4_WIREGUARD_PORT:-51820}
  CLUSTER_WORKER_NODE_4_NETWORK_APPROVED=${AI_GATEWAY_WORKER_NODE_4_NETWORK_APPROVED:-false}
  CLUSTER_WORKER_NODE_4_ENABLED=${AI_GATEWAY_WORKER_NODE_4_ENABLED:-false}
  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST
  export CLUSTER_WORKER_NODE_4_HOST CLUSTER_WORKER_NODE_4_NAME CLUSTER_WORKER_NODE_4_SITE
  export CLUSTER_WORKER_NODE_4_WIREGUARD_PORT CLUSTER_WORKER_NODE_4_NETWORK_APPROVED
  export CLUSTER_WORKER_NODE_4_ENABLED

  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
    [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || {
      echo "cluster hosts must use root@host syntax" >&2
      return 1
    }
  done
  if [[ -n $CLUSTER_WORKER_NODE_4_HOST ]]; then
    [[ $CLUSTER_WORKER_NODE_4_HOST =~ ^root@[A-Za-z0-9.-]+$ ]] || {
      echo "dedicated Worker node 4 host must use root@host syntax" >&2
      return 1
    }
    [[ ${#CLUSTER_WORKER_NODE_4_NAME} -le 63 &&
       $CLUSTER_WORKER_NODE_4_NAME =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] || {
      echo "dedicated Worker node 4 name must be a Kubernetes DNS label" >&2
      return 1
    }
    [[ $CLUSTER_WORKER_NODE_4_SITE == ningbo || $CLUSTER_WORKER_NODE_4_SITE == hongkong ]] || {
      echo "dedicated Worker node 4 site must be ningbo or hongkong" >&2
      return 1
    }
    [[ $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT =~ ^[0-9]+$ &&
       $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT -ge 1 &&
       $CLUSTER_WORKER_NODE_4_WIREGUARD_PORT -le 65535 ]] || {
      echo "dedicated Worker node 4 WireGuard port is invalid" >&2
      return 1
    }
    [[ $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == true ||
       $CLUSTER_WORKER_NODE_4_NETWORK_APPROVED == false ]] || {
      echo "dedicated Worker node 4 network approval must be true or false" >&2
      return 1
    }
    [[ $CLUSTER_WORKER_NODE_4_ENABLED == true || $CLUSTER_WORKER_NODE_4_ENABLED == false ]] || {
      echo "dedicated Worker node 4 enabled flag must be true or false" >&2
      return 1
    }
  elif [[ -n $CLUSTER_WORKER_NODE_4_NAME || -n $CLUSTER_WORKER_NODE_4_SITE ]]; then
    echo "dedicated Worker node 4 metadata requires AI_GATEWAY_WORKER_NODE_4_HOST" >&2
    return 1
  fi
  [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || {
    echo 'invalid OSS region or bucket' >&2
    return 1
  }
  [[ -f $cluster_ssh_key && ! -L $cluster_ssh_key ]] || {
    echo "missing cluster SSH key: $cluster_ssh_key" >&2
    return 1
  }
}

cluster_ssh() {
  local host=$1
  shift
  prepare_cluster_ssh_control_dir
  ssh \
    -i "$cluster_ssh_key" \
    -o BatchMode=yes \
    -o ConnectionAttempts=3 \
    -o ConnectTimeout=10 \
    -o ControlMaster=auto \
    -o ControlPersist=120 \
    -o "ControlPath=$cluster_ssh_control_dir/%C" \
    -o ServerAliveInterval=15 \
    -o ServerAliveCountMax=3 \
    "$host" "$@"
}

cluster_scp() {
  prepare_cluster_ssh_control_dir
  scp \
    -i "$cluster_ssh_key" \
    -o BatchMode=yes \
    -o ConnectionAttempts=3 \
    -o ConnectTimeout=10 \
    -o ControlMaster=auto \
    -o ControlPersist=120 \
    -o "ControlPath=$cluster_ssh_control_dir/%C" \
    "$@"
}

require_commands() {
  local command
  for command in "$@"; do
    command -v "$command" >/dev/null 2>&1 || {
      echo "$command is required" >&2
      return 1
    }
  done
}

assert_private_file() {
  local path=$1
  local mode
  [[ -f $path && ! -L $path ]] || {
    echo "missing private file: $path" >&2
    return 1
  }
  if [[ $(uname -s) == Darwin ]]; then
    mode=$(stat -f '%Lp' "$path")
  else
    mode=$(stat -c '%a' "$path")
  fi
  [[ $mode == 600 ]] || {
    echo "private file must have mode 0600: $path" >&2
    return 1
  }
}