easyai-ai-gateway/deploy/kubernetes/easyai-ai-gateway-cluster-release

#!/usr/bin/env bash
set -euo pipefail

[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; }

config_file=${AI_GATEWAY_CLUSTER_RELEASE_CONFIG:-/etc/easyai-ai-gateway-cluster-release.conf}
manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs}
[[ -f $config_file && ! -L $config_file ]] || { echo 'missing cluster release config' >&2; exit 1; }
[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; }
# shellcheck source=/dev/null
source "$config_file"

: "${RELEASES_DIR:?}"
: "${NAMESPACE:=easyai}"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
  echo 'invalid cluster release configuration' >&2
  exit 1
}
for capacity_value in \
  "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
  "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
  "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
  "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
  "$AI_GATEWAY_DATABASE_MAX_CONNS" \
  "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
  "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
  [[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
    echo 'worker capacity configuration must contain positive integers' >&2
    exit 1
  }
done
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
   AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
   AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
   AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
   AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
   AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
   AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
  echo 'worker capacity configuration exceeds the release safety bounds' >&2
  exit 1
}
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
  echo 'invalid public base URL' >&2
  exit 1
}
for command in "$K3S_BIN" curl flock install jq node; do
  command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
done

kubectl=("$K3S_BIN" kubectl)
install -d -m 0700 "$RELEASES_DIR"
exec 9>"$RELEASES_DIR/.release.lock"
flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; }

manifest_get() {
  node "$manifest_tool" get "$1" "$2"
}

current_to_file() {
  local output=$1
  "${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
    -o jsonpath='{.data.manifest\.json}' >"$output"
  [[ -s $output ]]
  node "$manifest_tool" validate "$output" >/dev/null
}

wait_for_url() {
  local label=$1
  local url=$2
  local expected=${3:-}
  local body
  for _ in $(seq 1 60); do
    if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then
      if [[ -z $expected || $body == *"$expected"* ]]; then
        echo "[cluster-release] verified $label"
        return 0
      fi
    fi
    sleep 2
  done
  echo "release probe failed: $label ($url)" >&2
  return 1
}

verify_site() {
  local site=$1
  local address
  local api_port
  local web_port
  case $site in
    hongkong)
      address=10.77.0.2
      api_port=31089
      web_port=31179
      ;;
    ningbo)
      address=10.77.0.1
      api_port=31088
      web_port=31178
      ;;
    *) return 1 ;;
  esac
  wait_for_url "$site API health" "http://$address:$api_port/api/v1/healthz" easyai-ai-gateway
  wait_for_url "$site API readiness" "http://$address:$api_port/api/v1/readyz" '"ok":true'
  wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
}

rollout_worker_site() {
  local site=$1
  local api_image=$2
  local replicas
  case $site in
    ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
    hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
    *) return 1 ;;
  esac
  "${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
    "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
    "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
    "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
    "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
    "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
  "${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
    --replicas="$replicas"
  if [[ -n $api_image ]]; then
    "${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
      "worker=$api_image"
  fi
  "${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
  [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
    -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
  [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
    -o jsonpath='{.status.readyReplicas}') == "$replicas" ]]
}

rollout_api_capacity_site() {
  local site=$1
  "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
    "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
    "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS"
  "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
}

apply_capacity_config() {
  rollout_worker_site hongkong ""
  rollout_worker_site ningbo ""
  rollout_api_capacity_site hongkong
  rollout_api_capacity_site ningbo
  verify_site hongkong
  verify_site ningbo
  wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
  echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
}

rollout_site() {
  local site=$1
  local api_image=$2
  local web_image=$3
  local api_changed=$4
  local web_changed=$5
  if [[ $api_changed == true ]]; then
    "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
      "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
      "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS"
    "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
      "api=$api_image"
    "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
  fi
  if [[ $web_changed == true ]]; then
    "${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
      "web=$web_image"
    "${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s
  fi
  verify_site "$site"
}

run_backup() {
  local source_sha=$1
  local backup=easyai-pre-release-${source_sha:0:12}
  cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
apiVersion: postgresql.cnpg.io/v1
kind: Backup
metadata:
  name: $backup
  namespace: $NAMESPACE
spec:
  cluster:
    name: easyai-postgres
  method: plugin
  pluginConfiguration:
    name: barman-cloud.cloudnative-pg.io
EOF
  for _ in $(seq 1 120); do
    phase=$("${kubectl[@]}" get backup "$backup" -n "$NAMESPACE" \
      -o jsonpath='{.status.phase}' 2>/dev/null || true)
    [[ ${phase,,} == completed ]] && return 0
    [[ ${phase,,} == failed ]] && break
    sleep 5
  done
  echo "pre-migration backup failed: $backup phase=${phase:-unknown}" >&2
  return 1
}

run_migrator() {
  local source_sha=$1
  local api_image=$2
  local job=easyai-migrator-${source_sha:0:12}
  "${kubectl[@]}" delete job "$job" -n "$NAMESPACE" \
    --ignore-not-found=true --wait=true >/dev/null
  cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
  name: $job
  namespace: $NAMESPACE
  labels:
    easyai.io/release: $source_sha
spec:
  backoffLimit: 0
  ttlSecondsAfterFinished: 86400
  template:
    spec:
      restartPolicy: Never
      imagePullSecrets:
        - name: easyai-registry
      nodeSelector:
        easyai.io/site: ningbo
      containers:
        - name: migrator
          image: $api_image
          command: ["/app/easyai-ai-gateway-migrate"]
          envFrom:
            - secretRef:
                name: easyai-ai-gateway-runtime
EOF
  "${kubectl[@]}" wait --for=condition=complete "job/$job" \
    -n "$NAMESPACE" --timeout=300s
}

record_current() {
  local manifest=$1
  local action=$2
  local started_at=$3
  local source_sha recorded temporary
  source_sha=$(manifest_get "$manifest" sourceSha)
  recorded=$RELEASES_DIR/$source_sha.json
  temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX")
  rm -f -- "$temporary"
  RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
  RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \
  RELEASE_DEPLOY_ACTION=$action \
    node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null
  install -m 0600 "$temporary" "$recorded"
  "${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
    --from-file=manifest.json="$recorded" --dry-run=client -o yaml |
    "${kubectl[@]}" apply -f - >/dev/null
  rm -f -- "$temporary"
}

activate_manifest() {
  local manifest=$1
  local action=${2:-deploy}
  local source_sha base_sha api_image web_image migrations_changed components
  local api_changed=false web_changed=false current_file previous_api previous_web started_at
  [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
  node "$manifest_tool" validate "$manifest" >/dev/null
  source_sha=$(manifest_get "$manifest" sourceSha)
  base_sha=$(manifest_get "$manifest" baseReleaseSha)
  api_image=$(manifest_get "$manifest" images.api)
  web_image=$(manifest_get "$manifest" images.web)
  migrations_changed=$(manifest_get "$manifest" migrationsChanged)
  components=$(manifest_get "$manifest" components)
  [[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || {
    echo 'release images must be digest-pinned' >&2
    return 1
  }
  [[ $components == *'"api"'* ]] && api_changed=true
  [[ $components == *'"web"'* ]] && web_changed=true
  if [[ $action == rollback ]]; then
    api_changed=true
    web_changed=true
  fi

  current_file=$(mktemp "$RELEASES_DIR/.current.XXXXXX")
  if current_to_file "$current_file"; then
    current_sha=$(manifest_get "$current_file" sourceSha)
    previous_api=$(manifest_get "$current_file" images.api)
    previous_web=$(manifest_get "$current_file" images.web)
    if [[ $action == deploy && $current_sha != "$base_sha" ]]; then
      echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2
      rm -f -- "$current_file"
      return 1
    fi
  else
    if [[ $action == deploy && -n $base_sha ]]; then
      echo 'cluster release status is unavailable for a non-bootstrap manifest' >&2
      rm -f -- "$current_file"
      return 1
    fi
    previous_api=
    previous_web=
  fi
  started_at=$(date +%s)

  if [[ $action == deploy && $migrations_changed == true ]]; then
    run_backup "$source_sha"
    run_migrator "$source_sha" "$api_image"
  fi

  if { [[ $api_changed != true ]] ||
       { rollout_worker_site hongkong "$api_image" &&
         rollout_worker_site ningbo "$api_image"; }; } &&
     rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
     rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
     wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
    :
  else
    if [[ -n $previous_api && -n $previous_web ]]; then
      echo '[cluster-release] restoring previous application images' >&2
      if [[ $api_changed == true ]]; then
        rollout_worker_site hongkong "$previous_api" || true
        rollout_worker_site ningbo "$previous_api" || true
      fi
      rollout_site hongkong "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
      rollout_site ningbo "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
    fi
    rm -f -- "$current_file"
    return 1
  fi

  record_current "$manifest" "$action" "$started_at"
  rm -f -- "$current_file"
  echo "production_release=PASS action=$action release=$source_sha"
}

case ${1:-} in
  status)
    [[ $# -eq 1 ]] || exit 64
    status_file=$(mktemp "$RELEASES_DIR/.status.XXXXXX")
    current_to_file "$status_file"
    cat "$status_file"
    rm -f -- "$status_file"
    ;;
  adopt)
    [[ $# -eq 2 ]] || exit 64
    node "$manifest_tool" validate "$2" >/dev/null
    api_image=$(manifest_get "$2" images.api)
    web_image=$(manifest_get "$2" images.web)
    for site in ningbo hongkong; do
      [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
        -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
    done
    [[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
      -o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]]
    [[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \
      -o jsonpath='{.spec.template.spec.containers[?(@.name=="web")].image}') == "$web_image" ]]
    record_current "$2" deploy "$(date +%s)"
    ;;
  deploy)
    [[ $# -eq 2 ]] || exit 64
    activate_manifest "$2" deploy
    ;;
  rollback)
    [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64
    target=$RELEASES_DIR/$2.json
    [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
    activate_manifest "$target" rollback
    ;;
  capacity)
    [[ $# -eq 1 ]] || exit 64
    apply_capacity_config
    ;;
  *)
    echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity}' >&2
    exit 64
    ;;
esac