easyai-ai-gateway/deploy/manual/easyai-ai-gateway-release

#!/usr/bin/env bash
set -euo pipefail

[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; }

config_file=${AI_GATEWAY_RELEASE_CONFIG:-/etc/easyai-ai-gateway-release.conf}
manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs}
[[ -f $config_file && ! -L $config_file ]] || { echo 'missing release config' >&2; exit 1; }
[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; }
# shellcheck source=/dev/null
source "$config_file"

: "${DEPLOY_DIR:?}"
: "${COMPOSE_FILE:?}"
: "${RELEASES_DIR:?}"
: "${BACKUP_DIR:?}"
: "${PUBLIC_BASE_URL:?}"
: "${API_PORT:?}"
: "${WEB_PORT:?}"
: "${POSTGRES_SERVICE:?}"
: "${POSTGRES_USER:?}"
: "${POSTGRES_DATABASE:?}"
: "${BACKUP_RETENTION:?}"

[[ $DEPLOY_DIR == /* && $COMPOSE_FILE == /* && $RELEASES_DIR == /* && $BACKUP_DIR == /* ]] || {
  echo 'release paths must be absolute' >&2
  exit 1
}
[[ $BACKUP_RETENTION =~ ^[1-9][0-9]*$ ]] || { echo 'invalid BACKUP_RETENTION' >&2; exit 1; }
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { echo 'invalid PUBLIC_BASE_URL' >&2; exit 1; }
[[ -d $DEPLOY_DIR && -f $COMPOSE_FILE ]] || { echo 'deployment directory is unavailable' >&2; exit 1; }

for command in docker curl node flock install; do
  command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
done
docker info >/dev/null 2>&1 || { echo 'Docker Engine is unavailable' >&2; exit 1; }
docker compose version >/dev/null 2>&1 || { echo 'Docker Compose v2 is required' >&2; exit 1; }

install -d -m 0700 "$RELEASES_DIR" "$BACKUP_DIR"
exec 9>"$RELEASES_DIR/.release.lock"
flock -n 9 || { echo 'another release operation is running' >&2; exit 1; }

compose=(docker compose --project-directory "$DEPLOY_DIR" -f "$COMPOSE_FILE")
current_manifest=$RELEASES_DIR/current.json

manifest_get() {
  node "$manifest_tool" get "$1" "$2"
}

validate_image() {
  local image=$1
  local source_sha=$2
  local require_revision=$3
  local revision architecture

  [[ $image =~ @sha256:[0-9a-f]{64}$ ]] || { echo "image is not digest-pinned: $image" >&2; return 1; }
  docker pull --platform linux/amd64 "$image"
  architecture=$(docker image inspect "$image" --format '{{.Os}}/{{.Architecture}}')
  [[ $architecture == linux/amd64 ]] || { echo "unexpected image architecture: $architecture" >&2; return 1; }
  if [[ $require_revision == true ]]; then
    revision=$(docker image inspect "$image" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')
    [[ $revision == "$source_sha" ]] || { echo "image revision does not match release: $image" >&2; return 1; }
  fi
}

wait_for_url() {
  local label=$1
  local url=$2
  local expected=${3:-}
  local body=

  for _ in $(seq 1 60); do
    if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then
      if [[ -z $expected || $body == *"$expected"* ]]; then
        echo "[release] verified $label"
        return 0
      fi
    fi
    sleep 2
  done
  echo "release probe failed: $label ($url)" >&2
  return 1
}

verify_runtime() {
  wait_for_url 'internal API health' "http://127.0.0.1:$API_PORT/api/v1/healthz" 'easyai-ai-gateway'
  wait_for_url 'internal API readiness' "http://127.0.0.1:$API_PORT/api/v1/readyz" '"ok":true'
  wait_for_url 'internal OpenAPI' "http://127.0.0.1:$API_PORT/api/v1/openapi.json" '"/api/v1/chat/completions"'
  wait_for_url 'Web API reverse proxy' "http://127.0.0.1:$WEB_PORT/api/v1/healthz" 'easyai-ai-gateway'
  wait_for_url 'Web application' "http://127.0.0.1:$WEB_PORT/" 'EasyAI AI Gateway'
  wait_for_url 'public API health' "$PUBLIC_BASE_URL/api/v1/healthz" 'easyai-ai-gateway'
  wait_for_url 'public API readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
  wait_for_url 'public OpenAPI' "$PUBLIC_BASE_URL/api/v1/openapi.json" '"/api/v1/chat/completions"'
}

activate_images() {
  local api_image=$1
  local web_image=$2
  export AI_GATEWAY_API_IMAGE=$api_image
  export AI_GATEWAY_WEB_IMAGE=$web_image
  "${compose[@]}" up -d --no-build --pull always api web
}

restore_previous() {
  local previous=$1
  [[ -f $previous ]] || return 1
  local previous_api previous_web
  previous_api=$(manifest_get "$previous" images.api)
  previous_web=$(manifest_get "$previous" images.web)
  echo '[release] restoring previous application images' >&2
  activate_images "$previous_api" "$previous_web"
  verify_runtime
}

backup_database() {
  local source_sha=$1
  local temp_backup=$BACKUP_DIR/.${source_sha}.dump.tmp
  local final_backup=$BACKUP_DIR/${source_sha}.dump

  umask 077
  "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
    pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -Fc >"$temp_backup"
  [[ -s $temp_backup ]] || { echo 'database backup is empty' >&2; rm -f "$temp_backup"; return 1; }
  "${compose[@]}" exec -T "$POSTGRES_SERVICE" pg_restore -l <"$temp_backup" >/dev/null
  mv "$temp_backup" "$final_backup"
  mapfile -t expired < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name '*.dump' -printf '%T@ %p\n' | sort -rn | awk -v keep="$BACKUP_RETENTION" 'NR > keep { sub(/^[^ ]+ /, ""); print }')
  if [[ ${#expired[@]} -gt 0 ]]; then
    rm -f -- "${expired[@]}"
  fi
  echo "[release] verified database backup: $final_backup"
}

record_current() {
  local manifest=$1
  local action=$2
  local started_at=$3
  local source_sha recorded temp
  source_sha=$(manifest_get "$manifest" sourceSha)
  recorded=$RELEASES_DIR/$source_sha.json
  temp=$RELEASES_DIR/.${source_sha}.json.tmp
  rm -f "$temp"
  RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
  RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \
  RELEASE_DEPLOY_ACTION=$action \
    node "$manifest_tool" record-deployment "$manifest" "$temp" >/dev/null
  install -m 0600 "$temp" "$recorded"
  install -m 0600 "$recorded" "$current_manifest.tmp"
  mv "$current_manifest.tmp" "$current_manifest"
  rm -f "$temp"
}

deploy_manifest() {
  local manifest=$1
  local action=${2:-deploy}
  local source_sha base_sha api_image web_image migrations_changed current_sha
  local changed_components api_changed=false web_changed=false
  local previous_copy=
  local started_at

  [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
  node "$manifest_tool" validate "$manifest"
  source_sha=$(manifest_get "$manifest" sourceSha)
  base_sha=$(manifest_get "$manifest" baseReleaseSha)
  api_image=$(manifest_get "$manifest" images.api)
  web_image=$(manifest_get "$manifest" images.web)
  migrations_changed=$(manifest_get "$manifest" migrationsChanged)
  changed_components=$(manifest_get "$manifest" components)
  [[ $changed_components == *'"api"'* ]] && api_changed=true
  [[ $changed_components == *'"web"'* ]] && web_changed=true

  if [[ $action == deploy ]]; then
    if [[ -f $current_manifest ]]; then
      current_sha=$(manifest_get "$current_manifest" sourceSha)
      [[ $current_sha == "$base_sha" ]] || {
        echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2
        return 1
      }
    else
      [[ -z $base_sha ]] || { echo 'non-bootstrap manifest has no production base' >&2; return 1; }
    fi
  fi

  validate_image "$api_image" "$source_sha" "$api_changed"
  validate_image "$web_image" "$source_sha" "$web_changed"

  if [[ -f $current_manifest ]]; then
    previous_copy=$(mktemp "$RELEASES_DIR/.previous.XXXXXX")
    install -m 0600 "$current_manifest" "$previous_copy"
  fi
  started_at=$(date +%s)

  if [[ $action == deploy && $migrations_changed == true ]]; then
    backup_database "$source_sha"
    export AI_GATEWAY_API_IMAGE=$api_image
    export AI_GATEWAY_WEB_IMAGE=$web_image
    "${compose[@]}" run --rm --no-deps migrator
  fi

  if ! activate_images "$api_image" "$web_image" || ! verify_runtime; then
    if [[ -n $previous_copy ]]; then
      restore_previous "$previous_copy" || echo 'automatic application rollback also failed' >&2
    fi
    rm -f "$previous_copy"
    return 1
  fi

  record_current "$manifest" "$action" "$started_at"
  rm -f "$previous_copy"
  echo "production_release=PASS action=$action release=$source_sha"
}

case ${1:-} in
  status)
    [[ $# -eq 1 ]] || exit 64
    [[ -f $current_manifest ]] || { echo 'no production release manifest' >&2; exit 1; }
    node "$manifest_tool" validate "$current_manifest" >/dev/null
    cat "$current_manifest"
    ;;
  deploy)
    [[ $# -eq 2 ]] || exit 64
    deploy_manifest "$2" deploy
    ;;
  rollback)
    [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64
    target=$RELEASES_DIR/$2.json
    [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
    deploy_manifest "$target" rollback
    ;;
  *)
    echo 'usage: easyai-ai-gateway-release {status|deploy <manifest>|rollback <sha>}' >&2
    exit 64
    ;;
esac