easyai-ai-gateway/deploy/manual/easyai-ai-gateway-release

#!/usr/bin/env bash
set -euo pipefail

[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; }

config_file=${AI_GATEWAY_RELEASE_CONFIG:-/etc/easyai-ai-gateway-release.conf}
manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs}
[[ -f $config_file && ! -L $config_file ]] || { echo 'missing release config' >&2; exit 1; }
[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; }
# shellcheck source=/dev/null
source "$config_file"

: "${DEPLOY_DIR:?}"
: "${COMPOSE_FILE:?}"
: "${RELEASES_DIR:?}"
: "${BACKUP_DIR:?}"
: "${PUBLIC_BASE_URL:?}"
: "${API_PORT:?}"
: "${WEB_PORT:?}"
: "${POSTGRES_SERVICE:?}"
: "${POSTGRES_USER:?}"
: "${POSTGRES_DATABASE:?}"
: "${BACKUP_RETENTION:?}"

[[ $DEPLOY_DIR == /* && $COMPOSE_FILE == /* && $RELEASES_DIR == /* && $BACKUP_DIR == /* ]] || {
  echo 'release paths must be absolute' >&2
  exit 1
}
[[ $BACKUP_RETENTION =~ ^[1-9][0-9]*$ ]] || { echo 'invalid BACKUP_RETENTION' >&2; exit 1; }
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { echo 'invalid PUBLIC_BASE_URL' >&2; exit 1; }
[[ -d $DEPLOY_DIR && -f $COMPOSE_FILE ]] || { echo 'deployment directory is unavailable' >&2; exit 1; }

for command in docker curl node flock install sha256sum date wc; do
  command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
done
docker info >/dev/null 2>&1 || { echo 'Docker Engine is unavailable' >&2; exit 1; }
docker compose version >/dev/null 2>&1 || { echo 'Docker Compose v2 is required' >&2; exit 1; }

install -d -m 0700 "$RELEASES_DIR" "$BACKUP_DIR"
exec 9>"$RELEASES_DIR/.release.lock"
flock -n 9 || { echo 'another release operation is running' >&2; exit 1; }

compose=(docker compose --project-directory "$DEPLOY_DIR" -f "$COMPOSE_FILE")
current_manifest=$RELEASES_DIR/current.json

manifest_get() {
  node "$manifest_tool" get "$1" "$2"
}

validate_image() {
  local image=$1
  local source_sha=$2
  local require_revision=$3
  local revision architecture

  [[ $image =~ @sha256:[0-9a-f]{64}$ ]] || { echo "image is not digest-pinned: $image" >&2; return 1; }
  docker pull --platform linux/amd64 "$image"
  architecture=$(docker image inspect "$image" --format '{{.Os}}/{{.Architecture}}')
  [[ $architecture == linux/amd64 ]] || { echo "unexpected image architecture: $architecture" >&2; return 1; }
  if [[ $require_revision == true ]]; then
    revision=$(docker image inspect "$image" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')
    [[ $revision == "$source_sha" ]] || { echo "image revision does not match release: $image" >&2; return 1; }
  fi
}

wait_for_url() {
  local label=$1
  local url=$2
  local expected=${3:-}
  local body=

  for _ in $(seq 1 60); do
    if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then
      if [[ -z $expected || $body == *"$expected"* ]]; then
        echo "[release] verified $label"
        return 0
      fi
    fi
    sleep 2
  done
  echo "release probe failed: $label ($url)" >&2
  return 1
}

verify_runtime() {
  wait_for_url 'internal API health' "http://127.0.0.1:$API_PORT/api/v1/healthz" 'easyai-ai-gateway'
  wait_for_url 'internal API readiness' "http://127.0.0.1:$API_PORT/api/v1/readyz" '"ok":true'
  wait_for_url 'internal OpenAPI' "http://127.0.0.1:$API_PORT/api/v1/openapi.json" '"/api/v1/chat/completions"'
  wait_for_url 'Web API reverse proxy' "http://127.0.0.1:$WEB_PORT/api/v1/healthz" 'easyai-ai-gateway'
  wait_for_url 'Web application' "http://127.0.0.1:$WEB_PORT/" 'EasyAI AI Gateway'
  wait_for_url 'public API health' "$PUBLIC_BASE_URL/api/v1/healthz" 'easyai-ai-gateway'
  wait_for_url 'public API readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
  wait_for_url 'public OpenAPI' "$PUBLIC_BASE_URL/api/v1/openapi.json" '"/api/v1/chat/completions"'
}

activate_images() {
  local api_image=$1
  local web_image=$2
  export AI_GATEWAY_API_IMAGE=$api_image
  export AI_GATEWAY_WEB_IMAGE=$web_image
  "${compose[@]}" up -d --no-build --pull always api web
}

restore_previous() {
  local previous=$1
  [[ -f $previous ]] || return 1
  local previous_api previous_web
  previous_api=$(manifest_get "$previous" images.api)
  previous_web=$(manifest_get "$previous" images.web)
  echo '[release] restoring previous application images' >&2
  activate_images "$previous_api" "$previous_web"
  verify_runtime
}

backup_database() {
  local source_sha=$1
  local temp_backup=$BACKUP_DIR/.${source_sha}.dump.tmp
  local final_backup=$BACKUP_DIR/${source_sha}.dump
  local checksum_file=$BACKUP_DIR/${source_sha}.dump.sha256
  local exclusions_file=$BACKUP_DIR/${source_sha}.excluded-tables
  local restore_database=easyai_restore_${source_sha:0:12}_$$
  local source_counts restored_counts
  local backup_started_at backup_seconds backup_bytes restore_started_at restore_seconds
  local restore_failed=false
  local count_sql="SELECT (SELECT count(*) FROM gateway_wallet_transactions)::text || ':' || (SELECT count(*) FROM gateway_upload_assets)::text || ':' || (SELECT count(*) FROM gateway_cloned_voices)::text;"
  local task_count_sql='SELECT (SELECT count(*) FROM gateway_tasks) + (SELECT count(*) FROM gateway_task_attempts) + (SELECT count(*) FROM gateway_task_events);'
  local -a excluded_tables=(
    gateway_tasks
    gateway_task_attempts
    gateway_task_events
    gateway_task_callback_outbox
    gateway_task_param_preprocessing_logs
    gateway_task_message_refs
    gateway_concurrency_leases
    gateway_rate_limit_reservations
    gateway_response_chains
    settlement_outbox
    river_job
  )
  local -a exclude_args=()
  local table
  for table in "${excluded_tables[@]}"; do
    exclude_args+=("--exclude-table-data=$table")
  done

  umask 077
  backup_started_at=$(date +%s)
  "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
    pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -Fc "${exclude_args[@]}" >"$temp_backup"
  backup_seconds=$(( $(date +%s) - backup_started_at ))
  backup_bytes=$(wc -c <"$temp_backup")
  [[ -s $temp_backup ]] || { echo 'database backup is empty' >&2; rm -f "$temp_backup"; return 1; }
  if (( backup_bytes > 1073741824 || backup_seconds > 300 )); then
    echo "database backup control target exceeded: bytes=$backup_bytes seconds=$backup_seconds" >&2
    rm -f "$temp_backup"
    return 1
  fi
  "${compose[@]}" exec -T "$POSTGRES_SERVICE" pg_restore -l <"$temp_backup" >/dev/null

  restore_started_at=$(date +%s)
  "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
    dropdb -U "$POSTGRES_USER" --if-exists "$restore_database"
  "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
    createdb -U "$POSTGRES_USER" "$restore_database"
  if ! "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
      pg_restore -U "$POSTGRES_USER" -d "$restore_database" --section=pre-data <"$temp_backup"; then
    restore_failed=true
  fi
  if [[ $restore_failed == false ]] && ! "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
      pg_restore -U "$POSTGRES_USER" -d "$restore_database" --section=data <"$temp_backup"; then
    restore_failed=true
  fi
  if [[ $restore_failed == false ]]; then
    if ! "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
      psql -U "$POSTGRES_USER" -d "$restore_database" -v ON_ERROR_STOP=1 -c \
      'UPDATE gateway_upload_assets SET task_id = NULL; UPDATE gateway_cloned_voices SET source_task_id = NULL, source_attempt_id = NULL;'; then
      restore_failed=true
    fi
  fi
  if [[ $restore_failed == false ]] && ! "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
      pg_restore -U "$POSTGRES_USER" -d "$restore_database" --section=post-data <"$temp_backup"; then
    restore_failed=true
  fi
  if [[ $restore_failed == false ]]; then
    if ! source_counts=$("${compose[@]}" exec -T "$POSTGRES_SERVICE" \
      psql -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -At -v ON_ERROR_STOP=1 -c \
      "$count_sql"); then
      restore_failed=true
    fi
    if ! restored_counts=$("${compose[@]}" exec -T "$POSTGRES_SERVICE" \
      psql -U "$POSTGRES_USER" -d "$restore_database" -At -v ON_ERROR_STOP=1 -c \
      "$count_sql"); then
      restore_failed=true
    fi
    [[ $source_counts == "$restored_counts" ]] || restore_failed=true
  fi
  if [[ $restore_failed == false ]]; then
    if ! restored_counts=$("${compose[@]}" exec -T "$POSTGRES_SERVICE" \
      psql -U "$POSTGRES_USER" -d "$restore_database" -At -v ON_ERROR_STOP=1 -c \
      "$task_count_sql"); then
      restore_failed=true
    fi
    [[ $restored_counts == 0 ]] || restore_failed=true
  fi
  "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
    dropdb -U "$POSTGRES_USER" --if-exists "$restore_database"
  restore_seconds=$(( $(date +%s) - restore_started_at ))
  if (( restore_seconds > 300 )); then
    restore_failed=true
  fi
  if [[ $restore_failed == true ]]; then
    echo "database backup restore verification failed: seconds=$restore_seconds" >&2
    rm -f "$temp_backup"
    return 1
  fi

  mv "$temp_backup" "$final_backup"
  sha256sum "$final_backup" >"$checksum_file"
  printf '%s\n' "${excluded_tables[@]}" >"$exclusions_file"
  mapfile -t expired < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name '*.dump' -printf '%T@ %p\n' | sort -rn | awk -v keep="$BACKUP_RETENTION" 'NR > keep { sub(/^[^ ]+ /, ""); print }')
  if [[ ${#expired[@]} -gt 0 ]]; then
    for table in "${expired[@]}"; do
      rm -f -- "$table" "$table.sha256" "${table%.dump}.excluded-tables"
    done
  fi
  echo "[release] verified restorable task-history-free database backup: $final_backup bytes=$backup_bytes backup_seconds=$backup_seconds restore_seconds=$restore_seconds"
}

record_current() {
  local manifest=$1
  local action=$2
  local started_at=$3
  local source_sha recorded temp
  source_sha=$(manifest_get "$manifest" sourceSha)
  recorded=$RELEASES_DIR/$source_sha.json
  temp=$RELEASES_DIR/.${source_sha}.json.tmp
  rm -f "$temp"
  RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
  RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \
  RELEASE_DEPLOY_ACTION=$action \
    node "$manifest_tool" record-deployment "$manifest" "$temp" >/dev/null
  install -m 0600 "$temp" "$recorded"
  install -m 0600 "$recorded" "$current_manifest.tmp"
  mv "$current_manifest.tmp" "$current_manifest"
  rm -f "$temp"
}

deploy_manifest() {
  local manifest=$1
  local action=${2:-deploy}
  local source_sha base_sha api_image web_image migrations_changed current_sha
  local changed_components api_changed=false web_changed=false
  local previous_copy=
  local started_at

  [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
  node "$manifest_tool" validate "$manifest"
  source_sha=$(manifest_get "$manifest" sourceSha)
  base_sha=$(manifest_get "$manifest" baseReleaseSha)
  api_image=$(manifest_get "$manifest" images.api)
  web_image=$(manifest_get "$manifest" images.web)
  migrations_changed=$(manifest_get "$manifest" migrationsChanged)
  changed_components=$(manifest_get "$manifest" components)
  [[ $changed_components == *'"api"'* ]] && api_changed=true
  [[ $changed_components == *'"web"'* ]] && web_changed=true

  if [[ $action == deploy ]]; then
    if [[ -f $current_manifest ]]; then
      current_sha=$(manifest_get "$current_manifest" sourceSha)
      [[ $current_sha == "$base_sha" ]] || {
        echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2
        return 1
      }
    else
      [[ -z $base_sha ]] || { echo 'non-bootstrap manifest has no production base' >&2; return 1; }
    fi
  fi

  validate_image "$api_image" "$source_sha" "$api_changed"
  validate_image "$web_image" "$source_sha" "$web_changed"

  if [[ -f $current_manifest ]]; then
    previous_copy=$(mktemp "$RELEASES_DIR/.previous.XXXXXX")
    install -m 0600 "$current_manifest" "$previous_copy"
  fi
  started_at=$(date +%s)

  if [[ $action == deploy && $migrations_changed == true ]]; then
    backup_database "$source_sha"
    export AI_GATEWAY_API_IMAGE=$api_image
    export AI_GATEWAY_WEB_IMAGE=$web_image
    "${compose[@]}" run --rm --no-deps migrator
  fi

  if ! activate_images "$api_image" "$web_image" || ! verify_runtime; then
    if [[ -n $previous_copy ]]; then
      restore_previous "$previous_copy" || echo 'automatic application rollback also failed' >&2
    fi
    rm -f "$previous_copy"
    return 1
  fi

  record_current "$manifest" "$action" "$started_at"
  rm -f "$previous_copy"
  echo "production_release=PASS action=$action release=$source_sha"
}

case ${1:-} in
  status)
    [[ $# -eq 1 ]] || exit 64
    [[ -f $current_manifest ]] || { echo 'no production release manifest' >&2; exit 1; }
    node "$manifest_tool" validate "$current_manifest" >/dev/null
    cat "$current_manifest"
    ;;
  deploy)
    [[ $# -eq 2 ]] || exit 64
    deploy_manifest "$2" deploy
    ;;
  rollback)
    [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64
    target=$RELEASES_DIR/$2.json
    [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
    deploy_manifest "$target" rollback
    ;;
  *)
    echo 'usage: easyai-ai-gateway-release {status|deploy <manifest>|rollback <sha>}' >&2
    exit 64
    ;;
esac