#!/usr/bin/env bash set -euo pipefail [[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; } config_file=${AI_GATEWAY_RELEASE_CONFIG:-/etc/easyai-ai-gateway-release.conf} manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs} [[ -f $config_file && ! -L $config_file ]] || { echo 'missing release config' >&2; exit 1; } [[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; } # shellcheck source=/dev/null source "$config_file" : "${DEPLOY_DIR:?}" : "${COMPOSE_FILE:?}" : "${RELEASES_DIR:?}" : "${BACKUP_DIR:?}" : "${PUBLIC_BASE_URL:?}" : "${API_PORT:?}" : "${WEB_PORT:?}" : "${POSTGRES_SERVICE:?}" : "${POSTGRES_USER:?}" : "${POSTGRES_DATABASE:?}" : "${BACKUP_RETENTION:?}" [[ $DEPLOY_DIR == /* && $COMPOSE_FILE == /* && $RELEASES_DIR == /* && $BACKUP_DIR == /* ]] || { echo 'release paths must be absolute' >&2 exit 1 } [[ $BACKUP_RETENTION =~ ^[1-9][0-9]*$ ]] || { echo 'invalid BACKUP_RETENTION' >&2; exit 1; } [[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { echo 'invalid PUBLIC_BASE_URL' >&2; exit 1; } [[ -d $DEPLOY_DIR && -f $COMPOSE_FILE ]] || { echo 'deployment directory is unavailable' >&2; exit 1; } for command in docker curl node flock install; do command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; } done docker info >/dev/null 2>&1 || { echo 'Docker Engine is unavailable' >&2; exit 1; } docker compose version >/dev/null 2>&1 || { echo 'Docker Compose v2 is required' >&2; exit 1; } install -d -m 0700 "$RELEASES_DIR" "$BACKUP_DIR" exec 9>"$RELEASES_DIR/.release.lock" flock -n 9 || { echo 'another release operation is running' >&2; exit 1; } compose=(docker compose --project-directory "$DEPLOY_DIR" -f "$COMPOSE_FILE") current_manifest=$RELEASES_DIR/current.json manifest_get() { node "$manifest_tool" get "$1" "$2" } validate_image() { local image=$1 local source_sha=$2 local require_revision=$3 local revision architecture [[ $image =~ @sha256:[0-9a-f]{64}$ ]] || { echo "image is not digest-pinned: $image" >&2; return 1; } docker pull --platform linux/amd64 "$image" architecture=$(docker image inspect "$image" --format '{{.Os}}/{{.Architecture}}') [[ $architecture == linux/amd64 ]] || { echo "unexpected image architecture: $architecture" >&2; return 1; } if [[ $require_revision == true ]]; then revision=$(docker image inspect "$image" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}') [[ $revision == "$source_sha" ]] || { echo "image revision does not match release: $image" >&2; return 1; } fi } wait_for_url() { local label=$1 local url=$2 local expected=${3:-} local body= for _ in $(seq 1 60); do if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then if [[ -z $expected || $body == *"$expected"* ]]; then echo "[release] verified $label" return 0 fi fi sleep 2 done echo "release probe failed: $label ($url)" >&2 return 1 } verify_runtime() { wait_for_url 'internal API health' "http://127.0.0.1:$API_PORT/api/v1/healthz" 'easyai-ai-gateway' wait_for_url 'internal API readiness' "http://127.0.0.1:$API_PORT/api/v1/readyz" '"ok":true' wait_for_url 'internal OpenAPI' "http://127.0.0.1:$API_PORT/api/v1/openapi.json" '"/api/v1/chat/completions"' wait_for_url 'Web API reverse proxy' "http://127.0.0.1:$WEB_PORT/api/v1/healthz" 'easyai-ai-gateway' wait_for_url 'Web application' "http://127.0.0.1:$WEB_PORT/" 'EasyAI AI Gateway' wait_for_url 'public API health' "$PUBLIC_BASE_URL/api/v1/healthz" 'easyai-ai-gateway' wait_for_url 'public API readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true' wait_for_url 'public OpenAPI' "$PUBLIC_BASE_URL/api/v1/openapi.json" '"/api/v1/chat/completions"' } activate_images() { local api_image=$1 local web_image=$2 export AI_GATEWAY_API_IMAGE=$api_image export AI_GATEWAY_WEB_IMAGE=$web_image "${compose[@]}" up -d --no-build --pull always api web } restore_previous() { local previous=$1 [[ -f $previous ]] || return 1 local previous_api previous_web previous_api=$(manifest_get "$previous" images.api) previous_web=$(manifest_get "$previous" images.web) echo '[release] restoring previous application images' >&2 activate_images "$previous_api" "$previous_web" verify_runtime } backup_database() { local source_sha=$1 local temp_backup=$BACKUP_DIR/.${source_sha}.dump.tmp local final_backup=$BACKUP_DIR/${source_sha}.dump umask 077 "${compose[@]}" exec -T "$POSTGRES_SERVICE" \ pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -Fc >"$temp_backup" [[ -s $temp_backup ]] || { echo 'database backup is empty' >&2; rm -f "$temp_backup"; return 1; } "${compose[@]}" exec -T "$POSTGRES_SERVICE" pg_restore -l <"$temp_backup" >/dev/null mv "$temp_backup" "$final_backup" mapfile -t expired < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name '*.dump' -printf '%T@ %p\n' | sort -rn | awk -v keep="$BACKUP_RETENTION" 'NR > keep { sub(/^[^ ]+ /, ""); print }') if [[ ${#expired[@]} -gt 0 ]]; then rm -f -- "${expired[@]}" fi echo "[release] verified database backup: $final_backup" } record_current() { local manifest=$1 local action=$2 local started_at=$3 local source_sha recorded temp source_sha=$(manifest_get "$manifest" sourceSha) recorded=$RELEASES_DIR/$source_sha.json temp=$RELEASES_DIR/.${source_sha}.json.tmp rm -f "$temp" RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \ RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \ RELEASE_DEPLOY_ACTION=$action \ node "$manifest_tool" record-deployment "$manifest" "$temp" >/dev/null install -m 0600 "$temp" "$recorded" install -m 0600 "$recorded" "$current_manifest.tmp" mv "$current_manifest.tmp" "$current_manifest" rm -f "$temp" } deploy_manifest() { local manifest=$1 local action=${2:-deploy} local source_sha base_sha api_image web_image migrations_changed current_sha local changed_components api_changed=false web_changed=false local previous_copy= local started_at [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; } node "$manifest_tool" validate "$manifest" source_sha=$(manifest_get "$manifest" sourceSha) base_sha=$(manifest_get "$manifest" baseReleaseSha) api_image=$(manifest_get "$manifest" images.api) web_image=$(manifest_get "$manifest" images.web) migrations_changed=$(manifest_get "$manifest" migrationsChanged) changed_components=$(manifest_get "$manifest" components) [[ $changed_components == *'"api"'* ]] && api_changed=true [[ $changed_components == *'"web"'* ]] && web_changed=true if [[ $action == deploy ]]; then if [[ -f $current_manifest ]]; then current_sha=$(manifest_get "$current_manifest" sourceSha) [[ $current_sha == "$base_sha" ]] || { echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2 return 1 } else [[ -z $base_sha ]] || { echo 'non-bootstrap manifest has no production base' >&2; return 1; } fi fi validate_image "$api_image" "$source_sha" "$api_changed" validate_image "$web_image" "$source_sha" "$web_changed" if [[ -f $current_manifest ]]; then previous_copy=$(mktemp "$RELEASES_DIR/.previous.XXXXXX") install -m 0600 "$current_manifest" "$previous_copy" fi started_at=$(date +%s) if [[ $action == deploy && $migrations_changed == true ]]; then backup_database "$source_sha" export AI_GATEWAY_API_IMAGE=$api_image export AI_GATEWAY_WEB_IMAGE=$web_image "${compose[@]}" run --rm --no-deps migrator fi if ! activate_images "$api_image" "$web_image" || ! verify_runtime; then if [[ -n $previous_copy ]]; then restore_previous "$previous_copy" || echo 'automatic application rollback also failed' >&2 fi rm -f "$previous_copy" return 1 fi record_current "$manifest" "$action" "$started_at" rm -f "$previous_copy" echo "production_release=PASS action=$action release=$source_sha" } case ${1:-} in status) [[ $# -eq 1 ]] || exit 64 [[ -f $current_manifest ]] || { echo 'no production release manifest' >&2; exit 1; } node "$manifest_tool" validate "$current_manifest" >/dev/null cat "$current_manifest" ;; deploy) [[ $# -eq 2 ]] || exit 64 deploy_manifest "$2" deploy ;; rollback) [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64 target=$RELEASES_DIR/$2.json [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; } deploy_manifest "$target" rollback ;; *) echo 'usage: easyai-ai-gateway-release {status|deploy |rollback }' >&2 exit 64 ;; esac