#!/usr/bin/env bash set -euo pipefail [[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; } config_file=${AI_GATEWAY_CLUSTER_RELEASE_CONFIG:-/etc/easyai-ai-gateway-cluster-release.conf} manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs} [[ -f $config_file && ! -L $config_file ]] || { echo 'missing cluster release config' >&2; exit 1; } [[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; } # shellcheck source=/dev/null source "$config_file" : "${RELEASES_DIR:?}" : "${NAMESPACE:=easyai}" : "${PUBLIC_BASE_URL:=https://ai.51easyai.com}" : "${K3S_BIN:=/usr/local/bin/k3s}" [[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || { echo 'invalid cluster release configuration' >&2 exit 1 } [[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { echo 'invalid public base URL' >&2 exit 1 } for command in "$K3S_BIN" curl flock install jq node; do command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; } done kubectl=("$K3S_BIN" kubectl) install -d -m 0700 "$RELEASES_DIR" exec 9>"$RELEASES_DIR/.release.lock" flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; } manifest_get() { node "$manifest_tool" get "$1" "$2" } current_to_file() { local output=$1 "${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \ -o jsonpath='{.data.manifest\.json}' >"$output" [[ -s $output ]] node "$manifest_tool" validate "$output" >/dev/null } wait_for_url() { local label=$1 local url=$2 local expected=${3:-} local body for _ in $(seq 1 60); do if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then if [[ -z $expected || $body == *"$expected"* ]]; then echo "[cluster-release] verified $label" return 0 fi fi sleep 2 done echo "release probe failed: $label ($url)" >&2 return 1 } verify_site() { local site=$1 local address case $site in hongkong) address=10.77.0.2 ;; ningbo) address=10.77.0.1 ;; *) return 1 ;; esac wait_for_url "$site API health" "http://$address:30088/api/v1/healthz" easyai-ai-gateway wait_for_url "$site API readiness" "http://$address:30088/api/v1/readyz" '"ok":true' wait_for_url "$site Web" "http://$address:30178/" 'EasyAI AI Gateway' } rollout_site() { local site=$1 local api_image=$2 local web_image=$3 local api_changed=$4 local web_changed=$5 if [[ $api_changed == true ]]; then "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \ "api=$api_image" "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s fi if [[ $web_changed == true ]]; then "${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \ "web=$web_image" "${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s fi verify_site "$site" } run_backup() { local source_sha=$1 local backup=easyai-pre-release-${source_sha:0:12} cat </dev/null apiVersion: postgresql.cnpg.io/v1 kind: Backup metadata: name: $backup namespace: $NAMESPACE spec: cluster: name: easyai-postgres method: plugin pluginConfiguration: name: barman-cloud.cloudnative-pg.io EOF for _ in $(seq 1 120); do phase=$("${kubectl[@]}" get backup "$backup" -n "$NAMESPACE" \ -o jsonpath='{.status.phase}' 2>/dev/null || true) [[ ${phase,,} == completed ]] && return 0 [[ ${phase,,} == failed ]] && break sleep 5 done echo "pre-migration backup failed: $backup phase=${phase:-unknown}" >&2 return 1 } run_migrator() { local source_sha=$1 local api_image=$2 local job=easyai-migrator-${source_sha:0:12} "${kubectl[@]}" delete job "$job" -n "$NAMESPACE" \ --ignore-not-found=true --wait=true >/dev/null cat </dev/null apiVersion: batch/v1 kind: Job metadata: name: $job namespace: $NAMESPACE labels: easyai.io/release: $source_sha spec: backoffLimit: 0 ttlSecondsAfterFinished: 86400 template: spec: restartPolicy: Never imagePullSecrets: - name: easyai-registry nodeSelector: easyai.io/site: ningbo containers: - name: migrator image: $api_image command: ["/app/easyai-ai-gateway-migrate"] envFrom: - secretRef: name: easyai-ai-gateway-runtime EOF "${kubectl[@]}" wait --for=condition=complete "job/$job" \ -n "$NAMESPACE" --timeout=300s } record_current() { local manifest=$1 local action=$2 local started_at=$3 local source_sha recorded temporary source_sha=$(manifest_get "$manifest" sourceSha) recorded=$RELEASES_DIR/$source_sha.json temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX") rm -f -- "$temporary" RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \ RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \ RELEASE_DEPLOY_ACTION=$action \ node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null install -m 0600 "$temporary" "$recorded" "${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \ --from-file=manifest.json="$recorded" --dry-run=client -o yaml | "${kubectl[@]}" apply -f - >/dev/null rm -f -- "$temporary" } activate_manifest() { local manifest=$1 local action=${2:-deploy} local source_sha base_sha api_image web_image migrations_changed components local api_changed=false web_changed=false current_file previous_api previous_web started_at [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; } node "$manifest_tool" validate "$manifest" >/dev/null source_sha=$(manifest_get "$manifest" sourceSha) base_sha=$(manifest_get "$manifest" baseReleaseSha) api_image=$(manifest_get "$manifest" images.api) web_image=$(manifest_get "$manifest" images.web) migrations_changed=$(manifest_get "$manifest" migrationsChanged) components=$(manifest_get "$manifest" components) [[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || { echo 'release images must be digest-pinned' >&2 return 1 } [[ $components == *'"api"'* ]] && api_changed=true [[ $components == *'"web"'* ]] && web_changed=true if [[ $action == rollback ]]; then api_changed=true web_changed=true fi current_file=$(mktemp "$RELEASES_DIR/.current.XXXXXX") if current_to_file "$current_file"; then current_sha=$(manifest_get "$current_file" sourceSha) previous_api=$(manifest_get "$current_file" images.api) previous_web=$(manifest_get "$current_file" images.web) if [[ $action == deploy && $current_sha != "$base_sha" ]]; then echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2 rm -f -- "$current_file" return 1 fi else if [[ $action == deploy && -n $base_sha ]]; then echo 'cluster release status is unavailable for a non-bootstrap manifest' >&2 rm -f -- "$current_file" return 1 fi previous_api= previous_web= fi started_at=$(date +%s) if [[ $action == deploy && $migrations_changed == true ]]; then run_backup "$source_sha" run_migrator "$source_sha" "$api_image" fi if ! rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" || ! rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" || ! wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then if [[ -n $previous_api && -n $previous_web ]]; then echo '[cluster-release] restoring previous application images' >&2 rollout_site hongkong "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true rollout_site ningbo "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true fi rm -f -- "$current_file" return 1 fi record_current "$manifest" "$action" "$started_at" rm -f -- "$current_file" echo "production_release=PASS action=$action release=$source_sha" } case ${1:-} in status) [[ $# -eq 1 ]] || exit 64 status_file=$(mktemp "$RELEASES_DIR/.status.XXXXXX") current_to_file "$status_file" cat "$status_file" rm -f -- "$status_file" ;; adopt) [[ $# -eq 2 ]] || exit 64 node "$manifest_tool" validate "$2" >/dev/null api_image=$(manifest_get "$2" images.api) web_image=$(manifest_get "$2" images.web) [[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \ -o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]] [[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \ -o jsonpath='{.spec.template.spec.containers[?(@.name=="web")].image}') == "$web_image" ]] record_current "$2" deploy "$(date +%s)" ;; deploy) [[ $# -eq 2 ]] || exit 64 activate_manifest "$2" deploy ;; rollback) [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64 target=$RELEASES_DIR/$2.json [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; } activate_manifest "$target" rollback ;; *) echo 'usage: easyai-ai-gateway-cluster-release {status|adopt |deploy |rollback }' >&2 exit 64 ;; esac