#!/usr/bin/env bash
set -euo pipefail

[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; }

config_file=${AI_GATEWAY_CLUSTER_RELEASE_CONFIG:-/etc/easyai-ai-gateway-cluster-release.conf}
manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs}
[[ -f $config_file && ! -L $config_file ]] || { echo 'missing cluster release config' >&2; exit 1; }
[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; }
# shellcheck source=/dev/null
source "$config_file"

: "${RELEASES_DIR:?}"
: "${NAMESPACE:=easyai}"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}"
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
: "${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB:=1536}"
: "${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES:=500}"
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_RIVER_MAX_CONNS:=8}"
: "${AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS:=30}"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
: "${AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS:=600}"
: "${AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT:=75}"
: "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}"
: "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}"
: "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}"
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS * 2 + 8))}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
  echo 'invalid cluster release configuration' >&2
  exit 1
}
[[ $DESIRED_STATE_DIR == /* &&
   -f $DESIRED_STATE_DIR/kustomization.yaml &&
   ! -L $DESIRED_STATE_DIR/kustomization.yaml ]] || {
  echo 'production desired-state bundle is unavailable' >&2
  exit 1
}
for capacity_value in \
  "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
  "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
  "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
  "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
  "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
  "$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB" \
  "$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES" \
  "$AI_GATEWAY_DATABASE_MAX_CONNS" \
  "$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
  "$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
  "$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
  "$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
  "$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
  "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
  "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
  "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
  "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
  "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
  "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
  "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
  "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
  "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
  "$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
  "$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
  "$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
  "$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
  "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET"; do
  [[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
    echo 'worker capacity configuration must contain positive integers' >&2
    exit 1
  }
done
[[ $AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET =~ ^[1-9][0-9]*$ ]] || {
  echo 'PostgreSQL non-Worker connection budget must be a positive integer' >&2
  exit 1
}
[[ $AI_GATEWAY_DATABASE_MIN_IDLE_CONNS =~ ^[0-9]+$ ]] || {
  echo 'database minimum idle connection configuration must be a non-negative integer' >&2
  exit 1
}
[[ $AI_GATEWAY_WORKER_AUTOSCALING_ENABLED == true ||
   $AI_GATEWAY_WORKER_AUTOSCALING_ENABLED == false ]] || {
  echo 'worker autoscaling flag must be true or false' >&2
  exit 1
}
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
   AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
   AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
   AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
   AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 &&
   AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT == AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT &&
   AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB >= 256 &&
   AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB <= 1900 &&
   AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES >= 100 &&
   AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES <= 1900 &&
   AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
   AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 &&
   AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS < AI_GATEWAY_DATABASE_MAX_CONNS &&
   AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS + AI_GATEWAY_DATABASE_RIVER_MAX_CONNS < AI_GATEWAY_DATABASE_MAX_CONNS &&
   AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS + AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS < AI_GATEWAY_API_DATABASE_MAX_CONNS &&
   AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS &&
   AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS &&
   AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS <= 3600 &&
   AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
   AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 &&
   AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO &&
   AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO &&
   AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG &&
   AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG &&
   AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 &&
   AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 &&
   AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT &&
   AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 &&
   AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 &&
   AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 &&
   AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET &&
   AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET +
     (AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) *
       AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || {
  echo 'worker capacity configuration exceeds the release safety bounds' >&2
  exit 1
}
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
  echo 'invalid public base URL' >&2
  exit 1
}
for command in "$K3S_BIN" curl flock install jq node sha256sum; do
  command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
done

kubectl=("$K3S_BIN" kubectl)
ACTIVE_RELEASE_SHA=
install -d -m 0700 "$RELEASES_DIR"
exec 9>"$RELEASES_DIR/.release.lock"
flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; }

manifest_get() {
  node "$manifest_tool" get "$1" "$2"
}

capacity_config_hash() {
  printf '%s\n' \
    "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
    "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
    "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
    "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
    "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
    "$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB" \
    "$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES" \
    "$AI_GATEWAY_DATABASE_MAX_CONNS" \
    "$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
    "$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
    "$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
    "$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
    "$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
    "$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
    "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
    "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
    "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
    "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
    "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
    "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
    "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
    "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
    "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
    "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
    "$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
    "$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
    "$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
    "$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
    "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
    "$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET" |
    sha256sum | awk '{print $1}'
}

record_capacity_config() {
  local config_hash
  config_hash=$(capacity_config_hash)
  "${kubectl[@]}" patch configmap easyai-gateway-release -n "$NAMESPACE" \
    --type merge -p "{\"data\":{\"configHash\":\"$config_hash\"}}" >/dev/null
  "${kubectl[@]}" annotate namespace "$NAMESPACE" \
    "easyai.io/config-hash=$config_hash" --overwrite >/dev/null
  for deployment in \
    easyai-api-ningbo easyai-api-hongkong \
    easyai-worker-ningbo easyai-worker-hongkong \
    easyai-capacity-controller; do
    "${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
      "easyai.io/config-hash=$config_hash" --overwrite >/dev/null
  done
}

current_to_file() {
  local output=$1
  "${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
    -o jsonpath='{.data.manifest\.json}' >"$output"
  [[ -s $output ]]
  node "$manifest_tool" validate "$output" >/dev/null
}

wait_for_url() {
  local label=$1
  local url=$2
  local expected=${3:-}
  local body
  for _ in $(seq 1 60); do
    if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then
      if [[ -z $expected || $body == *"$expected"* ]]; then
        echo "[cluster-release] verified $label"
        return 0
      fi
    fi
    sleep 2
  done
  echo "release probe failed: $label ($url)" >&2
  return 1
}

verify_site() {
  local site=$1
  local address
  local api_port
  local web_port
  case $site in
    hongkong)
      address=10.77.0.2
      api_port=31089
      web_port=31179
      ;;
    ningbo)
      address=10.77.0.1
      api_port=31088
      web_port=31178
      ;;
    *) return 1 ;;
  esac
  wait_for_url "$site API health" "http://$address:$api_port/api/v1/healthz" easyai-ai-gateway
  wait_for_url "$site API readiness" "http://$address:$api_port/api/v1/readyz" '"ok":true'
  wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
}

apply_desired_state() {
  local api_image=$1
  local web_image=$2
  local capacity_controller_replicas=${3:-2}
  local api_repository=${api_image%@*}
  local api_digest=${api_image#*@}
  local web_repository=${web_image%@*}
  local web_digest=${web_image#*@}
  local working_directory
  working_directory=$(mktemp -d)
  cp -a "$DESIRED_STATE_DIR/." "$working_directory/"
  sed -i \
    -e "/  - name: easyai-api/{n;s|    newName:.*|    newName: $api_repository|;n;s|    digest:.*|    digest: $api_digest|;}" \
    -e "/  - name: easyai-web/{n;s|    newName:.*|    newName: $web_repository|;n;s|    digest:.*|    digest: $web_digest|;}" \
    "$working_directory/kustomization.yaml"
  if [[ $capacity_controller_replicas == 0 ]]; then
    sed -i \
      '/^  name: easyai-capacity-controller$/,/^---$/s/^  replicas: 2$/  replicas: 0/' \
      "$working_directory/application.yaml"
  fi
  "${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \
    --field-manager=easyai-production-preflight \
    -k "$working_directory" >/dev/null
  "${kubectl[@]}" apply --server-side --force-conflicts \
    --field-manager=easyai-production-release \
    -k "$working_directory" >/dev/null
  rm -rf -- "$working_directory"
}

snapshot_application_deployments() {
  local output=$1
  "${kubectl[@]}" get deployments -n "$NAMESPACE" \
    easyai-api-ningbo easyai-api-hongkong \
    easyai-worker-ningbo easyai-worker-hongkong \
    easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong \
    -o json |
    jq '
      del(.metadata)
      | .items |= map(
          del(
            .metadata.annotations["deployment.kubernetes.io/revision"],
            .metadata.creationTimestamp,
            .metadata.generation,
            .metadata.managedFields,
            .metadata.resourceVersion,
            .metadata.uid,
            .status
          )
        )
    ' >"$output"
}

restore_application_deployments() {
  local snapshot=$1
  [[ -s $snapshot ]] || return 1
  "${kubectl[@]}" apply -f "$snapshot" >/dev/null
  for deployment in \
    easyai-api-ningbo easyai-api-hongkong \
    easyai-worker-ningbo easyai-worker-hongkong \
    easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong; do
    "${kubectl[@]}" rollout status "deployment/$deployment" \
      -n "$NAMESPACE" --timeout=300s || true
  done
}

rollout_worker_site() {
  local site=$1
  local api_image=$2
  local replicas
  case $site in
    ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
    hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
    *) return 1 ;;
  esac
  "${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
    "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
    "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
    "AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
    "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
    "AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
    "AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
    "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
    "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
    "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
    "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
    "AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \
    "AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
    "AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
    "AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
    "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
    "AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
    "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
    "AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
    "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA"
  "${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
    --containers=worker \
    --requests="cpu=${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB}Mi" \
    --limits='cpu=2,memory=2Gi'
  "${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
    --replicas="$replicas"
  if [[ -n $api_image ]]; then
    "${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
      "worker=$api_image"
  fi
  "${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
  [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
    -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
  [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
    -o jsonpath='{.status.readyReplicas}') == "$replicas" ]]
}

rollout_capacity_controller() {
  local api_image=${1:-}
  "${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \
    "AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
    "AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
    "AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
    "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
    "AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
    "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
    "AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
    "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
    "AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
    "AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
    "AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
    "AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT=$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
    "AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
    "AI_GATEWAY_NODE_CPU_TARGET_PERCENT=$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
    "AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
    "AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET" \
    "AI_GATEWAY_WORKER_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
    "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
    "AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
    "AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
  if [[ -n $api_image ]]; then
    "${kubectl[@]}" set image deployment/easyai-capacity-controller -n "$NAMESPACE" \
      "capacity-controller=$api_image"
  fi
  "${kubectl[@]}" rollout status deployment/easyai-capacity-controller -n "$NAMESPACE" --timeout=300s
}

rollout_api_capacity_site() {
  local site=$1
  "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
    "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
    "AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
    "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
    "AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
    "AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
    "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
    "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
    "AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
  "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
}

apply_capacity_config() {
  local capacity_controller_enabled
  ACTIVE_RELEASE_SHA=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
    -o jsonpath='{.data.releaseSha}' 2>/dev/null || true)
  capacity_controller_enabled=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
    -o jsonpath='{.data.capacityControllerEnabled}' 2>/dev/null || true)
  [[ $ACTIVE_RELEASE_SHA =~ ^[0-9a-f]{40}$ ]] || {
    echo 'current release SHA is unavailable for capacity update' >&2
    return 1
  }
  rollout_worker_site hongkong ""
  rollout_worker_site ningbo ""
  rollout_api_capacity_site hongkong
  rollout_api_capacity_site ningbo
  if [[ $capacity_controller_enabled == true ]]; then
    rollout_capacity_controller ""
  else
    "${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0
  fi
  verify_site hongkong
  verify_site ningbo
  wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
  record_capacity_config
  echo "production_capacity=PASS config_hash=$(capacity_config_hash) ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
}

rollout_site() {
  local site=$1
  local api_image=$2
  local web_image=$3
  local api_changed=$4
  local web_changed=$5
  if [[ $api_changed == true ]]; then
    "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
      "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
      "AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
      "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
      "AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
      "AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
      "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
      "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
    "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
      "api=$api_image"
    "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
  fi
  if [[ $web_changed == true ]]; then
    "${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
      "web=$web_image"
    "${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s
  fi
  verify_site "$site"
}

run_backup() {
  local source_sha=$1
  local backup=easyai-pre-release-${source_sha:0:12}
  cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
apiVersion: postgresql.cnpg.io/v1
kind: Backup
metadata:
  name: $backup
  namespace: $NAMESPACE
spec:
  cluster:
    name: easyai-postgres
  method: plugin
  pluginConfiguration:
    name: barman-cloud.cloudnative-pg.io
EOF
  for _ in $(seq 1 120); do
    phase=$("${kubectl[@]}" get backup "$backup" -n "$NAMESPACE" \
      -o jsonpath='{.status.phase}' 2>/dev/null || true)
    [[ ${phase,,} == completed ]] && return 0
    [[ ${phase,,} == failed ]] && break
    sleep 5
  done
  echo "pre-migration backup failed: $backup phase=${phase:-unknown}" >&2
  return 1
}

run_migrator() {
  local source_sha=$1
  local api_image=$2
  local job=easyai-migrator-${source_sha:0:12}
  "${kubectl[@]}" delete job "$job" -n "$NAMESPACE" \
    --ignore-not-found=true --wait=true >/dev/null
  cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
  name: $job
  namespace: $NAMESPACE
  labels:
    easyai.io/release: $source_sha
spec:
  backoffLimit: 0
  ttlSecondsAfterFinished: 86400
  template:
    spec:
      restartPolicy: Never
      imagePullSecrets:
        - name: easyai-registry
      nodeSelector:
        easyai.io/site: ningbo
      containers:
        - name: migrator
          image: $api_image
          command: ["/app/easyai-ai-gateway-migrate"]
          envFrom:
            - secretRef:
                name: easyai-ai-gateway-runtime
EOF
  "${kubectl[@]}" wait --for=condition=complete "job/$job" \
    -n "$NAMESPACE" --timeout=300s
}

verify_migration_version() {
  local expected=$1
  [[ $expected =~ ^[0-9]{4}_[a-z0-9][a-z0-9_]*$ ]] || {
    echo 'release manifest migration version is invalid' >&2
    return 1
  }
  local primary actual
  primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
    -o jsonpath='{.status.currentPrimary}')
  [[ $primary =~ ^easyai-postgres-[0-9]+$ ]] || {
    echo 'PostgreSQL primary is unavailable for migration verification' >&2
    return 1
  }
  actual=$("${kubectl[@]}" exec -n "$NAMESPACE" "$primary" -c postgres -- \
    psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At \
      -c 'SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1')
  [[ $actual == "$expected" ]] || {
    echo "production migration version mismatch: expected=$expected actual=${actual:-missing}" >&2
    return 1
  }
}

record_current() {
  local manifest=$1
  local action=$2
  local started_at=$3
  local source_sha recorded temporary capacity_controller_enabled=true
  if [[ $action == rollback ]]; then
    capacity_controller_enabled=false
  fi
  source_sha=$(manifest_get "$manifest" sourceSha)
  recorded=$RELEASES_DIR/$source_sha.json
  temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX")
  rm -f -- "$temporary"
  RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
  RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \
  RELEASE_DEPLOY_ACTION=$action \
    node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null
  install -m 0600 "$temporary" "$recorded"
  "${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
    --from-file=manifest.json="$recorded" \
    --from-literal=configHash="$(capacity_config_hash)" \
    --from-literal=capacityControllerEnabled="$capacity_controller_enabled" \
    --from-literal=releaseSha="$source_sha" \
    --dry-run=client -o yaml |
    "${kubectl[@]}" apply -f - >/dev/null
  "${kubectl[@]}" annotate namespace "$NAMESPACE" \
    "easyai.io/release=$source_sha" \
    "easyai.io/config-hash=$(capacity_config_hash)" \
    --overwrite >/dev/null
  for deployment in \
    easyai-api-ningbo easyai-api-hongkong \
    easyai-worker-ningbo easyai-worker-hongkong \
    easyai-capacity-controller \
    easyai-web-ningbo easyai-web-hongkong; do
    "${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
      "easyai.io/release=$source_sha" \
      "easyai.io/config-hash=$(capacity_config_hash)" \
      --overwrite >/dev/null
  done
  rm -f -- "$temporary"
}

record_certification() {
  local run_id=$1
  local capacity_profile=$2
  local config_hash=$3
  local instance_slots=$4
  local max_ningbo=$5
  local max_hongkong=$6
  [[ $run_id =~ ^[0-9a-f-]{36}$ &&
     $capacity_profile =~ ^P(24|28|32)$ &&
     $config_hash =~ ^[0-9a-f]{64}$ &&
     $instance_slots =~ ^[1-9][0-9]*$ &&
     $max_ningbo =~ ^[1-9][0-9]*$ &&
     $max_hongkong =~ ^[1-9][0-9]*$ ]] || {
    echo 'invalid production certification metadata' >&2
    return 1
  }
  local effective_config_hash
  effective_config_hash=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
    -o jsonpath='{.data.configHash}' 2>/dev/null || true)
  [[ $effective_config_hash == "$config_hash" ]] || {
    echo 'production certification config hash does not match the deployed release configuration' >&2
    return 1
  }
  local current_file source_sha recorded temporary capacity_controller_enabled
  current_file=$(mktemp "$RELEASES_DIR/.certification-current.XXXXXX")
  current_to_file "$current_file"
  source_sha=$(manifest_get "$current_file" sourceSha)
  recorded=$RELEASES_DIR/$source_sha.json
  temporary=$(mktemp "$RELEASES_DIR/.certification.XXXXXX")
  rm -f -- "$temporary"
  RELEASE_CERTIFIED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
  RELEASE_ACCEPTANCE_RUN_ID=$run_id \
  RELEASE_CONFIG_HASH=$config_hash \
  RELEASE_CAPACITY_PROFILE=$capacity_profile \
  RELEASE_WORKER_INSTANCE_SLOTS=$instance_slots \
  RELEASE_WORKER_MAX_REPLICAS_NINGBO=$max_ningbo \
  RELEASE_WORKER_MAX_REPLICAS_HONGKONG=$max_hongkong \
    node "$manifest_tool" record-certification "$current_file" "$temporary" >/dev/null
  install -m 0600 "$temporary" "$recorded"
  capacity_controller_enabled=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
    -o jsonpath='{.data.capacityControllerEnabled}' 2>/dev/null || true)
  "${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
    --from-file=manifest.json="$recorded" \
    --from-literal=configHash="$config_hash" \
    --from-literal=capacityControllerEnabled="${capacity_controller_enabled:-true}" \
    --from-literal=releaseSha="$source_sha" \
    --from-literal=certifiedRunId="$run_id" \
    --from-literal=certifiedCapacityProfile="$capacity_profile" \
    --dry-run=client -o yaml |
    "${kubectl[@]}" apply -f - >/dev/null
  "${kubectl[@]}" annotate namespace "$NAMESPACE" \
    "easyai.io/certified-run=$run_id" \
    "easyai.io/capacity-profile=$capacity_profile" \
    --overwrite >/dev/null
  rm -f -- "$temporary" "$current_file"
  echo "production_certification=PASS release=$source_sha run_id=$run_id capacity_profile=$capacity_profile config_hash=$config_hash"
}

activate_manifest() {
  local manifest=$1
  local action=${2:-deploy}
  local source_sha base_sha api_image web_image migrations_changed migration_version components
  local api_changed=false web_changed=false current_file deployment_snapshot previous_api previous_web started_at
  local capacity_controller_replicas
  [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
  node "$manifest_tool" validate "$manifest" >/dev/null
  source_sha=$(manifest_get "$manifest" sourceSha)
  ACTIVE_RELEASE_SHA=$source_sha
  base_sha=$(manifest_get "$manifest" baseReleaseSha)
  api_image=$(manifest_get "$manifest" images.api)
  web_image=$(manifest_get "$manifest" images.web)
  migrations_changed=$(manifest_get "$manifest" migrationsChanged)
  migration_version=$(manifest_get "$manifest" migrationVersion)
  components=$(manifest_get "$manifest" components)
  [[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || {
    echo 'release images must be digest-pinned' >&2
    return 1
  }
  [[ $components == *'"api"'* ]] && api_changed=true
  [[ $components == *'"web"'* ]] && web_changed=true
  if [[ $action == rollback ]]; then
    api_changed=true
    web_changed=true
  fi

  current_file=$(mktemp "$RELEASES_DIR/.current.XXXXXX")
  if current_to_file "$current_file"; then
    current_sha=$(manifest_get "$current_file" sourceSha)
    previous_api=$(manifest_get "$current_file" images.api)
    previous_web=$(manifest_get "$current_file" images.web)
    if [[ $action == deploy && $current_sha != "$base_sha" ]]; then
      echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2
      rm -f -- "$current_file"
      return 1
    fi
  else
    if [[ $action == deploy && -n $base_sha ]]; then
      echo 'cluster release status is unavailable for a non-bootstrap manifest' >&2
      rm -f -- "$current_file"
      return 1
    fi
    previous_api=
    previous_web=
  fi
  started_at=$(date +%s)
  deployment_snapshot=$(mktemp "$RELEASES_DIR/.deployments.XXXXXX")
  snapshot_application_deployments "$deployment_snapshot"

  if [[ $action == deploy && $migrations_changed == true ]]; then
    run_backup "$source_sha"
    run_migrator "$source_sha" "$api_image"
  fi
  if [[ $action == deploy ]]; then
    verify_migration_version "$migration_version"
  fi
  capacity_controller_replicas=2
  if [[ $action == rollback ]]; then
    capacity_controller_replicas=0
  fi
  if ! apply_desired_state "$api_image" "$web_image" "$capacity_controller_replicas"; then
    echo '[cluster-release] desired state apply failed; restoring exact deployment snapshot' >&2
    restore_application_deployments "$deployment_snapshot" || true
    rm -f -- "$deployment_snapshot" "$current_file"
    return 1
  fi
  if [[ $action == rollback ]]; then
    # Historical API images may predate the capacity-controller process role.
    # Rollbacks therefore restore static Worker replicas and leave the
    # controller at zero instead of ever starting an incompatible image.
    "${kubectl[@]}" scale deployment/easyai-capacity-controller \
      -n "$NAMESPACE" --replicas=0
  fi

  if { [[ $api_changed != true ]] ||
       { rollout_worker_site hongkong "$api_image" &&
         rollout_worker_site ningbo "$api_image" &&
         { [[ $action == rollback ]] ||
           rollout_capacity_controller "$api_image"; }; }; } &&
     rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
     rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
     wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
    :
  else
    if [[ -n $previous_api && -n $previous_web ]]; then
      echo '[cluster-release] restoring exact pre-release Deployment snapshot' >&2
      restore_application_deployments "$deployment_snapshot" || true
    fi
    rm -f -- "$deployment_snapshot" "$current_file"
    return 1
  fi

  record_current "$manifest" "$action" "$started_at"
  rm -f -- "$deployment_snapshot" "$current_file"
  echo "production_release=PASS action=$action release=$source_sha"
}

case ${1:-} in
  status)
    [[ $# -eq 1 ]] || exit 64
    status_file=$(mktemp "$RELEASES_DIR/.status.XXXXXX")
    current_to_file "$status_file"
    cat "$status_file"
    rm -f -- "$status_file"
    ;;
  adopt)
    [[ $# -eq 2 ]] || exit 64
    node "$manifest_tool" validate "$2" >/dev/null
    api_image=$(manifest_get "$2" images.api)
    web_image=$(manifest_get "$2" images.web)
    for site in ningbo hongkong; do
      [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
        -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
    done
    [[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \
      -o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]]
    [[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
      -o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]]
    [[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \
      -o jsonpath='{.spec.template.spec.containers[?(@.name=="web")].image}') == "$web_image" ]]
    record_current "$2" deploy "$(date +%s)"
    ;;
  deploy)
    [[ $# -eq 2 ]] || exit 64
    activate_manifest "$2" deploy
    ;;
  rollback)
    [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64
    target=$RELEASES_DIR/$2.json
    [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
    activate_manifest "$target" rollback
    ;;
  capacity)
    [[ $# -eq 1 ]] || exit 64
    apply_capacity_config
    ;;
  certify)
    [[ $# -eq 7 ]] || exit 64
    record_certification "$2" "$3" "$4" "$5" "$6" "$7"
    ;;
  *)
    echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity|certify <run-id> <profile> <config-hash> <slots> <max-ningbo> <max-hongkong>}' >&2
    exit 64
    ;;
esac