easyai-ai-gateway/scripts/cluster/promote-production-acceptance.sh

#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=scripts/cluster/common.sh
source "$script_dir/common.sh"
cluster_root=${cluster_root:?}

usage() {
  cat <<'EOF'
Usage:
  scripts/cluster/run-production-acceptance.sh \
    --promote dist/releases/<SHA>.json --run-id <production-run-id>

This is the only acceptance action that switches production traffic from
validation to live. It requires an already-passed overall acceptance report and
rechecks the release, digest, config, Run, and traffic-mode CAS fields.
EOF
}

[[ ${1:-} == --promote && ${3:-} == --run-id && $# -eq 4 ]] || {
  usage >&2
  exit 64
}
release_manifest=$2
run_id=$4
[[ -f $release_manifest && ! -L $release_manifest ]] || {
  echo 'release manifest must be a regular non-symlink file' >&2
  exit 1
}
[[ $run_id =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] || {
  echo 'run ID must be a lowercase UUID' >&2
  exit 64
}

load_cluster_env
require_commands curl git jq node openssl
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}"

node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)
api_digest=${api_image##*@}
[[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" &&
   -z $(git -C "$cluster_root" status --short) ]] || {
  echo 'manual promotion requires the clean release source at HEAD' >&2
  exit 1
}

report=${AI_GATEWAY_ACCEPTANCE_REPORT_DIR:-"$cluster_root/dist/acceptance/$run_id"}/acceptance-report.json
[[ -f $report && ! -L $report ]] || {
  echo "passed overall acceptance report is missing: $report" >&2
  exit 1
}
node "$cluster_root/scripts/acceptance/report.mjs" validate --input "$report" >/dev/null
[[ $(jq -r '.runId' "$report") == "$run_id" &&
   $(jq -r '.release.sourceSha' "$report") == "$release_sha" &&
   $(jq -r '.release.apiImageDigest' "$report") == "$api_digest" &&
   $(jq -r '.stages.onlineSimulation.status' "$report") == passed &&
   $(jq -r '.stages.realCanary.status' "$report") == passed &&
   $(jq -r '.promotion.ready' "$report") == true &&
   $(jq -r '.promotion.trafficMode' "$report") == validation ]] || {
  echo 'overall acceptance report is not ready for this manual promotion' >&2
  exit 1
}

namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai}
remote_release_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-cluster-release}
temporary_root=$(mktemp -d)
chmod 0700 "$temporary_root"
trap '[[ $temporary_root == /tmp/* || $temporary_root == /var/folders/* ]] && rm -rf -- "$temporary_root"' EXIT

remote_kubectl() {
  local command_text
  printf -v command_text '%q ' "$@"
  cluster_ssh "$CLUSTER_NINGBO_HOST" "k3s kubectl $command_text"
}

admin_request() {
  local method=$1 path=$2 body=${3:-} output=$4
  local service_ip service_port token_encoded body_encoded response status payload remote_script command
  service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
    -o 'jsonpath={.spec.clusterIP}')
  service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
    -o 'jsonpath={.spec.ports[0].port}')
  token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" | openssl base64 -A)
  body_encoded=$(printf '%s' "$body" | openssl base64 -A)
  # shellcheck disable=SC2016
  remote_script='
set -euo pipefail
token=$(printf "%s" "$4" | base64 -d)
body=$(printf "%s" "$5" | base64 -d)
args=(-sS --max-time 30 -w "\n%{http_code}" -X "$1" -H "Authorization: Bearer $token")
if [[ -n $body ]]; then
  args+=(-H "Content-Type: application/json" --data-binary "$body")
fi
curl "${args[@]}" "http://$2:$3$6"
'
  printf -v command 'bash -c %q -- %q %q %q %q %q %q' \
    "$remote_script" "$method" "$service_ip" "$service_port" \
    "$token_encoded" "$body_encoded" "$path"
  response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$command")
  status=${response##*$'\n'}
  payload=${response%$'\n'*}
  printf '%s' "$payload" >"$output"
  [[ $status =~ ^2[0-9][0-9]$ ]] || {
    echo "acceptance control API failed: method=$method path=$path status=$status" >&2
    return 1
  }
}

current_manifest=$temporary_root/current.json
cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_release_helper status" >"$current_manifest"
node "$cluster_root/scripts/release-manifest.mjs" validate "$current_manifest" >/dev/null
[[ $(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" sourceSha) == "$release_sha" &&
   $(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" images.api) == "$api_image" ]] || {
  echo 'production release changed after acceptance' >&2
  exit 1
}
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do
  [[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \
    -o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
done

mode_file=$temporary_root/mode.json
run_file=$temporary_root/run.json
promotion_file=$temporary_root/promotion.json
admin_request GET /api/admin/system/acceptance/traffic-mode '' "$mode_file"
admin_request GET "/api/admin/system/acceptance/runs/$run_id" '' "$run_file"
[[ $(jq -r '.mode' "$mode_file") == validation &&
   $(jq -r '.runId' "$mode_file") == "$run_id" &&
   $(jq -r '.releaseSha' "$mode_file") == "$release_sha" &&
   $(jq -r '.apiImageDigest' "$mode_file") == "$api_digest" &&
   $(jq -r '.status' "$run_file") == passed ]] || {
  echo 'production acceptance Run or traffic-mode CAS changed before promotion' >&2
  exit 1
}
config_hash=$(jq -r '.config.validationConfigHash' "$run_file")
profiles=$(jq -c '.config.validationCapacityProfiles' "$run_file")
[[ $config_hash =~ ^[0-9a-f]{64}$ && $(jq 'type == "array" and length > 0' <<<"$profiles") == true ]]
[[ $(remote_kubectl get configmap easyai-gateway-release -n "$namespace" \
  -o 'jsonpath={.data.configHash}') == "$config_hash" ]] || {
  echo 'production configuration changed after acceptance' >&2
  exit 1
}
body=$(jq -cn \
  --argjson revision "$(jq -r '.revision' "$mode_file")" \
  --arg releaseSha "$release_sha" \
  --arg apiDigest "$api_digest" \
  --arg configHash "$config_hash" \
  --argjson capacityProfiles "$profiles" \
  '{
    revision:$revision,
    releaseSha:$releaseSha,
    apiImageDigest:$apiDigest,
    workerImageDigest:$apiDigest,
    configHash:$configHash,
    capacityProfiles:$capacityProfiles
  }')
admin_request POST "/api/admin/system/acceptance/runs/$run_id/promote" "$body" "$promotion_file"
[[ $(jq -r '.mode' "$promotion_file") == live &&
   $(jq -r '.runId' "$promotion_file") == "$run_id" ]] || {
  echo 'live promotion CAS did not complete' >&2
  exit 1
}
node "$cluster_root/scripts/acceptance/report.mjs" mark-promoted \
  --input "$report" --output "$report" >/dev/null
echo "production_acceptance_promotion=PASS run_id=$run_id release=$release_sha traffic_mode=live"
exec "$script_dir/monitor-production-release.sh" \
  --run-id "$run_id" --release-manifest "$release_manifest"