#!/usr/bin/env bash set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=scripts/cluster/common.sh source "$script_dir/common.sh" cluster_root=${cluster_root:?} usage() { cat <<'EOF' Usage: scripts/cluster/run-production-acceptance.sh \ --promote dist/releases/.json --run-id This is the only acceptance action that switches production traffic from validation to live. It requires an already-passed overall acceptance report and rechecks the release, digest, config, Run, and traffic-mode CAS fields. EOF } [[ ${1:-} == --promote && ${3:-} == --run-id && $# -eq 4 ]] || { usage >&2 exit 64 } release_manifest=$2 run_id=$4 [[ -f $release_manifest && ! -L $release_manifest ]] || { echo 'release manifest must be a regular non-symlink file' >&2 exit 1 } [[ $run_id =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] || { echo 'run ID must be a lowercase UUID' >&2 exit 64 } load_cluster_env require_commands curl git jq node openssl : "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}" node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha) api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api) api_digest=${api_image##*@} [[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" && -z $(git -C "$cluster_root" status --short) ]] || { echo 'manual promotion requires the clean release source at HEAD' >&2 exit 1 } report=${AI_GATEWAY_ACCEPTANCE_REPORT_DIR:-"$cluster_root/dist/acceptance/$run_id"}/acceptance-report.json [[ -f $report && ! -L $report ]] || { echo "passed overall acceptance report is missing: $report" >&2 exit 1 } node "$cluster_root/scripts/acceptance/report.mjs" validate --input "$report" >/dev/null [[ $(jq -r '.runId' "$report") == "$run_id" && $(jq -r '.release.sourceSha' "$report") == "$release_sha" && $(jq -r '.release.apiImageDigest' "$report") == "$api_digest" && $(jq -r '.stages.onlineSimulation.status' "$report") == passed && $(jq -r '.stages.realCanary.status' "$report") == passed && $(jq -r '.promotion.ready' "$report") == true && $(jq -r '.promotion.trafficMode' "$report") == validation ]] || { echo 'overall acceptance report is not ready for this manual promotion' >&2 exit 1 } namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai} remote_release_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-cluster-release} temporary_root=$(mktemp -d) chmod 0700 "$temporary_root" trap '[[ $temporary_root == /tmp/* || $temporary_root == /var/folders/* ]] && rm -rf -- "$temporary_root"' EXIT remote_kubectl() { local command_text printf -v command_text '%q ' "$@" cluster_ssh "$CLUSTER_NINGBO_HOST" "k3s kubectl $command_text" } admin_request() { local method=$1 path=$2 body=${3:-} output=$4 local service_ip service_port token_encoded body_encoded response status payload remote_script command service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \ -o 'jsonpath={.spec.clusterIP}') service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \ -o 'jsonpath={.spec.ports[0].port}') token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" | openssl base64 -A) body_encoded=$(printf '%s' "$body" | openssl base64 -A) # shellcheck disable=SC2016 remote_script=' set -euo pipefail token=$(printf "%s" "$4" | base64 -d) body=$(printf "%s" "$5" | base64 -d) args=(-sS --max-time 30 -w "\n%{http_code}" -X "$1" -H "Authorization: Bearer $token") if [[ -n $body ]]; then args+=(-H "Content-Type: application/json" --data-binary "$body") fi curl "${args[@]}" "http://$2:$3$6" ' printf -v command 'bash -c %q -- %q %q %q %q %q %q' \ "$remote_script" "$method" "$service_ip" "$service_port" \ "$token_encoded" "$body_encoded" "$path" response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$command") status=${response##*$'\n'} payload=${response%$'\n'*} printf '%s' "$payload" >"$output" [[ $status =~ ^2[0-9][0-9]$ ]] || { echo "acceptance control API failed: method=$method path=$path status=$status" >&2 return 1 } } current_manifest=$temporary_root/current.json cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_release_helper status" >"$current_manifest" node "$cluster_root/scripts/release-manifest.mjs" validate "$current_manifest" >/dev/null [[ $(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" sourceSha) == "$release_sha" && $(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" images.api) == "$api_image" ]] || { echo 'production release changed after acceptance' >&2 exit 1 } for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do [[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \ -o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]] done mode_file=$temporary_root/mode.json run_file=$temporary_root/run.json promotion_file=$temporary_root/promotion.json admin_request GET /api/admin/system/acceptance/traffic-mode '' "$mode_file" admin_request GET "/api/admin/system/acceptance/runs/$run_id" '' "$run_file" [[ $(jq -r '.mode' "$mode_file") == validation && $(jq -r '.runId' "$mode_file") == "$run_id" && $(jq -r '.releaseSha' "$mode_file") == "$release_sha" && $(jq -r '.apiImageDigest' "$mode_file") == "$api_digest" && $(jq -r '.status' "$run_file") == passed ]] || { echo 'production acceptance Run or traffic-mode CAS changed before promotion' >&2 exit 1 } config_hash=$(jq -r '.config.validationConfigHash' "$run_file") profiles=$(jq -c '.config.validationCapacityProfiles' "$run_file") [[ $config_hash =~ ^[0-9a-f]{64}$ && $(jq 'type == "array" and length > 0' <<<"$profiles") == true ]] [[ $(remote_kubectl get configmap easyai-gateway-release -n "$namespace" \ -o 'jsonpath={.data.configHash}') == "$config_hash" ]] || { echo 'production configuration changed after acceptance' >&2 exit 1 } body=$(jq -cn \ --argjson revision "$(jq -r '.revision' "$mode_file")" \ --arg releaseSha "$release_sha" \ --arg apiDigest "$api_digest" \ --arg configHash "$config_hash" \ --argjson capacityProfiles "$profiles" \ '{ revision:$revision, releaseSha:$releaseSha, apiImageDigest:$apiDigest, workerImageDigest:$apiDigest, configHash:$configHash, capacityProfiles:$capacityProfiles }') admin_request POST "/api/admin/system/acceptance/runs/$run_id/promote" "$body" "$promotion_file" [[ $(jq -r '.mode' "$promotion_file") == live && $(jq -r '.runId' "$promotion_file") == "$run_id" ]] || { echo 'live promotion CAS did not complete' >&2 exit 1 } node "$cluster_root/scripts/acceptance/report.mjs" mark-promoted \ --input "$report" --output "$report" >/dev/null echo "production_acceptance_promotion=PASS run_id=$run_id release=$release_sha traffic_mode=live" exec "$script_dir/monitor-production-release.sh" \ --run-id "$run_id" --release-manifest "$release_manifest"