#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=scripts/cluster/common.sh
source "$script_dir/common.sh"
cluster_root=${cluster_root:?}

usage() {
  cat <<'EOF'
Usage:
  scripts/cluster/run-production-acceptance.sh \
    --execute dist/releases/<SHA>.json \
    --local-report dist/acceptance/local/<run-id>/acceptance-report.json
  scripts/cluster/run-production-acceptance.sh \
    --execute dist/releases/<SHA>.json \
    --skip-local-acceptance
  scripts/cluster/run-production-acceptance.sh \
    --execute-single-node dist/releases/<SHA>.json
  scripts/cluster/run-production-acceptance.sh \
    --promote dist/releases/<SHA>.json --run-id <production-run-id>

This is a destructive production acceptance workflow. It pauses new live tasks,
changes Worker capacity, creates paid real canaries, and deletes one Worker Pod
during each recovery phase. It also grants the dedicated acceptance API Key
access to the selected production candidates. A failed run deliberately keeps
traffic in validation mode and restores only the last stable capacity profile.
Successful execute runs also remain in validation. Only the separate --promote
action performs the final CAS switch to live after manual confirmation.
The explicit --skip-local-acceptance form records both local stages as skipped
with a user-directed waiver. It never records them as passed and preserves all
online simulation, real canary, resource, consistency, and release CAS gates.

The --execute-single-node form is a Ningbo-only diagnostic. It keeps production
traffic in validation, routes acceptance candidates only to the protocol
emulator, never runs real-canary, and leaves the Run non-promotable.

Required private environment values:
  AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN, or AI_GATEWAY_ONLINE_ACCOUNT/PASSWORD

The dedicated acceptance identity, API Key, two node gateways, synthetic real
canary images, and production snapshot transport are bootstrapped when their
AI_GATEWAY_ACCEPTANCE_* overrides are omitted.

Optional model overrides (otherwise selected from current production candidates):
  AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL
  AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL
  AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME
  AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS
  AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY
  AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT
  AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS
  AI_GATEWAY_ACCEPTANCE_ETCD_BACKEND_COMMIT_MAX_MS
  AI_GATEWAY_ACCEPTANCE_ETCD_WAL_FSYNC_MAX_MS
EOF
}

if [[ ${1:-} == --promote ]]; then
  exec "$script_dir/promote-production-acceptance.sh" "$@"
fi

skip_local_acceptance=false
single_node_acceptance=false
local_acceptance_report=
if [[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]]; then
  release_manifest=$2
  local_acceptance_report=$4
elif [[ ${1:-} == --execute && ${3:-} == --skip-local-acceptance && $# -eq 3 ]]; then
  release_manifest=$2
  skip_local_acceptance=true
elif [[ ${1:-} == --execute-single-node && $# -eq 2 ]]; then
  release_manifest=$2
  skip_local_acceptance=true
  single_node_acceptance=true
else
  usage >&2
  exit 64
fi
[[ -f $release_manifest && ! -L $release_manifest ]] || {
  echo 'release manifest must be a regular file' >&2
  exit 1
}
if [[ $skip_local_acceptance != true ]]; then
  [[ -f $local_acceptance_report && ! -L $local_acceptance_report ]] || {
    echo 'local acceptance report must be a regular file' >&2
    exit 1
  }
fi

load_cluster_env
require_commands curl git go jq node openssl sed shasum

if [[ $single_node_acceptance == true ]]; then
  AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO=1
  AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG=0
  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=1
  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=0
  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO=1
  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG=0
  AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS=${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:-16}
  AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB=512
  AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS=30
fi

: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:=}"
: "${AI_GATEWAY_ACCEPTANCE_USER_ID:=}"
: "${AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL:=}"
: "${AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS:=31}"
: "${AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS:=8}"
: "${AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS:=4}"
: "${AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS:=300}"
: "${AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE:=8}"
: "${AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY:=128}"
: "${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:=32}"
: "${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB:=1536}"
: "${AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES:=500}"
: "${AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT:=65}"
: "${AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS:=60}"
: "${AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_TIMEOUT_SECONDS:=240}"
: "${AI_GATEWAY_ACCEPTANCE_ETCD_BACKEND_COMMIT_MAX_MS:=50}"
: "${AI_GATEWAY_ACCEPTANCE_ETCD_WAL_FSYNC_MAX_MS:=15}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG}"
: "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}"
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=4}"
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=4}"
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS:=96}"
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=4 8 12 16 24 32 40 48}"
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}"
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}"
: "${AI_GATEWAY_ONLINE_ACCOUNT:=}"
: "${AI_GATEWAY_ONLINE_PASSWORD:=}"
: "${AI_GATEWAY_ONLINE_BASE_URL:=}"
if [[ -z $AI_GATEWAY_ACCEPTANCE_GATEWAYS ]]; then
  if [[ $single_node_acceptance == true ]]; then
    AI_GATEWAY_ACCEPTANCE_GATEWAYS='http://10.77.0.1:18089'
  else
    AI_GATEWAY_ACCEPTANCE_GATEWAYS="https://${CLUSTER_NINGBO_HOST#root@},https://${CLUSTER_HONGKONG_HOST#root@}"
    : "${AI_GATEWAY_DEPLOY_DOMAIN:?}"
    AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:-$AI_GATEWAY_DEPLOY_DOMAIN}
  fi
fi
single_node_video_slots=()
if [[ $single_node_acceptance == true ]]; then
  [[ $AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS =~ ^[1-9][0-9]*$ &&
     $AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS -le 128 &&
     $AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS =~ ^[1-9][0-9]*$ &&
     $AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS -le 128 &&
     $AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS =~ ^[1-9][0-9]*$ ]] || {
    echo 'single-node baseline, GEMINI slots, and GEMINI requests must be positive bounded integers' >&2
    exit 1
  }
  read -r -a single_node_video_slots <<<"$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS"
  (( ${#single_node_video_slots[@]} > 0 )) || {
    echo 'AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS must contain at least one slot count' >&2
    exit 1
  }
  for single_node_slot in "${single_node_video_slots[@]}"; do
    [[ $single_node_slot =~ ^[1-9][0-9]*$ && $single_node_slot -le 128 ]] || {
      echo 'single-node video slot counts must be integers between 1 and 128' >&2
      exit 1
    }
  done
fi
[[ $AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS -le 256 ]] || {
  echo 'AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS must be between 1 and 256' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS -le 3600 ]] || {
  echo 'AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS must be between 1 and 3600' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE -le 32 ]] || {
  echo 'AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE must be between 1 and 32' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS -lt 28 &&
   $AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS -lt $((AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS - 4)) ]] || {
  echo 'acceptance River database pool sizes must be positive integers' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY -le 1024 ]] || {
  echo 'AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY must be between 1 and 1024' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS -le 128 ]] || {
  echo 'AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS must be between 1 and 128' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB -ge 256 &&
   $AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB -le 1900 ]] || {
  echo 'AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB must be between 256 and 1900' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES -ge 100 &&
   $AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES -le 1900 ]] || {
  echo 'AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES must be between 100 and 1900' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -ge 50 &&
   $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -lt 80 ]] || {
  echo 'AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT must be between 50 and 79' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS -ge 30 &&
   $AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS -le 300 &&
   $AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_TIMEOUT_SECONDS =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_TIMEOUT_SECONDS -ge $AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS &&
   $AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_TIMEOUT_SECONDS -le 900 ]] || {
  echo 'post-rollout stability window must be 30..300 seconds and timeout must be window..900 seconds' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_ETCD_BACKEND_COMMIT_MAX_MS =~ ^[1-9][0-9]*$ &&
   $AI_GATEWAY_ACCEPTANCE_ETCD_WAL_FSYNC_MAX_MS =~ ^[1-9][0-9]*$ ]] || {
  echo 'etcd latency gates must be positive integer milliseconds' >&2
  exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO =~ ^[0-9]+$ &&
   $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO =~ ^[0-9]+$ &&
   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[0-9]+$ &&
   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
   $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
   $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG &&
   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
   $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG -le 16 ]] || {
  echo 'acceptance baseline Worker topology or per-site autoscaling bounds are invalid' >&2
  exit 1
}
if [[ $single_node_acceptance == true ]]; then
  [[ $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 1 &&
     $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 0 ]] || {
    echo 'single-node acceptance requires one Ningbo Worker and zero Hong Kong Workers' >&2
    exit 1
  }
else
  (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) || {
    echo 'production acceptance requires exactly two baseline Workers' >&2
    exit 1
  }
fi

node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)
[[ $release_sha =~ ^[0-9a-f]{40}$ && $api_image =~ @sha256:[0-9a-f]{64}$ ]] || {
  echo 'release manifest is not full-SHA and digest pinned' >&2
  exit 1
}
api_digest=${api_image##*@}
worker_digest=$api_digest
acceptance_group_key=production-acceptance-${release_sha:0:12}
if [[ $skip_local_acceptance != true ]]; then
  node "$cluster_root/scripts/acceptance/report.mjs" validate \
    --input "$local_acceptance_report" >/dev/null
  [[ $(jq -r '.release.sourceSha' "$local_acceptance_report") == "$release_sha" &&
     $(jq -r '.release.apiImageDigest' "$local_acceptance_report") == "$api_digest" &&
     $(jq -r '.stages.localNative.status' "$local_acceptance_report") == passed &&
     $(jq -r '.stages.amd64Artifact.status' "$local_acceptance_report") == passed &&
     $(jq -r '.stages.onlineSimulation.status' "$local_acceptance_report") == not-run &&
     $(jq -r '.promotion.ready' "$local_acceptance_report") == false ]] || {
    echo 'local acceptance report does not match this release or has invalid stage state' >&2
    exit 1
  }
fi
if [[ $single_node_acceptance != true ]]; then
  [[ -z $(git -C "$cluster_root" status --short) ]] || {
    echo 'production acceptance requires a clean release working copy' >&2
    exit 1
  }
else
  echo 'single_node_acceptance_tool=working_tree diagnostic_only=true certification=false'
fi
acceptance_tool_sha=$(git -C "$cluster_root" rev-parse HEAD)
if [[ $acceptance_tool_sha != "$release_sha" ]]; then
  git -C "$cluster_root" merge-base --is-ancestor "$release_sha" "$acceptance_tool_sha" || {
    echo 'production release must be an ancestor of the acceptance tool HEAD' >&2
    exit 1
  }
  if [[ $single_node_acceptance != true ]]; then
    acceptance_tool_delta=$(node "$cluster_root/scripts/release-components.mjs" \
      "$release_sha" "$acceptance_tool_sha")
    [[ $(jq -r '.components' <<<"$acceptance_tool_delta") == none &&
       $(jq -r '.migrationsChanged' <<<"$acceptance_tool_delta") == false ]] || {
      echo 'acceptance tool HEAD contains runtime or migration changes beyond the production release' >&2
      exit 1
    }
    echo "acceptance_tool_delta=PASS release=$release_sha tool_sha=$acceptance_tool_sha runtime_changes=false"
  fi
fi

namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai}
remote_release_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-cluster-release}
[[ $namespace =~ ^[a-z0-9-]+$ && $remote_release_helper =~ ^/[A-Za-z0-9._/-]+$ ]] || {
  echo 'invalid namespace or remote release helper' >&2
  exit 1
}
[[ $namespace == easyai ]] || {
  echo 'the production acceptance manifests currently require namespace easyai' >&2
  exit 1
}

temporary_root=$(mktemp -d)
chmod 0700 "$temporary_root"
current_manifest=$temporary_root/current.json
run_token=$(openssl rand -hex 32)
run_id=
report_root=
stable_profile=P24
active_profile=P24
single_node_stable_slots=4
failure_reason=
failure_gate_id=
failure_recorded=false
acceptance_admin_base=
acceptance_admin_fallback_base=
acceptance_group_id=
acceptance_participants_json='[]'
AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
acceptance_load_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-load-linux-amd64
acceptance_emulator_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-emulator-linux-amd64
acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
current_production_snapshot=$temporary_root/current-production-snapshot.json
local_snapshot_config_hash=
local_snapshot_sha256=
if [[ $skip_local_acceptance != true ]]; then
  local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report")
  local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report")
fi
active_load_pid=
active_pressure_pid=
remote_load_drivers_installed=false
capacity_profiles_payload=
image_stable_throughput=
image_admitted_throughput=
video_stable_throughput=
video_admitted_throughput=
certified_max_replicas_ningbo=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
runtime_observation_started_at=
single_node_emulator_url=
single_node_emulator_installed=false

cleanup() {
  local status=$?
  trap - EXIT
  if [[ -n $active_load_pid ]]; then
    kill "$active_load_pid" >/dev/null 2>&1 || true
    wait "$active_load_pid" >/dev/null 2>&1 || true
  fi
  if [[ -n $active_pressure_pid ]]; then
    kill "$active_pressure_pid" >/dev/null 2>&1 || true
    wait "$active_pressure_pid" >/dev/null 2>&1 || true
  fi
  if [[ $remote_load_drivers_installed == true ]]; then
    cleanup_remote_load_drivers >/dev/null 2>&1 || true
  fi
  if [[ $single_node_emulator_installed == true ]]; then
    cleanup_single_node_emulator >/dev/null 2>&1 || true
  fi
  if [[ $single_node_acceptance == true ]]; then
    cleanup_single_node_kubernetes_resources >/dev/null 2>&1 || true
  fi
  if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
    set +e
    if [[ $single_node_acceptance == true ]]; then
      apply_single_node_capacity "$single_node_stable_slots" >/dev/null 2>&1
    else
      apply_capacity_profile "$stable_profile" >/dev/null 2>&1
    fi
    [[ -n $failure_gate_id ]] || failure_gate_id=workflow_unexpected_exit
    mark_run_failed "${failure_reason:-acceptance workflow exited unexpectedly}"
    set -e
  fi
  rm -rf -- "$temporary_root"
  exit "$status"
}
trap cleanup EXIT
trap 'failure_gate_id=workflow_interrupted; failure_reason="production acceptance interrupted by signal"; exit 130' HUP INT TERM

bootstrap_acceptance_admin_token() {
  if [[ -n $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN ]]; then
    echo 'acceptance_admin_identity=PASS source=explicit_token'
    return 0
  fi
  [[ -n $AI_GATEWAY_ONLINE_ACCOUNT && -n $AI_GATEWAY_ONLINE_PASSWORD &&
     $AI_GATEWAY_ONLINE_BASE_URL == https://* ]] || {
    echo 'acceptance admin token or HTTPS online account credentials are required' >&2
    return 1
  }
  local response=$temporary_root/admin-login.json
  local body status
  body=$(jq -cn \
    --arg account "$AI_GATEWAY_ONLINE_ACCOUNT" \
    --arg password "$AI_GATEWAY_ONLINE_PASSWORD" \
    '{account:$account,password:$password}')
  status=$(curl -sS --max-time 15 -o "$response" -w '%{http_code}' \
    -H 'Content-Type: application/json' --data-binary "$body" \
    "${AI_GATEWAY_ONLINE_BASE_URL%/}/api/v1/auth/login")
  chmod 0600 "$response"
  [[ $status == 200 &&
     $(jq '(.user.role // []) as $roles | if ($roles|type)=="array" then ($roles|index("manager")) != null else $roles == "manager" end' "$response") == true ]] || {
    echo "acceptance Manager login failed: status=$status" >&2
    return 1
  }
  AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN=$(jq -r '.accessToken // empty' "$response")
  [[ -n $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN ]] || {
    echo 'acceptance Manager login did not return an access token' >&2
    return 1
  }
  export AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN
  echo 'acceptance_admin_identity=PASS source=short_lived_manager_login'
}

bootstrap_acceptance_admin_token

(
  cd "$cluster_root/apps/api"
  env -u AI_GATEWAY_TEST_DATABASE_URL go build \
    -trimpath -o "$acceptance_load_binary" ./cmd/acceptance-load
  env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
    -trimpath -o "$acceptance_load_linux_binary" ./cmd/acceptance-load
  if [[ $single_node_acceptance == true ]]; then
    env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
      -trimpath -o "$acceptance_emulator_linux_binary" ./cmd/acceptance-emulator
  fi
  env -u AI_GATEWAY_TEST_DATABASE_URL go build \
    -trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
)
export_production_snapshot() {
  local output=$1
  if [[ -n $AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL ]]; then
    AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \
      "$acceptance_snapshot_binary" export \
        --release-sha "$release_sha" \
        --output "$output" >/dev/null
  else
    local attempt temporary_output
    temporary_output=$output.remote
    rm -f -- "$temporary_output"
    for attempt in 1 2 3; do
      if cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$temporary_output" <<'REMOTE'
set -euo pipefail
release_sha=$1
namespace=easyai
pod=$(k3s kubectl get pods -n "$namespace" \
  -l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
  --field-selector status.phase=Running \
  -o 'jsonpath={.items[0].metadata.name}')
[[ -n $pod && $pod == easyai-api-ningbo-* ]]
[[ $(k3s kubectl get pod "$pod" -n "$namespace" \
  -o 'jsonpath={.status.containerStatuses[0].ready}') == true ]]
snapshot=/tmp/easyai-production-acceptance-snapshot-$release_sha.json
cleanup() {
  if [[ -n ${pod:-} ]]; then
    k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
  fi
}
trap cleanup EXIT
k3s kubectl exec -n "$namespace" "$pod" -- \
  /app/easyai-ai-gateway-acceptance-snapshot export \
    --release-sha "$release_sha" --output "$snapshot" >/dev/null
k3s kubectl exec -n "$namespace" "$pod" -- cat "$snapshot"
REMOTE
      then
        mv -- "$temporary_output" "$output"
        break
      fi
      rm -f -- "$temporary_output"
      if (( attempt == 3 )); then
        echo 'production snapshot export failed after three read-only attempts' >&2
        return 1
      fi
      echo "production snapshot export attempt $attempt failed; retrying read-only export" >&2
      sleep "$attempt"
    done
  fi
  chmod 0600 "$output"
  "$acceptance_snapshot_binary" validate --input "$output" >/dev/null
}

export_production_snapshot "$current_production_snapshot"
if [[ $skip_local_acceptance == true ]]; then
  local_acceptance_report=$temporary_root/online-waiver-base.json
  node "$cluster_root/scripts/acceptance/report.mjs" build-online-waiver \
    --release-sha "$release_sha" \
    --api-digest "$api_digest" \
    --snapshot "$current_production_snapshot" \
    --waiver-reason user_directed_online_only_acceptance \
    --output "$local_acceptance_report" >/dev/null
  local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report")
  local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report")
else
  [[ $(jq -r '.source.configHash' "$current_production_snapshot") == "$local_snapshot_config_hash" ]] || {
    echo 'production candidate configuration changed after the local snapshot; rerun local acceptance' >&2
    exit 1
  }
fi

remote_kubectl() {
  local command_text
  printf -v command_text '%q ' "$@"
  cluster_ssh "$CLUSTER_NINGBO_HOST" "k3s kubectl $command_text"
}

database_query() {
  local sql=$1
  local primary
  primary=$(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.currentPrimary}')
  [[ $primary =~ ^easyai-postgres-[0-9]+$ ]]
  remote_kubectl exec -n "$namespace" "$primary" -c postgres -- \
    psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql"
}

bootstrap_acceptance_primary_identity() {
  if [[ -n $AI_GATEWAY_ACCEPTANCE_API_KEY || -n $AI_GATEWAY_ACCEPTANCE_API_KEY_ID ||
        -n $AI_GATEWAY_ACCEPTANCE_USER_ID ]]; then
    [[ -n $AI_GATEWAY_ACCEPTANCE_API_KEY &&
       $AI_GATEWAY_ACCEPTANCE_API_KEY_ID =~ ^[0-9a-f-]{36}$ &&
       $AI_GATEWAY_ACCEPTANCE_USER_ID =~ ^[0-9a-f-]{36}$ ]] || {
      echo 'explicit acceptance primary identity variables must be provided together' >&2
      return 1
    }
    echo 'acceptance_primary_identity=PASS source=explicit'
    return 0
  fi
  local material
  material=$(database_query "
WITH primary_user AS (
  INSERT INTO gateway_users (
    user_key, source, username, display_name, roles, auth_profile, metadata, status
  )
  VALUES (
    'production-acceptance-primary', 'gateway', 'production-acceptance-primary',
    'Production Acceptance Primary', '[\"user\"]'::jsonb, '{}'::jsonb,
    '{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb, 'active'
  )
  ON CONFLICT (user_key) DO UPDATE
  SET source=EXCLUDED.source,
      username=EXCLUDED.username,
      display_name=EXCLUDED.display_name,
      roles=EXCLUDED.roles,
      auth_profile=EXCLUDED.auth_profile,
      metadata=EXCLUDED.metadata,
      status='active',
      deleted_at=NULL,
      updated_at=now()
  RETURNING id
), existing_key AS (
  SELECT key.id, key.gateway_user_id, key.key_secret
  FROM gateway_api_keys key
  JOIN primary_user ON primary_user.id=key.gateway_user_id
  WHERE key.name='Production Acceptance Primary'
    AND key.status='active'
    AND key.deleted_at IS NULL
    AND COALESCE(key.key_secret, '') <> ''
  ORDER BY key.created_at
  LIMIT 1
), new_key_material AS (
  SELECT primary_user.id AS gateway_user_id,
         'sk-gw-' || encode(gen_random_bytes(32), 'hex') AS secret
  FROM primary_user
  WHERE NOT EXISTS (SELECT 1 FROM existing_key)
), inserted_key AS (
  INSERT INTO gateway_api_keys (
    gateway_user_id, user_id, key_prefix, key_secret, key_hash, name, scopes,
    rate_limit_policy, quota_policy, metadata, status
  )
  SELECT material.gateway_user_id,
         material.gateway_user_id::text,
         left(material.secret, 18),
         material.secret,
         crypt(material.secret, gen_salt('bf', 10)),
         'Production Acceptance Primary',
         '[\"chat\",\"embedding\",\"rerank\",\"image\",\"image_vectorize\",\"video\",\"video_enhance\",\"music\",\"audio\",\"voice_clone\"]'::jsonb,
         '{}'::jsonb,
         '{}'::jsonb,
         '{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb,
         'active'
  FROM new_key_material material
  RETURNING id, gateway_user_id, key_secret
), selected_key AS (
  SELECT * FROM existing_key
  UNION ALL
  SELECT * FROM inserted_key
), wallet AS (
  INSERT INTO gateway_wallet_accounts (
    gateway_user_id, user_id, currency, balance, total_recharged, metadata, status
  )
  SELECT primary_user.id,
         primary_user.id::text,
         'resource',
         1000000000::numeric,
         1000000000::numeric,
         '{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb,
         'active'
  FROM primary_user
  ON CONFLICT (gateway_user_id, currency) DO UPDATE
  SET balance=GREATEST(gateway_wallet_accounts.balance, EXCLUDED.balance),
      total_recharged=GREATEST(gateway_wallet_accounts.total_recharged, EXCLUDED.total_recharged),
      metadata=EXCLUDED.metadata,
      status='active',
      updated_at=now()
  RETURNING gateway_user_id
)
SELECT primary_user.id::text || E'\\t' || selected_key.id::text || E'\\t' || selected_key.key_secret
FROM primary_user
CROSS JOIN selected_key
CROSS JOIN (SELECT count(*) FROM wallet) wallet_write_barrier;")
  [[ $material == *$'\t'* ]] || {
    echo 'acceptance primary identity bootstrap returned an invalid payload' >&2
    return 1
  }
  IFS=$'\t' read -r AI_GATEWAY_ACCEPTANCE_USER_ID \
    AI_GATEWAY_ACCEPTANCE_API_KEY_ID AI_GATEWAY_ACCEPTANCE_API_KEY <<<"$material"
  [[ $AI_GATEWAY_ACCEPTANCE_USER_ID =~ ^[0-9a-f-]{36}$ &&
     $AI_GATEWAY_ACCEPTANCE_API_KEY_ID =~ ^[0-9a-f-]{36}$ &&
     $AI_GATEWAY_ACCEPTANCE_API_KEY =~ ^sk-gw-[0-9a-f]{64}$ ]] || {
    echo 'acceptance primary identity bootstrap failed validation' >&2
    return 1
  }
  AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
  export AI_GATEWAY_ACCEPTANCE_USER_ID AI_GATEWAY_ACCEPTANCE_API_KEY_ID
  export AI_GATEWAY_ACCEPTANCE_API_KEY AI_GATEWAY_ACCEPTANCE_API_KEYS
  echo 'acceptance_primary_identity=PASS source=idempotent_database_bootstrap isolated=true'
}

ensure_acceptance_real_images() {
  if [[ -n $AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS ]]; then
    [[ $(awk -F',' '{print NF}' <<<"$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS") -ge 3 ]] || {
      echo 'explicit acceptance real image URLs must contain at least three entries' >&2
      return 1
    }
    echo 'acceptance_real_images=PASS source=explicit count=3_or_more'
    return 0
  fi
  [[ $AI_GATEWAY_ONLINE_BASE_URL == https://* ]] || {
    echo 'HTTPS online base URL is required to upload synthetic real canary images' >&2
    return 1
  }
  local fixture_root=$temporary_root/real-canary-images
  local response url index
  local -a urls=()
  install -d -m 0700 "$fixture_root"
  node "$cluster_root/scripts/acceptance/generate-reference-images.mjs" "$fixture_root" >/dev/null
  for index in 1 2 3; do
    response=$fixture_root/upload-$index.json
    chmod 0600 "$fixture_root/production-acceptance-reference-$index.png"
    {
      printf 'silent\nshow-error\nfail-with-body\nmax-time = 120\n'
      printf 'header = "Authorization: Bearer %s"\n' "$AI_GATEWAY_ACCEPTANCE_API_KEY"
      printf 'form = "source=production-acceptance"\n'
      printf 'form = "file=@%s;type=image/png"\n' \
        "$fixture_root/production-acceptance-reference-$index.png"
      printf 'url = "%s/api/v1/files/upload"\n' "${AI_GATEWAY_ONLINE_BASE_URL%/}"
    } | curl --config - >"$response"
    chmod 0600 "$response"
    url=$(jq -r '.url // .data.url // empty' "$response")
    if [[ $url == /* ]]; then
      url="${AI_GATEWAY_ONLINE_BASE_URL%/}$url"
    fi
    [[ $url == https://* ]] || {
      echo "synthetic acceptance image upload $index did not return an HTTPS URL" >&2
      return 1
    }
    urls+=("$url")
  done
  AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(IFS=,; printf '%s' "${urls[*]}")
  export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS
  echo 'acceptance_real_images=PASS source=synthetic_gateway_upload count=3 dimensions=512x512'
}

admin_request() {
  local method=$1
  local path=$2
  local body=${3:-}
  local output=$4
  local status response payload token_encoded body_encoded has_body=false
  local service_ip service_port fallback_service_ip fallback_service_port
  local remote_script remote_command fallback_used=false
  if [[ -z $acceptance_admin_base ]]; then
    service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
      -o 'jsonpath={.spec.clusterIP}')
    service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
      -o 'jsonpath={.spec.ports[0].port}')
    [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ ]] || {
      echo 'acceptance admin Service has an invalid cluster address' >&2
      return 1
    }
    acceptance_admin_base=http://$service_ip:$service_port
    if [[ ${single_node_acceptance:-false} != true ]]; then
      fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
        -o 'jsonpath={.spec.clusterIP}')
      fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
        -o 'jsonpath={.spec.ports[0].port}')
      [[ $fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || {
        echo 'acceptance fallback admin Service has an invalid cluster address' >&2
        return 1
      }
      acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port
    fi
  fi
  token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" |
    openssl base64 -A)
  body_encoded=$(printf '%s' "$body" | openssl base64 -A)
  [[ -n $body ]] && has_body=true
  # shellcheck disable=SC2016 # This script is quoted for execution on the remote node.
  remote_script='
set -euo pipefail
admin_base=$1
method=$2
path=$3
token=$(printf "%s" "$4" | base64 -d)
body=$(printf "%s" "$5" | base64 -d)
has_body=$6
if [[ $has_body == true ]]; then
  curl -sS --max-time 30 -w "\n%{http_code}" -X "$method" \
    -H "Authorization: Bearer $token" \
    -H "Content-Type: application/json" \
    --data-binary "$body" "$admin_base$path"
else
  curl -sS --max-time 30 -w "\n%{http_code}" -X "$method" \
    -H "Authorization: Bearer $token" "$admin_base$path"
fi
'
  printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
    "$remote_script" "$acceptance_admin_base" "$method" "$path" \
    "$token_encoded" "$body_encoded" "$has_body"
  if ! response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_command"); then
    if [[ $method != GET && $path != /api/admin/system/acceptance/runs/*/finish ]]; then
      echo "acceptance control API transport failed without safe retry: method=$method path=$path" >&2
      return 1
    fi
    if [[ ${single_node_acceptance:-false} == true ]]; then
      echo "acceptance control API transport failed on Ningbo: method=$method path=$path" >&2
      return 1
    fi
    printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
      "$remote_script" "$acceptance_admin_fallback_base" "$method" "$path" \
      "$token_encoded" "$body_encoded" "$has_body"
    if ! response=$(cluster_ssh "$CLUSTER_HONGKONG_HOST" "$remote_command"); then
      echo "acceptance control API transport failed on both sites: method=$method path=$path" >&2
      return 1
    fi
    fallback_used=true
  fi
  status=${response##*$'\n'}
  payload=${response%$'\n'*}
  if [[ $fallback_used == true && $status == 409 &&
        $path == /api/admin/system/acceptance/runs/*/finish ]]; then
    local expected_status verification_path verification_response verification_status
    expected_status=$(jq -r 'if .passed == true then "passed" else "failed" end' <<<"$body")
    verification_path=${path%/finish}
    printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
      "$remote_script" "$acceptance_admin_fallback_base" GET "$verification_path" \
      "$token_encoded" '' false
    if verification_response=$(cluster_ssh "$CLUSTER_HONGKONG_HOST" "$remote_command"); then
      verification_status=${verification_response##*$'\n'}
      payload=${verification_response%$'\n'*}
      if [[ $verification_status == 200 &&
            $(jq -r '.status // empty' <<<"$payload") == "$expected_status" ]]; then
        status=200
      fi
    fi
  fi
  printf '%s' "$payload" >"$output"
  [[ $status =~ ^2[0-9][0-9]$ ]] || {
    echo "acceptance control API failed: method=$method path=$path status=$status" >&2
    return 1
  }
}

verify_release_cas() {
  local verify_capacity=${1:-true}
  cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_release_helper status" >"$current_manifest"
  node "$cluster_root/scripts/release-manifest.mjs" validate "$current_manifest" >/dev/null
  local current_sha current_api
  current_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" sourceSha)
  current_api=$(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" images.api)
  [[ $current_sha == "$release_sha" && $current_api == "$api_image" ]] || {
    echo "production release CAS mismatch: expected=$release_sha current=$current_sha" >&2
    return 1
  }
  for deployment in easyai-api-ningbo easyai-worker-ningbo; do
    [[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \
      -o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
  done
  [[ $verify_capacity == true ]] || return 0
  [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
    local expected_replicas eligible_nodes eligible_count pod_report
    local running_count invalid_count node4_count used_nodes
    case $CLUSTER_WORKER_NODE_4_SITE in
      ningbo) expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO ;;
      hongkong) expected_replicas=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG ;;
    esac
    eligible_nodes=$(remote_kubectl get nodes \
      -l "easyai.io/site=$CLUSTER_WORKER_NODE_4_SITE,easyai.io/worker=true" \
      -o json | jq -c '[.items[].metadata.name]')
    eligible_count=$(jq 'length' <<<"$eligible_nodes")
    jq -e --arg node "$CLUSTER_WORKER_NODE_4_NAME" 'index($node) != null' \
      <<<"$eligible_nodes" >/dev/null
    pod_report=$(remote_kubectl get pods -n "$namespace" \
      -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$CLUSTER_WORKER_NODE_4_SITE" -o json)
    running_count=$(jq '[.items[] | select(.status.phase=="Running")] | length' <<<"$pod_report")
    invalid_count=$(jq --argjson nodes "$eligible_nodes" \
      '[.items[] | select(.status.phase=="Running") | select(.spec.nodeName as $node | $nodes | index($node) | not)] | length' \
      <<<"$pod_report")
    node4_count=$(jq --arg node "$CLUSTER_WORKER_NODE_4_NAME" \
      '[.items[] | select(.status.phase=="Running" and .spec.nodeName==$node)] | length' <<<"$pod_report")
    used_nodes=$(jq '[.items[] | select(.status.phase=="Running") | .spec.nodeName] | unique | length' \
      <<<"$pod_report")
    [[ $running_count == "$expected_replicas" && $invalid_count == 0 ]] || {
      echo "Worker placement is outside eligible nodes: site=$CLUSTER_WORKER_NODE_4_SITE running_pods=$running_count expected=$expected_replicas invalid=$invalid_count" >&2
      return 1
    }
    if (( expected_replicas > 0 && node4_count == 0 )); then
      echo "dedicated Worker node has no baseline pod: site=$CLUSTER_WORKER_NODE_4_SITE node=$CLUSTER_WORKER_NODE_4_NAME" >&2
      return 1
    fi
    if (( expected_replicas >= 2 && eligible_count >= 2 && used_nodes < 2 )); then
      echo "Worker baseline is not distributed across physical nodes: site=$CLUSTER_WORKER_NODE_4_SITE used_nodes=$used_nodes" >&2
      return 1
    fi
  fi
}

read_node_memory_percent() {
  local node=$1
  local attempt output memory_percent
  for attempt in 1 2 3; do
    if output=$(remote_kubectl top node "$node" --no-headers 2>/dev/null); then
      memory_percent=$(awk '{value=$5; sub(/%$/, "", value); print value}' <<<"$output")
      if [[ $memory_percent =~ ^[0-9]+$ ]]; then
        printf '%s\n' "$memory_percent"
        return 0
      fi
    fi
    (( attempt == 3 )) || sleep 2
  done
  echo "Worker node memory collection failed after retries: node=$node attempts=3" >&2
  return 2
}

verify_resource_preconditions() {
  local report=$temporary_root/resource-preconditions.csv
  local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
  worker_nodes=
  if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then
    worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo,easyai.io/worker=true' \
      -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
    worker_nodes+=$'\n'
  fi
  if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then
    worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong,easyai.io/worker=true' \
      -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
  fi
  worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u)
  los_angeles_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=los-angeles' \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
  [[ -n $worker_nodes && -n $los_angeles_nodes ]]
  invalid_mixed_workloads=$(remote_kubectl get pods -A -o json |
    jq --argjson nodes "$(jq -Rn --arg value "$worker_nodes" '$value | split("\n") | map(select(length>0))')" '
      [.items[]
        | select(.spec.nodeName as $name | $nodes | index($name))
        | select(.metadata.name | test("staging|runner|(^|-)ci(-|$)"; "i"))
        | select(any(.spec.containers[];
            (.resources.requests.memory // "") == "" or
            (.resources.limits.memory // "") == "" or
            (.resources.requests.cpu // "") == "" or
            (.resources.limits.cpu // "") == ""))]
      | length')
  [[ $invalid_mixed_workloads == 0 ]] || {
    echo 'Worker-site staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2
    return 1
  }
  witness_workloads=$(remote_kubectl get pods -A -o json |
    jq --argjson nodes "$(jq -Rn --arg value "$los_angeles_nodes" '$value | split("\n") | map(select(length>0))')" '
      [.items[]
        | select(.spec.nodeName as $name | $nodes | index($name))
        | select(
            .metadata.labels["app.kubernetes.io/name"] == "easyai-worker" or
            .metadata.labels["app.kubernetes.io/name"] == "easyai-acceptance-emulator" or
            .metadata.labels["app.kubernetes.io/name"] == "easyai-capacity-controller"
          )]
      | length')
  [[ $witness_workloads == 0 ]] || {
    echo 'Los Angeles witness node is carrying a Worker, controller, or acceptance emulator' >&2
    return 1
  }
  echo 'timestamp,node,memory_percent,precondition_percent' >"$report"
  local sample node memory_percent
  for sample in $(seq 1 90); do
    while read -r node; do
      [[ -n $node ]] || continue
      if ! memory_percent=$(read_node_memory_percent "$node"); then
        return 2
      fi
      [[ $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
        echo "Worker node memory precondition failed: node=$node memory_percent=$memory_percent limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
        return 1
      }
      printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
        "$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report"
    done <<<"$worker_nodes"
    (( sample == 90 )) || sleep 10
  done
  cp "$report" "$report_root/resource-preconditions.csv"
}

wait_for_existing_tasks_to_drain() {
  local deadline=$((SECONDS + 900))
  local active
  while (( SECONDS < deadline )); do
    active=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id IS NULL AND status IN ('queued','running');")
    if [[ $active == 0 ]]; then
      return 0
    fi
    echo "waiting_for_existing_tasks=$active"
    sleep 5
  done
  echo 'existing production tasks did not drain within 15 minutes' >&2
  return 1
}

snapshot_pre_acceptance_capacity() {
  local output=$report_root/pre-acceptance-capacity.json
  remote_kubectl get \
    deployment/easyai-api-ningbo \
    deployment/easyai-worker-ningbo \
    deployment/easyai-capacity-controller \
    configmap/easyai-ai-gateway-config \
    configmap/easyai-gateway-release \
    -n "$namespace" -o json |
    jq '
      .items |= map(
        del(
          .metadata.creationTimestamp,
          .metadata.generation,
          .metadata.managedFields,
          .metadata.resourceVersion,
          .metadata.uid,
          .status
        )
      )
    ' >"$output"
  chmod 0600 "$output"
  jq -e '.kind == "List" and (.items | length) == 7' "$output" >/dev/null
}

wait_for_terminal_acceptance_tasks_to_drain() {
  local deadline=$((SECONDS + 300))
  local state active side_effects connections max_connections
  while (( SECONDS < deadline )); do
    state=$(database_query "
SELECT
  (SELECT count(*)
   FROM gateway_tasks task
   JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
   WHERE run.status IN ('failed', 'aborted')
     AND task.status IN ('queued', 'running'))||','||
  (
    (SELECT count(*)
     FROM settlement_outbox settlement
     JOIN gateway_tasks task ON task.id=settlement.task_id
     JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
     WHERE run.status IN ('failed', 'aborted')
       AND settlement.status IN ('pending', 'processing', 'retryable_failed'))
    +
    (SELECT count(*)
     FROM gateway_task_callback_outbox callback
     JOIN gateway_tasks task ON task.id=callback.task_id
     JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
     WHERE run.status IN ('failed', 'aborted')
       AND callback.status <> 'delivered')
  )||','||
  (SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
  (SELECT setting FROM pg_settings WHERE name='max_connections');")
    IFS=',' read -r active side_effects connections max_connections <<<"$state"
    if [[ $active == 0 && $side_effects == 0 ]] &&
      (( connections * 2 < max_connections )); then
      return 0
    fi
    echo "waiting_for_terminal_acceptance_tasks=$active side_effects=$side_effects database_connections=$connections"
    sleep 2
  done
  echo "terminal acceptance state did not drain within 5 minutes: active=$active side_effects=$side_effects database_connections=$connections" >&2
  return 1
}

ensure_acceptance_user_group() {
  local groups_response=$temporary_root/user-groups.json
  local group_response=$temporary_root/acceptance-user-group.json
  local users_response=$temporary_root/users.json
  local user_response=$temporary_root/acceptance-user.json
  local group_id group_body user_body isolation_state
  admin_request GET /api/admin/user-groups '' "$groups_response"
  group_id=$(jq -r --arg groupKey "$acceptance_group_key" '
    [.items[] | select(.groupKey == $groupKey) | .id]
    | if length == 1 then .[0] else empty end
  ' "$groups_response")
  group_body=$(jq -cn --arg groupKey "$acceptance_group_key" '{
    groupKey:$groupKey,
    name:"Production Acceptance",
    description:"Dedicated production-isomorphic acceptance traffic",
    source:"gateway",
    priority:1,
    rechargeDiscountPolicy:{},
    billingDiscountPolicy:{},
    rateLimitPolicy:{rules:[]},
    quotaPolicy:{},
    metadata:{purpose:"production_acceptance",isolated:true},
    status:"active"
  }')
  if [[ -z $group_id ]]; then
    [[ $(jq --arg groupKey "$acceptance_group_key" \
      '[.items[] | select(.groupKey == $groupKey)] | length' "$groups_response") == 0 ]] || {
      echo 'acceptance user group lookup was ambiguous' >&2
      return 1
    }
    admin_request POST /api/admin/user-groups "$group_body" "$group_response"
    group_id=$(jq -r '.id // empty' "$group_response")
  else
    [[ $group_id =~ ^[0-9a-f-]{36}$ ]] || {
      echo 'acceptance user group API returned an invalid group ID' >&2
      return 1
    }
    isolation_state=$(database_query "
SELECT
  (SELECT count(*)
   FROM gateway_users
   WHERE default_user_group_id='$group_id'::uuid
     AND id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
     AND COALESCE(metadata->>'purpose','') <> 'production_acceptance_shard')
  ||':'||
  (SELECT count(*)
   FROM gateway_api_keys key
   JOIN gateway_users user_record ON user_record.id=key.gateway_user_id
   WHERE key.user_group_id='$group_id'::uuid
     AND key.gateway_user_id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
     AND COALESCE(user_record.metadata->>'purpose','') <> 'production_acceptance_shard');")
    [[ $isolation_state == 0:0 ]] || {
      echo 'acceptance user group is referenced by non-acceptance identities' >&2
      return 1
    }
    admin_request PATCH "/api/admin/user-groups/$group_id" \
      "$group_body" "$group_response"
  fi
  [[ $group_id =~ ^[0-9a-f-]{36}$ ]] || {
    echo 'acceptance user group API returned an invalid group ID' >&2
    return 1
  }
  acceptance_group_id=$group_id

  admin_request GET /api/admin/users '' "$users_response"
  user_body=$(jq -c \
    --arg userID "$AI_GATEWAY_ACCEPTANCE_USER_ID" \
    --arg groupID "$group_id" '
      [.items[] | select(.id == $userID)]
      | if length != 1 then empty else .[0] end
      | {
          userKey:(.userKey // ""),
          source:(.source // "gateway"),
          externalUserId:(.externalUserId // ""),
          username:(.username // ""),
          displayName:(.displayName // ""),
          email:(.email // ""),
          phone:(.phone // ""),
          avatarUrl:(.avatarUrl // ""),
          gatewayTenantId:(.gatewayTenantId // ""),
          tenantId:(.tenantId // ""),
          tenantKey:(.tenantKey // ""),
          defaultUserGroupId:$groupID,
          roles:(.roles // []),
          authProfile:(.authProfile // {}),
          metadata:(.metadata // {}),
          status:(.status // "active")
        }
    ' "$users_response")
  [[ -n $user_body && $(jq -r '.username // empty' <<<"$user_body") != '' ]] || {
    echo 'dedicated acceptance user was not found through the control API' >&2
    return 1
  }
  admin_request PATCH "/api/admin/users/$AI_GATEWAY_ACCEPTANCE_USER_ID" \
    "$user_body" "$user_response"

  isolation_state=$(database_query "
SELECT
  (SELECT count(*)
   FROM gateway_users
   WHERE id='$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
     AND default_user_group_id='$group_id'::uuid)
  ||':'||
  (SELECT count(*)
   FROM gateway_api_keys
   WHERE id='$AI_GATEWAY_ACCEPTANCE_API_KEY_ID'::uuid
     AND gateway_user_id='$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
     AND COALESCE(user_group_id,'$group_id'::uuid)='$group_id'::uuid
     AND status='active'
     AND deleted_at IS NULL)
  ||':'||
  (SELECT count(*)
   FROM gateway_api_keys
   WHERE gateway_user_id='$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
     AND status='active'
     AND deleted_at IS NULL)
  ||':'||
  (SELECT count(*)
   FROM gateway_user_groups
   WHERE id='$group_id'::uuid
     AND status='active'
     AND NOT EXISTS (
       SELECT 1
       FROM jsonb_array_elements(
         COALESCE(rate_limit_policy->'rules','[]'::jsonb)
       ) rule
       WHERE rule->>'metric' IN ('concurrent','queue_size')
     ))
  ||':'||
  (SELECT count(*)
   FROM gateway_users
   WHERE default_user_group_id='$group_id'::uuid
     AND id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
     AND COALESCE(metadata->>'purpose','') <> 'production_acceptance_shard')
  ||':'||
  (SELECT count(*)
   FROM gateway_api_keys key
   JOIN gateway_users user_record ON user_record.id=key.gateway_user_id
   WHERE key.user_group_id='$group_id'::uuid
     AND key.gateway_user_id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
     AND COALESCE(user_record.metadata->>'purpose','') <> 'production_acceptance_shard');")
  [[ $isolation_state == 1:1:1:1:0:0 ]] || {
    echo 'dedicated acceptance identity isolation verification failed' >&2
    return 1
  }
  echo 'acceptance_user_group=PASS rate_limit=production_candidates active_keys=1 isolated=true'
}

ensure_acceptance_identity_shards() {
  local identity_material key_count participant_count
  [[ $acceptance_group_id =~ ^[0-9a-f-]{36}$ ]] || {
    echo 'acceptance user group must be prepared before identity shards' >&2
    return 1
  }
  identity_material=$(database_query "
WITH primary_user AS (
  SELECT roles
  FROM gateway_users
  WHERE id='$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
    AND status='active'
    AND deleted_at IS NULL
), desired AS (
  SELECT ordinal,
         'production-acceptance-shard-' || lpad(ordinal::text, 3, '0') AS user_key
  FROM generate_series(1, $((AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS - 1))) ordinal
), shard_users AS (
  INSERT INTO gateway_users (
    user_key, source, username, display_name, default_user_group_id,
    roles, auth_profile, metadata, status
  )
  SELECT desired.user_key,
         'gateway',
         desired.user_key,
         'Production Acceptance Shard ' || lpad(desired.ordinal::text, 3, '0'),
         '$acceptance_group_id'::uuid,
         primary_user.roles,
         '{}'::jsonb,
         jsonb_build_object(
           'purpose', 'production_acceptance_shard',
           'ordinal', desired.ordinal,
           'isolated', true
         ),
         'active'
  FROM desired
  CROSS JOIN primary_user
  ON CONFLICT (user_key) DO UPDATE
  SET default_user_group_id=EXCLUDED.default_user_group_id,
      roles=EXCLUDED.roles,
      metadata=EXCLUDED.metadata,
      status='active',
      deleted_at=NULL,
      updated_at=now()
  RETURNING id, user_key
), primary_key AS (
  SELECT id, gateway_user_id, key_secret, scopes, rate_limit_policy, quota_policy
  FROM gateway_api_keys
  WHERE id='$AI_GATEWAY_ACCEPTANCE_API_KEY_ID'::uuid
    AND gateway_user_id='$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
    AND status='active'
    AND deleted_at IS NULL
    AND COALESCE(key_secret, '') <> ''
), existing_keys AS (
  UPDATE gateway_api_keys key
  SET user_group_id='$acceptance_group_id'::uuid,
      updated_at=now()
  FROM shard_users shard
  WHERE key.id=(
    SELECT selected.id
    FROM gateway_api_keys selected
    WHERE selected.gateway_user_id=shard.id
      AND selected.name='Production Acceptance Shard'
      AND selected.status='active'
      AND selected.deleted_at IS NULL
      AND COALESCE(selected.key_secret, '') <> ''
    ORDER BY selected.created_at
    LIMIT 1
  )
  RETURNING key.id, key.gateway_user_id, key.key_secret
), new_key_material AS (
  SELECT shard.id AS gateway_user_id,
         'sk-gw-' || encode(gen_random_bytes(32), 'hex') AS secret
  FROM shard_users shard
  WHERE NOT EXISTS (
    SELECT 1 FROM existing_keys existing
    WHERE existing.gateway_user_id=shard.id
  )
), inserted_keys AS (
  INSERT INTO gateway_api_keys (
    gateway_user_id, user_id, key_prefix, key_secret, key_hash, name, scopes,
    user_group_id, rate_limit_policy, quota_policy, status
  )
  SELECT material.gateway_user_id,
         material.gateway_user_id::text,
         left(material.secret, 18),
         material.secret,
         crypt(material.secret, gen_salt('bf', 10)),
         'Production Acceptance Shard',
         template.scopes,
         '$acceptance_group_id'::uuid,
         template.rate_limit_policy,
         template.quota_policy,
         'active'
  FROM new_key_material material
  CROSS JOIN primary_key template
  RETURNING id, gateway_user_id, key_secret
), shard_keys AS (
  SELECT * FROM existing_keys
  UNION ALL
  SELECT * FROM inserted_keys
), wallets AS (
  INSERT INTO gateway_wallet_accounts (
    gateway_user_id, user_id, currency, balance, total_recharged, metadata, status
  )
  SELECT shard.id,
         shard.id::text,
         'resource',
         1000000000::numeric,
         1000000000::numeric,
         '{\"purpose\":\"production_acceptance_shard\",\"isolated\":true}'::jsonb,
         'active'
  FROM shard_users shard
  ON CONFLICT (gateway_user_id, currency) DO UPDATE
  SET balance=GREATEST(gateway_wallet_accounts.balance, EXCLUDED.balance),
      total_recharged=GREATEST(gateway_wallet_accounts.total_recharged, EXCLUDED.total_recharged),
      metadata=EXCLUDED.metadata,
      status='active',
      updated_at=now()
  RETURNING gateway_user_id
), participants AS (
  SELECT 0 AS ordinal, key.id, key.gateway_user_id, key.key_secret
  FROM primary_key key
  UNION ALL
  SELECT row_number() OVER (ORDER BY shard.user_key)::int,
         key.id,
         key.gateway_user_id,
         key.key_secret
  FROM shard_keys key
  JOIN shard_users shard ON shard.id=key.gateway_user_id
)
SELECT jsonb_agg(
         jsonb_build_object(
           'apiKeyId', participant.id,
           'userId', participant.gateway_user_id
         )
         ORDER BY participant.ordinal
       )::text
       || E'\\t' ||
       string_agg(participant.key_secret, ',' ORDER BY participant.ordinal)
FROM participants participant
CROSS JOIN (SELECT count(*) FROM wallets) wallet_write_barrier;")
  [[ $identity_material == *$'\t'* ]] || {
    echo 'acceptance identity shard provisioning returned an invalid payload' >&2
    return 1
  }
  acceptance_participants_json=${identity_material%%$'\t'*}
  AI_GATEWAY_ACCEPTANCE_API_KEYS=${identity_material#*$'\t'}
  participant_count=$(jq 'length' <<<"$acceptance_participants_json")
  key_count=$(awk -F',' '{print NF}' <<<"$AI_GATEWAY_ACCEPTANCE_API_KEYS")
  [[ $participant_count == "$AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS" &&
     $key_count == "$AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS" ]] || {
    echo 'acceptance identity shard count verification failed' >&2
    return 1
  }
  export AI_GATEWAY_ACCEPTANCE_API_KEYS
  echo "acceptance_identity_shards=PASS count=$participant_count isolated_wallets=true"
}

select_acceptance_models() {
  if [[ -z $AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL ]]; then
    AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(database_query "
SELECT b.invocation_name
FROM platform_models model
JOIN integration_platforms platform ON platform.id=model.platform_id
JOIN base_model_catalog b ON b.id=model.base_model_id
WHERE platform.status='enabled' AND platform.deleted_at IS NULL
  AND model.enabled=true
  AND lower(platform.provider) IN ('gemini','google-gemini','gemini-openai')
  AND model.model_type @> '[\"image_edit\"]'::jsonb
ORDER BY
  CASE
    WHEN lower(b.invocation_name) = 'gemini-3.1-flash-image' THEN 0
    WHEN lower(b.invocation_name) LIKE '%gemini%image%'
      AND lower(b.invocation_name) NOT LIKE '%preview%' THEN 1
    WHEN lower(b.invocation_name) LIKE '%gemini%image%' THEN 2
    ELSE 3
  END,
  platform.priority, model.created_at
LIMIT 1;")
  fi
  if [[ -z $AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL ]]; then
    AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$(database_query "
SELECT b.invocation_name
FROM platform_models model
JOIN integration_platforms platform ON platform.id=model.platform_id
JOIN base_model_catalog b ON b.id=model.base_model_id
WHERE platform.status='enabled' AND platform.deleted_at IS NULL
  AND model.enabled=true
  AND model.model_type @> '[\"omni_video\"]'::jsonb
  AND COALESCE(
    NULLIF(model.capability_override #>> '{omni_video,max_images}','')::int,
    NULLIF(model.capabilities #>> '{omni_video,max_images}','')::int,
    NULLIF(b.capabilities #>> '{omni_video,max_images}','')::int,
    0
  ) >= 9
ORDER BY
  CASE
    WHEN lower(b.invocation_name) LIKE '%seedance%2%fast%' THEN 0
    WHEN lower(b.invocation_name) LIKE '%seedance%2%' THEN 1
    ELSE 2
  END,
  platform.priority, model.created_at
LIMIT 1;")
  fi
  [[ -n $AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL && -n $AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL ]] || {
    echo 'no enabled Gemini image-edit or omni-video max_images>=9 model was found' >&2
    return 1
  }
  [[ $AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL =~ ^[A-Za-z0-9._:/+-]+$ &&
     $AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL =~ ^[A-Za-z0-9._:/+-]+$ ]] || {
    echo 'acceptance model invocation names contain unsupported characters' >&2
    return 1
  }
  export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL
}

ensure_acceptance_model_access() {
  local resources response body missing expected
  response=$temporary_root/model-access.json
  resources=$(database_query "
WITH selected_candidates AS (
  SELECT DISTINCT platform.id AS platform_id, model.id AS platform_model_id, model.base_model_id
  FROM platform_models model
  JOIN integration_platforms platform ON platform.id=model.platform_id
  JOIN base_model_catalog base_model ON base_model.id=model.base_model_id
  WHERE platform.status='enabled'
    AND platform.deleted_at IS NULL
    AND model.enabled=true
    AND base_model.invocation_name IN (
      '$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL',
      '$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL'
    )
), resources AS (
  SELECT 'platform'::text AS resource_type, platform_id AS resource_id
  FROM selected_candidates
  UNION
  SELECT 'platform_model', platform_model_id
  FROM selected_candidates
  UNION
  SELECT 'base_model', base_model_id
  FROM selected_candidates
  WHERE base_model_id IS NOT NULL
)
SELECT COALESCE(
  jsonb_agg(
    jsonb_build_object(
      'resourceType', resource_type,
      'resourceId', resource_id,
      'priority', 100,
      'minPermissionLevel', 0,
      'status', 'active'
    )
    ORDER BY resource_type, resource_id
  ),
  '[]'::jsonb
)::text
FROM resources;")
  expected=$(jq 'length' <<<"$resources")
  (( expected > 0 )) || {
    echo 'selected acceptance models did not resolve to production candidate resources' >&2
    return 1
  }
  body=$(jq -cn \
    --arg subjectID "$acceptance_group_id" \
    --argjson resources "$resources" \
    '{
      subjectType:"user_group",
      subjectId:$subjectID,
      effect:"allow",
      upsertResources:$resources,
      deleteResources:[]
    }')
  admin_request POST /api/admin/access-rules/batch "$body" "$response"
  missing=$(database_query "
WITH selected_candidates AS (
  SELECT DISTINCT platform.id AS platform_id, model.id AS platform_model_id, model.base_model_id
  FROM platform_models model
  JOIN integration_platforms platform ON platform.id=model.platform_id
  JOIN base_model_catalog base_model ON base_model.id=model.base_model_id
  WHERE platform.status='enabled'
    AND platform.deleted_at IS NULL
    AND model.enabled=true
    AND base_model.invocation_name IN (
      '$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL',
      '$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL'
    )
), resources AS (
  SELECT 'platform'::text AS resource_type, platform_id AS resource_id
  FROM selected_candidates
  UNION
  SELECT 'platform_model', platform_model_id
  FROM selected_candidates
  UNION
  SELECT 'base_model', base_model_id
  FROM selected_candidates
  WHERE base_model_id IS NOT NULL
)
SELECT count(*)
FROM resources resource
WHERE NOT EXISTS (
  SELECT 1
  FROM gateway_access_rules rule
  WHERE rule.subject_type='user_group'
    AND rule.subject_id='$acceptance_group_id'::uuid
    AND rule.effect='allow'
    AND rule.status='active'
    AND rule.resource_type=resource.resource_type
    AND rule.resource_id=resource.resource_id
);")
  [[ $missing == 0 ]] || {
    echo "acceptance user group is missing $missing selected candidate access rules" >&2
    return 1
  }
  echo "acceptance_model_access=PASS resources=$expected"
}

cleanup_single_node_emulator() {
  [[ $run_token =~ ^[0-9a-f]{64}$ ]] || return 0
  local suffix=${run_token:0:12}
  local binary_path=/root/easyai-acceptance-emulator-$suffix
  local pid_path=$binary_path.pid
  local log_path=$binary_path.log
  cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$binary_path" "$pid_path" "$log_path" <<'REMOTE' || true
set -euo pipefail
binary_path=$1
pid_path=$2
log_path=$3
if [[ -f $pid_path && ! -L $pid_path ]]; then
  pid=$(<"$pid_path")
  if [[ $pid =~ ^[0-9]+$ && -e /proc/$pid/exe && $(readlink -f "/proc/$pid/exe") == "$binary_path" ]]; then
    kill -TERM "$pid"
    for _ in {1..20}; do
      kill -0 "$pid" >/dev/null 2>&1 || break
      sleep 0.1
    done
  fi
fi
for path in "$binary_path" "$pid_path" "$log_path"; do
  [[ ! -e $path ]] || unlink "$path"
done
REMOTE
  single_node_emulator_installed=false
}

start_single_node_emulator() {
  [[ $run_token =~ ^[0-9a-f]{64}$ ]]
  local suffix=${run_token:0:12}
  local binary_path=/root/easyai-acceptance-emulator-$suffix
  local pid_path=$binary_path.pid
  local log_path=$binary_path.log
  local local_digest remote_digest node_ip
  local_digest=$(shasum -a 256 "$acceptance_emulator_linux_binary" | awk '{print $1}')
  cluster_scp "$acceptance_emulator_linux_binary" "$CLUSTER_NINGBO_HOST:$binary_path" >/dev/null
  remote_digest=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "chmod 0755 '$binary_path'; sha256sum '$binary_path'" | awk '{print $1}')
  [[ $remote_digest == "$local_digest" ]]
  single_node_emulator_installed=true
  cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$binary_path" "$pid_path" "$log_path" <<'REMOTE'
set -euo pipefail
binary_path=$1
pid_path=$2
log_path=$3
if ss -ltn '( sport = :18090 )' | tail -n +2 | grep -q .; then
  echo 'single-node emulator port 18090 is already in use' >&2
  exit 1
fi
nohup env HTTP_ADDR=:18090 "$binary_path" </dev/null >"$log_path" 2>&1 &
pid=$!
printf '%s\n' "$pid" >"$pid_path"
chmod 0600 "$pid_path" "$log_path"
for _ in {1..50}; do
  if curl -fsS --max-time 2 http://127.0.0.1:18090/healthz >/dev/null; then
    exit 0
  fi
  kill -0 "$pid" >/dev/null 2>&1 || exit 1
  sleep 0.2
done
exit 1
REMOTE
  node_ip=$(remote_kubectl get node easyai-ningbo -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')
  [[ $node_ip =~ ^[0-9a-fA-F:.]+$ ]]
  single_node_emulator_url=http://$node_ip:18090
  remote_kubectl exec -n "$namespace" deployment/easyai-api-ningbo -- \
    wget -qO- "$single_node_emulator_url/healthz" >/dev/null
  echo 'single_node_protocol_emulator=PASS placement=ningbo-host digest_verified=true pod_reachable=true'
}

deploy_protocol_emulator() {
  sed "s|image: easyai-api|image: $api_image|" \
    "$cluster_root/deploy/kubernetes/acceptance/protocol-emulator.yaml" |
    cluster_ssh "$CLUSTER_NINGBO_HOST" 'k3s kubectl apply -f -' >/dev/null
  if [[ $single_node_acceptance == true ]]; then
    remote_kubectl patch deployment easyai-acceptance-emulator -n "$namespace" \
      --type=json -p='[{"op":"replace","path":"/spec/template/spec/nodeSelector","value":{"kubernetes.io/hostname":"easyai-ningbo"}}]' >/dev/null
    remote_kubectl patch deployment easyai-acceptance-callback-collector -n "$namespace" \
      --type=json -p='[{"op":"replace","path":"/spec/template/spec/nodeSelector","value":{"kubernetes.io/hostname":"easyai-ningbo"}}]' >/dev/null
  fi
  remote_kubectl rollout status deployment/easyai-acceptance-emulator \
    -n "$namespace" --timeout=300s
  remote_kubectl rollout status deployment/easyai-acceptance-callback-collector \
    -n "$namespace" --timeout=300s
  if [[ $single_node_acceptance == true ]]; then
    start_single_node_emulator
  fi
}

is_release_ancestor() {
  local ancestor=$1
  local descendant=$2
  [[ $ancestor =~ ^[0-9a-f]{40}$ && $descendant =~ ^[0-9a-f]{40}$ ]] &&
    git -C "$cluster_root" merge-base --is-ancestor "$ancestor" "$descendant"
}

create_and_activate_run() {
  local create_response=$temporary_root/create-run.json
  local activate_response=$temporary_root/activate-run.json
  local traffic_response=$temporary_root/pre-activate-traffic.json
  local previous_run_response=$temporary_root/previous-run.json
  local body emulator_url='http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090'
  if [[ $single_node_acceptance == true ]]; then
    [[ $single_node_emulator_url == http://* ]]
    emulator_url=$single_node_emulator_url
  fi
  body=$(jq -cn \
    --arg releaseSha "$release_sha" \
    --arg apiDigest "$api_digest" \
    --arg workerDigest "$worker_digest" \
    --arg apiKeyID "$AI_GATEWAY_ACCEPTANCE_API_KEY_ID" \
    --arg userID "$AI_GATEWAY_ACCEPTANCE_USER_ID" \
    --arg token "$run_token" \
    --arg emulatorURL "$emulator_url" \
    --arg callbackURL 'http://easyai-acceptance-callback-collector.easyai.svc.cluster.local:8091/callbacks' \
    --argjson participants "$acceptance_participants_json" \
    '{
      releaseSha: $releaseSha,
      apiImageDigest: $apiDigest,
      workerImageDigest: $workerDigest,
      apiKeyId: $apiKeyID,
      userId: $userID,
      token: $token,
      emulatorBaseUrl: $emulatorURL,
      callbackUrl: $callbackURL,
      capacityProfile: "P24",
      config: {
        workloads: ["gemini_image_edit", "multi_reference_video"],
        participants: $participants
      }
    }')
  admin_request POST /api/admin/system/acceptance/runs "$body" "$create_response"
  run_id=$(jq -r '.id // empty' "$create_response")
  [[ $run_id =~ ^[0-9a-f-]{36}$ ]] || {
    echo 'acceptance control API returned an invalid Run ID' >&2
    return 1
  }
  report_root=${AI_GATEWAY_ACCEPTANCE_REPORT_DIR:-"$cluster_root/dist/acceptance/$run_id"}
  mkdir -p "$report_root"
  chmod 0700 "$report_root"

  admin_request GET /api/admin/system/acceptance/traffic-mode '' "$traffic_response"
  local traffic_mode previous_run_id previous_status
  local traffic_release traffic_api_digest traffic_worker_digest
  local previous_release previous_api_digest previous_worker_digest
  traffic_mode=$(jq -r '.mode // empty' "$traffic_response")
  if [[ $traffic_mode == validation ]]; then
    previous_run_id=$(jq -r '.runId // empty' "$traffic_response")
    [[ $previous_run_id =~ ^[0-9a-f-]{36}$ && $previous_run_id != "$run_id" ]] || {
      echo 'validation mode does not reference a different previous Run' >&2
      return 1
    }
    admin_request GET "/api/admin/system/acceptance/runs/$previous_run_id" '' \
      "$previous_run_response"
    previous_status=$(jq -r '.status // empty' "$previous_run_response")
    [[ $previous_status == failed ]] || {
      echo "refusing to replace previous acceptance Run in status $previous_status" >&2
      return 1
    }
    traffic_release=$(jq -r '.releaseSha // empty' "$traffic_response")
    traffic_api_digest=$(jq -r '.apiImageDigest // empty' "$traffic_response")
    traffic_worker_digest=$(jq -r '.workerImageDigest // empty' "$traffic_response")
    previous_release=$(jq -r '.releaseSha // empty' "$previous_run_response")
    previous_api_digest=$(jq -r '.apiImageDigest // empty' "$previous_run_response")
    previous_worker_digest=$(jq -r '.workerImageDigest // empty' "$previous_run_response")
    [[ $traffic_release == "$previous_release" &&
       $traffic_api_digest == "$previous_api_digest" &&
       $traffic_worker_digest == "$previous_worker_digest" ]] || {
      echo 'previous failed Run does not match the active validation CAS fields' >&2
      return 1
    }
    if [[ $traffic_release == "$release_sha" ]]; then
      [[ $traffic_api_digest == "$api_digest" &&
         $traffic_worker_digest == "$worker_digest" ]] || {
        echo 'current-release validation mode has unexpected image digests' >&2
        return 1
      }
    elif ! is_release_ancestor "$traffic_release" "$release_sha"; then
      echo 'refusing to replace validation mode owned by a non-ancestor release' >&2
      return 1
    fi
    # ActivateAcceptanceRun replaces a drained failed Run while holding the
    # traffic-mode row lock. Formal traffic therefore never observes a brief
    # live interval between two validation Runs.
    wait_for_terminal_acceptance_tasks_to_drain
  elif [[ $traffic_mode != live ]]; then
    echo "unsupported production traffic mode before acceptance: $traffic_mode" >&2
    return 1
  fi

  admin_request POST "/api/admin/system/acceptance/runs/$run_id/activate" '' "$activate_response"
  [[ $(jq -r '.mode' "$activate_response") == validation &&
     $(jq -r '.runId' "$activate_response") == "$run_id" ]]
}

merge_partial_acceptance_report() {
  local summary=$1
  local output=$2
  if [[ -e $output ]]; then
    [[ -f $output && ! -L $output ]] || {
      echo "existing partial acceptance report is not a regular file: $output" >&2
      return 1
    }
    node "$cluster_root/scripts/acceptance/report.mjs" validate \
      --input "$output" >/dev/null
    return 0
  fi
  node "$cluster_root/scripts/acceptance/report.mjs" merge-production \
    --local-report "$local_acceptance_report" \
    --production-summary "$summary" \
    --output "$output" >/dev/null
}

mark_run_failed() {
  local reason=$1
  local response=$temporary_root/failed-run.json
  local body
  [[ -n $run_id ]] || return 0
  local artifact_names task_count gemini_count video_count queue_state
  task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;" 2>/dev/null || echo 0)
  gemini_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND model='$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL';" 2>/dev/null || echo 0)
  video_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND model='$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL';" 2>/dev/null || echo 0)
  queue_state=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;" 2>/dev/null || echo '0:0')
  [[ $task_count =~ ^[0-9]+$ ]] || task_count=0
  [[ $gemini_count =~ ^[0-9]+$ ]] || gemini_count=0
  [[ $video_count =~ ^[0-9]+$ ]] || video_count=0
  [[ $queue_state =~ ^[0-9]+:[0-9]+$ ]] || queue_state=0:0
  artifact_names=$(find "$report_root" -maxdepth 1 -type f -exec basename {} \; 2>/dev/null |
    LC_ALL=C sort | jq -Rsc 'split("\n") | map(select(length > 0))')
  body=$(jq -cn \
    --arg reason "$reason" \
    --arg gateId "${failure_gate_id:-unknown_gate}" \
    --arg activeProfile "$active_profile" \
    --arg queueState "$queue_state" \
    --argjson taskCount "$task_count" \
    --argjson artifacts "$artifact_names" \
    '{
      passed:false,
      failureReason:$reason,
      report:{
        gateId:$gateId,
        activeCapacityProfile:$activeProfile,
        tasks:$taskCount,
        queueAtFailure:$queueState,
        completedArtifacts:$artifacts
      }
    }')
  jq -n \
    --arg runId "$run_id" \
    --arg releaseSha "$release_sha" \
    --arg apiImageDigest "$api_digest" \
    --arg snapshotConfigHash "$local_snapshot_config_hash" \
    --arg snapshotSha256 "$local_snapshot_sha256" \
    --arg stableCapacityProfile "$stable_profile" \
    --arg gateId "${failure_gate_id:-unknown_gate}" \
    --arg reason "$reason" \
    --arg queueState "$queue_state" \
    --argjson taskCount "$task_count" \
    --argjson geminiCount "$gemini_count" \
    --argjson videoCount "$video_count" \
    --argjson artifacts "$artifact_names" \
    '{
      runId:$runId,
      releaseSha:$releaseSha,
      apiImageDigest:$apiImageDigest,
      snapshotConfigHash:$snapshotConfigHash,
      snapshotSha256:$snapshotSha256,
      stableCapacityProfile:$stableCapacityProfile,
      tasks:$taskCount,
      geminiTasks:$geminiCount,
      videoTasks:$videoCount,
      queueAtFailure:$queueState,
      realCanaryPassed:false,
      failureReason:$reason,
      completedArtifacts:$artifacts,
      gates:[
        {
          id:$gateId,
          passed:false,
          detail:$reason
        }
      ],
      passed:false
    }' >"$report_root/summary.partial.json"
  chmod 0600 "$report_root/summary.partial.json"
  merge_partial_acceptance_report \
    "$report_root/summary.partial.json" \
    "$report_root/acceptance-report.partial.json" || true
  if admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$body" "$response"; then
    failure_recorded=true
    if ! wait_for_terminal_acceptance_tasks_to_drain; then
      echo 'failed acceptance Run did not drain all safe tasks and outbox work' >&2
    fi
  fi
}

fail_acceptance_gate() {
  failure_gate_id=$1
  failure_reason=$2
  echo "acceptance gate failed: gate_id=$failure_gate_id reason=$failure_reason" >&2
  return 1
}

apply_capacity_profile() {
  local profile=$1
  local slots pool media global baseline_replicas
  local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
  local min_idle=4
  local max_idle_seconds=$AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS
  case $profile in
    P24) slots=24; pool=32; media=24 ;;
    P28) slots=28; pool=36; media=28 ;;
    P32) slots=32; pool=40; media=32 ;;
    *) return 1 ;;
  esac
  baseline_replicas=$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG))
  global=$((slots * baseline_replicas))
  local command_text capacity_output
  printf -v command_text \
    'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
    "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
    "$slots" "$global" "$global" "$AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE" "$pool" "$api_pool" \
    "$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" "$AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS" \
    "$min_idle" "$max_idle_seconds" \
    "$media" "$AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY" "$((slots * 2))" \
    "$AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB" \
    "$AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES" \
    "$remote_release_helper"
  capacity_output=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$command_text")
  grep -Fq 'production_capacity=PASS' <<<"$capacity_output"
  active_profile=$profile
}

capture_etcd_latency_counters() {
  local output=$1
  local site host metrics backend_sum backend_count wal_sum wal_count
  echo 'site,backend_sum,backend_count,wal_sum,wal_count' >"$output"
  for site in ningbo hongkong losangeles; do
    case $site in
      ningbo) host=$CLUSTER_NINGBO_HOST ;;
      hongkong) host=$CLUSTER_HONGKONG_HOST ;;
      losangeles) host=$CLUSTER_LOS_ANGELES_HOST ;;
    esac
    metrics=$(cluster_ssh "$host" 'curl -fsS --max-time 5 http://127.0.0.1:2381/metrics') || return 1
    backend_sum=$(awk '$1=="etcd_disk_backend_commit_duration_seconds_sum" {print $2}' <<<"$metrics")
    backend_count=$(awk '$1=="etcd_disk_backend_commit_duration_seconds_count" {print $2}' <<<"$metrics")
    wal_sum=$(awk '$1=="etcd_disk_wal_fsync_duration_seconds_sum" {print $2}' <<<"$metrics")
    wal_count=$(awk '$1=="etcd_disk_wal_fsync_duration_seconds_count" {print $2}' <<<"$metrics")
    [[ $backend_sum =~ ^[0-9.]+$ && $backend_count =~ ^[0-9]+$ &&
       $wal_sum =~ ^[0-9.]+$ && $wal_count =~ ^[0-9]+$ ]] || return 1
    echo "$site,$backend_sum,$backend_count,$wal_sum,$wal_count" >>"$output"
  done
}

control_plane_hard_errors_since() {
  local since=$1
  local host total=0 count pod
  [[ $since =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
    count=$(cluster_ssh "$host" \
      "journalctl -u k3s --since '$since' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /http: Handler timeout/ || /rejected connection on peer endpoint.*(timeout|reset)/ || /etcdserver: request timed out/ || /leader changed/ {count++} END {print count+0}'") || return 1
    [[ $count =~ ^[0-9]+$ ]] || return 1
    total=$((total + count))
  done
  while read -r pod; do
    [[ -n $pod ]] || continue
    count=$(remote_kubectl logs "$pod" -n "$namespace" -c postgres --since-time="$since" 2>/dev/null |
      awk 'BEGIN {IGNORECASE=1}
        /API server connectivity issue/ ||
        /Instance connectivity error/ ||
        /terminating walreceiver due to timeout/ ||
        /synchronous commit.*cancel/ {count++}
        END {print count+0}') || return 1
    [[ $count =~ ^[0-9]+$ ]] || return 1
    total=$((total + count))
  done < <(remote_kubectl get pods -n "$namespace" \
    -l 'cnpg.io/cluster=easyai-postgres' \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
  echo "$total"
}

evaluate_etcd_latency_window() {
  local before=$1
  local after=$2
  local output=$3
  echo 'site,backend_commit_average_ms,wal_fsync_average_ms' >"$output"
  awk -F',' \
    -v backend_limit="$AI_GATEWAY_ACCEPTANCE_ETCD_BACKEND_COMMIT_MAX_MS" \
    -v wal_limit="$AI_GATEWAY_ACCEPTANCE_ETCD_WAL_FSYNC_MAX_MS" '
      NR==FNR {
        if (FNR>1) {
          backend_sum[$1]=$2; backend_count[$1]=$3
          wal_sum[$1]=$4; wal_count[$1]=$5
        }
        next
      }
      FNR==1 {next}
      {
        backend_delta_count=$3-backend_count[$1]
        wal_delta_count=$5-wal_count[$1]
        if (backend_delta_count<=0 || wal_delta_count<=0) {failed=1; next}
        backend_ms=($2-backend_sum[$1])*1000/backend_delta_count
        wal_ms=($4-wal_sum[$1])*1000/wal_delta_count
        printf "%s,%.3f,%.3f\n", $1, backend_ms, wal_ms
        if (backend_ms>=backend_limit || wal_ms>=wal_limit) failed=1
      }
      END {exit failed}
    ' "$before" "$after" >>"$output"
}

wait_for_post_rollout_stability() {
  local profile=$1
  local profile_slug deadline attempt=0 since before after latency_report errors
  profile_slug=$(printf '%s' "$profile" | tr '[:upper:]' '[:lower:]')
  latency_report=$report_root/$profile_slug-control-plane-stability.csv
  deadline=$((SECONDS + AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_TIMEOUT_SECONDS))
  while (( SECONDS + AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS <= deadline )); do
    attempt=$((attempt + 1))
    since=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
    before=$temporary_root/etcd-before-$profile_slug-$attempt.csv
    after=$temporary_root/etcd-after-$profile_slug-$attempt.csv
    capture_etcd_latency_counters "$before" || continue
    sleep "$AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS"
    capture_etcd_latency_counters "$after" || continue
    errors=$(control_plane_hard_errors_since "$since" || echo 1)
    if [[ $errors == 0 ]] && evaluate_etcd_latency_window "$before" "$after" "$latency_report" &&
       verify_control_plane_ready_now; then
      runtime_observation_started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
      echo "post_rollout_stability=PASS profile=$profile window_seconds=$AI_GATEWAY_ACCEPTANCE_POST_ROLLOUT_STABILITY_SECONDS hard_errors=0"
      return 0
    fi
    echo "post-rollout control plane not stable: profile=$profile attempt=$attempt hard_errors=${errors:-unknown}" >&2
  done
  return 1
}

verify_control_plane_ready_now() {
  local host ready_output sync_state memory_percent
  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
    ready_output=$(cluster_ssh "$host" "k3s kubectl get --raw='/readyz?verbose'")
    grep -Fq '[+]etcd ok' <<<"$ready_output" || return 1
  done
  [[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.readyInstances}') == 2 ]] || return 1
  sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
  [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] || return 1
  while read -r _node _cpu _cpu_percent _memory memory_percent; do
    memory_percent=${memory_percent%\%}
    [[ $memory_percent =~ ^[0-9]+$ ]] || return 1
    (( memory_percent < 80 )) || return 1
  done < <(remote_kubectl top nodes --no-headers)
}

capacity_controller_status() {
  local pod status
  while read -r pod; do
    [[ -n $pod ]] || continue
    if status=$(remote_kubectl exec -n "$namespace" "$pod" -- \
      wget -qO- http://127.0.0.1:8088/status 2>/dev/null) &&
      jq -e '.leader == true and (.lastError // "") == ""' >/dev/null <<<"$status"; then
      printf '%s\n' "$status"
      return 0
    fi
  done < <(remote_kubectl get pods -n "$namespace" \
    -l 'app.kubernetes.io/name=easyai-capacity-controller' \
    -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}')
  return 1
}

apply_autoscaling_profile() {
  local profile=$1
  local max_replicas_ningbo=${2:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO}
  local max_replicas_hongkong=${3:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG}
  local slots pool media global
  [[ $max_replicas_ningbo =~ ^[0-9]+$ &&
     $max_replicas_hongkong =~ ^[0-9]+$ &&
     $((max_replicas_ningbo + max_replicas_hongkong)) -ge 1 ]] || return 1
  (( max_replicas_ningbo <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
     max_replicas_hongkong <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG )) || return 1
  case $profile in
    P24) slots=24; pool=32 ;;
    P28) slots=28; pool=36 ;;
    P32) slots=32; pool=40 ;;
    *) return 1 ;;
  esac
  media=$slots
  global=$((slots * (max_replicas_ningbo + max_replicas_hongkong)))
  local command_text capacity_output
  printf -v command_text \
    'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
    "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
    "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
    "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
    "$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
    "$max_replicas_ningbo" \
    "$max_replicas_hongkong" \
    "$slots" "$global" "$global" "$AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE" "$pool" \
    "$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS" \
    "$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" \
    "$AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS" \
    "$AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS" \
    "$media" "$AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY" "$((slots * 2))" \
    "$AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB" \
    "$AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES" \
    "$remote_release_helper"
  capacity_output=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$command_text")
  grep -Fq 'production_capacity=PASS' <<<"$capacity_output"
}

autoscaling_phase_observed_max=2

run_autoscaling_topology_phase() {
  local phase=$1
  local target_ningbo=$2
  local target_hongkong=$3
  local observations=$4
  local load_report=$report_root/autoscaling-"$phase"-load.json
  local status deadline load_pid load_status=0
  local ningbo_replicas hongkong_replicas current_total
  local reached=false drained=false resource_max

  run_load_profile video-throughput "$load_report" &
  load_pid=$!
  active_load_pid=$load_pid
  deadline=$((SECONDS + 420))
  while (( SECONDS < deadline )); do
    status=$(capacity_controller_status || true)
    if [[ -n $status ]]; then
      ningbo_replicas=$(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
        -o 'jsonpath={.spec.replicas}')
      hongkong_replicas=$(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
        -o 'jsonpath={.spec.replicas}')
      current_total=$((ningbo_replicas + hongkong_replicas))
      (( current_total > autoscaling_phase_observed_max )) &&
        autoscaling_phase_observed_max=$current_total
      resource_max=$(jq '[.plan.sites[].resourceMax] | add // 0' <<<"$status")
      printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \
        "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$phase" \
        "$ningbo_replicas" "$hongkong_replicas" \
        "$(jq -r '.plan.rawDesired // 0' <<<"$status")" \
        "$(jq -r '.plan.desiredTotal // 0' <<<"$status")" \
        "$(jq -r '.plan.frozenReason // ""' <<<"$status")" "$resource_max" >>"$observations"
      if (( ningbo_replicas == target_ningbo && hongkong_replicas == target_hongkong )); then
        reached=true
        break
      fi
    fi
    kill -0 "$load_pid" >/dev/null 2>&1 || break
    sleep 5
  done
  wait "$load_pid" || load_status=$?
  active_load_pid=
  (( load_status == 0 )) || return "$load_status"
  [[ $reached == true ]] || {
    echo "capacity controller did not reach topology $target_ningbo+$target_hongkong during $phase" >&2
    return 1
  }

  deadline=$((SECONDS + 780))
  while (( SECONDS < deadline )); do
    ningbo_replicas=$(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
      -o 'jsonpath={.spec.replicas}')
    hongkong_replicas=$(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
      -o 'jsonpath={.spec.replicas}')
    status=$(capacity_controller_status || true)
    resource_max=$(jq '[.plan.sites[].resourceMax] | add // 0' <<<"${status:-{}}")
    printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \
      "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$phase-drain" \
      "$ningbo_replicas" "$hongkong_replicas" \
      "$(jq -r '.plan.rawDesired // 0' <<<"${status:-{}}")" \
      "$(jq -r '.plan.desiredTotal // 0' <<<"${status:-{}}")" \
      "$(jq -r '.plan.frozenReason // ""' <<<"${status:-{}}")" "$resource_max" >>"$observations"
    if (( ningbo_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO &&
          hongkong_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG )); then
      drained=true
      break
    fi
    sleep 10
  done
  [[ $drained == true ]] || {
    echo "capacity controller did not safely drain $phase to configured minimum topology" >&2
    return 1
  }
  local unsafe_residue
  unsafe_residue=$(database_query "
SELECT count(*)
FROM gateway_worker_instances worker
WHERE worker.status IN ('draining','stale')
  AND (
    EXISTS (
      SELECT 1 FROM gateway_tasks task
      WHERE task.locked_by=worker.instance_id AND task.status='running'
    )
    OR EXISTS (
      SELECT 1
      FROM gateway_tasks task
      JOIN gateway_concurrency_leases lease ON lease.task_id=task.id
      WHERE task.locked_by=worker.instance_id AND lease.released_at IS NULL
    )
  );")
  [[ $unsafe_residue == 0 ]] || {
    echo "capacity controller left task or lease residue after draining $phase" >&2
    return 1
  }
}

run_autoscaling_acceptance() {
  local report=$report_root/autoscaling.json
  local observations=$report_root/autoscaling-observations.csv
  local status deadline resource_max_hongkong resource_max_ningbo
  local tested_south_pool_scale_out=false tested_1_plus_2=false tested_2_plus_2=false
  local requested_hongkong_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG
  local requested_ningbo_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO
  (( requested_hongkong_max > 2 )) && requested_hongkong_max=2
  (( requested_ningbo_max > 2 )) && requested_ningbo_max=2
  autoscaling_phase_observed_max=$((
    AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO +
    AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG
  ))
  echo 'timestamp,phase,ningbo_replicas,hongkong_replicas,raw_desired,desired_total,frozen_reason,resource_max_total' >"$observations"

  apply_autoscaling_profile "$stable_profile" "$requested_ningbo_max" "$requested_hongkong_max"
  deadline=$((SECONDS + 180))
  while (( SECONDS < deadline )); do
    status=$(capacity_controller_status || true)
    [[ -n $status ]] && break
    sleep 2
  done
  [[ -n ${status:-} ]] || {
    echo 'capacity controller did not elect a healthy leader' >&2
    return 1
  }
  resource_max_hongkong=$(jq -r \
    '[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
  if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
        AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG < AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
        requested_hongkong_max >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
        resource_max_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG )); then
    run_autoscaling_topology_phase south-pool-scale-out \
      "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
      "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" "$observations"
    tested_south_pool_scale_out=true
    certified_max_replicas_ningbo=0
    certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
  elif (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then
    run_autoscaling_topology_phase 1-plus-2 1 2 "$observations"
    tested_1_plus_2=true
    certified_max_replicas_hongkong=2
  fi

  if [[ $tested_1_plus_2 == true && $requested_ningbo_max -ge 2 ]]; then
    apply_autoscaling_profile "$stable_profile" 2 2
    deadline=$((SECONDS + 180))
    status=
    while (( SECONDS < deadline )); do
      status=$(capacity_controller_status || true)
      [[ -n $status ]] && break
      sleep 2
    done
    [[ -n $status ]]
    resource_max_ningbo=$(jq -r \
      '[.plan.sites[] | select(.site == "ningbo") | .resourceMax][0] // 0' <<<"$status")
    resource_max_hongkong=$(jq -r \
      '[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
    if (( resource_max_ningbo >= 2 && resource_max_hongkong >= 2 )); then
      run_autoscaling_topology_phase 2-plus-2 2 2 "$observations"
      tested_2_plus_2=true
      certified_max_replicas_ningbo=2
      certified_max_replicas_hongkong=2
    fi
  fi

  jq -n \
    --arg profile "$stable_profile" \
    --argjson baselineNingbo "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
    --argjson baselineHongkong "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
    --argjson testedSouthPoolScaleOut "$tested_south_pool_scale_out" \
    --argjson testedOnePlusTwo "$tested_1_plus_2" \
    --argjson testedTwoPlusTwo "$tested_2_plus_2" \
    --argjson observedMaxReplicas "$autoscaling_phase_observed_max" \
    '{
      passed:true,
      drained:true,
      capacityProfile:$profile,
      baseline:{ningbo:$baselineNingbo,hongkong:$baselineHongkong},
      topologies:{
        "1+1":($baselineNingbo == 1 and $baselineHongkong == 1),
        "0+1_to_0+2":$testedSouthPoolScaleOut,
        "1+2":$testedOnePlusTwo,
        "2+2":$testedTwoPlusTwo
      },
      observedMaxReplicas:$observedMaxReplicas
    }' >"$report"
  apply_capacity_profile "$stable_profile"
  verify_runtime_gates
}

calculate_capacity_profiles() {
  local profile_slug instance_slots max_running config_hash
  local -a image_reports video_reports
  profile_slug=$(printf '%s' "$stable_profile" | tr '[:upper:]' '[:lower:]')
  image_reports=("$report_root"/"$profile_slug"-*-gemini-*.json)
  video_reports=("$report_root"/"$profile_slug"-*-video-*.json)
  [[ -f ${image_reports[0]} && -f ${video_reports[0]} ]]
  image_stable_throughput=$(jq -s '
    [.[] | .phases[]?
      | select(.completed > 0 and .durationMs > 0)
      | (.completed * 1000 / .durationMs)] | min
  ' "${image_reports[@]}")
  video_stable_throughput=$(jq -s '
    [.[] | .phases[]?
      | select(.completed > 0 and .durationMs > 0)
      | (.completed * 1000 / .durationMs)] | min
  ' "${video_reports[@]}")
  image_admitted_throughput=$(awk -v value="$image_stable_throughput" 'BEGIN {printf "%.6f", value * 0.8}')
  video_admitted_throughput=$(awk -v value="$video_stable_throughput" 'BEGIN {printf "%.6f", value * 0.8}')
  awk -v image="$image_admitted_throughput" -v video="$video_admitted_throughput" \
    'BEGIN {exit !(image > 0 && video > 0)}'
  case $stable_profile in
    P24) instance_slots=24 ;;
    P28) instance_slots=28 ;;
    P32) instance_slots=32 ;;
  esac
  max_running=$((instance_slots * (
    certified_max_replicas_ningbo + certified_max_replicas_hongkong
  )))
  config_hash=$(remote_kubectl get configmap easyai-gateway-release -n "$namespace" \
    -o 'jsonpath={.data.configHash}')
  [[ $config_hash =~ ^[0-9a-f]{64}$ ]]
  capacity_profiles_payload=$report_root/capacity-profiles.json
  jq -n \
    --arg configHash "$config_hash" \
    --arg capacityProfile "$stable_profile" \
    --arg imageModel "$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL" \
    --arg videoModel "$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
    --argjson imageStable "$image_stable_throughput" \
    --argjson imageAdmitted "$image_admitted_throughput" \
    --argjson videoStable "$video_stable_throughput" \
    --argjson videoAdmitted "$video_admitted_throughput" \
    --argjson maxRunning "$max_running" \
    '{
      configHash:$configHash,
      capacityProfiles:[
        {
          kind:"image",
          model:$imageModel,
          stableThroughputPerSecond:$imageStable,
          admittedThroughputPerSecond:$imageAdmitted,
          maxRunning:$maxRunning,
          maxQueued:([($imageAdmitted * 60 | floor), $maxRunning] | max),
          maxPredictedWaitSeconds:60,
          metadata:{
            capacityProfile:$capacityProfile,
            repetitions:3,
            loadRunner:"external"
          }
        },
        {
          kind:"video",
          model:$videoModel,
          stableThroughputPerSecond:$videoStable,
          admittedThroughputPerSecond:$videoAdmitted,
          maxRunning:$maxRunning,
          maxQueued:([($videoAdmitted * 120 | floor), $maxRunning] | max),
          maxPredictedWaitSeconds:120,
          metadata:{
            capacityProfile:$capacityProfile,
            repetitions:3,
            loadRunner:"external"
          }
        }
      ]
    }' >"$capacity_profiles_payload"
}

stage_capacity_profiles() {
  local response=$temporary_root/staged-capacity-profiles.json
  calculate_capacity_profiles
  admin_request POST \
    "/api/admin/system/acceptance/runs/$run_id/capacity-profiles" \
    "$(cat "$capacity_profiles_payload")" \
    "$response"
  [[ $(jq -r '.status' "$response") == running ]]
}

run_mixed_load_profile() {
  local profile=$1
  local duration=$2
  local image_rate=$3
  local video_rate=$4
  local report_path=$5
  local timeout_value=$6
  local site_image_rate site_video_rate
  site_image_rate=$(awk -v value="$image_rate" 'BEGIN {printf "%.9f", value/2}')
  site_video_rate=$(awk -v value="$video_rate" 'BEGIN {printf "%.9f", value/2}')
  run_distributed_load_profile "$profile" "$report_path" \
    -duration "$duration" \
    -image-rate "$site_image_rate" \
    -video-rate "$site_video_rate" \
    -timeout "$timeout_value"
}

run_mixed_soak_and_overload() {
  local historical_mix image_count video_count total_count image_weight video_weight total_rate
  local soak_image_rate soak_video_rate overload_image_rate overload_video_rate
  historical_mix=$(database_query "
SELECT
  count(*) FILTER (WHERE kind IN ('images.generations','images.edits','images.vectorize'))||','||
  count(*) FILTER (WHERE kind IN ('videos.generations','videos.upscales'))
FROM gateway_tasks
WHERE acceptance_run_id IS NULL
  AND created_at >= now()-interval '7 days';")
  IFS=',' read -r image_count video_count <<<"$historical_mix"
  total_count=$((image_count + video_count))
  if (( total_count < 100 || image_count == 0 || video_count == 0 )); then
    image_weight=0.5
    video_weight=0.5
  else
    image_weight=$(awk -v image="$image_count" -v total="$total_count" 'BEGIN {printf "%.6f", image/total}')
    video_weight=$(awk -v video="$video_count" -v total="$total_count" 'BEGIN {printf "%.6f", video/total}')
  fi
  total_rate=$(awk \
    -v image="$image_admitted_throughput" \
    -v video="$video_admitted_throughput" \
    -v image_weight="$image_weight" \
    -v video_weight="$video_weight" '
    BEGIN {
      image_capacity=image/image_weight
      video_capacity=video/video_weight
      if (image_capacity<video_capacity) print image_capacity
      else print video_capacity
    }')
  soak_image_rate=$(awk -v total="$total_rate" -v weight="$image_weight" 'BEGIN {printf "%.6f", total*weight}')
  soak_video_rate=$(awk -v total="$total_rate" -v weight="$video_weight" 'BEGIN {printf "%.6f", total*weight}')
  overload_image_rate=$(awk -v value="$soak_image_rate" 'BEGIN {printf "%.6f", value*1.2}')
  overload_video_rate=$(awk -v value="$soak_video_rate" 'BEGIN {printf "%.6f", value*1.2}')
  jq -n \
    --argjson imageCount "$image_count" \
    --argjson videoCount "$video_count" \
    --argjson imageWeight "$image_weight" \
    --argjson videoWeight "$video_weight" \
    --argjson soakImageRate "$soak_image_rate" \
    --argjson soakVideoRate "$soak_video_rate" \
    --argjson overloadImageRate "$overload_image_rate" \
    --argjson overloadVideoRate "$overload_video_rate" \
    '{
      historicalWindowDays:7,
      historicalImageTasks:$imageCount,
      historicalVideoTasks:$videoCount,
      imageWeight:$imageWeight,
      videoWeight:$videoWeight,
      soakImageRatePerSecond:$soakImageRate,
      soakVideoRatePerSecond:$soakVideoRate,
      overloadImageRatePerSecond:$overloadImageRate,
      overloadVideoRatePerSecond:$overloadVideoRate
    }' >"$report_root/mixed-workload-profile.json"

  local pressure_report=$report_root/mixed-soak-pressure.csv
  local pressure_stop=$temporary_root/mixed-pressure-stop
  local pressure_pid pressure_status=0 status=0
  local expected_slots
  case $stable_profile in
    P24) expected_slots=24 ;;
    P28) expected_slots=28 ;;
    P32) expected_slots=32 ;;
  esac
  rm -f -- "$pressure_stop"
  sample_pressure "$pressure_report" "$pressure_stop" &
  pressure_pid=$!
  active_pressure_pid=$pressure_pid
  run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \
    mixed-soak "$AI_GATEWAY_ACCEPTANCE_SOAK_DURATION" \
    "$soak_image_rate" "$soak_video_rate" \
    "$report_root/mixed-soak.json" 3h || status=$?
  if (( status == 0 )); then
    run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \
      mixed-overload "$AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION" \
      "$overload_image_rate" "$overload_video_rate" \
      "$report_root/mixed-overload.json" 30m || status=$?
  fi
  touch "$pressure_stop"
  wait "$pressure_pid" || pressure_status=$?
  active_pressure_pid=
  (( status == 0 && pressure_status == 0 )) || {
    (( status != 0 )) && return "$status"
    return "$pressure_status"
  }
  jq -e '.phases[0].throttled == 0 and .phases[0].unexpected5xx == 0' \
    "$report_root/mixed-soak.json" >/dev/null
  jq -e '.phases[0].throttled > 0 and .phases[0].unexpected5xx == 0' \
    "$report_root/mixed-overload.json" >/dev/null
  verify_pressure_report "$pressure_report" "$expected_slots"
  verify_runtime_gates
}

activate_certified_autoscaling() {
  local deadline status resource_max test_min_ningbo test_min_hongkong
  test_min_ningbo=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO
  test_min_hongkong=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG
  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
  apply_autoscaling_profile \
    "$stable_profile" \
    "$certified_max_replicas_ningbo" \
    "$certified_max_replicas_hongkong"
  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$test_min_ningbo
  AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$test_min_hongkong
  deadline=$((SECONDS + 180))
  while (( SECONDS < deadline )); do
    status=$(capacity_controller_status || true)
    if [[ -n $status ]]; then
      resource_max=$(jq '[.plan.sites[].resourceMax] | add // 0' <<<"$status")
      if (( resource_max >= 2 )); then
        break
      fi
    fi
    sleep 2
  done
  [[ -n ${status:-} && ${resource_max:-0} -ge 2 ]]
  certified_max_replicas_ningbo=$(jq -r \
    '[.plan.sites[] | select(.site == "ningbo") | .resourceMax][0] // 0' <<<"$status")
  certified_max_replicas_hongkong=$(jq -r \
    '[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
  (( certified_max_replicas_ningbo >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
     certified_max_replicas_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
     certified_max_replicas_ningbo + certified_max_replicas_hongkong >= 2 ))
  jq -n \
    --arg profile "$stable_profile" \
    --argjson ningbo "$certified_max_replicas_ningbo" \
    --argjson hongkong "$certified_max_replicas_hongkong" \
    --argjson total "$resource_max" \
    '{
      capacityProfile:$profile,
      sites:{ningbo:{resourceMax:$ningbo},hongkong:{resourceMax:$hongkong}},
      resourceMaxTotal:$total
    }' >"$report_root/certified-site-capacity.json"
  [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
  [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
    -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
  stage_capacity_profiles
  verify_runtime_gates
}

remote_load_binary_path() {
  [[ $run_id =~ ^[0-9a-f-]{36}$ ]]
  printf '/root/easyai-acceptance-load-%s' "$run_id"
}

remote_load_env_path() {
  local site=$1
  [[ $site == ningbo || $site == hongkong ]]
  [[ $run_id =~ ^[0-9a-f-]{36}$ ]]
  printf '/root/easyai-acceptance-load-%s-%s.env' "$run_id" "$site"
}

cleanup_remote_load_drivers() {
  [[ $run_id =~ ^[0-9a-f-]{36}$ ]] || return 0
  local binary_path ningbo_env hongkong_env
  binary_path=$(remote_load_binary_path)
  ningbo_env=$(remote_load_env_path ningbo)
  hongkong_env=$(remote_load_env_path hongkong)
  cluster_ssh "$CLUSTER_NINGBO_HOST" \
    "pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$ningbo_env' ]] || unlink '$ningbo_env'" || true
  if [[ $single_node_acceptance != true ]]; then
    cluster_ssh "$CLUSTER_HONGKONG_HOST" \
      "pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$hongkong_env' ]] || unlink '$hongkong_env'" || true
  fi
  remote_load_drivers_installed=false
}

write_remote_load_env() {
  local output=$1
  local gateway=$2
  install -m 0600 /dev/null "$output"
  {
    printf 'AI_GATEWAY_ACCEPTANCE_GATEWAYS=%q\n' "$gateway"
    printf 'AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=%q\n' "$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME"
    printf 'AI_GATEWAY_ACCEPTANCE_EMULATOR_URL=%q\n' 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090'
    printf 'AI_GATEWAY_ACCEPTANCE_API_KEYS=%q\n' "$AI_GATEWAY_ACCEPTANCE_API_KEYS"
    printf 'AI_GATEWAY_ACCEPTANCE_RUN_ID=%q\n' "$run_id"
    printf 'AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=%q\n' "$run_token"
    printf 'AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=%q\n' "$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL"
    printf 'AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=%q\n' "$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL"
    printf 'AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=%q\n' "$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS"
  } >>"$output"
}

install_remote_load_drivers() {
  [[ $remote_load_drivers_installed == false ]] || return 0
  local binary_path local_digest remote_digest site host gateway env_file remote_env
  binary_path=$(remote_load_binary_path)
  local_digest=$(shasum -a 256 "$acceptance_load_linux_binary" | awk '{print $1}')
  local -a sites=(ningbo hongkong)
  local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST")
  local -a gateways=("https://${CLUSTER_NINGBO_HOST#root@}" "https://${CLUSTER_HONGKONG_HOST#root@}")
  for site_index in 0 1; do
    site=${sites[$site_index]}
    host=${hosts[$site_index]}
    gateway=${gateways[$site_index]}
    env_file=$temporary_root/remote-load-$site.env
    remote_env=$(remote_load_env_path "$site")
    write_remote_load_env "$env_file" "$gateway"
    cluster_scp "$acceptance_load_linux_binary" "$host:$binary_path" >/dev/null
    cluster_scp "$env_file" "$host:$remote_env" >/dev/null
    cluster_ssh "$host" "chmod 0755 '$binary_path'; chmod 0600 '$remote_env'; sha256sum '$binary_path'" >"$temporary_root/remote-load-$site.sha256"
    remote_digest=$(awk '{print $1}' "$temporary_root/remote-load-$site.sha256")
    [[ $remote_digest == "$local_digest" ]]
  done
  remote_load_drivers_installed=true
  echo 'acceptance_load_drivers=PASS sites=2 placement=k3s_external_host_processes digest_verified=true'
}

run_remote_load_site() {
  local site=$1
  local host=$2
  local profile=$3
  local artifact=$4
  local shard_index=$5
  shift 5
  [[ $site == ningbo || $site == hongkong ]]
  [[ $profile =~ ^[a-z-]+$ && $artifact =~ ^[a-z0-9-]+\.json$ ]]
  local binary_path remote_env remote_report local_report status=0
  binary_path=$(remote_load_binary_path)
  remote_env=$(remote_load_env_path "$site")
  remote_report=/root/easyai-acceptance-load-"$run_id"-"$artifact"
  local_report=$temporary_root/"$site"-"$artifact"
  cluster_ssh "$host" bash -s -- \
    "$binary_path" "$remote_env" "$profile" "$remote_report" "$shard_index" 2 "${artifact%.json}" "$@" \
    >"$temporary_root/$site-$artifact.stdout" <<'REMOTE' || status=$?
set -euo pipefail
binary_path=$1
env_file=$2
profile=$3
report=$4
shard_index=$5
shard_count=$6
execution_id=$7
shift 7
set -a
# shellcheck source=/dev/null
source "$env_file"
set +a
"$binary_path" -profile "$profile" -report "$report" \
  -shard-index "$shard_index" -shard-count "$shard_count" \
  -execution-id "$execution_id" "$@"
REMOTE
  cluster_scp "$host:$remote_report" "$local_report" >/dev/null || return 1
  cluster_ssh "$host" "[[ ! -e '$remote_report' ]] || unlink '$remote_report'" >/dev/null || true
  jq -e --arg runId "$run_id" --arg profile "$profile" \
    '.schemaVersion == "acceptance-load-report/v2" and .runId == $runId and .profile == $profile and .secretSafe == true' \
    "$local_report" >/dev/null || return 1
  return "$status"
}

merge_remote_load_reports() {
  local profile=$1
  local ningbo_report=$2
  local hongkong_report=$3
  local output=$4
  jq -s --arg profile "$profile" '
    def sum_field($name): [.[].phases[0][$name] // 0] | add;
    def max_field($name): [.[].phases[0][$name] // 0] | max;
    . as $reports
    | {
        schemaVersion:"acceptance-load-report/v2",
        runId:$reports[0].runId,
        profile:$profile,
        startedAt:([$reports[].startedAt] | min),
        finishedAt:([$reports[].finishedAt] | max),
        passed:([$reports[].passed] | all),
        phases:[{
          name:$profile,
          requests:sum_field("requests"),
          completed:sum_field("completed"),
          failed:sum_field("failed"),
          durationMs:max_field("durationMs"),
          submissionDurationMs:max_field("submissionDurationMs"),
          p50Ms:max_field("p50Ms"),
          p95Ms:max_field("p95Ms"),
          p99Ms:max_field("p99Ms"),
          decodedOutputBytes:sum_field("decodedOutputBytes"),
          outputSha256:([$reports[].phases[0].outputSha256 // ""] | map(select(length>0)) | first // ""),
          uniqueTaskIds:sum_field("uniqueTaskIds"),
          uniqueImageCombinations:max_field("uniqueImageCombinations"),
          forcedConversionTasks:sum_field("forcedConversionTasks"),
          throttled:sum_field("throttled"),
          unexpected5xx:sum_field("unexpected5xx"),
          offeredRatePerSecond:sum_field("offeredRatePerSecond")
        }],
        failure:([$reports[] | select(.passed == false) | .failure] | first // null),
        failureOperation:([$reports[] | select(.passed == false) | .failureOperation] | first // null),
        aggregation:{
          sites:["ningbo","hongkong"],
          percentileMethod:"conservative_max_of_site_percentiles",
          durationMethod:"max_site_duration",
          placement:"k3s_external_host_processes"
        },
        secretSafe:true
      }
    | with_entries(select(.value != null))
  ' "$ningbo_report" "$hongkong_report" >"$output"
  chmod 0600 "$output"
}

run_distributed_load_profile() {
  local profile=$1
  local report_path=$2
  shift 2
  install_remote_load_drivers
  local artifact base ningbo_report hongkong_report ningbo_pid hongkong_pid
  local ningbo_status=0 hongkong_status=0
  base=$(basename "$report_path" .json)
  artifact=$base.json
  run_remote_load_site ningbo "$CLUSTER_NINGBO_HOST" "$profile" "$artifact" 0 "$@" &
  ningbo_pid=$!
  run_remote_load_site hongkong "$CLUSTER_HONGKONG_HOST" "$profile" "$artifact" 1 "$@" &
  hongkong_pid=$!
  wait "$ningbo_pid" || ningbo_status=$?
  wait "$hongkong_pid" || hongkong_status=$?
  ningbo_report=$temporary_root/ningbo-$artifact
  hongkong_report=$temporary_root/hongkong-$artifact
  [[ -f $ningbo_report && -f $hongkong_report ]] || return 1
  install -m 0600 "$ningbo_report" "${report_path%.json}-ningbo.json"
  install -m 0600 "$hongkong_report" "${report_path%.json}-hongkong.json"
  merge_remote_load_reports "$profile" \
    "${report_path%.json}-ningbo.json" "${report_path%.json}-hongkong.json" "$report_path"
  (( ningbo_status == 0 && hongkong_status == 0 )) || return 1
  jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
}

run_load_profile() {
  local profile=$1
  local report_path=$2
  [[ $profile =~ ^[a-z-]+$ ]] || return 1
  if [[ $profile != real-canary ]]; then
    run_distributed_load_profile "$profile" "$report_path"
    return
  fi
  local load_stdout=$temporary_root/load-$profile-stdout.json
  AI_GATEWAY_ACCEPTANCE_GATEWAYS="$AI_GATEWAY_ACCEPTANCE_GATEWAYS" \
  AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME="$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" \
  AI_GATEWAY_ACCEPTANCE_EMULATOR_URL='http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' \
  AI_GATEWAY_ACCEPTANCE_API_KEYS="$AI_GATEWAY_ACCEPTANCE_API_KEYS" \
  AI_GATEWAY_ACCEPTANCE_RUN_ID="$run_id" \
  AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$run_token" \
  AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL" \
  AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
  AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS="$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" \
    "$acceptance_load_binary" -profile "$profile" -report "$report_path" \
      -execution-id "$(basename "$report_path" .json)" \
      >"$load_stdout"
  jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
}

kill_one_worker_during_recovery() {
  local deadline=$((SECONDS + 120))
  local owner pod site task_id
  while (( SECONDS < deadline )); do
    owner=$(database_query "
SELECT task.id::text||','||worker.pod_name||','||worker.site
FROM gateway_tasks task
JOIN gateway_worker_instances worker ON worker.instance_id=task.locked_by
WHERE task.acceptance_run_id='$run_id'::uuid
  AND task.status='running'
  AND task.request::text LIKE '%acceptance-long-recovery%'
  AND worker.status='active'
  AND worker.heartbeat_at > now()-interval '30 seconds'
ORDER BY task.updated_at ASC, task.id ASC
LIMIT 1;")
    if [[ -n $owner ]]; then
      break
    fi
    sleep 1
  done
  [[ -n $owner ]] || {
    echo 'recovery workload never obtained a live Worker owner' >&2
    return 1
  }
  IFS=',' read -r task_id pod site <<<"$owner"
  [[ $task_id =~ ^[0-9a-f-]{36}$ ]]
  [[ $site == ningbo || $site == hongkong ]]
  [[ $pod =~ ^easyai-worker-$site-[a-z0-9-]+$ ]]
  [[ $(remote_kubectl get pod "$pod" -n "$namespace" \
    -o 'jsonpath={.metadata.uid}') != "" ]]
  printf 'worker_failure_target_task=%s pod=%s site=%s\n' "$task_id" "$pod" "$site"
  remote_kubectl delete pod "$pod" -n "$namespace" --wait=false >/dev/null
  remote_kubectl rollout status "deployment/easyai-worker-$site" \
    -n "$namespace" --timeout=300s
}

run_recovery_profile() {
  local report_path=$1
  run_load_profile video-recovery "$report_path" &
  local load_pid=$!
  active_load_pid=$load_pid
  if ! kill_one_worker_during_recovery; then
    kill "$load_pid" >/dev/null 2>&1 || true
    wait "$load_pid" >/dev/null 2>&1 || true
    return 1
  fi
  wait "$load_pid"
  active_load_pid=
}

verify_wireguard() {
  local source_host=$1
  local target_ip=$2
  local max_rtt=$3
  local label=$4
  local output loss average handshakes now latest age
  output=$(cluster_ssh "$source_host" "ping -q -i 0.2 -c 10 -W 2 $target_ip")
  loss=$(awk -F',' '/packet loss/ {gsub(/[^0-9.]/, "", $3); print $3}' <<<"$output")
  average=$(awk -F'=' '/min\\/avg\\/max/ {split($2, values, \"/\"); gsub(/ /, "", values[2]); print values[2]}' <<<"$output")
  [[ -n $loss && -n $average ]]
  awk -v loss="$loss" -v average="$average" -v max="$max_rtt" \
    'BEGIN { exit !(loss < 1 && average < max) }'
  now=$(date +%s)
  latest=$(cluster_ssh "$source_host" "wg show all latest-handshakes | awk '{print \\$3}' | sort -nr | head -n 2")
  handshakes=$(wc -l <<<"$latest" | tr -d ' ')
  (( handshakes >= 2 ))
  while IFS= read -r timestamp; do
    [[ $timestamp =~ ^[0-9]+$ && $timestamp -gt 0 ]]
    age=$((now - timestamp))
    (( age < 180 ))
  done <<<"$latest"
  echo "wireguard=$label loss_percent=$loss average_rtt_ms=$average recent_handshakes=$handshakes"
}

verify_cluster_links() {
  verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.2 80 'ningbo_to_hongkong'
  verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.1 80 'hongkong_to_ningbo'
  verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.3 300 'ningbo_to_los_angeles'
  verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
  verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
  verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
    verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_worker_node_4'
    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.1 80 'worker_node_4_to_ningbo'
    verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_worker_node_4'
    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.2 80 'worker_node_4_to_hongkong'
    verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_worker_node_4'
    verify_wireguard "$CLUSTER_WORKER_NODE_4_HOST" 10.77.0.3 300 'worker_node_4_to_losangeles'
  fi
}

verify_control_plane_and_recent_logs() {
  local host ready_output workload pod error_count log_window
  log_window=--since=10m
  if [[ -n $runtime_observation_started_at ]]; then
    log_window=--since-time=$runtime_observation_started_at
  fi
  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
    ready_output=$(cluster_ssh "$host" "k3s kubectl get --raw='/readyz?verbose'")
    grep -Fq '[+]etcd ok' <<<"$ready_output" || return 1
    if [[ -n $runtime_observation_started_at ]]; then
      error_count=$(cluster_ssh "$host" \
        "journalctl -u k3s --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /http: Handler timeout/ || /rejected connection on peer endpoint.*(timeout|reset)/ || /etcdserver: request timed out/ || /leader changed/ {count++} END {print count+0}'")
      [[ $error_count == 0 ]] || return 1
    fi
  done
  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true && -n $runtime_observation_started_at ]]; then
    error_count=$(cluster_ssh "$CLUSTER_WORKER_NODE_4_HOST" \
      "journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
    [[ $error_count == 0 ]] || return 1
  fi
  for workload in api worker; do
    while read -r pod; do
      error_count=$(remote_kubectl logs "$pod" -n "$namespace" "$log_window" |
        awk 'BEGIN {IGNORECASE=1}
          /postgres_unavailable/ ||
          /postgres readiness.*(fail|unavailable|error)/ ||
          /leadership elector.*(error|fail)/ ||
          /concurrency lease.*(renew.*(error|fail)|lost)/ ||
          /task execution lease (lost|renewal failed)/ ||
          /upstream submission requires manual review/ {count++}
          END {print count+0}')
      [[ $error_count == 0 ]] || return 1
    done < <(remote_kubectl get pods -n "$namespace" \
      -l "app.kubernetes.io/name=easyai-$workload" \
      -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
  done
  while read -r pod; do
    error_count=$(remote_kubectl logs "$pod" -n "$namespace" -c postgres "$log_window" |
      awk 'BEGIN {IGNORECASE=1}
        /apply request took too long/ ||
        /dial tcp.*6443/ ||
        /API server connectivity issue/ ||
        /Instance connectivity error/ ||
        /terminating walreceiver due to timeout/ ||
        /synchronous commit.*cancel/ ||
        /canceling statement due to user request/ ||
        /canceling statement due to lock timeout/ ||
        /(liveness|readiness).*fail/ {count++}
        END {print count+0}')
    [[ $error_count == 0 ]] || return 1
  done < <(remote_kubectl get pods -n "$namespace" \
    -l 'cnpg.io/cluster=easyai-postgres' \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
}

workload_metrics_for_site() {
  local workload=$1
  local site=$2
  local host pod_ip node_name replicas targets
  [[ $workload == api || $workload == worker ]]
  [[ $site == ningbo || $site == hongkong ]]
  targets=$(remote_kubectl get pods -n "$namespace" \
    -l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json |
    jq -r '.items[]
      | select(.metadata.deletionTimestamp == null)
      | select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))
      | [.status.podIP,.spec.nodeName]
      | @tsv') || return 1
  if [[ -z $targets && $workload == worker ]]; then
    replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \
      -o 'jsonpath={.spec.replicas}') || return 1
    [[ $replicas =~ ^[0-9]+$ ]] || return 1
    (( replicas > 0 )) || return 2
  fi
  [[ -n $targets ]] || return 1
  while IFS=$'\t' read -r pod_ip node_name; do
    [[ $pod_ip =~ ^[0-9a-fA-F:.]+$ && $node_name =~ ^[a-z0-9-]+$ ]] || return 1
    case $node_name in
      easyai-ningbo) host=$CLUSTER_NINGBO_HOST ;;
      easyai-hongkong) host=$CLUSTER_HONGKONG_HOST ;;
      easyai-los-angeles) host=$CLUSTER_LOS_ANGELES_HOST ;;
      *)
        if [[ -n $CLUSTER_WORKER_NODE_4_NAME &&
              $node_name == "$CLUSTER_WORKER_NODE_4_NAME" ]]; then
          host=$CLUSTER_WORKER_NODE_4_HOST
        else
          echo "metrics target node has no SSH mapping: node=$node_name" >&2
          return 1
        fi
        ;;
    esac
    cluster_ssh "$host" "curl -fsS --max-time 10 http://$pod_ip:8088/metrics" \
      </dev/null || return 1
  done <<<"$targets"
}

metric_min() {
  local metric=$1
  awk -v metric="$metric" '
    $1==metric && (!seen || $2<value) {value=$2; seen=1}
    END {if (seen) print value}
  '
}

metric_max() {
  local metric=$1
  awk -v metric="$metric" '
    $1==metric && (!seen || $2>value) {value=$2; seen=1}
    END {if (seen) print value}
  '
}

metric_sum() {
  local metric=$1
  awk -v metric="$metric" '
    $1==metric {value+=$2; seen=1}
    END {if (seen) print value}
  '
}

run_with_pressure_monitor() {
  local pressure_pid=$1
  shift
  local load_pid load_status=0
  "$@" &
  load_pid=$!
  active_load_pid=$load_pid
  while kill -0 "$load_pid" >/dev/null 2>&1; do
    if ! kill -0 "$pressure_pid" >/dev/null 2>&1; then
      kill "$load_pid" >/dev/null 2>&1 || true
      wait "$load_pid" >/dev/null 2>&1 || true
      active_load_pid=
      echo 'pressure sampler exited before the active load completed' >&2
      return 1
    fi
    sleep "${AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS:-2}"
  done
  wait "$load_pid" || load_status=$?
  active_load_pid=
  return "$load_status"
}

pressure_row_live_gate_reason() {
  local pressure_row=$1
  awk -F',' '
    $4 >= 900 {print "oldest_wait_seconds=" $4 " limit=900"; exit}
    $6 <= 0 {print "postgres_max_connections_unavailable"; exit}
    $5 * 4 >= $6 * 3 {print "postgres_connections=" $5 " limit_75_percent_of=" $6; exit}
    $7 >= 85 {print "node_memory_percent=" $7 " hard_limit=85"; exit}
    $8 >= 1536 {print "api_pod_memory_mib=" $8 " limit=1536"; exit}
    $14 > 0 {print "lease_renewal_failures=" $14; exit}
    $15 > 0 {print "lease_lost=" $15; exit}
    $16 < 1 || $16 > 2 {print "active_instances=" $16 " expected=1..2"; exit}
    $28 >= 1536 {print "worker_memory_mib=" $28 " limit=1536"; exit}
  ' <<<"$pressure_row"
}

pressure_row_passes_live_gates() {
  [[ -z $(pressure_row_live_gate_reason "$1") ]]
}

current_max_pod_memory_mib() {
  remote_kubectl top pods -n "$namespace" \
    -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers |
    awk '{
      value=$3
      if (value ~ /Gi$/) {sub(/Gi$/, "", value); value*=1024}
      else if (value ~ /Mi$/) {sub(/Mi$/, "", value)}
      else if (value ~ /Ki$/) {sub(/Ki$/, "", value); value/=1024}
      if (value>max) max=value
    } END {printf "%.0f", max+0}'
}

current_max_worker_resources() {
  remote_kubectl top pods -n "$namespace" \
    -l 'app.kubernetes.io/name=easyai-worker' --no-headers |
    awk '{
      cpu=$2
      memory=$3
      if (cpu ~ /n$/) {sub(/n$/, "", cpu); cpu/=1000000}
      else if (cpu ~ /u$/) {sub(/u$/, "", cpu); cpu/=1000}
      else if (cpu ~ /m$/) {sub(/m$/, "", cpu)}
      else {cpu*=1000}
      if (memory ~ /Gi$/) {sub(/Gi$/, "", memory); memory*=1024}
      else if (memory ~ /Mi$/) {sub(/Mi$/, "", memory)}
      else if (memory ~ /Ki$/) {sub(/Ki$/, "", memory); memory/=1024}
      if (cpu>max_cpu) max_cpu=cpu
      if (memory>max_memory) max_memory=memory
    } END {printf "%.0f,%.0f", max_memory+0, max_cpu+0}'
}

wait_for_memory_recovery() {
  local baseline_mib=$1
  local deadline=$((SECONDS + 120))
  local current_mib
  while (( SECONDS < deadline )); do
    current_mib=$(current_max_pod_memory_mib)
    if (( current_mib <= baseline_mib + 256 )); then
      return 0
    fi
    sleep 5
  done
  echo "Pod memory did not return near baseline: baseline_mib=$baseline_mib current_mib=$current_mib" >&2
  return 1
}

verify_runtime_gates() {
  local expected_slots expected_global expected_pool expected_worker_execution_pool expected_api_execution_pool
  local expected_ready_nodes=3
  local expected_critical_pool=4
  local expected_api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
  local expected_worker_river_pool=$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS
  local expected_api_river_pool=$AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS
  case $active_profile in
    P24) expected_slots=24; expected_pool=32; expected_global=48 ;;
    P28) expected_slots=28; expected_pool=36; expected_global=56 ;;
    P32) expected_slots=32; expected_pool=40; expected_global=64 ;;
  esac
  expected_worker_execution_pool=$((expected_pool - expected_critical_pool - expected_worker_river_pool))
  expected_api_execution_pool=$((expected_api_pool - expected_critical_pool - expected_api_river_pool))
  if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
    expected_ready_nodes=4
  fi
  [[ $(remote_kubectl get nodes -o json |
    jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') == "$expected_ready_nodes" ]]
  [[ $(remote_kubectl get nodes -o json |
    jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]]
  [[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.readyInstances}') == 2 ]]
  local sync_state
  sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
  [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]]
  verify_control_plane_and_recent_logs

  local unhealthy_containers pod_memory
  unhealthy_containers=$(remote_kubectl get pods -n "$namespace" -o json |
    jq '[.items[]
      | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or
               .metadata.labels["app.kubernetes.io/name"] == "easyai-worker")
      | .status.containerStatuses[]?
      | select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or
               .state.terminated.reason == "OOMKilled")] | length')
  [[ $unhealthy_containers == 0 ]]
  while read -r _node _cpu _cpu_percent _memory memory_percent; do
    memory_percent=${memory_percent%\%}
    [[ $memory_percent =~ ^[0-9]+$ ]]
    (( memory_percent < 80 ))
  done < <(remote_kubectl top nodes --no-headers)
  while read -r _pod _cpu pod_memory; do
    case $pod_memory in
      *Gi) pod_memory=$(awk -v value="${pod_memory%Gi}" 'BEGIN { printf "%.0f", value * 1024 }') ;;
      *Mi) pod_memory=${pod_memory%Mi} ;;
      *Ki) pod_memory=$(awk -v value="${pod_memory%Ki}" 'BEGIN { printf "%.0f", value / 1024 }') ;;
      *) return 1 ;;
    esac
    [[ $pod_memory =~ ^[0-9]+$ ]]
    (( pod_memory < 1536 ))
  done < <(remote_kubectl top pods -n "$namespace" \
    -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers)
  local pod_name pod_anon_mib
  while read -r pod_name; do
    pod_anon_mib=$(remote_kubectl exec -n "$namespace" "$pod_name" -- \
      cat /sys/fs/cgroup/memory.stat |
      awk '$1=="anon" {printf "%.0f", $2/1048576}')
    [[ $pod_anon_mib =~ ^[0-9]+$ ]]
    (( pod_anon_mib < 1536 ))
  done < <(remote_kubectl get pods -n "$namespace" -o json |
    jq -r '.items[]
      | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or
               .metadata.labels["app.kubernetes.io/name"] == "easyai-worker")
      | .metadata.name')

  local completion_deadline=$((SECONDS + 180))
  local incomplete side_effects
  while (( SECONDS < completion_deadline )); do
    incomplete=$(database_query "
SELECT count(*)
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid
  AND (
    status <> 'succeeded'
    OR billing_status <> 'settled'
  );")
    side_effects=$(database_query "
SELECT
  (SELECT count(*)
   FROM settlement_outbox settlement
   WHERE settlement.task_id IN (
     SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
   )
     AND settlement.status <> 'completed')
  +
  (SELECT count(*)
   FROM gateway_tasks task
   WHERE task.acceptance_run_id='$run_id'::uuid
     AND NOT EXISTS (
       SELECT 1
       FROM gateway_task_callback_outbox callback
       WHERE callback.task_id=task.id
         AND callback.status='delivered'
     ));")
    if [[ $incomplete == 0 && $side_effects == 0 ]]; then
      break
    fi
    sleep 2
  done
  [[ $incomplete == 0 && $side_effects == 0 ]]

  local queue_state duplicates settlements wallet_duplicates missing_billings callbacks connections max_connections
  local active_workers stale_workers
  queue_state=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
  [[ $queue_state == 0:0 ]]
  duplicates=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) duplicate;")
  [[ $duplicates == 0 ]]
  settlements=$(database_query "SELECT count(*) FROM (SELECT task_id,action FROM settlement_outbox WHERE task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY task_id,action HAVING count(*)>1) duplicate;")
  [[ $settlements == 0 ]]
  wallet_duplicates=$(database_query "
SELECT count(*)
FROM (
  SELECT reference_id,transaction_type
  FROM gateway_wallet_transactions
  WHERE reference_type='gateway_task'
    AND reference_id IN (
      SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
    )
  GROUP BY reference_id,transaction_type
  HAVING count(*)>1
) duplicate;")
  missing_billings=$(database_query "
SELECT count(*)
FROM gateway_tasks task
WHERE task.acceptance_run_id='$run_id'::uuid
  AND NOT EXISTS (
    SELECT 1
    FROM gateway_wallet_transactions transaction
    WHERE transaction.reference_type='gateway_task'
      AND transaction.reference_id=task.id::text
      AND transaction.transaction_type='task_billing'
  );")
  [[ $wallet_duplicates == 0 && $missing_billings == 0 ]]
  callbacks=$(database_query "
SELECT count(*)
FROM gateway_task_callback_outbox
WHERE task_id IN (
  SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
)
  AND status <> 'delivered';")
  [[ $callbacks == 0 ]]
  active_workers=$(database_query "
SELECT count(*)
FROM gateway_worker_instances
WHERE status='active'
  AND heartbeat_at > now()-interval '30 seconds';")
  stale_workers=$(database_query "
SELECT count(*)
FROM gateway_worker_instances
WHERE status='active'
  AND (
    heartbeat_at <= now()-interval '30 seconds'
    OR desired_capacity <> $expected_slots
    OR capacity_limit <> $expected_slots
    OR allocated_capacity > $expected_slots
  );")
  [[ $active_workers == 2 && $stale_workers == 0 ]]
  connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
  max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
  (( connections * 4 < max_connections * 3 ))

  local pod metrics desired current global allocated active_instances pool_max acquired idle
  local desired_max current_max global_max pool_max_min metrics_status
  local critical_pool_max critical_pool_min critical_acquired critical_idle
  local river_pool_max river_pool_min river_acquired river_idle
  local lease_failures lease_lost
  for site in ningbo hongkong; do
    metrics=$(workload_metrics_for_site api "$site")
    pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics")
    acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics")
    idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics")
    critical_pool_max=$(awk '$1=="easyai_gateway_postgres_critical_pool_max_connections" {print $2}' <<<"$metrics")
    river_pool_max=$(awk '$1=="easyai_gateway_postgres_river_pool_max_connections" {print $2}' <<<"$metrics")
    river_acquired=$(awk '$1=="easyai_gateway_postgres_river_pool_acquired_connections" {print $2}' <<<"$metrics")
    river_idle=$(awk '$1=="easyai_gateway_postgres_river_pool_idle_connections" {print $2}' <<<"$metrics")
    [[ ${pool_max%.*} == "$expected_api_execution_pool" ]]
    [[ ${critical_pool_max%.*} == "$expected_critical_pool" ]]
    [[ ${river_pool_max%.*} == "$expected_api_river_pool" ]]
    if [[ ${acquired%.*} == "$expected_api_execution_pool" && ${idle%.*} == 0 ]]; then
      echo "API connection pool saturated for site $site" >&2
      return 1
    fi
    if [[ ${river_acquired%.*} == "$expected_api_river_pool" && ${river_idle%.*} == 0 ]]; then
      echo "API River connection pool saturated for site $site" >&2
      return 1
    fi
  done
  for _ in 1 2 3 4 5 6; do
    for site in ningbo hongkong; do
      metrics_status=0
      metrics=$(workload_metrics_for_site worker "$site") || metrics_status=$?
      if (( metrics_status == 2 )); then
        continue
      fi
      (( metrics_status == 0 )) || return 1
      desired=$(metric_min easyai_gateway_async_worker_desired_capacity <<<"$metrics")
      desired_max=$(metric_max easyai_gateway_async_worker_desired_capacity <<<"$metrics")
      current=$(metric_min easyai_gateway_async_worker_capacity <<<"$metrics")
      current_max=$(metric_max easyai_gateway_async_worker_capacity <<<"$metrics")
      global=$(metric_min easyai_gateway_worker_global_capacity <<<"$metrics")
      global_max=$(metric_max easyai_gateway_worker_global_capacity <<<"$metrics")
      allocated=$(metric_max easyai_gateway_worker_allocated_capacity <<<"$metrics")
      active_instances=$(metric_min easyai_gateway_worker_active_instances <<<"$metrics")
      pool_max=$(metric_max easyai_gateway_postgres_pool_max_connections <<<"$metrics")
      pool_max_min=$(metric_min easyai_gateway_postgres_pool_max_connections <<<"$metrics")
      acquired=$(metric_max easyai_gateway_postgres_pool_acquired_connections <<<"$metrics")
      idle=$(metric_min easyai_gateway_postgres_pool_idle_connections <<<"$metrics")
      lease_failures=$(metric_sum 'easyai_gateway_concurrency_lease_renewals_total{outcome="failure"}' <<<"$metrics")
      lease_lost=$(metric_sum 'easyai_gateway_concurrency_lease_renewals_total{outcome="lost"}' <<<"$metrics")
      [[ ${desired%.*} == "$expected_global" && ${desired_max%.*} == "$expected_global" ]]
      [[ ${current%.*} == "$expected_slots" && ${current_max%.*} == "$expected_slots" ]]
      [[ ${global%.*} == "$expected_global" && ${global_max%.*} == "$expected_global" ]]
      [[ ${allocated%.*} -le "$expected_slots" ]]
      [[ ${active_instances%.*} == 2 ]]
      [[ ${pool_max%.*} == "$expected_worker_execution_pool" &&
         ${pool_max_min%.*} == "$expected_worker_execution_pool" ]]
      critical_pool_max=$(metric_max easyai_gateway_postgres_critical_pool_max_connections <<<"$metrics")
      critical_pool_min=$(metric_min easyai_gateway_postgres_critical_pool_max_connections <<<"$metrics")
      critical_acquired=$(metric_max easyai_gateway_postgres_critical_pool_acquired_connections <<<"$metrics")
      critical_idle=$(metric_min easyai_gateway_postgres_critical_pool_idle_connections <<<"$metrics")
      river_pool_max=$(metric_max easyai_gateway_postgres_river_pool_max_connections <<<"$metrics")
      river_pool_min=$(metric_min easyai_gateway_postgres_river_pool_max_connections <<<"$metrics")
      river_acquired=$(metric_max easyai_gateway_postgres_river_pool_acquired_connections <<<"$metrics")
      river_idle=$(metric_min easyai_gateway_postgres_river_pool_idle_connections <<<"$metrics")
      [[ ${critical_pool_max%.*} == "$expected_critical_pool" &&
         ${critical_pool_min%.*} == "$expected_critical_pool" ]]
      [[ ${river_pool_max%.*} == "$expected_worker_river_pool" &&
         ${river_pool_min%.*} == "$expected_worker_river_pool" ]]
      [[ ${lease_failures%.*} == 0 && ${lease_lost%.*} == 0 ]]
      if [[ ${acquired%.*} == "$expected_worker_execution_pool" && ${idle%.*} == 0 ]]; then
        echo "Worker connection pool saturated for site $site" >&2
        return 1
      fi
      if [[ ${critical_acquired%.*} == "$expected_critical_pool" && ${critical_idle%.*} == 0 ]]; then
        echo "Worker critical connection pool saturated for site $site" >&2
        return 1
      fi
      if [[ ${river_acquired%.*} == "$expected_worker_river_pool" && ${river_idle%.*} == 0 ]]; then
        echo "Worker River connection pool saturated for site $site" >&2
        return 1
      fi
    done
    sleep 5
  done
  verify_cluster_links
  curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/healthz >/dev/null
  curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz >/dev/null
}

sample_pressure() {
  local output=$1
  local stop_file=$2
  local failure_file=${3:-}
  local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics
  local consecutive_failures=0 missing_fields metrics_status pressure_row
  echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output"
  while [[ ! -f $stop_file ]]; do
    if ! database_state=$(database_query "
SELECT
  count(*) FILTER (WHERE status='queued')||','||
  count(*) FILTER (WHERE status='running')||','||
  COALESCE(EXTRACT(EPOCH FROM (now()-(min(created_at) FILTER (WHERE status='queued')))),0)::bigint||','||
  (SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
  (SELECT setting FROM pg_settings WHERE name='max_connections')
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid;"); then
      database_state=
    fi
    if ! node_memory_percent=$(remote_kubectl top nodes --no-headers 2>/dev/null |
      awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}'); then
      node_memory_percent=
    fi
    if ! pod_memory_mib=$(current_max_pod_memory_mib 2>/dev/null); then
      pod_memory_mib=
    fi
    if ! worker_resources=$(current_max_worker_resources 2>/dev/null); then
      worker_resources=
    fi
    if ! pool_state=$(
      for workload in api worker; do
        for site in ningbo hongkong; do
          metrics_status=0
          metrics=$(workload_metrics_for_site "$workload" "$site") || metrics_status=$?
          if (( metrics_status == 2 )); then
            continue
          fi
          (( metrics_status == 0 )) || exit 1
          awk -v workload="$workload" '
            BEGIN {idle=999999; critical_idle=999999; river_idle=999999; active=999999}
            $1=="easyai_gateway_postgres_pool_acquired_connections" && (!acquired_seen || $2>acquired) {acquired=$2; acquired_seen=1}
            $1=="easyai_gateway_postgres_pool_max_connections" && (!pool_max_seen || $2>pool_max) {pool_max=$2; pool_max_seen=1}
            $1=="easyai_gateway_postgres_pool_idle_connections" && $2<idle {idle=$2}
            $1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty+=$2}
            $1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled+=$2}
            $1=="easyai_gateway_postgres_critical_pool_acquired_connections" && (!critical_acquired_seen || $2>critical_acquired) {critical_acquired=$2; critical_acquired_seen=1}
            $1=="easyai_gateway_postgres_critical_pool_max_connections" && (!critical_max_seen || $2>critical_max) {critical_max=$2; critical_max_seen=1}
            $1=="easyai_gateway_postgres_critical_pool_idle_connections" && $2<critical_idle {critical_idle=$2}
            $1=="easyai_gateway_postgres_critical_pool_empty_acquire_total" {critical_empty+=$2}
            $1=="easyai_gateway_postgres_critical_pool_canceled_acquire_total" {critical_canceled+=$2}
            $1=="easyai_gateway_postgres_river_pool_acquired_connections" && (!river_acquired_seen || $2>river_acquired) {river_acquired=$2; river_acquired_seen=1}
            $1=="easyai_gateway_postgres_river_pool_max_connections" && (!river_max_seen || $2>river_max) {river_max=$2; river_max_seen=1}
            $1=="easyai_gateway_postgres_river_pool_idle_connections" && $2<river_idle {river_idle=$2}
            $1=="easyai_gateway_postgres_river_pool_empty_acquire_total" {river_empty+=$2}
            $1=="easyai_gateway_postgres_river_pool_canceled_acquire_total" {river_canceled+=$2}
            index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure+=$2}
            index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost+=$2}
            $1=="easyai_gateway_worker_active_instances" && $2<active {active=$2}
            $1=="easyai_gateway_worker_allocated_capacity" && $2>allocated {allocated=$2}
            END {
              if (workload=="api") {
                failure=0
                lost=0
                active=999999
                allocated=0
              }
              if (idle==999999) idle=0
              if (critical_idle==999999) critical_idle=0
              if (river_idle==999999) river_idle=0
              print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated "," critical_acquired "," critical_max "," critical_idle "," critical_empty "," critical_canceled "," river_acquired "," river_max "," river_idle "," river_empty "," river_canceled
            }
          ' <<<"$metrics"
        done
      done |
        awk -F',' '
          NR==1 {
            acquired=$1; pool_max=$2; idle=$3; empty=$4; canceled=$5
            failure=$6; lost=$7; active=$8; allocated=$9
            critical_acquired=$10; critical_max=$11; critical_idle=$12
            critical_empty=$13; critical_canceled=$14
            river_acquired=$15; river_max=$16; river_idle=$17
            river_empty=$18; river_canceled=$19
            next
          }
          {
            if ($1>acquired) acquired=$1
            if ($2>pool_max) pool_max=$2
            if ($3<idle) idle=$3
            empty+=$4
            canceled+=$5
            failure+=$6
            lost+=$7
            if ($8<active) active=$8
            if ($9>allocated) allocated=$9
            if ($10>critical_acquired) critical_acquired=$10
            if ($11>critical_max) critical_max=$11
            if ($12<critical_idle) critical_idle=$12
            critical_empty+=$13
            critical_canceled+=$14
            if ($15>river_acquired) river_acquired=$15
            if ($16>river_max) river_max=$16
            if ($17<river_idle) river_idle=$17
            river_empty+=$18
            river_canceled+=$19
          }
          END {
            if (critical_idle==999999) critical_idle=0
            print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated "," critical_acquired "," critical_max "," critical_idle "," critical_empty "," critical_canceled "," river_acquired "," river_max "," river_idle "," river_empty "," river_canceled
          }
        '
    ) 2>/dev/null; then
      pool_state=
    fi
    missing_fields=
    [[ -n $database_state ]] || missing_fields+=database_state,
    [[ -n $node_memory_percent ]] || missing_fields+=node_memory_percent,
    [[ -n $pod_memory_mib ]] || missing_fields+=pod_memory_mib,
    [[ -n $worker_resources ]] || missing_fields+=worker_resources,
    [[ -n $pool_state ]] || missing_fields+=pool_state,
    if [[ -z $database_state || -z $node_memory_percent ||
          -z $pod_memory_mib || -z $worker_resources || -z $pool_state ]]; then
      ((consecutive_failures += 1))
      echo "pressure sampling unavailable: attempt=$consecutive_failures fields=${missing_fields%,}" >&2
      if (( consecutive_failures >= 3 )); then
        echo 'pressure sampling failed three consecutive times' >&2
        return 1
      fi
      sleep 2
      continue
    fi
    consecutive_failures=0
    printf -v pressure_row '%s,%s,%s,%s,%s,%s' \
      "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$database_state" "$node_memory_percent" \
      "$pod_memory_mib" "$pool_state" "$worker_resources"
    printf '%s\n' "$pressure_row" >>"$output"
    if ! pressure_row_passes_live_gates "$pressure_row"; then
      local gate_reason
      gate_reason=$(pressure_row_live_gate_reason "$pressure_row")
      if [[ -n $failure_file ]]; then
        printf '%s\n' "$gate_reason" >"$failure_file"
      fi
      echo "pressure sampling crossed a live resource or lease hard gate: $gate_reason" >&2
      return 1
    fi
    sleep 5
  done
}

verify_pressure_report() {
  local report_path=$1
  local expected_slots=$2
  awk -F',' '
    NR == 1 { next }
    {
      samples++
      if ($14 > 0 || $15 > 0) {
        failed=1
      }
      if ($9 >= $10 && $11 == 0) {
        saturated++
      } else {
        saturated=0
      }
      if ($18 >= $19 && $20 == 0) {
        critical_saturated++
      } else {
        critical_saturated=0
      }
      if ($23 >= $24 && $25 == 0) {
        river_saturated++
      } else {
        river_saturated=0
      }
      if (samples > 1 && $12 > previous_empty) {
        empty_growth++
      } else {
        empty_growth=0
      }
      if (samples > 1 && $21 > previous_critical_empty) {
        critical_empty_growth++
      } else {
        critical_empty_growth=0
      }
      if (samples > 1 && $26 > previous_river_empty) {
        river_empty_growth++
      } else {
        river_empty_growth=0
      }
      if (samples > 1 && ($13 > previous_canceled ||
          $22 > previous_critical_canceled || $27 > previous_river_canceled)) {
        failed=1
      }
      previous_empty=$12
      previous_canceled=$13
      previous_critical_empty=$21
      previous_critical_canceled=$22
      previous_river_empty=$26
      previous_river_canceled=$27
      if ($4 >= 900 || $6 <= 0 || $5 * 4 >= $6 * 3 || $7 >= 80 || $8 >= 1536 ||
          $10 <= 0 || $28 >= 1536 || saturated >= 6 || critical_saturated >= 6 ||
          river_saturated >= 6 || empty_growth >= 6 || critical_empty_growth >= 6 ||
          river_empty_growth >= 6 || $16 < 1 || $16 > 2 || $17 > slots ||
          $19 != 4 || $24 <= 0) {
        failed=1
      }
    }
    END {
      exit !(samples > 0 && failed == 0)
    }
  ' slots="$expected_slots" "$report_path"
}

certify_worker_resource_requests() {
  local profile_slug
  profile_slug=$(printf '%s' "$stable_profile" | tr '[:upper:]' '[:lower:]')
  local memory_values=$temporary_root/worker-memory-values
  local cpu_values=$temporary_root/worker-cpu-values
  awk -F',' 'FNR > 1 && $28 ~ /^[0-9]+([.][0-9]+)?$/ {print $28}' \
    "$report_root"/"$profile_slug"-*-pressure.csv | sort -n >"$memory_values"
  awk -F',' 'FNR > 1 && $29 ~ /^[0-9]+([.][0-9]+)?$/ {print $29}' \
    "$report_root"/"$profile_slug"-*-pressure.csv | sort -n >"$cpu_values"
  local memory_count cpu_count memory_rank cpu_rank memory_p99 cpu_p99
  memory_count=$(wc -l <"$memory_values" | tr -d ' ')
  cpu_count=$(wc -l <"$cpu_values" | tr -d ' ')
  (( memory_count > 0 && cpu_count > 0 ))
  memory_rank=$(( (memory_count * 99 + 99) / 100 ))
  cpu_rank=$(( (cpu_count * 99 + 99) / 100 ))
  memory_p99=$(sed -n "${memory_rank}p" "$memory_values")
  cpu_p99=$(sed -n "${cpu_rank}p" "$cpu_values")
  AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB=$(awk -v value="$memory_p99" '
    BEGIN {
      request=int(value*1.2+0.999)
      if (request<256) request=256
      print request
    }')
  AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES=$(awk -v value="$cpu_p99" '
    BEGIN {
      request=int(value*1.2+0.999)
      if (request<100) request=100
      print request
    }')
  (( AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB <= 1900 &&
     AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES <= 1900 )) || {
    echo "certified Worker request exceeds the 2 GiB/2 CPU Pod budget: memory_mib=$AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB cpu_millicores=$AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES" >&2
    return 1
  }
  jq -n \
    --arg capacityProfile "$stable_profile" \
    --argjson memoryP99MiB "$memory_p99" \
    --argjson cpuP99Millicores "$cpu_p99" \
    --argjson memoryRequestMiB "$AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB" \
    --argjson cpuRequestMillicores "$AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES" \
    '{
      capacityProfile:$capacityProfile,
      workerMemoryP99MiB:$memoryP99MiB,
      workerCPUP99Millicores:$cpuP99Millicores,
      workerMemoryRequestMiB:$memoryRequestMiB,
      workerCPURequestMillicores:$cpuRequestMillicores,
      safetyMultiplier:1.2
    }' >"$report_root/certified-worker-resources.json"
}

apply_single_node_capacity() {
  local slots=$1
  [[ $slots =~ ^[0-9]+$ ]] && (( slots >= 4 && slots <= 48 )) || return 1
  local worker_pool=$((slots + 12))
  local target_outstanding=$((slots * 2))
  local config_patch
  config_patch=$(jq -nc \
    --arg slots "$slots" \
    --arg workerPool "$worker_pool" \
    --arg targetOutstanding "$target_outstanding" \
    '{data:{
      AI_GATEWAY_WORKER_REPLICAS_NINGBO:"1",
      AI_GATEWAY_WORKER_REPLICAS_HONGKONG:"0",
      AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:"1",
      AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:"0",
      AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:"1",
      AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:"0",
      AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
      AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
      AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
      AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$slots,
      AI_GATEWAY_WORKER_DATABASE_MAX_CONNS:$workerPool,
      AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:$slots,
      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:$slots,
      AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$targetOutstanding
    }}')
  remote_kubectl label node easyai-ningbo easyai.io/worker=true --overwrite >/dev/null
  remote_kubectl scale deployment/easyai-capacity-controller -n "$namespace" --replicas=0 >/dev/null
  remote_kubectl patch configmap easyai-ai-gateway-config -n "$namespace" \
    --type=merge -p "$config_patch" >/dev/null
  remote_kubectl set env deployment/easyai-worker-ningbo -n "$namespace" \
    "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$slots" \
    "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$slots" \
    "AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$slots" \
    "AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=$AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE" \
    "AI_GATEWAY_DATABASE_MAX_CONNS=$worker_pool" \
    'AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4' \
    "AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" \
    'AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4' \
    'AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30' \
    "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$slots" \
    "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$slots" \
    'AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=2' >/dev/null
  remote_kubectl set resources deployment/easyai-worker-ningbo -n "$namespace" \
    --containers=worker \
    --requests="cpu=${AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB}Mi" \
    --limits='cpu=2,memory=2Gi' >/dev/null
  remote_kubectl scale deployment/easyai-worker-ningbo -n "$namespace" --replicas=1 >/dev/null
  remote_kubectl rollout status deployment/easyai-worker-ningbo -n "$namespace" --timeout=300s
  remote_kubectl set env deployment/easyai-api-ningbo -n "$namespace" \
    "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$slots" \
    "AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$slots" >/dev/null
  remote_kubectl rollout status deployment/easyai-api-ningbo -n "$namespace" --timeout=300s

  local deadline=$((SECONDS + 90)) allocation
  while (( SECONDS < deadline )); do
    allocation=$(database_query "
SELECT count(*)||':'||
  COALESCE(max(hard_capacity_limit),0)||':'||
  COALESCE(max(allocated_capacity),0)||':'||
  COALESCE(max(safe_capacity),0)||':'||
  COALESCE(max(pressure_state),'unknown')
FROM gateway_worker_instances
WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';")
    if single_node_capacity_ready "$allocation" "$slots"; then
      local metrics_ready=false
      for _ in {1..30}; do
        if remote_kubectl top node easyai-ningbo --no-headers >/dev/null 2>&1 &&
          [[ -n $(remote_kubectl top pods -n "$namespace" \
            -l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo' \
            --no-headers 2>/dev/null) ]]; then
          metrics_ready=true
          break
        fi
        sleep 2
      done
      [[ $metrics_ready == true ]] || {
        echo 'single-node metrics did not become ready after Worker rollout' >&2
        return 1
      }
      active_profile=S$slots
      local active_instances hard_limit allocated safe_capacity pressure_state
      IFS=':' read -r active_instances hard_limit allocated safe_capacity pressure_state <<<"$allocation"
      echo "single_node_capacity=PASS hard_limit=$slots allocated=$allocated safe_capacity=$safe_capacity pressure_state=$pressure_state worker_pool=$worker_pool active_instances=1"
      return 0
    fi
    sleep 2
  done
  echo "single-node Worker capacity did not converge: expected=active:1 hard_limit:$slots allocated_and_safe:1..$slots pressure:not-critical actual=${allocation:-missing}" >&2
  return 1
}

single_node_capacity_ready() {
  local state=$1
  local expected_hard_limit=$2
  local active_instances hard_limit allocated safe_capacity pressure_state extra
  IFS=':' read -r active_instances hard_limit allocated safe_capacity pressure_state extra <<<"$state"
  [[ -z $extra &&
     $active_instances =~ ^[0-9]+$ && $active_instances -eq 1 &&
     $hard_limit =~ ^[0-9]+$ && $hard_limit -eq $expected_hard_limit &&
     $allocated =~ ^[0-9]+$ && $allocated -ge 1 && $allocated -le $expected_hard_limit &&
     $safe_capacity =~ ^[0-9]+$ && $safe_capacity -ge 1 && $safe_capacity -le $expected_hard_limit &&
     $pressure_state != critical ]]
}

install_single_node_load_driver() {
  [[ $remote_load_drivers_installed == false ]] || return 0
  local binary_path env_file remote_env local_digest remote_digest
  binary_path=$(remote_load_binary_path)
  remote_env=$(remote_load_env_path ningbo)
  env_file=$temporary_root/remote-load-ningbo.env
  write_remote_load_env "$env_file" "$AI_GATEWAY_ACCEPTANCE_GATEWAYS"
  local_digest=$(shasum -a 256 "$acceptance_load_linux_binary" | awk '{print $1}')
  cluster_scp "$acceptance_load_linux_binary" "$CLUSTER_NINGBO_HOST:$binary_path" >/dev/null
  cluster_scp "$env_file" "$CLUSTER_NINGBO_HOST:$remote_env" >/dev/null
  remote_digest=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \
    "chmod 0755 '$binary_path'; chmod 0600 '$remote_env'; sha256sum '$binary_path'" | awk '{print $1}')
  [[ $remote_digest == "$local_digest" ]]
  remote_load_drivers_installed=true
  echo 'acceptance_load_driver=PASS sites=1 placement=ningbo-host digest_verified=true'
}

run_single_node_load_profile() {
  local profile=$1
  local report_path=$2
  local requests=$3
  install_single_node_load_driver
  local binary_path remote_env artifact remote_report status=0
  binary_path=$(remote_load_binary_path)
  remote_env=$(remote_load_env_path ningbo)
  artifact=$(basename "$report_path")
  remote_report=/root/easyai-acceptance-load-"$run_id"-"$artifact"
  cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- \
    "$binary_path" "$remote_env" "$profile" "$remote_report" "${artifact%.json}" "$requests" \
    >"$temporary_root/$artifact.stdout" <<'REMOTE' || status=$?
set -euo pipefail
binary_path=$1
env_file=$2
profile=$3
report=$4
execution_id=$5
requests=$6
[[ ! -e $report ]] || unlink "$report"
set -a
# shellcheck source=/dev/null
source "$env_file"
set +a
"$binary_path" -profile "$profile" -report "$report" \
  -shard-index 0 -shard-count 1 -execution-id "$execution_id" \
  -requests "$requests"
REMOTE
  cluster_scp "$CLUSTER_NINGBO_HOST:$remote_report" "$report_path" >/dev/null || return 1
  cluster_ssh "$CLUSTER_NINGBO_HOST" "[[ ! -e '$remote_report' ]] || unlink '$remote_report'" >/dev/null || true
  chmod 0600 "$report_path"
  jq -e --arg runId "$run_id" --arg profile "$profile" \
    '.schemaVersion == "acceptance-load-report/v2" and .runId == $runId and .profile == $profile and .secretSafe == true' \
    "$report_path" >/dev/null || return 1
  (( status == 0 )) && jq -e '.passed == true' "$report_path" >/dev/null
}

sample_single_node_pressure() {
  local output=$1
  local stop_file=$2
  local failure_file=$3
  echo 'timestamp,queued,running,db_connections,db_max_connections,active_instances,allocated_capacity,node_memory_percent,worker_memory_mib,worker_cpu_millicores,restarts' >"$output"
  while [[ ! -f $stop_file ]]; do
    local state node_memory worker_resources restarts row
    state=$(database_query "
SELECT
  count(*) FILTER (WHERE status='queued')||','||
  count(*) FILTER (WHERE status='running')||','||
  (SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
  (SELECT setting FROM pg_settings WHERE name='max_connections')||','||
  (SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now()-interval '30 seconds')||','||
  (SELECT COALESCE(max(allocated_capacity),0) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now()-interval '30 seconds')
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid;") || return 1
    node_memory=$(remote_kubectl top node easyai-ningbo --no-headers |
      awk '{value=$5; sub(/%$/, "", value); print value}') || return 1
    worker_resources=$(remote_kubectl top pods -n "$namespace" \
      -l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo' --no-headers |
      awk '{
        cpu=$2; memory=$3
        if (cpu ~ /n$/) {sub(/n$/, "", cpu); cpu/=1000000}
        else if (cpu ~ /u$/) {sub(/u$/, "", cpu); cpu/=1000}
        else if (cpu ~ /m$/) {sub(/m$/, "", cpu)}
        else {cpu*=1000}
        if (memory ~ /Gi$/) {sub(/Gi$/, "", memory); memory*=1024}
        else if (memory ~ /Mi$/) {sub(/Mi$/, "", memory)}
        else if (memory ~ /Ki$/) {sub(/Ki$/, "", memory); memory/=1024}
        print int(memory+0) "," int(cpu+0)
      }') || return 1
    restarts=$(remote_kubectl get pods -n "$namespace" \
      -l 'app.kubernetes.io/part-of=easyai-ai-gateway' -o json |
      jq '[.items[].status.containerStatuses[]?.restartCount] | add // 0') || return 1
    row="$(date -u '+%Y-%m-%dT%H:%M:%SZ'),$state,$node_memory,$worker_resources,$restarts"
    printf '%s\n' "$row" >>"$output"
    if ! awk -F',' '($5<=0)||($4*4>=$5*3)||($6!=1)||($7<1)||($8>=85)||($9>=1536)||($11>0){exit 1}' <<<"$row"; then
      printf '%s\n' "$row" >"$failure_file"
      return 1
    fi
    sleep 1
  done
}

record_single_node_result() {
  local workload=$1
  local slots=$2
  local load_report=$3
  local pressure_report=$4
  local summary=$5
  local peaks
  peaks=$(awk -F',' '
    NR>1 {
      samples++
      if ($2>queued) queued=$2
      if ($3>running) running=$3
      if ($4>db) db=$4
      if ($7>allocated) allocated=$7
      if ($8>node_memory) node_memory=$8
      if ($9>worker_memory) worker_memory=$9
      if ($10>worker_cpu) worker_cpu=$10
    }
    END {printf "%d,%d,%d,%d,%d,%d,%d,%d",samples,queued,running,db,allocated,node_memory,worker_memory,worker_cpu}
  ' "$pressure_report")
  IFS=',' read -r samples peak_queued peak_running peak_db peak_allocated peak_node_memory peak_worker_memory peak_worker_cpu <<<"$peaks"
  jq -n \
    --arg workload "$workload" \
    --argjson slots "$slots" \
    --argjson samples "$samples" \
    --argjson peakQueued "$peak_queued" \
    --argjson peakRunning "$peak_running" \
    --argjson peakDatabaseConnections "$peak_db" \
    --argjson peakAllocatedCapacity "$peak_allocated" \
    --argjson peakNodeMemoryPercent "$peak_node_memory" \
    --argjson peakWorkerMemoryMiB "$peak_worker_memory" \
    --argjson peakWorkerCPUMillicores "$peak_worker_cpu" \
    --slurpfile load "$load_report" \
    '{
      workload:$workload,
      configuredSlots:$slots,
      samples:$samples,
      peakQueued:$peakQueued,
      peakRunning:$peakRunning,
      peakDatabaseConnections:$peakDatabaseConnections,
      peakAllocatedCapacity:$peakAllocatedCapacity,
      peakNodeMemoryPercent:$peakNodeMemoryPercent,
      peakWorkerMemoryMiB:$peakWorkerMemoryMiB,
      peakWorkerCPUMillicores:$peakWorkerCPUMillicores,
      load:$load[0]
    }' >"$summary"
  chmod 0600 "$summary"
  if (( peak_allocated < slots || peak_running < slots )); then
    echo "single-node target concurrency was not observed: configured=$slots peak_allocated=$peak_allocated peak_running=$peak_running" >&2
    return 1
  fi
}

cleanup_single_node_kubernetes_resources() {
  remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \
    -n "$namespace" --ignore-not-found >/dev/null || true
  remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \
    -n "$namespace" --ignore-not-found >/dev/null || true
}

run_single_node_profile() {
  local workload=$1
  local profile=$2
  local slots=$3
  local requests=$4
  local prefix=$report_root/single-node-$workload-s$slots
  local load_report=$prefix-load.json
  local pressure_report=$prefix-pressure.csv
  local pressure_failure=$prefix-pressure.failure.csv
  local pressure_stop=$temporary_root/single-pressure-stop
  local sampler_pid status=0 sampler_status=0
  rm -f -- "$pressure_stop" "$pressure_failure"
  sample_single_node_pressure "$pressure_report" "$pressure_stop" "$pressure_failure" &
  sampler_pid=$!
  active_pressure_pid=$sampler_pid
  run_with_pressure_monitor "$sampler_pid" \
    run_single_node_load_profile "$profile" "$load_report" "$requests" || status=$?
  touch "$pressure_stop"
  wait "$sampler_pid" || sampler_status=$?
  active_pressure_pid=
  (( status == 0 && sampler_status == 0 )) || {
    [[ -s $pressure_failure ]] && cp "$pressure_failure" "$report_root/last-pressure-failure.csv"
    return 1
  }
  record_single_node_result "$workload" "$slots" "$load_report" "$pressure_report" \
    "$prefix-summary.json"
}

finish_single_node_diagnostic() {
  local diagnostic_status=$1
  local reason=$2
  local emulator_report=$report_root/single-node-emulator-report.json
  local callback_report=$report_root/single-node-callback-report.json
  local finish_response=$temporary_root/single-node-finish.json
  if [[ $single_node_emulator_installed == true ]]; then
    cluster_ssh "$CLUSTER_NINGBO_HOST" \
      'curl -fsS --max-time 10 http://127.0.0.1:18090/report' >"$emulator_report" || true
  else
    remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-emulator -- \
      wget -qO- http://127.0.0.1:8090/report >"$emulator_report" || true
  fi
  remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-callback-collector -- \
    wget -qO- http://127.0.0.1:8091/report >"$callback_report" || true
  local task_state
  task_state=$(database_query "
SELECT count(*)||':'||count(*) FILTER (WHERE status='succeeded')||':'||count(*) FILTER (WHERE status IN ('queued','running'))
FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
  local results_json=$temporary_root/single-node-results.json
  local -a result_files=()
  shopt -s nullglob
  result_files=("$report_root"/single-node-*-summary.json)
  shopt -u nullglob
  if (( ${#result_files[@]} > 0 )); then
    jq -s '.' "${result_files[@]}" >"$results_json"
  else
    printf '[]\n' >"$results_json"
  fi
  jq -n \
    --arg runId "$run_id" \
    --arg releaseSha "$release_sha" \
    --arg status "$diagnostic_status" \
    --arg reason "$reason" \
    --arg taskState "$task_state" \
    --argjson stableSlots "$single_node_stable_slots" \
    --slurpfile results "$results_json" \
    '{
      schemaVersion:"acceptance-single-node-diagnostic/v1",
      runId:$runId,
      releaseSha:$releaseSha,
      site:"ningbo",
      mode:"acceptance-emulator-only",
      realUpstreamRequests:0,
      certification:false,
      promotable:false,
      status:$status,
      reason:$reason,
      taskState:$taskState,
      stableVideoSlots:$stableSlots,
      results:$results[0]
    }' >"$report_root/single-node-summary.json"
  chmod 0600 "$report_root/single-node-summary.json"
  local finish_body
  finish_body=$(jq -cn \
    --arg reason "single-node diagnostic completed; real upstream and certification intentionally skipped" \
    --arg status "$diagnostic_status" \
    --argjson stableSlots "$single_node_stable_slots" \
    '{passed:false,failureReason:$reason,report:{diagnosticStatus:$status,site:"ningbo",realUpstreamRequests:0,certification:false,stableVideoSlots:$stableSlots}}')
  admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response"
  failure_recorded=true
  cleanup_single_node_emulator
  cleanup_single_node_kubernetes_resources
}

run_single_node_acceptance() {
  wait_for_existing_tasks_to_drain
  single_node_stable_slots=$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS
  apply_single_node_capacity "$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS"
  if ! run_single_node_profile gemini-multi-image gemini-multi-image \
    "$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS" \
    "$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS"; then
    finish_single_node_diagnostic failed 'GEMINI multi-image diagnostic failed or crossed a resource gate'
    return 1
  fi
  local slots requests
  for slots in "${single_node_video_slots[@]}"; do
    apply_single_node_capacity "$slots" || break
    requests=$((slots * 6))
    (( requests < 64 )) && requests=64
    if ! run_single_node_profile image-video video-capacity "$slots" "$requests"; then
      break
    fi
    single_node_stable_slots=$slots
  done
  apply_single_node_capacity "$single_node_stable_slots"
  finish_single_node_diagnostic completed 'single-node emulator-only capacity ladder completed'
  echo "single_node_acceptance=PASS_DIAGNOSTIC run_id=$run_id stable_video_slots=$single_node_stable_slots traffic_mode=validation certification=false report=$report_root/single-node-summary.json"
}

run_capacity_round() {
  local profile=$1
  local repetition=$2
  local profile_slug
  profile_slug=$(printf '%s' "$profile" | tr '[:upper:]' '[:lower:]')
  local prefix=$report_root/$profile_slug-$repetition
  local pressure_report=$prefix-pressure.csv
  local pressure_failure=$prefix-pressure.failure.txt
  local pressure_stop=$temporary_root/pressure-stop
  local pressure_pid status=0 pressure_status=0
  local baseline_memory_mib
  local expected_slots
  case $profile in
    P24) expected_slots=24 ;;
    P28) expected_slots=28 ;;
    P32) expected_slots=32 ;;
  esac
  baseline_memory_mib=$(current_max_pod_memory_mib)
  rm -f -- "$pressure_stop"
  rm -f -- "$pressure_failure"
  sample_pressure "$pressure_report" "$pressure_stop" "$pressure_failure" &
  pressure_pid=$!
  active_pressure_pid=$pressure_pid
  run_with_pressure_monitor "$pressure_pid" \
    run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$?
  if (( status == 0 )); then
    run_with_pressure_monitor "$pressure_pid" \
      run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?
  fi
  if (( status == 0 )); then
    run_with_pressure_monitor "$pressure_pid" \
      run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?
  fi
  if (( status == 0 )); then
    run_with_pressure_monitor "$pressure_pid" \
      run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?
  fi
  if (( status == 0 )); then
    run_with_pressure_monitor "$pressure_pid" \
      run_recovery_profile "$prefix-video-recovery.json" || status=$?
  fi
  touch "$pressure_stop"
  wait "$pressure_pid" || pressure_status=$?
  active_pressure_pid=
  (( status == 0 && pressure_status == 0 )) || {
    if [[ -s $pressure_failure ]]; then
      failure_gate_id=pressure_live_gate
      failure_reason=$(<"$pressure_failure")
    fi
    (( status != 0 )) && return "$status"
    return "$pressure_status"
  }
  verify_pressure_report "$pressure_report" "$expected_slots"
  verify_runtime_gates
  wait_for_memory_recovery "$baseline_memory_mib"
}

finish_and_report() {
  local emulator_report=$report_root/emulator-report.json
  local callback_report=$report_root/callback-report.json
  local finish_response=$temporary_root/finish.json
  local mode_response=$temporary_root/mode.json
  remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-emulator -- \
    wget -qO- http://127.0.0.1:8090/report >"$emulator_report"
  remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-callback-collector -- \
    wget -qO- http://127.0.0.1:8091/report >"$callback_report"
  local task_count video_task_count gemini_task_count submissions duplicates callbacks
  local missing_idempotency duplicate_submissions
  local gemini_requests forced_tasks forced_requests verified_conversions compact_payloads
  local gemini_invalid video_invalid reference_count_3 reference_count_6 reference_count_9
  local reference_role_count first_frame_role_count last_frame_role_count
  local unique_image_hashes unique_video_tasks
  local unique_gemini_input_hashes direct_gemini_input_assets
  local duplicate_remote_tasks duplicate_terminal_events duplicate_settlements
  local duplicate_wallet_transactions duplicate_callback_rows pending_settlements pending_callbacks
  local missing_billings
  task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
  video_task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='videos.generations' AND run_mode='acceptance';")
  gemini_task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND run_mode='acceptance';")
  forced_tasks=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND request::text LIKE '%acceptance-force-conversion%';")
  compact_payloads=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND (pg_column_size(request) >= 1048576 OR pg_column_size(result) >= 1048576);")
  unique_gemini_input_hashes=$(database_query "
SELECT count(DISTINCT request #>> '{image,assetRef,sha256}')
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid
  AND kind='images.edits'
  AND run_mode='acceptance';")
  direct_gemini_input_assets=$(database_query "
SELECT count(*)
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid
  AND kind='images.edits'
  AND run_mode='acceptance'
  AND request #>> '{image,assetRef,storageProvider}' = 'aliyun_oss_direct';")
  submissions=$(jq -r '.videoSubmissions' "$emulator_report")
  duplicates=$(jq -r '.duplicates' "$callback_report")
  callbacks=$(jq -r '.deliveries' "$callback_report")
  [[ $(jq -r '.invalid' "$callback_report") == 0 ]] ||
    fail_acceptance_gate callback_protocol 'callback collector observed an invalid event'
  gemini_requests=$(jq -r '.geminiRequests' "$emulator_report")
  forced_requests=$(jq -r '.forcedConversionRequests' "$emulator_report")
  verified_conversions=$(jq -r '.verifiedConversions' "$emulator_report")
  missing_idempotency=$(jq -r '.missingIdempotencyKeys' "$emulator_report")
  duplicate_submissions=$(jq -r '.duplicateSubmissionAttempts' "$emulator_report")
  gemini_invalid=$(jq -r '.geminiInvalid // 0' "$emulator_report")
  video_invalid=$(jq -r '.videoInvalid // 0' "$emulator_report")
  reference_count_3=$(jq -r '.videoReferenceCounts["3"] // 0' "$emulator_report")
  reference_count_6=$(jq -r '.videoReferenceCounts["6"] // 0' "$emulator_report")
  reference_count_9=$(jq -r '.videoReferenceCounts["9"] // 0' "$emulator_report")
  reference_role_count=$(jq -r '.videoRoleCounts.reference_image // 0' "$emulator_report")
  first_frame_role_count=$(jq -r '.videoRoleCounts.first_frame // 0' "$emulator_report")
  last_frame_role_count=$(jq -r '.videoRoleCounts.last_frame // 0' "$emulator_report")
  unique_image_hashes=$(jq -r '.uniqueImageHashes // 0' "$emulator_report")
  unique_video_tasks=$(jq -r '.uniqueVideoTasks // 0' "$emulator_report")
  [[ $submissions == "$video_task_count" && $gemini_requests == "$gemini_task_count" ]] ||
    fail_acceptance_gate protocol_task_counts 'emulator and database task counts differ'
  [[ $forced_requests == "$forced_tasks" && $verified_conversions == "$forced_tasks" ]] ||
    fail_acceptance_gate media_conversion_counts 'forced image conversion counts differ'
  [[ $duplicates == 0 && $duplicate_submissions == 0 &&
     $missing_idempotency == 0 && $compact_payloads == 0 ]] ||
    fail_acceptance_gate idempotency_and_materialization \
      'duplicate submission, missing idempotency key, or inline database media detected'
  [[ $unique_gemini_input_hashes == "$gemini_task_count" &&
     $direct_gemini_input_assets == "$gemini_task_count" ]] ||
    fail_acceptance_gate gemini_input_materialization \
      'Gemini inputs were reused across executions or bypassed direct OSS materialization'
  [[ $gemini_invalid == 0 && $video_invalid == 0 ]] ||
    fail_acceptance_gate provider_protocol_validity 'Gemini or Volces protocol validation failed'
  [[ $reference_count_3 -gt 0 && $reference_count_6 -gt 0 && $reference_count_9 -gt 0 ]] ||
    fail_acceptance_gate video_reference_distribution '3/6/9 reference-image coverage is incomplete'
  [[ $((reference_count_3 + reference_count_6 + reference_count_9)) -eq $submissions ]] ||
    fail_acceptance_gate video_reference_accounting 'reference-image distribution does not cover every video'
  [[ $reference_role_count -gt 0 && $first_frame_role_count -gt 0 && $last_frame_role_count -gt 0 ]] ||
    fail_acceptance_gate video_reference_roles 'reference, first-frame, and last-frame roles were not all observed'
  [[ $unique_image_hashes -ge 12 && $unique_video_tasks -eq $submissions ]] ||
    fail_acceptance_gate media_content_uniqueness 'media hashes or unique remote video tasks are incomplete'
  [[ $callbacks -gt 0 ]] ||
    fail_acceptance_gate callback_delivery 'no acceptance callback was delivered'
  duplicate_remote_tasks=$(database_query "
SELECT count(*)
FROM (
  SELECT remote_task_id
  FROM gateway_tasks
  WHERE acceptance_run_id='$run_id'::uuid
    AND remote_task_id IS NOT NULL
  GROUP BY remote_task_id
  HAVING count(*) > 1
) duplicate;")
  duplicate_terminal_events=$(database_query "
SELECT count(*)
FROM (
  SELECT task_id,event_type
  FROM gateway_task_events
  WHERE task_id IN (
    SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
  )
    AND event_type IN ('task.completed','task.failed','task.cancelled')
  GROUP BY task_id,event_type
  HAVING count(*) > 1
) duplicate;")
  duplicate_settlements=$(database_query "
SELECT count(*)
FROM (
  SELECT task_id,action
  FROM settlement_outbox
  WHERE task_id IN (
    SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
  )
  GROUP BY task_id,action
  HAVING count(*) > 1
) duplicate;")
  duplicate_wallet_transactions=$(database_query "
SELECT count(*)
FROM (
  SELECT reference_id,transaction_type
  FROM gateway_wallet_transactions
  WHERE reference_type='gateway_task'
    AND reference_id IN (
      SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
    )
  GROUP BY reference_id,transaction_type
  HAVING count(*) > 1
) duplicate;")
  duplicate_callback_rows=$(database_query "
SELECT count(*)
FROM (
  SELECT task_id,seq,callback_url
  FROM gateway_task_callback_outbox
  WHERE task_id IN (
    SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
  )
  GROUP BY task_id,seq,callback_url
  HAVING count(*) > 1
) duplicate;")
  pending_settlements=$(database_query "
SELECT count(*)
FROM settlement_outbox
WHERE task_id IN (
  SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
)
  AND status <> 'completed';")
  pending_callbacks=$(database_query "
SELECT count(*)
FROM gateway_task_callback_outbox
WHERE task_id IN (
  SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid
)
  AND status <> 'delivered';")
  missing_billings=$(database_query "
SELECT count(*)
FROM gateway_tasks task
WHERE task.acceptance_run_id='$run_id'::uuid
  AND NOT EXISTS (
    SELECT 1
    FROM gateway_wallet_transactions transaction
    WHERE transaction.reference_type='gateway_task'
      AND transaction.reference_id=task.id::text
      AND transaction.transaction_type='task_billing'
  );")
  [[ $duplicate_remote_tasks == 0 && $duplicate_terminal_events == 0 ]] ||
    fail_acceptance_gate remote_and_terminal_uniqueness 'duplicate remote tasks or terminal events detected'
  [[ $duplicate_settlements == 0 && $duplicate_wallet_transactions == 0 ]] ||
    fail_acceptance_gate billing_uniqueness 'duplicate settlements or wallet transactions detected'
  [[ $duplicate_callback_rows == 0 && $pending_settlements == 0 ]] ||
    fail_acceptance_gate outbox_completion 'duplicate callbacks or pending settlements remain'
  [[ $pending_callbacks == 0 && $missing_billings == 0 ]] ||
    fail_acceptance_gate billing_and_callback_completion 'pending callbacks or missing billings remain'
  admin_request GET /api/admin/system/acceptance/traffic-mode '' "$mode_response"
  verify_release_cas
  local config_hash
  [[ -n $capacity_profiles_payload && -f $capacity_profiles_payload ]] ||
    fail_acceptance_gate capacity_profile_artifact 'staged capacity profile artifact is missing'
  config_hash=$(jq -r '.configHash' "$capacity_profiles_payload")
  [[ $(remote_kubectl get configmap easyai-gateway-release -n "$namespace" \
    -o 'jsonpath={.data.configHash}') == "$config_hash" ]] ||
    fail_acceptance_gate capacity_profile_config_cas 'staged capacity profile config hash changed'
  [[ $(jq -r '.mode' "$mode_response") == validation &&
     $(jq -r '.runId' "$mode_response") == "$run_id" ]] ||
    fail_acceptance_gate validation_mode_cas 'traffic mode changed before acceptance completion'
  jq -n \
    --arg runId "$run_id" \
    --arg releaseSha "$release_sha" \
    --arg apiImageDigest "$api_digest" \
    --arg workerImageDigest "$worker_digest" \
    --arg stableCapacityProfile "$stable_profile" \
    --arg configHash "$config_hash" \
    --arg snapshotConfigHash "$local_snapshot_config_hash" \
    --arg snapshotSha256 "$local_snapshot_sha256" \
    --argjson imageStableThroughput "$image_stable_throughput" \
    --argjson imageAdmittedThroughput "$image_admitted_throughput" \
    --argjson videoStableThroughput "$video_stable_throughput" \
    --argjson videoAdmittedThroughput "$video_admitted_throughput" \
    --argjson tasks "$task_count" \
    --argjson geminiTasks "$gemini_task_count" \
    --argjson videoTasks "$video_task_count" \
    --argjson upstreamSubmissions "$submissions" \
    --argjson callbackDeliveries "$callbacks" \
    --argjson missingIdempotencyKeys "$missing_idempotency" \
    --argjson duplicateSubmissionAttempts "$duplicate_submissions" \
    --argjson geminiInvalid "$gemini_invalid" \
    --argjson videoInvalid "$video_invalid" \
    --argjson referenceCount3 "$reference_count_3" \
    --argjson referenceCount6 "$reference_count_6" \
    --argjson referenceCount9 "$reference_count_9" \
    --argjson referenceRoleCount "$reference_role_count" \
    --argjson firstFrameRoleCount "$first_frame_role_count" \
    --argjson lastFrameRoleCount "$last_frame_role_count" \
    --argjson uniqueImageHashes "$unique_image_hashes" \
    --argjson uniqueVideoTasks "$unique_video_tasks" \
    --argjson uniqueGeminiInputHashes "$unique_gemini_input_hashes" \
    --argjson directGeminiInputAssets "$direct_gemini_input_assets" \
    --argjson duplicateRemoteTasks "$duplicate_remote_tasks" \
    --argjson duplicateTerminalEvents "$duplicate_terminal_events" \
    --argjson duplicateSettlements "$duplicate_settlements" \
    --argjson duplicateWalletTransactions "$duplicate_wallet_transactions" \
    --argjson duplicateCallbackRows "$duplicate_callback_rows" \
    --argjson pendingSettlements "$pending_settlements" \
    --argjson pendingCallbacks "$pending_callbacks" \
    --argjson missingBillings "$missing_billings" \
    '{
      runId:$runId,
      releaseSha:$releaseSha,
      apiImageDigest:$apiImageDigest,
      workerImageDigest:$workerImageDigest,
      snapshotConfigHash:$snapshotConfigHash,
      snapshotSha256:$snapshotSha256,
      configHash:$configHash,
      stableCapacityProfile:$stableCapacityProfile,
      imageStableThroughputPerSecond:$imageStableThroughput,
      imageAdmittedThroughputPerSecond:$imageAdmittedThroughput,
      videoStableThroughputPerSecond:$videoStableThroughput,
      videoAdmittedThroughputPerSecond:$videoAdmittedThroughput,
      tasks:$tasks,
      geminiTasks:$geminiTasks,
      videoTasks:$videoTasks,
      upstreamSubmissions:$upstreamSubmissions,
      callbackDeliveries:$callbackDeliveries,
      missingIdempotencyKeys:$missingIdempotencyKeys,
      duplicateSubmissionAttempts:$duplicateSubmissionAttempts,
      protocolValidation:{
        geminiInvalid:$geminiInvalid,
        videoInvalid:$videoInvalid,
        videoReferenceCounts:{
          "3":$referenceCount3,
          "6":$referenceCount6,
          "9":$referenceCount9
        },
        videoRoleCounts:{
          reference_image:$referenceRoleCount,
          first_frame:$firstFrameRoleCount,
          last_frame:$lastFrameRoleCount
        },
        uniqueImageHashes:$uniqueImageHashes,
        uniqueVideoTasks:$uniqueVideoTasks,
        uniqueGeminiInputHashes:$uniqueGeminiInputHashes,
        directGeminiInputAssets:$directGeminiInputAssets
      },
      databaseConsistency:{
        duplicateRemoteTasks:$duplicateRemoteTasks,
        duplicateTerminalEvents:$duplicateTerminalEvents,
        duplicateSettlements:$duplicateSettlements,
        duplicateWalletTransactions:$duplicateWalletTransactions,
        duplicateCallbackRows:$duplicateCallbackRows,
        pendingSettlements:$pendingSettlements,
        pendingCallbacks:$pendingCallbacks,
        missingBillings:$missingBillings
      },
      realCanaryPassed:true,
      certifiedProfile:{
        name:$stableCapacityProfile,
        configHash:$configHash,
        imageStableThroughputPerSecond:$imageStableThroughput,
        imageAdmittedThroughputPerSecond:$imageAdmittedThroughput,
        videoStableThroughputPerSecond:$videoStableThroughput,
        videoAdmittedThroughputPerSecond:$videoAdmittedThroughput
      },
      gates:[
        {id:"protocol_and_media_integrity",passed:true},
        {id:"database_exactly_once",passed:true},
        {id:"runtime_resources",passed:true},
        {id:"real_media_canary",passed:true}
      ],
      promotionRequested:false,
      awaitingManualPromotion:true,
      passed:true
    }' >"$report_root/summary.json"
  local finish_body
  finish_body=$(jq -cn \
    --argjson tasks "$task_count" \
    --argjson videos "$video_task_count" \
    --argjson submissions "$submissions" \
    --arg profile "$stable_profile" \
    '{passed:true,report:{tasks:$tasks,videos:$videos,upstreamSubmissions:$submissions,stableCapacityProfile:$profile,queueFinal:0}}')
  admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response"
  local certification_command certification_output instance_slots
  case $stable_profile in
    P24) instance_slots=24 ;;
    P28) instance_slots=28 ;;
    P32) instance_slots=32 ;;
  esac
  printf -v certification_command '%q certify %q %q %q %q %q %q' \
    "$remote_release_helper" \
    "$run_id" \
    "$stable_profile" \
    "$config_hash" \
    "$instance_slots" \
    "$certified_max_replicas_ningbo" \
    "$certified_max_replicas_hongkong"
  certification_output=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$certification_command")
  grep -Fq 'production_certification=PASS' <<<"$certification_output"
  node "$cluster_root/scripts/acceptance/report.mjs" merge-production \
    --local-report "$local_acceptance_report" \
    --production-summary "$report_root/summary.json" \
    --output "$report_root/acceptance-report.json" >/dev/null
  node "$cluster_root/scripts/acceptance/report.mjs" validate \
    --input "$report_root/acceptance-report.json" >/dev/null
  [[ $(jq -r '.promotion.ready' "$report_root/acceptance-report.json") == true ]]
}

verify_release_cas false
bootstrap_acceptance_primary_identity
ensure_acceptance_user_group
ensure_acceptance_identity_shards
if [[ $single_node_acceptance != true ]]; then
  ensure_acceptance_real_images
fi
select_acceptance_models
ensure_acceptance_model_access
deploy_protocol_emulator
create_and_activate_run
if [[ $single_node_acceptance == true ]]; then
  run_single_node_acceptance
  exit $?
fi
wait_for_existing_tasks_to_drain
snapshot_pre_acceptance_capacity
if ! apply_capacity_profile P24; then
  failure_gate_id=baseline_capacity_apply
  failure_reason='failed to apply the drained P24 baseline topology'
  mark_run_failed "$failure_reason"
  exit 1
fi
if ! wait_for_post_rollout_stability P24; then
  failure_gate_id=baseline_control_plane_stability
  failure_reason='P24 baseline topology did not stabilize after rollout'
  mark_run_failed "$failure_reason"
  exit 1
fi
verify_release_cas
resource_precondition_status=0
verify_resource_preconditions || resource_precondition_status=$?
if (( resource_precondition_status != 0 )); then
  if (( resource_precondition_status == 2 )); then
    failure_gate_id=resource_precondition_transport
    failure_reason='production resource precondition metrics could not be collected after bounded retries'
  else
    failure_gate_id=resource_preconditions
    failure_reason='production resource preconditions failed'
  fi
  mark_run_failed "$failure_reason"
  exit 1
fi
for profile in P24 P28 P32; do
  if ! apply_capacity_profile "$profile"; then
    failure_gate_id=capacity_profile_apply
    failure_reason="failed to apply $profile"
    break
  fi
  if ! wait_for_post_rollout_stability "$profile"; then
    failure_gate_id=post_rollout_control_plane_stability
    failure_reason="$profile control plane did not stabilize after rollout"
    break
  fi
  profile_passed=true
  for repetition in 1 2 3; do
    if ! run_capacity_round "$profile" "$repetition"; then
      profile_passed=false
      [[ -n $failure_gate_id ]] || failure_gate_id=capacity_round
      [[ -n $failure_reason ]] || failure_reason="$profile repetition $repetition failed"
      break
    fi
  done
  if [[ $profile_passed != true ]]; then
    apply_capacity_profile "$stable_profile" || true
    break
  fi
  stable_profile=$profile
done

if [[ -n $failure_reason ]]; then
  apply_capacity_profile "$stable_profile" || true
  mark_run_failed "$failure_reason"
  echo "production_acceptance=FAIL run_id=$run_id reason=$failure_reason traffic_mode=validation restored_profile=$stable_profile" >&2
  exit 1
fi

if ! certify_worker_resource_requests; then
  failure_gate_id=worker_resource_certification
  failure_reason='worker p99 resource certification failed'
elif ! apply_capacity_profile "$stable_profile"; then
  failure_gate_id=certified_resource_apply
  failure_reason='failed to apply certified Worker resource requests'
elif ! verify_runtime_gates; then
  failure_gate_id=certified_resource_runtime
  failure_reason='certified Worker resource rollout failed runtime gates'
elif ! run_autoscaling_acceptance; then
  failure_gate_id=worker_autoscaling
  failure_reason='Worker autoscaling, drain, or resource-budget acceptance failed'
elif ! stage_capacity_profiles; then
  failure_gate_id=capacity_profile_staging
  failure_reason='failed to stage certified capacity profiles for validation traffic'
elif ! run_mixed_soak_and_overload; then
  failure_gate_id=mixed_soak_overload
  failure_reason='80% mixed soak or 120% active-throttling acceptance failed'
elif ! activate_certified_autoscaling; then
  failure_gate_id=certified_autoscaling_activation
  failure_reason='failed to activate and CAS the certified elastic Worker profile'
fi
if [[ -n $failure_reason ]]; then
  apply_capacity_profile "$stable_profile" || true
  mark_run_failed "$failure_reason"
  echo "production_acceptance=FAIL run_id=$run_id reason=$failure_reason traffic_mode=validation restored_profile=$stable_profile" >&2
  exit 1
fi

if ! run_load_profile real-canary "$report_root/real-canary.json"; then
  failure_gate_id=real_media_canary
  failure_reason='real canary failed'
elif ! verify_runtime_gates; then
  failure_gate_id=post_canary_runtime
  failure_reason='post-canary runtime gates failed'
fi
if [[ -n $failure_reason ]]; then
  apply_capacity_profile "$stable_profile" || true
  mark_run_failed "$failure_reason"
  echo "production_acceptance=FAIL run_id=$run_id reason=$failure_reason traffic_mode=validation restored_profile=$stable_profile" >&2
  exit 1
fi
failure_gate_id=release_certification_or_report
failure_reason='release certification record or final acceptance report failed'
finish_and_report
failure_gate_id=
failure_reason=
remote_kubectl delete deployment easyai-acceptance-emulator -n "$namespace" --ignore-not-found >/dev/null ||
  echo 'warning: acceptance emulator Deployment cleanup failed' >&2
remote_kubectl delete service easyai-acceptance-emulator -n "$namespace" --ignore-not-found >/dev/null ||
  echo 'warning: acceptance emulator Service cleanup failed' >&2
remote_kubectl delete deployment easyai-acceptance-callback-collector -n "$namespace" --ignore-not-found >/dev/null ||
  echo 'warning: acceptance callback collector Deployment cleanup failed' >&2
remote_kubectl delete service easyai-acceptance-callback-collector -n "$namespace" --ignore-not-found >/dev/null ||
  echo 'warning: acceptance callback collector Service cleanup failed' >&2
echo "production_acceptance=PASS run_id=$run_id release=$release_sha stable_profile=$stable_profile traffic_mode=validation promotion=awaiting_manual_confirmation report=$report_root/acceptance-report.json"
