easyai-ai-gateway/scripts/acceptance/run-local-acceptance.sh

#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
repository_root=$(cd "$script_dir/../.." && pwd)
private_root="$repository_root/.local-secrets/acceptance"
state_root="$private_root/state"
context=k3d-easyai-acceptance-local
namespace=easyai
cluster_name=easyai-acceptance-local
runtime_path_file="$state_root/current-runtime-path"
snapshot="$state_root/snapshot.json"
load_binary="$state_root/easyai-ai-gateway-acceptance-load"
identity_file="$state_root/control-plane-identity.json"
active_load_pid=
active_worker_sampler_pid=
active_resource_sampler_pid=
active_provider_sampler_pid=
worker_sample_stop=
netem_active=false
provider_burst_configured=false
current_phase=initialization
stable_profile=
failure_gate_id=local_execution_incomplete
# shellcheck source=scripts/acceptance/local-control-plane.sh
source "$script_dir/local-control-plane.sh"
# shellcheck source=scripts/acceptance/provider-burst.sh
source "$script_dir/provider-burst.sh"

usage() {
  cat <<'EOF'
Usage:
  scripts/acceptance/run-local-acceptance.sh quick
  scripts/acceptance/run-local-acceptance.sh adaptive
  scripts/acceptance/run-local-acceptance.sh provider-burst
  scripts/acceptance/run-local-acceptance.sh full --release-manifest dist/releases/<SHA>.json
  scripts/acceptance/run-local-acceptance.sh artifact-smoke --release-manifest dist/releases/<SHA>.json

`adaptive` runs a three-Worker emulator-only concurrency ladder without changing
provider limits. `provider-burst` binds one video model to three emulator-only
platforms with concurrency limits 2/4/6 and validates burst queueing and routing.
`full` executes P24/P28/P32 three times, the fault matrix, autoscaling/drain,
80% soak, 120% overload, and exact linux/amd64 artifact smoke. The load process
runs outside K3s and splits requests 50/50 across both TLS entrances.
EOF
}

private_file() {
  local path=$1 mode
  [[ -f $path && ! -L $path ]] || return 1
  if [[ $(uname -s) == Darwin ]]; then
    mode=$(stat -f '%Lp' "$path")
  else
    mode=$(stat -c '%a' "$path")
  fi
  [[ $mode == 600 ]]
}

cleanup() {
  local status=$?
  trap - EXIT
  if [[ -n $active_load_pid ]]; then
    kill "$active_load_pid" >/dev/null 2>&1 || true
    wait "$active_load_pid" >/dev/null 2>&1 || true
  fi
  if [[ -n $active_worker_sampler_pid ]]; then
    [[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
    wait "$active_worker_sampler_pid" >/dev/null 2>&1 || true
  fi
  if [[ -n $active_resource_sampler_pid ]]; then
    [[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
    wait "$active_resource_sampler_pid" >/dev/null 2>&1 || true
  fi
  if [[ -n $active_provider_sampler_pid ]]; then
    [[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
    wait "$active_provider_sampler_pid" >/dev/null 2>&1 || true
  fi
  if [[ $netem_active == true ]]; then
    "$script_dir/network-fault.sh" reset >/dev/null 2>&1 || true
  fi
  if [[ $provider_burst_configured == true && -n ${runtime:-} ]]; then
    restore_provider_burst_platforms >/dev/null 2>&1 || true
  fi
  if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then
    restore_profile_best_effort "${stable_profile:-P24}"
    if [[ -f ${failure_gate_file:-} && ! -L ${failure_gate_file:-} ]]; then
      failure_gate_id=$(jq -r '.id' "$failure_gate_file" 2>/dev/null || printf '%s' "$failure_gate_id")
    fi
    local partial_args=(
      "$script_dir/report.mjs" build-local-partial
      --runtime "$runtime" \
      --snapshot "$snapshot" \
      --reports "$report_root" \
      --failure-phase "$current_phase" \
      --failure-gate "$failure_gate_id" \
      --output "$report_root/acceptance-report.partial.json"
    )
    if [[ -n $stable_profile ]]; then
      partial_args+=(--certified-profile "$stable_profile")
    fi
    if node "${partial_args[@]}" >/dev/null 2>&1; then
      echo "local_acceptance=FAILED phase=$current_phase partial_report=$report_root/acceptance-report.partial.json" >&2
    else
      echo "local_acceptance=FAILED phase=$current_phase partial_report=unavailable" >&2
    fi
  fi
  exit "$status"
}
trap cleanup EXIT
trap 'current_phase=signal_interrupted; exit 130' HUP INT TERM

require_local_cluster() {
  command -v kubectl >/dev/null 2>&1
  command -v jq >/dev/null 2>&1
  command -v node >/dev/null 2>&1
  [[ $(kubectl config get-contexts "$context" -o name) == "$context" ]]
  private_file "$runtime_path_file" && private_file "$snapshot"
  runtime=$(<"$runtime_path_file")
  private_file "$runtime" || {
    echo 'local acceptance runtime file is missing or not mode 0600' >&2
    exit 1
  }
  jq -e '.schemaVersion == "acceptance-runtime/v1"' "$runtime" >/dev/null
  run_id=$(jq -r '.runId' "$runtime")
  release_sha=$(jq -r '.releaseSha' "$runtime")
  report_root="$repository_root/dist/acceptance/local/$run_id"
  install -d -m 0700 "$report_root"
  failure_gate_file="$report_root/failure-gate.json"
  (
    cd "$repository_root/apps/api"
    env -u AI_GATEWAY_TEST_DATABASE_URL go build -trimpath \
      -o "$load_binary" ./cmd/acceptance-load
  )
}

record_failure_gate() {
  local id=$1 detail=$2 temporary
  [[ $id =~ ^[a-z0-9_]+$ ]]
  failure_gate_id=$id
  [[ -n ${failure_gate_file:-} ]] || return 0
  if [[ ! -e $failure_gate_file ]]; then
    temporary="${failure_gate_file}.tmp.$$"
    jq -n --arg id "$id" --arg detail "$detail" \
      '{schemaVersion:"acceptance-failure-gate/v1",id:$id,detail:$detail}' >"$temporary"
    chmod 0600 "$temporary"
    mv "$temporary" "$failure_gate_file"
  fi
}

fail_gate() {
  local id=$1 detail=$2
  record_failure_gate "$id" "$detail"
  echo "gate=$id $detail" >&2
  return 1
}

database_query() {
  local sql=$1 primary
  primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
    -o 'jsonpath={.status.currentPrimary}')
  kubectl --context "$context" -n "$namespace" exec "$primary" -c postgres -- \
    psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql"
}

load_environment() {
  acceptance_api_keys=$(jq -r '.apiKeys | join(",")' "$runtime")
  acceptance_run_token=$(jq -r '.runToken' "$runtime")
  acceptance_gemini_model=$(jq -r '.geminiModel' "$runtime")
  acceptance_video_model=$(jq -r '.videoModel' "$runtime")
  acceptance_emulator_url=$(jq -r '.emulatorBaseUrl' "$runtime")
}

run_load() {
  local profile=$1 report_path=$2
  shift 2
  local stdout_path="$report_path.stdout" stdout_temporary="${report_path}.stdout.tmp.$$"
  local load_pid load_status=0 unavailable_samples=0 infrastructure_failure=false
  [[ ! -e $report_path && ! -e $stdout_path && ! -e $stdout_temporary ]] ||
    fail_gate acceptance_report_exists "refusing to overwrite an existing load report"
  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
    fail_gate local_control_plane_restarted "local K3s server identity changed before load"
  AI_GATEWAY_ACCEPTANCE_GATEWAYS='https://127.0.0.1:18443,https://127.0.0.1:19443' \
  AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=gateway.easyai.local \
  AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE="$state_root/ca.crt" \
  AI_GATEWAY_ACCEPTANCE_EMULATOR_URL="$acceptance_emulator_url" \
  AI_GATEWAY_ACCEPTANCE_API_KEYS="$acceptance_api_keys" \
  AI_GATEWAY_ACCEPTANCE_RUN_ID="$run_id" \
  AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$acceptance_run_token" \
  AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \
  AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \
    "$load_binary" -profile "$profile" -report "$report_path" \
      -execution-id "$(basename "$report_path" .json)" "$@" \
      >"$stdout_temporary" &
  load_pid=$!
  active_load_pid=$load_pid
  while kill -0 "$load_pid" >/dev/null 2>&1; do
    if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then
      record_failure_gate local_control_plane_restarted "local K3s server restarted during load"
      infrastructure_failure=true
      kill "$load_pid" >/dev/null 2>&1 || true
      break
    fi
    if verify_local_apiserver_ready "$context"; then
      unavailable_samples=0
    else
      unavailable_samples=$((unavailable_samples + 1))
      if ((unavailable_samples >= 3)); then
        record_failure_gate local_control_plane_unavailable "Kubernetes API readiness failed for three consecutive samples"
        infrastructure_failure=true
        kill "$load_pid" >/dev/null 2>&1 || true
        break
      fi
    fi
    sleep 2
  done
  if wait "$load_pid"; then
    load_status=0
  else
    load_status=$?
  fi
  active_load_pid=
  if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then
    record_failure_gate local_control_plane_restarted "local K3s server identity changed at load completion"
    infrastructure_failure=true
  fi
  [[ ! -e $stdout_path ]] || fail_gate acceptance_report_exists "load stdout report already exists"
  mv "$stdout_temporary" "$stdout_path"
  chmod 0600 "$stdout_path"
  if [[ -f $report_path && ! -L $report_path ]]; then
    chmod 0600 "$report_path"
  fi
  if [[ $infrastructure_failure == true ]]; then
    return 1
  fi
  ((load_status == 0)) || return "$load_status"
  jq -e '.schemaVersion == "acceptance-load-report/v2" and .secretSafe == true and .passed == true' \
    "$report_path" >/dev/null
}

sample_worker_runtime() {
  local output=$1 stop_file=$2
  : >"$output"
  while [[ ! -e $stop_file ]]; do
    database_query "
SELECT jsonb_build_object(
  'sampledAt', to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"'),
  'workers', COALESCE(jsonb_agg(jsonb_build_object(
    'instanceId', instance_id,
    'allocatedCapacity', allocated_capacity,
    'safeCapacity', safe_capacity,
    'heavyCapacity', heavy_capacity,
    'activeTasks', active_tasks,
    'preparingTasks', preparing_tasks,
    'waitingUpstreamTasks', waiting_upstream_tasks,
    'finalizingTasks', finalizing_tasks,
    'pressureState', pressure_state,
    'pressureReason', pressure_reason
  ) ORDER BY instance_id), '[]'::jsonb)
)
FROM gateway_worker_instances
WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';" >>"$output" || true
    sleep 1
  done
}

wait_for_adaptive_workloads_settled() {
  local deadline=$((SECONDS + 180)) stable_samples=0 pod_state
  local api_total api_ready api_terminating worker_total worker_ready worker_terminating
  while ((SECONDS < deadline)); do
    pod_state=$(kubectl --context "$context" -n "$namespace" get pods \
      -l 'app.kubernetes.io/name in (easyai-api,easyai-worker)' -o json) || {
      stable_samples=0
      sleep 2
      continue
    }
    read -r api_total api_ready api_terminating worker_total worker_ready worker_terminating < <(
      jq -r '
        def total($name): [.items[] | select(.metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp == null)] | length;
        def ready($name): [.items[] | select(
          .metadata.labels["app.kubernetes.io/name"] == $name and
          .metadata.deletionTimestamp == null and
          .status.phase == "Running" and
          ((.status.containerStatuses // []) | length) > 0 and
          all(.status.containerStatuses[]; .ready == true)
        )] | length;
        def terminating($name): [.items[] | select(.metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp != null)] | length;
        [total("easyai-api"), ready("easyai-api"), terminating("easyai-api"), total("easyai-worker"), ready("easyai-worker"), terminating("easyai-worker")] | @tsv
      ' <<<"$pod_state"
    )
    if [[ $api_total == 2 && $api_ready == 2 && $api_terminating == 0 &&
      $worker_total == 3 && $worker_ready == 3 && $worker_terminating == 0 ]]; then
      stable_samples=$((stable_samples + 1))
      if ((stable_samples >= 3)); then
        return 0
      fi
    else
      stable_samples=0
    fi
    sleep 2
  done
  fail_gate adaptive_workload_settle_timeout \
    "adaptive workloads did not settle at two Ready APIs and three Ready Workers without terminating Pods"
}

configure_adaptive_workers() {
  local hard_limit=${AI_GATEWAY_LOCAL_ADAPTIVE_HARD_LIMIT:-32}
  local global_limit=$((hard_limit * 3))
  [[ $hard_limit =~ ^[0-9]+$ ]] && ((hard_limit >= 4 && hard_limit <= 256))
  kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
    --type=merge -p "$(jq -cn --arg hard "$hard_limit" --arg global "$global_limit" '{data:{
      AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
      AI_GATEWAY_WORKER_LOAD_MODE:"adaptive",
      AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$hard,
      AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$global,
      AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global
    }}')" >/dev/null
  kubectl --context "$context" -n "$namespace" rollout restart \
    deployment/easyai-capacity-controller >/dev/null
  kubectl --context "$context" -n "$namespace" rollout status \
    deployment/easyai-capacity-controller --timeout=10m
  kubectl --context "$context" -n "$namespace" scale deployment/easyai-worker-ningbo --replicas=2 >/dev/null
  kubectl --context "$context" -n "$namespace" scale deployment/easyai-worker-hongkong --replicas=1 >/dev/null
  for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
    kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
      AI_GATEWAY_WORKER_LOAD_MODE=adaptive \
      AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$hard_limit" >/dev/null
    kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
  done
  for deployment in easyai-api-ningbo easyai-api-hongkong; do
    kubectl --context "$context" -n "$namespace" rollout restart deployment/"$deployment" >/dev/null
    kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
  done
  wait_for_adaptive_workloads_settled
  local ready
  ready=$(kubectl --context "$context" -n "$namespace" get pods -l app.kubernetes.io/name=easyai-worker -o json |
    jq '[.items[] | select(.status.phase=="Running" and any(.status.containerStatuses[]?; .ready==true))] | length')
  [[ $ready == 3 ]] || fail_gate adaptive_worker_count "adaptive validation requires exactly three Ready Workers"
}

run_adaptive_level() {
  local concurrency=$1 passed=true etcd_healthy=true
  local gemini_report="$report_root/adaptive-load-${concurrency}-gemini.json"
  local video_report="$report_root/adaptive-load-${concurrency}-video.json"
  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
    fail_gate local_control_plane_restarted "local K3s server identity changed before adaptive level $concurrency"
  record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
    fail_gate local_control_plane_identity "failed to record adaptive level $concurrency control-plane baseline"
  if ! run_load gemini-multi-image "$gemini_report" -requests "$concurrency"; then
    passed=false
  fi
  if [[ $passed == true ]] && ! run_load video-throughput "$video_report" -requests "$concurrency"; then
    passed=false
  fi
  local critical
  critical=$(database_query "SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND pressure_state='critical' AND heartbeat_at > now() - interval '30 seconds';")
  if ((critical > 0)); then
    passed=false
  fi
  if ! verify_local_etcd_runtime_logs "$cluster_name" "$identity_file"; then
    passed=false
    etcd_healthy=false
    record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || return 1
  fi
  jq -n --argjson concurrency "$concurrency" --argjson passed "$passed" \
    --argjson etcdHealthy "$etcd_healthy" \
    --arg gemini "$(basename "$gemini_report")" --arg video "$(basename "$video_report")" \
    '{concurrency:$concurrency,passed:$passed,etcdHealthy:$etcdHealthy,reports:[$gemini,$video]}' \
    >"$report_root/adaptive-level-${concurrency}.json"
  chmod 0600 "$report_root/adaptive-level-${concurrency}.json"
  [[ $passed == true ]]
}

build_adaptive_report() {
  local output=$1 samples=$2 resources=$3 highest=$4 failed=${5:-0}
  local levels workers distribution routing limits concurrency_peaks leaks duplicates sample_summary resource_summary phases
  levels=$(jq -s 'sort_by(.concurrency)' "$report_root"/adaptive-level-*.json)
  workers=$(database_query "
SELECT COALESCE(jsonb_agg(jsonb_build_object(
  'instanceId',instance_id,'podName',pod_name,'site',site,'status',status,
  'allocatedCapacity',allocated_capacity,'hardCapacityLimit',hard_capacity_limit,
  'safeCapacity',safe_capacity,'heavyCapacity',heavy_capacity,
  'activeTasks',active_tasks,'preparingTasks',preparing_tasks,
  'waitingUpstreamTasks',waiting_upstream_tasks,'finalizingTasks',finalizing_tasks,
  'pressureState',pressure_state,'pressureReason',pressure_reason,
  'loadSampledAt',load_sampled_at
) ORDER BY site,instance_id),'[]'::jsonb)
FROM gateway_worker_instances
WHERE heartbeat_at > now() - interval '30 seconds';")
  distribution=$(database_query "
SELECT COALESCE(jsonb_agg(jsonb_build_object('workerInstanceId',worker_id,'tasks',tasks) ORDER BY worker_id),'[]'::jsonb)
FROM (
  SELECT COALESCE(
           regexp_replace(job.attempted_by[cardinality(job.attempted_by)], '-exec-[^-]+-[^-]+$', ''),
           'unclaimed'
         ) worker_id,
         count(*) tasks
  FROM gateway_tasks task
  LEFT JOIN river_job job ON job.id = task.river_job_id
  WHERE task.acceptance_run_id='$run_id'::uuid
  GROUP BY worker_id
) grouped;")
  routing=$(database_query "
SELECT COALESCE(jsonb_agg(jsonb_build_object(
  'platformId',platform_id,'platformModelId',platform_model_id,'attempts',attempts
) ORDER BY platform_id,platform_model_id),'[]'::jsonb)
FROM (SELECT platform_id::text,platform_model_id::text,count(*) attempts
      FROM gateway_task_attempts WHERE task_id IN
        (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)
      GROUP BY platform_id,platform_model_id) grouped;")
  limits=$(database_query "
SELECT COALESCE(jsonb_agg(jsonb_build_object(
  'scopeType',scope_type,'scopeKey',scope_key,'metric',metric,
  'limit',limit_value,'used',used_value,'reserved',reserved_value,
  'windowStart',window_start,'resetAt',reset_at
) ORDER BY scope_type,scope_key,metric,window_start),'[]'::jsonb)
FROM gateway_rate_limit_counters
WHERE scope_key LIKE 'acceptance:$run_id:%';")
  concurrency_peaks=$(database_query "
WITH events AS (
  SELECT scope_type,scope_key,limit_value,acquired_at event_at,lease_value delta FROM gateway_concurrency_leases
  WHERE scope_key LIKE 'acceptance:$run_id:%'
  UNION ALL
  SELECT scope_type,scope_key,limit_value,LEAST(COALESCE(released_at,expires_at),expires_at),-lease_value
  FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%'
), points AS (
  SELECT scope_type,scope_key,limit_value,
         SUM(delta) OVER (PARTITION BY scope_type,scope_key ORDER BY event_at,delta ROWS UNBOUNDED PRECEDING) active
  FROM events
), peaks AS (
  SELECT scope_type,scope_key,MAX(limit_value) limit_value,COALESCE(MAX(active),0) peak
  FROM points GROUP BY scope_type,scope_key
)
SELECT COALESCE(jsonb_agg(jsonb_build_object('scopeType',scope_type,'scopeKey',scope_key,'limit',limit_value,'peak',peak)
       ORDER BY scope_type,scope_key),'[]'::jsonb) FROM peaks;")
  leaks=$(database_query "
SELECT jsonb_build_object(
  'activeConcurrencyLeases',(SELECT count(*) FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%' AND released_at IS NULL AND expires_at>now()),
  'activeRateReservations',(SELECT count(*) FROM gateway_rate_limit_reservations WHERE scope_key LIKE 'acceptance:$run_id:%' AND status='reserved'),
  'activeRiverJobs',(SELECT count(*) FROM river_job job JOIN gateway_tasks task ON task.river_job_id=job.id WHERE task.acceptance_run_id='$run_id'::uuid AND job.state IN ('available','pending','retryable','running','scheduled'))
);")
  duplicates=$(database_query "
SELECT jsonb_build_object(
  'remoteTaskIds',(SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) duplicate),
  'billingTransactions',(SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) duplicate)
);")
  sample_summary=$(jq -s '[.[].workers[]] | group_by(.instanceId) | map({
    instanceId:.[0].instanceId,
    peakActiveTasks:(map(.activeTasks)|max),
    peakPreparingTasks:(map(.preparingTasks)|max),
    peakWaitingUpstreamTasks:(map(.waitingUpstreamTasks)|max),
    peakFinalizingTasks:(map(.finalizingTasks)|max),
    minSafeCapacity:(map(.safeCapacity)|min),
    maxSafeCapacity:(map(.safeCapacity)|max),
    pressureStates:(map(.pressureState)|unique)
  })' "$samples")
  resource_summary=$(jq -Rn '
    def number_without($suffix): rtrimstr($suffix) | tonumber;
    def memory_mib:
      if endswith("Gi") then number_without("Gi") * 1024
      elif endswith("Mi") then number_without("Mi")
      elif endswith("Ki") then number_without("Ki") / 1024
      else tonumber end;
    [inputs | split(",") | select(.[0] != "timestamp") | {
      scope:.[1],name:.[2],cpu:.[3],memory:.[4]
    }]
    | group_by([.scope,.name])
    | map(if .[0].scope == "node" then {
        scope:"node",name:.[0].name,
        maxCpuPercent:(map(.cpu | number_without("%")) | max),
        maxMemoryPercent:(map(.memory | number_without("%")) | max)
      } else {
        scope:"pod",name:.[0].name,
        maxCpuMillicores:(map(.cpu | if endswith("m") then number_without("m") else tonumber * 1000 end) | max),
        maxMemoryMiB:(map(.memory | memory_mib) | max)
      } end)' <"$resources")
  phases=$(jq -s '[.[] | .phases[]]' "$report_root"/adaptive-load-*.json)
  local passed=true bottleneck=configured_probe_ceiling
  if ((highest < 1)); then passed=false; fi
  if ((failed > 0)); then bottleneck=worker_or_gateway_limit; fi
  if jq -e 'any(.[]; .etcdHealthy == false)' <<<"$levels" >/dev/null; then
    bottleneck=local_control_plane_limit
  fi
  if ! jq -e 'length == 3' <<<"$workers" >/dev/null; then passed=false; fi
  if ! jq -e 'all(.[]; ((.used + .reserved) <= .limit))' <<<"$limits" >/dev/null; then passed=false; fi
  if ! jq -e 'all(.[]; (.limit == null or .peak <= .limit))' <<<"$concurrency_peaks" >/dev/null; then passed=false; fi
  if ! jq -e '.activeConcurrencyLeases==0 and .activeRateReservations==0 and .activeRiverJobs==0' <<<"$leaks" >/dev/null; then passed=false; fi
  if ! jq -e '.remoteTaskIds==0 and .billingTransactions==0' <<<"$duplicates" >/dev/null; then passed=false; fi
  jq -n \
    --arg runId "$run_id" --arg startedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
    --arg bottleneck "$bottleneck" --argjson passed "$passed" \
    --argjson highest "$highest" --argjson failed "$failed" \
    --argjson levels "$levels" --argjson phases "$phases" \
    --argjson workers "$workers" --argjson distribution "$distribution" \
    --argjson routing "$routing" --argjson limits "$limits" \
    --argjson concurrencyPeaks "$concurrency_peaks" --argjson workerPeaks "$sample_summary" \
    --argjson resourcePeaks "$resource_summary" \
    --argjson leaks "$leaks" --argjson duplicates "$duplicates" \
    '{
      schemaVersion:"acceptance-load-report/v2",runId:$runId,profile:"adaptive-capacity",
      startedAt:$startedAt,finishedAt:(now|todateiso8601),passed:$passed,secretSafe:true,
      phases:$phases,
      capacity:{levels:$levels,highestStableConcurrency:$highest,firstFailedConcurrency:$failed,bottleneck:$bottleneck},
      cluster:{workers:$workers,workerPeaks:$workerPeaks,taskDistribution:$distribution,
        platformRouting:$routing,rateLimits:$limits,concurrencyPeaks:$concurrencyPeaks,
        resourcePeaks:$resourcePeaks,leaks:$leaks,duplicates:$duplicates}
    }' >"$output"
  chmod 0600 "$output"
  jq -e '.passed==true and .secretSafe==true' "$output" >/dev/null
}

sample_resources() {
  local output=$1 stop_file=$2
  printf 'timestamp,scope,name,cpu,memory\n' >"$output"
  while [[ ! -e $stop_file ]]; do
    kubectl --context "$context" top nodes --no-headers 2>/dev/null |
      awk -v timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
        '{print timestamp ",node," $1 "," $3 "," $5}' >>"$output" || true
    kubectl --context "$context" -n "$namespace" top pods --no-headers 2>/dev/null |
      awk -v timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
        '{print timestamp ",pod," $1 "," $2 "," $3}' >>"$output" || true
    sleep 5
  done
}

verify_hard_gates() {
  local unhealthy ready sync_state connections max_connections queue duplicate_remote duplicate_billing
  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
    fail_gate local_control_plane_restarted "local K3s server identity changed"
  verify_local_apiserver_ready "$context" ||
    fail_gate local_control_plane_unavailable "Kubernetes API readiness failed"
  verify_local_node_capacity "$context" ||
    fail_gate local_node_capacity_mismatch "node allocatable resources drifted from the acceptance envelope"
  verify_local_etcd_runtime_logs "$cluster_name" "$identity_file" ||
    fail_gate local_etcd_latency "etcd emitted a severe latency or timeout signal during the run"
  ready=$(kubectl --context "$context" get nodes -o json |
    jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length')
  [[ $ready == 3 ]] || fail_gate local_nodes_not_ready "not all three local K3s nodes are Ready"
  [[ $(kubectl --context "$context" get nodes -o json |
    jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] ||
    fail_gate local_node_memory_pressure "a local K3s node reports MemoryPressure"
  [[ $(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
    -o 'jsonpath={.status.readyInstances}') == 2 ]] ||
    fail_gate local_postgres_not_ready "local PostgreSQL is not 2/2 Ready"
  sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
  [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] ||
    fail_gate local_postgres_not_synchronous "local PostgreSQL lost its synchronous replica"
  unhealthy=$(kubectl --context "$context" -n "$namespace" get pods -o json |
    jq '[.items[]
      | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or
               .metadata.labels["app.kubernetes.io/name"] == "easyai-worker")
      | .status.containerStatuses[]?
      | select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or
               .state.terminated.reason == "OOMKilled")] | length')
  [[ $unhealthy == 0 ]] || fail_gate gateway_pod_restarted "an API or Worker container restarted or was OOMKilled"
  while read -r _node _cpu _cpu_percent _memory memory_percent; do
    memory_percent=${memory_percent%\%}
    [[ $memory_percent =~ ^[0-9]+$ ]]
    (( memory_percent < 80 )) || fail_gate local_node_memory_target "local node memory reached the 80 percent target"
  done < <(kubectl --context "$context" top nodes --no-headers)
  while read -r _pod _cpu memory; do
    case $memory in
      *Gi) memory=$(awk -v value="${memory%Gi}" 'BEGIN {printf "%.0f", value*1024}') ;;
      *Mi) memory=${memory%Mi} ;;
      *Ki) memory=$(awk -v value="${memory%Ki}" 'BEGIN {printf "%.0f", value/1024}') ;;
      *) return 1 ;;
    esac
    (( memory < 1536 )) || fail_gate gateway_pod_memory_limit "an API or Worker Pod reached 1.5 GiB RSS"
  done < <(kubectl --context "$context" -n "$namespace" top pods \
    -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers)
  connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
  max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
  (( connections < 150 && connections * 4 < max_connections * 3 )) ||
    fail_gate postgres_connection_budget "PostgreSQL client sessions exceeded the acceptance budget"
  queue=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
  [[ $queue == 0:0 ]] || fail_gate acceptance_queue_not_drained "acceptance queue did not return to zero"
  duplicate_remote=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) d;")
  duplicate_billing=$(database_query "SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) d;")
  [[ $duplicate_remote == 0 ]] || fail_gate duplicate_upstream_submission "duplicate remote task IDs were detected"
  [[ $duplicate_billing == 0 ]] || fail_gate duplicate_billing "duplicate billing transactions were detected"
  kubectl --context "$context" -n "$namespace" exec deployment/easyai-acceptance-callback-collector -- \
    wget -qO- http://127.0.0.1:8091/report |
    jq -e '.duplicates == 0 and .invalid == 0' >/dev/null ||
    fail_gate callback_validation_failed "callback collector detected duplicate or invalid callbacks"
}

apply_profile() {
  local profile=$1 slots pool
  case $profile in
    P24) slots=24; pool=32 ;;
    P28) slots=28; pool=36 ;;
    P32) slots=32; pool=40 ;;
    *) return 64 ;;
  esac
  local global=$((slots * 2))
  kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
    --type=merge -p "$(jq -cn \
      --arg slots "$slots" --arg global "$global" \
      '{data:{
        AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
        AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
        AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
        AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global,
        AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$global
      }}')" >/dev/null
  kubectl --context "$context" -n "$namespace" scale \
    deployment/easyai-worker-ningbo deployment/easyai-worker-hongkong --replicas=1 >/dev/null
  for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
    kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
      AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
      AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
      AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \
      AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null
    kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
  done
  for deployment in easyai-api-ningbo easyai-api-hongkong easyai-capacity-controller; do
    kubectl --context "$context" -n "$namespace" rollout restart deployment/"$deployment" >/dev/null
    kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
  done
}

restore_profile_best_effort() {
  local profile=$1 slots pool global
  case $profile in
    P24) slots=24; pool=32 ;;
    P28) slots=28; pool=36 ;;
    P32) slots=32; pool=40 ;;
    *) return ;;
  esac
  global=$((slots * 2))
  kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
    --type=merge -p "$(jq -cn \
      --arg slots "$slots" --arg global "$global" \
      '{data:{
        AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
        AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
        AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
        AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global,
        AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$global
      }}')" >/dev/null 2>&1 || true
  kubectl --context "$context" -n "$namespace" scale \
    deployment/easyai-worker-ningbo deployment/easyai-worker-hongkong \
    --replicas=1 >/dev/null 2>&1 || true
  for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
    kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
      AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
      AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
      AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \
      AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null 2>&1 || true
  done
}

run_profile_round() {
  local profile=$1 repetition=$2 prefix
  prefix="$report_root/${profile,,}-$repetition"
  local stop_file="$prefix.resources.stop"
  sample_resources "$prefix.resources.csv" "$stop_file" &
  local sampler=$!
  run_load gemini-baseline "$prefix-gemini-baseline.json"
  run_load gemini-large "$prefix-gemini-large.json"
  run_load gemini-peak "$prefix-gemini-peak.json"
  run_load video-throughput "$prefix-video-throughput.json"
  run_recovery "$prefix-video-recovery.json"
  touch "$stop_file"
  wait "$sampler"
  verify_hard_gates
}

wait_for_recovery_owner() {
  local deadline=$((SECONDS + 120)) owner
  while (( SECONDS < deadline )); do
    owner=$(database_query "
SELECT task.id::text||','||worker.pod_name||','||worker.site
FROM gateway_tasks task
JOIN gateway_worker_instances worker ON worker.instance_id=task.locked_by
WHERE task.acceptance_run_id='$run_id'::uuid
  AND task.status='running'
  AND task.request::text LIKE '%acceptance-long-recovery%'
  AND worker.status='active'
ORDER BY task.updated_at
LIMIT 1;")
    [[ -z $owner ]] || { printf '%s\n' "$owner"; return; }
    sleep 1
  done
  return 1
}

run_recovery() {
  local report_path=$1 owner task_id pod site
  run_load video-recovery "$report_path" &
  active_load_pid=$!
  owner=$(wait_for_recovery_owner)
  IFS=',' read -r task_id pod site <<<"$owner"
  [[ $task_id =~ ^[0-9a-f-]{36}$ && $pod == easyai-worker-"$site"-* ]]
  kubectl --context "$context" -n "$namespace" delete pod "$pod" \
    --force --grace-period=0 --wait=false >/dev/null
  kubectl --context "$context" -n "$namespace" rollout status \
    deployment/easyai-worker-"$site" --timeout=10m
  wait "$active_load_pid"
  active_load_pid=
}

run_fault_matrix() {
  local duration=${AI_GATEWAY_LOCAL_WEAK_LINK_DURATION:-5m}
  "$script_dir/network-fault.sh" weak-link
  netem_active=true
  run_load mixed-soak "$report_root/fault-weak-link.json" \
    -duration "$duration" -image-rate 1 -video-rate 1
  "$script_dir/network-fault.sh" reset
  netem_active=false

  run_load video-throughput "$report_root/fault-upstream-outage.json" &
  active_load_pid=$!
  sleep 5
  "$script_dir/network-fault.sh" upstream-outage
  netem_active=true
  sleep 10
  "$script_dir/network-fault.sh" reset
  netem_active=false
  wait "$active_load_pid"
  active_load_pid=

  run_load video-recovery "$report_root/fault-database-outage.json" &
  active_load_pid=$!
  sleep 10
  "$script_dir/network-fault.sh" database-outage hongkong
  netem_active=true
  sleep 30
  "$script_dir/network-fault.sh" reset
  netem_active=false
  wait "$active_load_pid"
  active_load_pid=

  local leader
  leader=$(kubectl --context "$context" -n "$namespace" get pods \
    -l app.kubernetes.io/name=easyai-capacity-controller -o name |
    while read -r pod; do
      status=$(kubectl --context "$context" -n "$namespace" exec "$pod" -- \
        wget -qO- http://127.0.0.1:8088/status)
      [[ $(jq -r '.leader' <<<"$status") == true ]] && { printf '%s\n' "${pod#pod/}"; break; }
    done)
  [[ -n $leader ]]
  kubectl --context "$context" -n "$namespace" delete pod "$leader" --wait=false >/dev/null
  kubectl --context "$context" -n "$namespace" rollout status \
    deployment/easyai-capacity-controller --timeout=5m
  verify_hard_gates
}

run_autoscaling() {
  local profile=$1 slots
  slots=${profile#P}
  kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
    --type=merge -p "$(jq -cn --arg global "$((slots * 4))" \
      '{data:{
        AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"true",
        AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:"1",
        AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:"1",
        AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:"2",
        AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:"2",
        AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global
      }}')" >/dev/null
  kubectl --context "$context" -n "$namespace" rollout restart \
    deployment/easyai-capacity-controller >/dev/null
  kubectl --context "$context" -n "$namespace" rollout status \
    deployment/easyai-capacity-controller --timeout=5m
  run_load video-throughput "$report_root/autoscaling-load.json" &
  active_load_pid=$!
  local deadline=$((SECONDS + 420)) total=2
  while (( SECONDS < deadline )); do
    total=$(kubectl --context "$context" -n "$namespace" get deployment \
      easyai-worker-ningbo easyai-worker-hongkong \
      -o json | jq '[.items[].spec.replicas] | add')
    (( total >= 3 )) && break
    sleep 5
  done
  (( total >= 3 ))
  wait "$active_load_pid"
  active_load_pid=
  deadline=$((SECONDS + 780))
  while (( SECONDS < deadline )); do
    total=$(kubectl --context "$context" -n "$namespace" get deployment \
      easyai-worker-ningbo easyai-worker-hongkong \
      -o json | jq '[.items[].spec.replicas] | add')
    (( total == 2 )) && break
    sleep 10
  done
  (( total == 2 ))
  verify_hard_gates
}

artifact_smoke() {
  local manifest=$1
  node "$repository_root/scripts/release-manifest.mjs" validate "$manifest" >/dev/null
  local manifest_sha api_image web_image api_digest artifact_report
  manifest_sha=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" sourceSha)
  api_image=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.api)
  web_image=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.web)
  api_digest=${api_image##*@}
  [[ $manifest_sha == "$release_sha" && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
  docker pull --platform linux/amd64 "$api_image" >/dev/null
  docker pull --platform linux/amd64 "$web_image" >/dev/null
  "$private_root/tools/k3d" image import -c easyai-acceptance-local "$api_image" "$web_image"
  for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo \
    easyai-worker-hongkong easyai-capacity-controller easyai-acceptance-emulator \
    easyai-acceptance-callback-collector; do
    kubectl --context "$context" -n "$namespace" set image deployment/"$deployment" \
      "*=$api_image" >/dev/null
    kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=15m
  done
  for deployment in easyai-acceptance-edge-ningbo easyai-acceptance-edge-hongkong; do
    kubectl --context "$context" -n "$namespace" set image deployment/"$deployment" \
      "*=$web_image" >/dev/null
    kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
  done
  kubectl --context "$context" -n "$namespace" delete job easyai-artifact-migrate \
    --ignore-not-found --wait=true >/dev/null
  cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
  name: easyai-artifact-migrate
  namespace: easyai
spec:
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: migrate
          image: $api_image
          command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
          envFrom:
            - secretRef:
                name: easyai-ai-gateway-runtime
EOF
  kubectl --context "$context" -n "$namespace" wait \
    --for=condition=complete job/easyai-artifact-migrate --timeout=10m >/dev/null
  run_load simulated-smoke "$report_root/artifact-simulated-smoke.json"
  verify_hard_gates
  artifact_report="$report_root/artifact-smoke.json"
  jq -n \
    --arg releaseSha "$manifest_sha" \
    --arg apiImageDigest "$api_digest" \
    --arg webImageDigest "${web_image##*@}" \
    '{
      schemaVersion:"acceptance-artifact-smoke/v1",
      releaseSha:$releaseSha,
      apiImageDigest:$apiImageDigest,
      webImageDigest:$webImageDigest,
      architecture:"linux/amd64",
      migrationSmoke:true,
      startupSmoke:true,
      mediaSmoke:true,
      passed:true,
      secretSafe:true
    }' >"$artifact_report"
  chmod 0600 "$artifact_report"
}

quick() {
  current_phase=quick
  run_load simulated-smoke "$report_root/quick.json"
  verify_hard_gates
  echo "local_acceptance_quick=PASS run_id=$run_id report=$report_root/quick.json"
}

adaptive() {
  local hard_limit=${AI_GATEWAY_LOCAL_ADAPTIVE_HARD_LIMIT:-32}
  local max_probe=${AI_GATEWAY_LOCAL_ADAPTIVE_MAX_CONCURRENCY:-$((hard_limit * 3))}
  local level=4 highest=0 failed=0 low high mid
  local samples="$report_root/adaptive-worker-samples.ndjson"
  local resources="$report_root/adaptive-resources.csv"
  worker_sample_stop="$report_root/adaptive-worker-samples.stop"
  [[ ! -e $samples && ! -e $resources && ! -e $worker_sample_stop ]] || fail_gate acceptance_report_exists "adaptive evidence already exists"
  current_phase=adaptive_workers
  configure_adaptive_workers
  sample_worker_runtime "$samples" "$worker_sample_stop" &
  active_worker_sampler_pid=$!
  sample_resources "$resources" "$worker_sample_stop" &
  active_resource_sampler_pid=$!
  while ((level <= max_probe)); do
    current_phase="adaptive_capacity_$level"
    if run_adaptive_level "$level"; then
      highest=$level
      level=$((level * 2))
    else
      failed=$level
      break
    fi
  done
  if ((failed > 0 && highest > 0)); then
    low=$((highest + 1))
    high=$((failed - 1))
    while ((low <= high)); do
      mid=$(((low + high) / 2))
      current_phase="adaptive_capacity_refine_$mid"
      if run_adaptive_level "$mid"; then
        highest=$mid
        low=$((mid + 1))
      else
        failed=$mid
        high=$((mid - 1))
      fi
    done
  fi
  touch "$worker_sample_stop"
  wait "$active_worker_sampler_pid"
  wait "$active_resource_sampler_pid"
  active_worker_sampler_pid=
  active_resource_sampler_pid=
  worker_sample_stop=
  local ready_workers registered_workers
  ready_workers=$(kubectl --context "$context" -n "$namespace" get pods \
    -l app.kubernetes.io/name=easyai-worker -o json |
    jq '[.items[] | select(.status.phase=="Running" and any(.status.containerStatuses[]?; .ready==true))] | length')
  registered_workers=$(database_query "SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';")
  [[ $ready_workers == 3 && $registered_workers == 3 ]] ||
    fail_gate adaptive_worker_count "adaptive validation requires three Ready and registered Workers throughout the final gate"
  verify_hard_gates
  current_phase=adaptive_report
  build_adaptive_report "$report_root/adaptive-capacity.json" "$samples" "$resources" "$highest" "$failed"
  echo "local_acceptance_adaptive=PASS run_id=$run_id highest_stable_concurrency=$highest first_failed_concurrency=$failed report=$report_root/adaptive-capacity.json"
}

full() {
  local manifest=$1 profile repetition
  quick
  for profile in P24 P28 P32; do
    current_phase="capacity_${profile,,}_apply"
    apply_profile "$profile"
    for repetition in 1 2 3; do
      current_phase="capacity_${profile,,}_round_$repetition"
      run_profile_round "$profile" "$repetition"
    done
    stable_profile=$profile
  done
  current_phase=fault_matrix
  run_fault_matrix
  current_phase=autoscaling_and_drain
  run_autoscaling "$stable_profile"
  current_phase=certified_soak
  run_load mixed-soak "$report_root/mixed-soak.json" \
    -duration "${AI_GATEWAY_LOCAL_SOAK_DURATION:-2h}" \
    -image-rate "${AI_GATEWAY_LOCAL_CERTIFIED_IMAGE_RATE:-1}" \
    -video-rate "${AI_GATEWAY_LOCAL_CERTIFIED_VIDEO_RATE:-1}"
  current_phase=overload_shedding
  run_load mixed-overload "$report_root/mixed-overload.json" \
    -duration "${AI_GATEWAY_LOCAL_OVERLOAD_DURATION:-10m}" \
    -image-rate "${AI_GATEWAY_LOCAL_OVERLOAD_IMAGE_RATE:-2}" \
    -video-rate "${AI_GATEWAY_LOCAL_OVERLOAD_VIDEO_RATE:-2}"
  current_phase=amd64_artifact_smoke
  artifact_smoke "$manifest"
  current_phase=final_report
  node "$script_dir/report.mjs" build-local \
    --runtime "$runtime" \
    --snapshot "$snapshot" \
    --reports "$report_root" \
    --artifact "$report_root/artifact-smoke.json" \
    --api-digest "$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.api | sed 's/.*@//')" \
    --certified-profile "$stable_profile" \
    --output "$report_root/acceptance-report.json"
  echo "local_acceptance_full=PASS run_id=$run_id certified_profile=$stable_profile report=$report_root/acceptance-report.json"
}

command=${1:-}
shift || true
case $command in
  quick|adaptive|provider-burst)
    [[ $# -eq 0 ]] || { usage >&2; exit 64; }
    ;;
  artifact-smoke|full)
    [[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; }
    ;;
  *)
    usage >&2
    exit 64
    ;;
esac
"$script_dir/local-cluster.sh" new-run
require_local_cluster
load_environment
case $command in
  quick)
    quick
    ;;
  adaptive)
    adaptive
    ;;
  provider-burst)
    provider_burst
    ;;
  artifact-smoke)
    artifact_smoke "$2"
    ;;
  full)
    full "$2"
    ;;
esac