#!/usr/bin/env bash set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) repository_root=$(cd "$script_dir/../.." && pwd) private_root="$repository_root/.local-secrets/acceptance" state_root="$private_root/state" context=k3d-easyai-acceptance-local namespace=easyai cluster_name=easyai-acceptance-local runtime_path_file="$state_root/current-runtime-path" snapshot="$state_root/snapshot.json" load_binary="$state_root/easyai-ai-gateway-acceptance-load" identity_file="$state_root/control-plane-identity.json" active_load_pid= active_worker_sampler_pid= active_resource_sampler_pid= active_provider_sampler_pid= worker_sample_stop= netem_active=false provider_burst_configured=false current_phase=initialization stable_profile= failure_gate_id=local_execution_incomplete # shellcheck source=scripts/acceptance/local-control-plane.sh source "$script_dir/local-control-plane.sh" # shellcheck source=scripts/acceptance/provider-burst.sh source "$script_dir/provider-burst.sh" usage() { cat <<'EOF' Usage: scripts/acceptance/run-local-acceptance.sh quick scripts/acceptance/run-local-acceptance.sh adaptive scripts/acceptance/run-local-acceptance.sh provider-burst scripts/acceptance/run-local-acceptance.sh full --release-manifest dist/releases/.json scripts/acceptance/run-local-acceptance.sh artifact-smoke --release-manifest dist/releases/.json `adaptive` runs a three-Worker emulator-only concurrency ladder without changing provider limits. `provider-burst` binds one video model to three emulator-only platforms with concurrency limits 2/4/6 and validates burst queueing and routing. `full` executes P24/P28/P32 three times, the fault matrix, autoscaling/drain, 80% soak, 120% overload, and exact linux/amd64 artifact smoke. The load process runs outside K3s and splits requests 50/50 across both TLS entrances. EOF } private_file() { local path=$1 mode [[ -f $path && ! -L $path ]] || return 1 if [[ $(uname -s) == Darwin ]]; then mode=$(stat -f '%Lp' "$path") else mode=$(stat -c '%a' "$path") fi [[ $mode == 600 ]] } cleanup() { local status=$? trap - EXIT if [[ -n $active_load_pid ]]; then kill "$active_load_pid" >/dev/null 2>&1 || true wait "$active_load_pid" >/dev/null 2>&1 || true fi if [[ -n $active_worker_sampler_pid ]]; then [[ -z $worker_sample_stop ]] || touch "$worker_sample_stop" wait "$active_worker_sampler_pid" >/dev/null 2>&1 || true fi if [[ -n $active_resource_sampler_pid ]]; then [[ -z $worker_sample_stop ]] || touch "$worker_sample_stop" wait "$active_resource_sampler_pid" >/dev/null 2>&1 || true fi if [[ -n $active_provider_sampler_pid ]]; then [[ -z $worker_sample_stop ]] || touch "$worker_sample_stop" wait "$active_provider_sampler_pid" >/dev/null 2>&1 || true fi if [[ $netem_active == true ]]; then "$script_dir/network-fault.sh" reset >/dev/null 2>&1 || true fi if [[ $provider_burst_configured == true && -n ${runtime:-} ]]; then restore_provider_burst_platforms >/dev/null 2>&1 || true fi if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then restore_profile_best_effort "${stable_profile:-P24}" if [[ -f ${failure_gate_file:-} && ! -L ${failure_gate_file:-} ]]; then failure_gate_id=$(jq -r '.id' "$failure_gate_file" 2>/dev/null || printf '%s' "$failure_gate_id") fi local partial_args=( "$script_dir/report.mjs" build-local-partial --runtime "$runtime" \ --snapshot "$snapshot" \ --reports "$report_root" \ --failure-phase "$current_phase" \ --failure-gate "$failure_gate_id" \ --output "$report_root/acceptance-report.partial.json" ) if [[ -n $stable_profile ]]; then partial_args+=(--certified-profile "$stable_profile") fi if node "${partial_args[@]}" >/dev/null 2>&1; then echo "local_acceptance=FAILED phase=$current_phase partial_report=$report_root/acceptance-report.partial.json" >&2 else echo "local_acceptance=FAILED phase=$current_phase partial_report=unavailable" >&2 fi fi exit "$status" } trap cleanup EXIT trap 'current_phase=signal_interrupted; exit 130' HUP INT TERM require_local_cluster() { command -v kubectl >/dev/null 2>&1 command -v jq >/dev/null 2>&1 command -v node >/dev/null 2>&1 [[ $(kubectl config get-contexts "$context" -o name) == "$context" ]] private_file "$runtime_path_file" && private_file "$snapshot" runtime=$(<"$runtime_path_file") private_file "$runtime" || { echo 'local acceptance runtime file is missing or not mode 0600' >&2 exit 1 } jq -e '.schemaVersion == "acceptance-runtime/v1"' "$runtime" >/dev/null run_id=$(jq -r '.runId' "$runtime") release_sha=$(jq -r '.releaseSha' "$runtime") report_root="$repository_root/dist/acceptance/local/$run_id" install -d -m 0700 "$report_root" failure_gate_file="$report_root/failure-gate.json" ( cd "$repository_root/apps/api" env -u AI_GATEWAY_TEST_DATABASE_URL go build -trimpath \ -o "$load_binary" ./cmd/acceptance-load ) } record_failure_gate() { local id=$1 detail=$2 temporary [[ $id =~ ^[a-z0-9_]+$ ]] failure_gate_id=$id [[ -n ${failure_gate_file:-} ]] || return 0 if [[ ! -e $failure_gate_file ]]; then temporary="${failure_gate_file}.tmp.$$" jq -n --arg id "$id" --arg detail "$detail" \ '{schemaVersion:"acceptance-failure-gate/v1",id:$id,detail:$detail}' >"$temporary" chmod 0600 "$temporary" mv "$temporary" "$failure_gate_file" fi } fail_gate() { local id=$1 detail=$2 record_failure_gate "$id" "$detail" echo "gate=$id $detail" >&2 return 1 } database_query() { local sql=$1 primary primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \ -o 'jsonpath={.status.currentPrimary}') kubectl --context "$context" -n "$namespace" exec "$primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql" } load_environment() { acceptance_api_keys=$(jq -r '.apiKeys | join(",")' "$runtime") acceptance_run_token=$(jq -r '.runToken' "$runtime") acceptance_gemini_model=$(jq -r '.geminiModel' "$runtime") acceptance_video_model=$(jq -r '.videoModel' "$runtime") acceptance_emulator_url=$(jq -r '.emulatorBaseUrl' "$runtime") } run_load() { local profile=$1 report_path=$2 shift 2 local stdout_path="$report_path.stdout" stdout_temporary="${report_path}.stdout.tmp.$$" local load_pid load_status=0 unavailable_samples=0 infrastructure_failure=false [[ ! -e $report_path && ! -e $stdout_path && ! -e $stdout_temporary ]] || fail_gate acceptance_report_exists "refusing to overwrite an existing load report" verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || fail_gate local_control_plane_restarted "local K3s server identity changed before load" AI_GATEWAY_ACCEPTANCE_GATEWAYS='https://127.0.0.1:18443,https://127.0.0.1:19443' \ AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=gateway.easyai.local \ AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE="$state_root/ca.crt" \ AI_GATEWAY_ACCEPTANCE_EMULATOR_URL="$acceptance_emulator_url" \ AI_GATEWAY_ACCEPTANCE_API_KEYS="$acceptance_api_keys" \ AI_GATEWAY_ACCEPTANCE_RUN_ID="$run_id" \ AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$acceptance_run_token" \ AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \ AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \ "$load_binary" -profile "$profile" -report "$report_path" \ -execution-id "$(basename "$report_path" .json)" "$@" \ >"$stdout_temporary" & load_pid=$! active_load_pid=$load_pid while kill -0 "$load_pid" >/dev/null 2>&1; do if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then record_failure_gate local_control_plane_restarted "local K3s server restarted during load" infrastructure_failure=true kill "$load_pid" >/dev/null 2>&1 || true break fi if verify_local_apiserver_ready "$context"; then unavailable_samples=0 else unavailable_samples=$((unavailable_samples + 1)) if ((unavailable_samples >= 3)); then record_failure_gate local_control_plane_unavailable "Kubernetes API readiness failed for three consecutive samples" infrastructure_failure=true kill "$load_pid" >/dev/null 2>&1 || true break fi fi sleep 2 done if wait "$load_pid"; then load_status=0 else load_status=$? fi active_load_pid= if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then record_failure_gate local_control_plane_restarted "local K3s server identity changed at load completion" infrastructure_failure=true fi [[ ! -e $stdout_path ]] || fail_gate acceptance_report_exists "load stdout report already exists" mv "$stdout_temporary" "$stdout_path" chmod 0600 "$stdout_path" if [[ -f $report_path && ! -L $report_path ]]; then chmod 0600 "$report_path" fi if [[ $infrastructure_failure == true ]]; then return 1 fi ((load_status == 0)) || return "$load_status" jq -e '.schemaVersion == "acceptance-load-report/v2" and .secretSafe == true and .passed == true' \ "$report_path" >/dev/null } sample_worker_runtime() { local output=$1 stop_file=$2 : >"$output" while [[ ! -e $stop_file ]]; do database_query " SELECT jsonb_build_object( 'sampledAt', to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"'), 'workers', COALESCE(jsonb_agg(jsonb_build_object( 'instanceId', instance_id, 'allocatedCapacity', allocated_capacity, 'safeCapacity', safe_capacity, 'heavyCapacity', heavy_capacity, 'activeTasks', active_tasks, 'preparingTasks', preparing_tasks, 'waitingUpstreamTasks', waiting_upstream_tasks, 'finalizingTasks', finalizing_tasks, 'pressureState', pressure_state, 'pressureReason', pressure_reason ) ORDER BY instance_id), '[]'::jsonb) ) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';" >>"$output" || true sleep 1 done } wait_for_adaptive_workloads_settled() { local deadline=$((SECONDS + 180)) stable_samples=0 pod_state local api_total api_ready api_terminating worker_total worker_ready worker_terminating while ((SECONDS < deadline)); do pod_state=$(kubectl --context "$context" -n "$namespace" get pods \ -l 'app.kubernetes.io/name in (easyai-api,easyai-worker)' -o json) || { stable_samples=0 sleep 2 continue } read -r api_total api_ready api_terminating worker_total worker_ready worker_terminating < <( jq -r ' def total($name): [.items[] | select(.metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp == null)] | length; def ready($name): [.items[] | select( .metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp == null and .status.phase == "Running" and ((.status.containerStatuses // []) | length) > 0 and all(.status.containerStatuses[]; .ready == true) )] | length; def terminating($name): [.items[] | select(.metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp != null)] | length; [total("easyai-api"), ready("easyai-api"), terminating("easyai-api"), total("easyai-worker"), ready("easyai-worker"), terminating("easyai-worker")] | @tsv ' <<<"$pod_state" ) if [[ $api_total == 2 && $api_ready == 2 && $api_terminating == 0 && $worker_total == 3 && $worker_ready == 3 && $worker_terminating == 0 ]]; then stable_samples=$((stable_samples + 1)) if ((stable_samples >= 3)); then return 0 fi else stable_samples=0 fi sleep 2 done fail_gate adaptive_workload_settle_timeout \ "adaptive workloads did not settle at two Ready APIs and three Ready Workers without terminating Pods" } configure_adaptive_workers() { local hard_limit=${AI_GATEWAY_LOCAL_ADAPTIVE_HARD_LIMIT:-32} local global_limit=$((hard_limit * 3)) [[ $hard_limit =~ ^[0-9]+$ ]] && ((hard_limit >= 4 && hard_limit <= 256)) kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \ --type=merge -p "$(jq -cn --arg hard "$hard_limit" --arg global "$global_limit" '{data:{ AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false", AI_GATEWAY_WORKER_LOAD_MODE:"adaptive", AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$hard, AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$global, AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global }}')" >/dev/null kubectl --context "$context" -n "$namespace" rollout restart \ deployment/easyai-capacity-controller >/dev/null kubectl --context "$context" -n "$namespace" rollout status \ deployment/easyai-capacity-controller --timeout=10m kubectl --context "$context" -n "$namespace" scale deployment/easyai-worker-ningbo --replicas=2 >/dev/null kubectl --context "$context" -n "$namespace" scale deployment/easyai-worker-hongkong --replicas=1 >/dev/null for deployment in easyai-worker-ningbo easyai-worker-hongkong; do kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \ AI_GATEWAY_WORKER_LOAD_MODE=adaptive \ AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$hard_limit" >/dev/null kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m done for deployment in easyai-api-ningbo easyai-api-hongkong; do kubectl --context "$context" -n "$namespace" rollout restart deployment/"$deployment" >/dev/null kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m done wait_for_adaptive_workloads_settled local ready ready=$(kubectl --context "$context" -n "$namespace" get pods -l app.kubernetes.io/name=easyai-worker -o json | jq '[.items[] | select(.status.phase=="Running" and any(.status.containerStatuses[]?; .ready==true))] | length') [[ $ready == 3 ]] || fail_gate adaptive_worker_count "adaptive validation requires exactly three Ready Workers" } run_adaptive_level() { local concurrency=$1 passed=true etcd_healthy=true local gemini_report="$report_root/adaptive-load-${concurrency}-gemini.json" local video_report="$report_root/adaptive-load-${concurrency}-video.json" verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || fail_gate local_control_plane_restarted "local K3s server identity changed before adaptive level $concurrency" record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || fail_gate local_control_plane_identity "failed to record adaptive level $concurrency control-plane baseline" if ! run_load gemini-multi-image "$gemini_report" -requests "$concurrency"; then passed=false fi if [[ $passed == true ]] && ! run_load video-throughput "$video_report" -requests "$concurrency"; then passed=false fi local critical critical=$(database_query "SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND pressure_state='critical' AND heartbeat_at > now() - interval '30 seconds';") if ((critical > 0)); then passed=false fi if ! verify_local_etcd_runtime_logs "$cluster_name" "$identity_file"; then passed=false etcd_healthy=false record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || return 1 fi jq -n --argjson concurrency "$concurrency" --argjson passed "$passed" \ --argjson etcdHealthy "$etcd_healthy" \ --arg gemini "$(basename "$gemini_report")" --arg video "$(basename "$video_report")" \ '{concurrency:$concurrency,passed:$passed,etcdHealthy:$etcdHealthy,reports:[$gemini,$video]}' \ >"$report_root/adaptive-level-${concurrency}.json" chmod 0600 "$report_root/adaptive-level-${concurrency}.json" [[ $passed == true ]] } build_adaptive_report() { local output=$1 samples=$2 resources=$3 highest=$4 failed=${5:-0} local levels workers distribution routing limits concurrency_peaks leaks duplicates sample_summary resource_summary phases levels=$(jq -s 'sort_by(.concurrency)' "$report_root"/adaptive-level-*.json) workers=$(database_query " SELECT COALESCE(jsonb_agg(jsonb_build_object( 'instanceId',instance_id,'podName',pod_name,'site',site,'status',status, 'allocatedCapacity',allocated_capacity,'hardCapacityLimit',hard_capacity_limit, 'safeCapacity',safe_capacity,'heavyCapacity',heavy_capacity, 'activeTasks',active_tasks,'preparingTasks',preparing_tasks, 'waitingUpstreamTasks',waiting_upstream_tasks,'finalizingTasks',finalizing_tasks, 'pressureState',pressure_state,'pressureReason',pressure_reason, 'loadSampledAt',load_sampled_at ) ORDER BY site,instance_id),'[]'::jsonb) FROM gateway_worker_instances WHERE heartbeat_at > now() - interval '30 seconds';") distribution=$(database_query " SELECT COALESCE(jsonb_agg(jsonb_build_object('workerInstanceId',worker_id,'tasks',tasks) ORDER BY worker_id),'[]'::jsonb) FROM ( SELECT COALESCE( regexp_replace(job.attempted_by[cardinality(job.attempted_by)], '-exec-[^-]+-[^-]+$', ''), 'unclaimed' ) worker_id, count(*) tasks FROM gateway_tasks task LEFT JOIN river_job job ON job.id = task.river_job_id WHERE task.acceptance_run_id='$run_id'::uuid GROUP BY worker_id ) grouped;") routing=$(database_query " SELECT COALESCE(jsonb_agg(jsonb_build_object( 'platformId',platform_id,'platformModelId',platform_model_id,'attempts',attempts ) ORDER BY platform_id,platform_model_id),'[]'::jsonb) FROM (SELECT platform_id::text,platform_model_id::text,count(*) attempts FROM gateway_task_attempts WHERE task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY platform_id,platform_model_id) grouped;") limits=$(database_query " SELECT COALESCE(jsonb_agg(jsonb_build_object( 'scopeType',scope_type,'scopeKey',scope_key,'metric',metric, 'limit',limit_value,'used',used_value,'reserved',reserved_value, 'windowStart',window_start,'resetAt',reset_at ) ORDER BY scope_type,scope_key,metric,window_start),'[]'::jsonb) FROM gateway_rate_limit_counters WHERE scope_key LIKE 'acceptance:$run_id:%';") concurrency_peaks=$(database_query " WITH events AS ( SELECT scope_type,scope_key,limit_value,acquired_at event_at,lease_value delta FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%' UNION ALL SELECT scope_type,scope_key,limit_value,LEAST(COALESCE(released_at,expires_at),expires_at),-lease_value FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%' ), points AS ( SELECT scope_type,scope_key,limit_value, SUM(delta) OVER (PARTITION BY scope_type,scope_key ORDER BY event_at,delta ROWS UNBOUNDED PRECEDING) active FROM events ), peaks AS ( SELECT scope_type,scope_key,MAX(limit_value) limit_value,COALESCE(MAX(active),0) peak FROM points GROUP BY scope_type,scope_key ) SELECT COALESCE(jsonb_agg(jsonb_build_object('scopeType',scope_type,'scopeKey',scope_key,'limit',limit_value,'peak',peak) ORDER BY scope_type,scope_key),'[]'::jsonb) FROM peaks;") leaks=$(database_query " SELECT jsonb_build_object( 'activeConcurrencyLeases',(SELECT count(*) FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%' AND released_at IS NULL AND expires_at>now()), 'activeRateReservations',(SELECT count(*) FROM gateway_rate_limit_reservations WHERE scope_key LIKE 'acceptance:$run_id:%' AND status='reserved'), 'activeRiverJobs',(SELECT count(*) FROM river_job job JOIN gateway_tasks task ON task.river_job_id=job.id WHERE task.acceptance_run_id='$run_id'::uuid AND job.state IN ('available','pending','retryable','running','scheduled')) );") duplicates=$(database_query " SELECT jsonb_build_object( 'remoteTaskIds',(SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) duplicate), 'billingTransactions',(SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) duplicate) );") sample_summary=$(jq -s '[.[].workers[]] | group_by(.instanceId) | map({ instanceId:.[0].instanceId, peakActiveTasks:(map(.activeTasks)|max), peakPreparingTasks:(map(.preparingTasks)|max), peakWaitingUpstreamTasks:(map(.waitingUpstreamTasks)|max), peakFinalizingTasks:(map(.finalizingTasks)|max), minSafeCapacity:(map(.safeCapacity)|min), maxSafeCapacity:(map(.safeCapacity)|max), pressureStates:(map(.pressureState)|unique) })' "$samples") resource_summary=$(jq -Rn ' def number_without($suffix): rtrimstr($suffix) | tonumber; def memory_mib: if endswith("Gi") then number_without("Gi") * 1024 elif endswith("Mi") then number_without("Mi") elif endswith("Ki") then number_without("Ki") / 1024 else tonumber end; [inputs | split(",") | select(.[0] != "timestamp") | { scope:.[1],name:.[2],cpu:.[3],memory:.[4] }] | group_by([.scope,.name]) | map(if .[0].scope == "node" then { scope:"node",name:.[0].name, maxCpuPercent:(map(.cpu | number_without("%")) | max), maxMemoryPercent:(map(.memory | number_without("%")) | max) } else { scope:"pod",name:.[0].name, maxCpuMillicores:(map(.cpu | if endswith("m") then number_without("m") else tonumber * 1000 end) | max), maxMemoryMiB:(map(.memory | memory_mib) | max) } end)' <"$resources") phases=$(jq -s '[.[] | .phases[]]' "$report_root"/adaptive-load-*.json) local passed=true bottleneck=configured_probe_ceiling if ((highest < 1)); then passed=false; fi if ((failed > 0)); then bottleneck=worker_or_gateway_limit; fi if jq -e 'any(.[]; .etcdHealthy == false)' <<<"$levels" >/dev/null; then bottleneck=local_control_plane_limit fi if ! jq -e 'length == 3' <<<"$workers" >/dev/null; then passed=false; fi if ! jq -e 'all(.[]; ((.used + .reserved) <= .limit))' <<<"$limits" >/dev/null; then passed=false; fi if ! jq -e 'all(.[]; (.limit == null or .peak <= .limit))' <<<"$concurrency_peaks" >/dev/null; then passed=false; fi if ! jq -e '.activeConcurrencyLeases==0 and .activeRateReservations==0 and .activeRiverJobs==0' <<<"$leaks" >/dev/null; then passed=false; fi if ! jq -e '.remoteTaskIds==0 and .billingTransactions==0' <<<"$duplicates" >/dev/null; then passed=false; fi jq -n \ --arg runId "$run_id" --arg startedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ --arg bottleneck "$bottleneck" --argjson passed "$passed" \ --argjson highest "$highest" --argjson failed "$failed" \ --argjson levels "$levels" --argjson phases "$phases" \ --argjson workers "$workers" --argjson distribution "$distribution" \ --argjson routing "$routing" --argjson limits "$limits" \ --argjson concurrencyPeaks "$concurrency_peaks" --argjson workerPeaks "$sample_summary" \ --argjson resourcePeaks "$resource_summary" \ --argjson leaks "$leaks" --argjson duplicates "$duplicates" \ '{ schemaVersion:"acceptance-load-report/v2",runId:$runId,profile:"adaptive-capacity", startedAt:$startedAt,finishedAt:(now|todateiso8601),passed:$passed,secretSafe:true, phases:$phases, capacity:{levels:$levels,highestStableConcurrency:$highest,firstFailedConcurrency:$failed,bottleneck:$bottleneck}, cluster:{workers:$workers,workerPeaks:$workerPeaks,taskDistribution:$distribution, platformRouting:$routing,rateLimits:$limits,concurrencyPeaks:$concurrencyPeaks, resourcePeaks:$resourcePeaks,leaks:$leaks,duplicates:$duplicates} }' >"$output" chmod 0600 "$output" jq -e '.passed==true and .secretSafe==true' "$output" >/dev/null } sample_resources() { local output=$1 stop_file=$2 printf 'timestamp,scope,name,cpu,memory\n' >"$output" while [[ ! -e $stop_file ]]; do kubectl --context "$context" top nodes --no-headers 2>/dev/null | awk -v timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ '{print timestamp ",node," $1 "," $3 "," $5}' >>"$output" || true kubectl --context "$context" -n "$namespace" top pods --no-headers 2>/dev/null | awk -v timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ '{print timestamp ",pod," $1 "," $2 "," $3}' >>"$output" || true sleep 5 done } verify_hard_gates() { local unhealthy ready sync_state connections max_connections queue duplicate_remote duplicate_billing verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || fail_gate local_control_plane_restarted "local K3s server identity changed" verify_local_apiserver_ready "$context" || fail_gate local_control_plane_unavailable "Kubernetes API readiness failed" verify_local_node_capacity "$context" || fail_gate local_node_capacity_mismatch "node allocatable resources drifted from the acceptance envelope" verify_local_etcd_runtime_logs "$cluster_name" "$identity_file" || fail_gate local_etcd_latency "etcd emitted a severe latency or timeout signal during the run" ready=$(kubectl --context "$context" get nodes -o json | jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') [[ $ready == 3 ]] || fail_gate local_nodes_not_ready "not all three local K3s nodes are Ready" [[ $(kubectl --context "$context" get nodes -o json | jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] || fail_gate local_node_memory_pressure "a local K3s node reports MemoryPressure" [[ $(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \ -o 'jsonpath={.status.readyInstances}') == 2 ]] || fail_gate local_postgres_not_ready "local PostgreSQL is not 2/2 Ready" sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;") [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] || fail_gate local_postgres_not_synchronous "local PostgreSQL lost its synchronous replica" unhealthy=$(kubectl --context "$context" -n "$namespace" get pods -o json | jq '[.items[] | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or .metadata.labels["app.kubernetes.io/name"] == "easyai-worker") | .status.containerStatuses[]? | select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or .state.terminated.reason == "OOMKilled")] | length') [[ $unhealthy == 0 ]] || fail_gate gateway_pod_restarted "an API or Worker container restarted or was OOMKilled" while read -r _node _cpu _cpu_percent _memory memory_percent; do memory_percent=${memory_percent%\%} [[ $memory_percent =~ ^[0-9]+$ ]] (( memory_percent < 80 )) || fail_gate local_node_memory_target "local node memory reached the 80 percent target" done < <(kubectl --context "$context" top nodes --no-headers) while read -r _pod _cpu memory; do case $memory in *Gi) memory=$(awk -v value="${memory%Gi}" 'BEGIN {printf "%.0f", value*1024}') ;; *Mi) memory=${memory%Mi} ;; *Ki) memory=$(awk -v value="${memory%Ki}" 'BEGIN {printf "%.0f", value/1024}') ;; *) return 1 ;; esac (( memory < 1536 )) || fail_gate gateway_pod_memory_limit "an API or Worker Pod reached 1.5 GiB RSS" done < <(kubectl --context "$context" -n "$namespace" top pods \ -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers) connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';") max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';") (( connections < 150 && connections * 4 < max_connections * 3 )) || fail_gate postgres_connection_budget "PostgreSQL client sessions exceeded the acceptance budget" queue=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;") [[ $queue == 0:0 ]] || fail_gate acceptance_queue_not_drained "acceptance queue did not return to zero" duplicate_remote=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) d;") duplicate_billing=$(database_query "SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) d;") [[ $duplicate_remote == 0 ]] || fail_gate duplicate_upstream_submission "duplicate remote task IDs were detected" [[ $duplicate_billing == 0 ]] || fail_gate duplicate_billing "duplicate billing transactions were detected" kubectl --context "$context" -n "$namespace" exec deployment/easyai-acceptance-callback-collector -- \ wget -qO- http://127.0.0.1:8091/report | jq -e '.duplicates == 0 and .invalid == 0' >/dev/null || fail_gate callback_validation_failed "callback collector detected duplicate or invalid callbacks" } apply_profile() { local profile=$1 slots pool case $profile in P24) slots=24; pool=32 ;; P28) slots=28; pool=36 ;; P32) slots=32; pool=40 ;; *) return 64 ;; esac local global=$((slots * 2)) kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \ --type=merge -p "$(jq -cn \ --arg slots "$slots" --arg global "$global" \ '{data:{ AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false", AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots, AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots, AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global, AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$global }}')" >/dev/null kubectl --context "$context" -n "$namespace" scale \ deployment/easyai-worker-ningbo deployment/easyai-worker-hongkong --replicas=1 >/dev/null for deployment in easyai-worker-ningbo easyai-worker-hongkong; do kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \ AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \ AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \ AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \ AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \ AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m done for deployment in easyai-api-ningbo easyai-api-hongkong easyai-capacity-controller; do kubectl --context "$context" -n "$namespace" rollout restart deployment/"$deployment" >/dev/null kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m done } restore_profile_best_effort() { local profile=$1 slots pool global case $profile in P24) slots=24; pool=32 ;; P28) slots=28; pool=36 ;; P32) slots=32; pool=40 ;; *) return ;; esac global=$((slots * 2)) kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \ --type=merge -p "$(jq -cn \ --arg slots "$slots" --arg global "$global" \ '{data:{ AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false", AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots, AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots, AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global, AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$global }}')" >/dev/null 2>&1 || true kubectl --context "$context" -n "$namespace" scale \ deployment/easyai-worker-ningbo deployment/easyai-worker-hongkong \ --replicas=1 >/dev/null 2>&1 || true for deployment in easyai-worker-ningbo easyai-worker-hongkong; do kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \ AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \ AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \ AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \ AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \ AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null 2>&1 || true done } run_profile_round() { local profile=$1 repetition=$2 prefix prefix="$report_root/${profile,,}-$repetition" local stop_file="$prefix.resources.stop" sample_resources "$prefix.resources.csv" "$stop_file" & local sampler=$! run_load gemini-baseline "$prefix-gemini-baseline.json" run_load gemini-large "$prefix-gemini-large.json" run_load gemini-peak "$prefix-gemini-peak.json" run_load video-throughput "$prefix-video-throughput.json" run_recovery "$prefix-video-recovery.json" touch "$stop_file" wait "$sampler" verify_hard_gates } wait_for_recovery_owner() { local deadline=$((SECONDS + 120)) owner while (( SECONDS < deadline )); do owner=$(database_query " SELECT task.id::text||','||worker.pod_name||','||worker.site FROM gateway_tasks task JOIN gateway_worker_instances worker ON worker.instance_id=task.locked_by WHERE task.acceptance_run_id='$run_id'::uuid AND task.status='running' AND task.request::text LIKE '%acceptance-long-recovery%' AND worker.status='active' ORDER BY task.updated_at LIMIT 1;") [[ -z $owner ]] || { printf '%s\n' "$owner"; return; } sleep 1 done return 1 } run_recovery() { local report_path=$1 owner task_id pod site run_load video-recovery "$report_path" & active_load_pid=$! owner=$(wait_for_recovery_owner) IFS=',' read -r task_id pod site <<<"$owner" [[ $task_id =~ ^[0-9a-f-]{36}$ && $pod == easyai-worker-"$site"-* ]] kubectl --context "$context" -n "$namespace" delete pod "$pod" \ --force --grace-period=0 --wait=false >/dev/null kubectl --context "$context" -n "$namespace" rollout status \ deployment/easyai-worker-"$site" --timeout=10m wait "$active_load_pid" active_load_pid= } run_fault_matrix() { local duration=${AI_GATEWAY_LOCAL_WEAK_LINK_DURATION:-5m} "$script_dir/network-fault.sh" weak-link netem_active=true run_load mixed-soak "$report_root/fault-weak-link.json" \ -duration "$duration" -image-rate 1 -video-rate 1 "$script_dir/network-fault.sh" reset netem_active=false run_load video-throughput "$report_root/fault-upstream-outage.json" & active_load_pid=$! sleep 5 "$script_dir/network-fault.sh" upstream-outage netem_active=true sleep 10 "$script_dir/network-fault.sh" reset netem_active=false wait "$active_load_pid" active_load_pid= run_load video-recovery "$report_root/fault-database-outage.json" & active_load_pid=$! sleep 10 "$script_dir/network-fault.sh" database-outage hongkong netem_active=true sleep 30 "$script_dir/network-fault.sh" reset netem_active=false wait "$active_load_pid" active_load_pid= local leader leader=$(kubectl --context "$context" -n "$namespace" get pods \ -l app.kubernetes.io/name=easyai-capacity-controller -o name | while read -r pod; do status=$(kubectl --context "$context" -n "$namespace" exec "$pod" -- \ wget -qO- http://127.0.0.1:8088/status) [[ $(jq -r '.leader' <<<"$status") == true ]] && { printf '%s\n' "${pod#pod/}"; break; } done) [[ -n $leader ]] kubectl --context "$context" -n "$namespace" delete pod "$leader" --wait=false >/dev/null kubectl --context "$context" -n "$namespace" rollout status \ deployment/easyai-capacity-controller --timeout=5m verify_hard_gates } run_autoscaling() { local profile=$1 slots slots=${profile#P} kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \ --type=merge -p "$(jq -cn --arg global "$((slots * 4))" \ '{data:{ AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"true", AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:"1", AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:"1", AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:"2", AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:"2", AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global }}')" >/dev/null kubectl --context "$context" -n "$namespace" rollout restart \ deployment/easyai-capacity-controller >/dev/null kubectl --context "$context" -n "$namespace" rollout status \ deployment/easyai-capacity-controller --timeout=5m run_load video-throughput "$report_root/autoscaling-load.json" & active_load_pid=$! local deadline=$((SECONDS + 420)) total=2 while (( SECONDS < deadline )); do total=$(kubectl --context "$context" -n "$namespace" get deployment \ easyai-worker-ningbo easyai-worker-hongkong \ -o json | jq '[.items[].spec.replicas] | add') (( total >= 3 )) && break sleep 5 done (( total >= 3 )) wait "$active_load_pid" active_load_pid= deadline=$((SECONDS + 780)) while (( SECONDS < deadline )); do total=$(kubectl --context "$context" -n "$namespace" get deployment \ easyai-worker-ningbo easyai-worker-hongkong \ -o json | jq '[.items[].spec.replicas] | add') (( total == 2 )) && break sleep 10 done (( total == 2 )) verify_hard_gates } artifact_smoke() { local manifest=$1 node "$repository_root/scripts/release-manifest.mjs" validate "$manifest" >/dev/null local manifest_sha api_image web_image api_digest artifact_report manifest_sha=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" sourceSha) api_image=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.api) web_image=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.web) api_digest=${api_image##*@} [[ $manifest_sha == "$release_sha" && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]] docker pull --platform linux/amd64 "$api_image" >/dev/null docker pull --platform linux/amd64 "$web_image" >/dev/null "$private_root/tools/k3d" image import -c easyai-acceptance-local "$api_image" "$web_image" for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo \ easyai-worker-hongkong easyai-capacity-controller easyai-acceptance-emulator \ easyai-acceptance-callback-collector; do kubectl --context "$context" -n "$namespace" set image deployment/"$deployment" \ "*=$api_image" >/dev/null kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=15m done for deployment in easyai-acceptance-edge-ningbo easyai-acceptance-edge-hongkong; do kubectl --context "$context" -n "$namespace" set image deployment/"$deployment" \ "*=$web_image" >/dev/null kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m done kubectl --context "$context" -n "$namespace" delete job easyai-artifact-migrate \ --ignore-not-found --wait=true >/dev/null cat </dev/null apiVersion: batch/v1 kind: Job metadata: name: easyai-artifact-migrate namespace: easyai spec: backoffLimit: 0 template: spec: restartPolicy: Never containers: - name: migrate image: $api_image command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"] envFrom: - secretRef: name: easyai-ai-gateway-runtime EOF kubectl --context "$context" -n "$namespace" wait \ --for=condition=complete job/easyai-artifact-migrate --timeout=10m >/dev/null run_load simulated-smoke "$report_root/artifact-simulated-smoke.json" verify_hard_gates artifact_report="$report_root/artifact-smoke.json" jq -n \ --arg releaseSha "$manifest_sha" \ --arg apiImageDigest "$api_digest" \ --arg webImageDigest "${web_image##*@}" \ '{ schemaVersion:"acceptance-artifact-smoke/v1", releaseSha:$releaseSha, apiImageDigest:$apiImageDigest, webImageDigest:$webImageDigest, architecture:"linux/amd64", migrationSmoke:true, startupSmoke:true, mediaSmoke:true, passed:true, secretSafe:true }' >"$artifact_report" chmod 0600 "$artifact_report" } quick() { current_phase=quick run_load simulated-smoke "$report_root/quick.json" verify_hard_gates echo "local_acceptance_quick=PASS run_id=$run_id report=$report_root/quick.json" } adaptive() { local hard_limit=${AI_GATEWAY_LOCAL_ADAPTIVE_HARD_LIMIT:-32} local max_probe=${AI_GATEWAY_LOCAL_ADAPTIVE_MAX_CONCURRENCY:-$((hard_limit * 3))} local level=4 highest=0 failed=0 low high mid local samples="$report_root/adaptive-worker-samples.ndjson" local resources="$report_root/adaptive-resources.csv" worker_sample_stop="$report_root/adaptive-worker-samples.stop" [[ ! -e $samples && ! -e $resources && ! -e $worker_sample_stop ]] || fail_gate acceptance_report_exists "adaptive evidence already exists" current_phase=adaptive_workers configure_adaptive_workers sample_worker_runtime "$samples" "$worker_sample_stop" & active_worker_sampler_pid=$! sample_resources "$resources" "$worker_sample_stop" & active_resource_sampler_pid=$! while ((level <= max_probe)); do current_phase="adaptive_capacity_$level" if run_adaptive_level "$level"; then highest=$level level=$((level * 2)) else failed=$level break fi done if ((failed > 0 && highest > 0)); then low=$((highest + 1)) high=$((failed - 1)) while ((low <= high)); do mid=$(((low + high) / 2)) current_phase="adaptive_capacity_refine_$mid" if run_adaptive_level "$mid"; then highest=$mid low=$((mid + 1)) else failed=$mid high=$((mid - 1)) fi done fi touch "$worker_sample_stop" wait "$active_worker_sampler_pid" wait "$active_resource_sampler_pid" active_worker_sampler_pid= active_resource_sampler_pid= worker_sample_stop= local ready_workers registered_workers ready_workers=$(kubectl --context "$context" -n "$namespace" get pods \ -l app.kubernetes.io/name=easyai-worker -o json | jq '[.items[] | select(.status.phase=="Running" and any(.status.containerStatuses[]?; .ready==true))] | length') registered_workers=$(database_query "SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';") [[ $ready_workers == 3 && $registered_workers == 3 ]] || fail_gate adaptive_worker_count "adaptive validation requires three Ready and registered Workers throughout the final gate" verify_hard_gates current_phase=adaptive_report build_adaptive_report "$report_root/adaptive-capacity.json" "$samples" "$resources" "$highest" "$failed" echo "local_acceptance_adaptive=PASS run_id=$run_id highest_stable_concurrency=$highest first_failed_concurrency=$failed report=$report_root/adaptive-capacity.json" } full() { local manifest=$1 profile repetition quick for profile in P24 P28 P32; do current_phase="capacity_${profile,,}_apply" apply_profile "$profile" for repetition in 1 2 3; do current_phase="capacity_${profile,,}_round_$repetition" run_profile_round "$profile" "$repetition" done stable_profile=$profile done current_phase=fault_matrix run_fault_matrix current_phase=autoscaling_and_drain run_autoscaling "$stable_profile" current_phase=certified_soak run_load mixed-soak "$report_root/mixed-soak.json" \ -duration "${AI_GATEWAY_LOCAL_SOAK_DURATION:-2h}" \ -image-rate "${AI_GATEWAY_LOCAL_CERTIFIED_IMAGE_RATE:-1}" \ -video-rate "${AI_GATEWAY_LOCAL_CERTIFIED_VIDEO_RATE:-1}" current_phase=overload_shedding run_load mixed-overload "$report_root/mixed-overload.json" \ -duration "${AI_GATEWAY_LOCAL_OVERLOAD_DURATION:-10m}" \ -image-rate "${AI_GATEWAY_LOCAL_OVERLOAD_IMAGE_RATE:-2}" \ -video-rate "${AI_GATEWAY_LOCAL_OVERLOAD_VIDEO_RATE:-2}" current_phase=amd64_artifact_smoke artifact_smoke "$manifest" current_phase=final_report node "$script_dir/report.mjs" build-local \ --runtime "$runtime" \ --snapshot "$snapshot" \ --reports "$report_root" \ --artifact "$report_root/artifact-smoke.json" \ --api-digest "$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.api | sed 's/.*@//')" \ --certified-profile "$stable_profile" \ --output "$report_root/acceptance-report.json" echo "local_acceptance_full=PASS run_id=$run_id certified_profile=$stable_profile report=$report_root/acceptance-report.json" } command=${1:-} shift || true case $command in quick|adaptive|provider-burst) [[ $# -eq 0 ]] || { usage >&2; exit 64; } ;; artifact-smoke|full) [[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; } ;; *) usage >&2 exit 64 ;; esac "$script_dir/local-cluster.sh" new-run require_local_cluster load_environment case $command in quick) quick ;; adaptive) adaptive ;; provider-burst) provider_burst ;; artifact-smoke) artifact_smoke "$2" ;; full) full "$2" ;; esac