#!/usr/bin/env bash # shellcheck disable=SC2034,SC2329 # Symbols are consumed by dynamically sourced functions. set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT script=$root/scripts/cluster/run-production-acceptance.sh # shellcheck source=scripts/cluster/common.sh source "$root/scripts/cluster/common.sh" ssh() { printf '%s\n' "$@" >"$tmp/ssh-arguments" } cluster_ssh_key=$tmp/test-key cluster_ssh_control_dir=$tmp/ssh-control cluster_ssh root@cluster.invalid true grep -Fxq 'ConnectionAttempts=3' "$tmp/ssh-arguments" grep -Fxq 'ControlMaster=auto' "$tmp/ssh-arguments" grep -Fxq 'ControlPersist=120' "$tmp/ssh-arguments" grep -Fxq "ControlPath=$cluster_ssh_control_dir/%C" "$tmp/ssh-arguments" [[ $(stat -f '%Lp' "$cluster_ssh_control_dir") == 700 ]] awk ' /^is_release_ancestor\(\)/ { capture = 1 } capture && /^create_and_activate_run\(\)/ { exit } capture { print } ' "$script" >"$tmp/is-release-ancestor.sh" # shellcheck source=/dev/null source "$tmp/is-release-ancestor.sh" awk ' /^workload_metrics_for_site\(\)/ { capture = 1 } capture && /^current_max_pod_memory_mib\(\)/ { exit } capture { print } ' "$script" >"$tmp/pressure-monitor.sh" # shellcheck source=/dev/null source "$tmp/pressure-monitor.sh" cluster_root=$tmp/repository git init -q "$cluster_root" git -C "$cluster_root" config user.name 'Acceptance Script Test' git -C "$cluster_root" config user.email 'acceptance-script-test@example.invalid' git -C "$cluster_root" commit --allow-empty -qm first first=$(git -C "$cluster_root" rev-parse HEAD) git -C "$cluster_root" commit --allow-empty -qm second git -C "$cluster_root" commit --allow-empty -qm third third=$(git -C "$cluster_root" rev-parse HEAD) is_release_ancestor "$first" "$third" if is_release_ancestor "$third" "$first"; then echo 'descendant release was accepted as an ancestor' >&2 exit 1 fi if is_release_ancestor invalid "$third"; then echo 'invalid release identifier was accepted as an ancestor' >&2 exit 1 fi remote_kubectl() { if [[ $* == *'get pods'* ]]; then printf '{"items":[]}' else printf 0 fi } cluster_ssh() { echo 'zero-replica Worker attempted to collect metrics' >&2 return 1 } namespace=easyai if workload_metrics_for_site worker ningbo; then echo 'zero-replica Worker site returned metrics success' >&2 exit 1 else metrics_status=$? fi [[ $metrics_status == 2 ]] remote_kubectl() { printf '%s\n' '{"items":[ {"metadata":{},"spec":{"nodeName":"easyai-hongkong"},"status":{"podIP":"10.42.3.10","conditions":[{"type":"Ready","status":"True"}]}}, {"metadata":{},"spec":{"nodeName":"easyai-hongkong-worker-2"},"status":{"podIP":"10.42.5.10","conditions":[{"type":"Ready","status":"True"}]}} ]}' } cluster_ssh() { while IFS= read -r _; do :; done printf '# target_host %s\n' "$1" printf 'test_metric 1\n' } CLUSTER_NINGBO_HOST=root@ningbo.invalid CLUSTER_HONGKONG_HOST=root@hongkong.invalid CLUSTER_LOS_ANGELES_HOST=root@losangeles.invalid CLUSTER_WORKER_NODE_4_NAME=easyai-hongkong-worker-2 CLUSTER_WORKER_NODE_4_HOST=root@hongkong-worker-2.invalid metrics=$(workload_metrics_for_site worker hongkong) grep -Fq '# target_host root@hongkong.invalid' <<<"$metrics" grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics" [[ $(grep -c '^test_metric 1$' <<<"$metrics") == 2 ]] [[ $(metric_min test_metric <<<"$metrics") == 1 ]] [[ $(metric_max test_metric <<<"$metrics") == 1 ]] [[ $(metric_sum test_metric <<<"$metrics") == 2 ]] passing_pressure_row='2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' pressure_row_passes_live_gates "$passing_pressure_row" for failed_pressure_row in \ '2026-08-01T00:00:00Z,0,0,901,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ '2026-08-01T00:00:00Z,0,0,900,150,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ '2026-08-01T00:00:00Z,0,0,900,149,200,80,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ '2026-08-01T00:00:00Z,0,0,900,149,200,79,1536,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,1,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,1,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,0,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,3,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1536,100'; do if pressure_row_passes_live_gates "$failed_pressure_row"; then echo "unsafe pressure row passed live gates: $failed_pressure_row" >&2 exit 1 fi done long_running_load() { while :; do sleep 1 done } (sleep 1; exit 1) & pressure_pid=$! AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS=1 active_load_pid= if run_with_pressure_monitor "$pressure_pid" long_running_load; then echo 'load continued after the pressure sampler failed' >&2 exit 1 fi wait "$pressure_pid" >/dev/null 2>&1 || true [[ -z $active_load_pid ]] echo 'production_acceptance_script_tests=PASS'