diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index f11ebd6..177abeb 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -132,10 +132,10 @@ fi : "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}" : "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}" : "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}" -: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=8}" -: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=8}" +: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=4}" +: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=4}" : "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS:=96}" -: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=8 12 16 24 32 40 48}" +: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=4 8 12 16 24 32 40 48}" : "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}" : "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}" : "${AI_GATEWAY_ONLINE_ACCOUNT:=}" @@ -383,6 +383,9 @@ cleanup() { if [[ $single_node_emulator_installed == true ]]; then cleanup_single_node_emulator >/dev/null 2>&1 || true fi + if [[ $single_node_acceptance == true ]]; then + cleanup_single_node_kubernetes_resources >/dev/null 2>&1 || true + fi if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then set +e if [[ $single_node_acceptance == true ]]; then @@ -3500,10 +3503,14 @@ apply_single_node_capacity() { local deadline=$((SECONDS + 90)) allocation while (( SECONDS < deadline )); do allocation=$(database_query " -SELECT count(*)||':'||COALESCE(max(allocated_capacity),0) +SELECT count(*)||':'|| + COALESCE(max(hard_capacity_limit),0)||':'|| + COALESCE(max(allocated_capacity),0)||':'|| + COALESCE(max(safe_capacity),0)||':'|| + COALESCE(max(pressure_state),'unknown') FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';") - if [[ $allocation == "1:$slots" ]]; then + if single_node_capacity_ready "$allocation" "$slots"; then local metrics_ready=false for _ in {1..30}; do if remote_kubectl top node easyai-ningbo --no-headers >/dev/null 2>&1 && @@ -3520,15 +3527,30 @@ WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';") return 1 } active_profile=S$slots - echo "single_node_capacity=PASS slots=$slots worker_pool=$worker_pool active_instances=1" + local active_instances hard_limit allocated safe_capacity pressure_state + IFS=':' read -r active_instances hard_limit allocated safe_capacity pressure_state <<<"$allocation" + echo "single_node_capacity=PASS hard_limit=$slots allocated=$allocated safe_capacity=$safe_capacity pressure_state=$pressure_state worker_pool=$worker_pool active_instances=1" return 0 fi sleep 2 done - echo "single-node Worker allocation did not converge: expected=1:$slots actual=${allocation:-missing}" >&2 + echo "single-node Worker capacity did not converge: expected=active:1 hard_limit:$slots allocated_and_safe:1..$slots pressure:not-critical actual=${allocation:-missing}" >&2 return 1 } +single_node_capacity_ready() { + local state=$1 + local expected_hard_limit=$2 + local active_instances hard_limit allocated safe_capacity pressure_state extra + IFS=':' read -r active_instances hard_limit allocated safe_capacity pressure_state extra <<<"$state" + [[ -z $extra && + $active_instances =~ ^[0-9]+$ && $active_instances -eq 1 && + $hard_limit =~ ^[0-9]+$ && $hard_limit -eq $expected_hard_limit && + $allocated =~ ^[0-9]+$ && $allocated -ge 1 && $allocated -le $expected_hard_limit && + $safe_capacity =~ ^[0-9]+$ && $safe_capacity -ge 1 && $safe_capacity -le $expected_hard_limit && + $pressure_state != critical ]] +} + install_single_node_load_driver() { [[ $remote_load_drivers_installed == false ]] || return 0 local binary_path env_file remote_env local_digest remote_digest @@ -3676,6 +3698,17 @@ record_single_node_result() { load:$load[0] }' >"$summary" chmod 0600 "$summary" + if (( peak_allocated < slots || peak_running < slots )); then + echo "single-node target concurrency was not observed: configured=$slots peak_allocated=$peak_allocated peak_running=$peak_running" >&2 + return 1 + fi +} + +cleanup_single_node_kubernetes_resources() { + remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \ + -n "$namespace" --ignore-not-found >/dev/null || true + remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \ + -n "$namespace" --ignore-not-found >/dev/null || true } run_single_node_profile() { @@ -3768,10 +3801,7 @@ FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;") admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response" failure_recorded=true cleanup_single_node_emulator - remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \ - -n "$namespace" --ignore-not-found >/dev/null || true - remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \ - -n "$namespace" --ignore-not-found >/dev/null || true + cleanup_single_node_kubernetes_resources } run_single_node_acceptance() { diff --git a/tests/release/production-acceptance-script-test.sh b/tests/release/production-acceptance-script-test.sh index b4ef8c8..37da5e8 100755 --- a/tests/release/production-acceptance-script-test.sh +++ b/tests/release/production-acceptance-script-test.sh @@ -67,6 +67,15 @@ awk ' # shellcheck source=/dev/null source "$tmp/admin-request.sh" +awk ' + /^single_node_capacity_ready\(\)/ { capture = 1 } + capture && /^install_single_node_load_driver\(\)/ { exit } + capture { print } +' "$script" >"$tmp/single-node-capacity.sh" + +# shellcheck source=/dev/null +source "$tmp/single-node-capacity.sh" + cluster_root=$tmp/repository git init -q "$cluster_root" git -C "$cluster_root" config user.name 'Acceptance Script Test' @@ -216,6 +225,23 @@ admin_request POST /api/admin/system/acceptance/runs/test-run/finish "$admin_bod [[ $(jq -r '.status' "$admin_output") == failed ]] [[ $(<"$admin_attempts") == 3 ]] +single_node_capacity_ready '1:8:4:4:normal' 8 +single_node_capacity_ready '1:8:8:8:busy' 8 +for invalid_capacity_state in \ + '0:8:4:4:normal' \ + '1:7:4:4:normal' \ + '1:8:0:4:normal' \ + '1:8:9:4:normal' \ + '1:8:4:0:normal' \ + '1:8:4:9:normal' \ + '1:8:4:4:critical' \ + '1:8:4:4:normal:extra'; do + if single_node_capacity_ready "$invalid_capacity_state" 8; then + echo "unsafe single-node capacity state passed: $invalid_capacity_state" >&2 + exit 1 + fi +done + passing_pressure_row='2026-08-01T00:00:00Z,0,0,899,149,200,84,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' pressure_row_passes_live_gates "$passing_pressure_row" [[ $(pressure_row_live_gate_reason '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100') == \