diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 6dc3777..d7bd6dd 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -43,7 +43,7 @@ release_manifest=$2 } load_cluster_env -require_commands curl git go jq node openssl sed +require_commands curl git jq node openssl sed : "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}" : "${AI_GATEWAY_ACCEPTANCE_API_KEY:?}" @@ -88,7 +88,6 @@ remote_release_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai temporary_root=$(mktemp -d) chmod 0700 "$temporary_root" current_manifest=$temporary_root/current.json -load_binary=$temporary_root/easyai-ai-gateway-acceptance-load run_token=$(openssl rand -hex 32) run_id= report_root= @@ -667,20 +666,68 @@ apply_capacity_profile() { run_load_profile() { local profile=$1 local report_path=$2 - ( - cd "$cluster_root/apps/api" - AI_GATEWAY_ACCEPTANCE_PROFILE=$profile \ - AI_GATEWAY_ACCEPTANCE_GATEWAYS=$AI_GATEWAY_ACCEPTANCE_GATEWAYS \ - AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME \ - AI_GATEWAY_ACCEPTANCE_EMULATOR_URL=http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090 \ - AI_GATEWAY_ACCEPTANCE_API_KEY=$AI_GATEWAY_ACCEPTANCE_API_KEY \ - AI_GATEWAY_ACCEPTANCE_RUN_ID=$run_id \ - AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=$run_token \ - AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL \ - AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL \ - AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS \ - "$load_binary" -profile "$profile" -report "$report_path" >/dev/null - ) + local load_pod remote_script + [[ $profile =~ ^[a-z-]+$ ]] || return 1 + load_pod=$(remote_kubectl get pods -n "$namespace" \ + -l 'app.kubernetes.io/name=easyai-acceptance-emulator' -o json | + jq -r '[.items[] + | select(any(.status.conditions[]?; .type=="Ready" and .status=="True")) + | .metadata.name][0] // empty') + [[ $load_pod =~ ^easyai-acceptance-emulator-[a-z0-9-]+$ ]] || { + echo 'acceptance load runner Pod is not Ready' >&2 + return 1 + } + # Feed private values through stdin so the API Key and Run Token never enter + # the Pod spec, remote command arguments, process list, or report. + # shellcheck disable=SC2016 # This script runs inside the release image. + remote_script=' +set -eu +read -r gateways_b64 +read -r tls_name_b64 +read -r emulator_b64 +read -r api_key_b64 +read -r run_id_b64 +read -r run_token_b64 +read -r gemini_model_b64 +read -r video_model_b64 +read -r image_urls_b64 +decode() { + printf "%s" "$1" | base64 -d +} +export AI_GATEWAY_ACCEPTANCE_GATEWAYS=$(decode "$gateways_b64") +export AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=$(decode "$tls_name_b64") +export AI_GATEWAY_ACCEPTANCE_EMULATOR_URL=$(decode "$emulator_b64") +export AI_GATEWAY_ACCEPTANCE_API_KEY=$(decode "$api_key_b64") +export AI_GATEWAY_ACCEPTANCE_RUN_ID=$(decode "$run_id_b64") +export AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=$(decode "$run_token_b64") +export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(decode "$gemini_model_b64") +export AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$(decode "$video_model_b64") +export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(decode "$image_urls_b64") +exec /app/easyai-ai-gateway-acceptance-load -profile "$1" +' + { + printf '%s' "$AI_GATEWAY_ACCEPTANCE_GATEWAYS" | openssl base64 -A + printf '\n' + printf '%s' "$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" | openssl base64 -A + printf '\n' + printf '%s' 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' | + openssl base64 -A + printf '\n' + printf '%s' "$AI_GATEWAY_ACCEPTANCE_API_KEY" | openssl base64 -A + printf '\n' + printf '%s' "$run_id" | openssl base64 -A + printf '\n' + printf '%s' "$run_token" | openssl base64 -A + printf '\n' + printf '%s' "$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL" | openssl base64 -A + printf '\n' + printf '%s' "$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" | openssl base64 -A + printf '\n' + printf '%s' "$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" | openssl base64 -A + printf '\n' + } | + remote_kubectl exec -i -n "$namespace" "$load_pod" -- \ + sh -c "$remote_script" -- "$profile" >"$report_path" } kill_one_worker_during_recovery() { @@ -1216,10 +1263,6 @@ ensure_acceptance_model_access deploy_protocol_emulator create_and_activate_run wait_for_existing_tasks_to_drain -( - cd "$cluster_root/apps/api" - go build -trimpath -o "$load_binary" ./cmd/acceptance-load -) for profile in P24 P28 P32; do if ! apply_capacity_profile "$profile"; then