diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 065a52d..371a5e4 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -243,6 +243,9 @@ deploy_protocol_emulator() { create_and_activate_run() { local create_response=$temporary_root/create-run.json local activate_response=$temporary_root/activate-run.json + local traffic_response=$temporary_root/pre-activate-traffic.json + local previous_run_response=$temporary_root/previous-run.json + local abort_response=$temporary_root/abort-previous-run.json local body body=$(jq -cn \ --arg releaseSha "$release_sha" \ @@ -274,8 +277,55 @@ create_and_activate_run() { report_root=${AI_GATEWAY_ACCEPTANCE_REPORT_DIR:-"$cluster_root/dist/acceptance/$run_id"} mkdir -p "$report_root" chmod 0700 "$report_root" + + admin_request GET /api/admin/system/acceptance/traffic-mode '' "$traffic_response" + local traffic_mode previous_run_id previous_status + traffic_mode=$(jq -r '.mode // empty' "$traffic_response") + if [[ $traffic_mode == validation ]]; then + previous_run_id=$(jq -r '.runId // empty' "$traffic_response") + [[ $previous_run_id =~ ^[0-9a-f-]{36}$ && $previous_run_id != "$run_id" ]] || { + echo 'validation mode does not reference a different previous Run' >&2 + return 1 + } + [[ $(jq -r '.releaseSha // empty' "$traffic_response") == "$release_sha" && + $(jq -r '.apiImageDigest // empty' "$traffic_response") == "$api_digest" && + $(jq -r '.workerImageDigest // empty' "$traffic_response") == "$worker_digest" ]] || { + echo 'refusing to replace validation mode for a different production release' >&2 + return 1 + } + admin_request GET "/api/admin/system/acceptance/runs/$previous_run_id" '' \ + "$previous_run_response" + previous_status=$(jq -r '.status // empty' "$previous_run_response") + [[ $previous_status == failed ]] || { + echo "refusing to replace previous acceptance Run in status $previous_status" >&2 + return 1 + } + body=$(jq -cn \ + --argjson revision "$(jq -r '.revision' "$traffic_response")" \ + --arg releaseSha "$release_sha" \ + --arg apiDigest "$api_digest" \ + --arg workerDigest "$worker_digest" \ + '{ + revision:$revision, + releaseSha:$releaseSha, + apiImageDigest:$apiDigest, + workerImageDigest:$workerDigest + }') + admin_request POST \ + "/api/admin/system/acceptance/runs/$previous_run_id/abort" \ + "$body" "$abort_response" + [[ $(jq -r '.mode // empty' "$abort_response") == live ]] || { + echo 'previous failed acceptance Run did not return traffic to live' >&2 + return 1 + } + elif [[ $traffic_mode != live ]]; then + echo "unsupported production traffic mode before acceptance: $traffic_mode" >&2 + return 1 + fi + admin_request POST "/api/admin/system/acceptance/runs/$run_id/activate" '' "$activate_response" - [[ $(jq -r '.mode' "$activate_response") == validation ]] + [[ $(jq -r '.mode' "$activate_response") == validation && + $(jq -r '.runId' "$activate_response") == "$run_id" ]] } mark_run_failed() {