diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 66e5a0c..4590746 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -12,8 +12,9 @@ Usage: This is a destructive production acceptance workflow. It pauses new live tasks, changes Worker capacity, creates paid real canaries, and deletes one Worker Pod -during each recovery phase. A failed run deliberately keeps traffic in -validation mode and restores only the last stable capacity profile. +during each recovery phase. It also grants the dedicated acceptance API Key +access to the selected production candidates. A failed run deliberately keeps +traffic in validation mode and restores only the last stable capacity profile. Required private environment values: AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN @@ -262,9 +263,114 @@ LIMIT 1;") echo 'no enabled Gemini image-edit or omni-video max_images>=9 model was found' >&2 return 1 } + [[ $AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL =~ ^[A-Za-z0-9._:/+-]+$ && + $AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL =~ ^[A-Za-z0-9._:/+-]+$ ]] || { + echo 'acceptance model invocation names contain unsupported characters' >&2 + return 1 + } export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL } +ensure_acceptance_model_access() { + local resources response body missing expected + response=$temporary_root/model-access.json + resources=$(database_query " +WITH selected_candidates AS ( + SELECT DISTINCT platform.id AS platform_id, model.id AS platform_model_id, model.base_model_id + FROM platform_models model + JOIN integration_platforms platform ON platform.id=model.platform_id + JOIN base_model_catalog base_model ON base_model.id=model.base_model_id + WHERE platform.status='enabled' + AND platform.deleted_at IS NULL + AND model.enabled=true + AND base_model.invocation_name IN ( + '$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL', + '$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL' + ) +), resources AS ( + SELECT 'platform'::text AS resource_type, platform_id AS resource_id + FROM selected_candidates + UNION + SELECT 'platform_model', platform_model_id + FROM selected_candidates + UNION + SELECT 'base_model', base_model_id + FROM selected_candidates + WHERE base_model_id IS NOT NULL +) +SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'resourceType', resource_type, + 'resourceId', resource_id, + 'priority', 100, + 'minPermissionLevel', 0, + 'status', 'active' + ) + ORDER BY resource_type, resource_id + ), + '[]'::jsonb +)::text +FROM resources;") + expected=$(jq 'length' <<<"$resources") + (( expected > 0 )) || { + echo 'selected acceptance models did not resolve to production candidate resources' >&2 + return 1 + } + body=$(jq -cn \ + --arg subjectID "$AI_GATEWAY_ACCEPTANCE_API_KEY_ID" \ + --argjson resources "$resources" \ + '{ + subjectType:"api_key", + subjectId:$subjectID, + effect:"allow", + upsertResources:$resources, + deleteResources:[] + }') + admin_request POST /api/admin/access-rules/batch "$body" "$response" + missing=$(database_query " +WITH selected_candidates AS ( + SELECT DISTINCT platform.id AS platform_id, model.id AS platform_model_id, model.base_model_id + FROM platform_models model + JOIN integration_platforms platform ON platform.id=model.platform_id + JOIN base_model_catalog base_model ON base_model.id=model.base_model_id + WHERE platform.status='enabled' + AND platform.deleted_at IS NULL + AND model.enabled=true + AND base_model.invocation_name IN ( + '$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL', + '$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL' + ) +), resources AS ( + SELECT 'platform'::text AS resource_type, platform_id AS resource_id + FROM selected_candidates + UNION + SELECT 'platform_model', platform_model_id + FROM selected_candidates + UNION + SELECT 'base_model', base_model_id + FROM selected_candidates + WHERE base_model_id IS NOT NULL +) +SELECT count(*) +FROM resources resource +WHERE NOT EXISTS ( + SELECT 1 + FROM gateway_access_rules rule + WHERE rule.subject_type='api_key' + AND rule.subject_id='$AI_GATEWAY_ACCEPTANCE_API_KEY_ID'::uuid + AND rule.effect='allow' + AND rule.status='active' + AND rule.resource_type=resource.resource_type + AND rule.resource_id=resource.resource_id +);") + [[ $missing == 0 ]] || { + echo "acceptance API Key is missing $missing selected candidate access rules" >&2 + return 1 + } + echo "acceptance_model_access=PASS resources=$expected" +} + deploy_protocol_emulator() { sed "s|image: easyai-api|image: $api_image|" \ "$cluster_root/deploy/kubernetes/acceptance/protocol-emulator.yaml" | @@ -964,6 +1070,7 @@ finish_and_promote() { verify_release_cas select_acceptance_models +ensure_acceptance_model_access deploy_protocol_emulator create_and_activate_run wait_for_existing_tasks_to_drain