|
|
|
@@ -30,12 +30,11 @@ with a user-directed waiver. It never records them as passed and preserves all
|
|
|
|
|
online simulation, real canary, resource, consistency, and release CAS gates.
|
|
|
|
|
|
|
|
|
|
Required private environment values:
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_API_KEY
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_API_KEY_ID
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_USER_ID
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_GATEWAYS
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN, or AI_GATEWAY_ONLINE_ACCOUNT/PASSWORD
|
|
|
|
|
|
|
|
|
|
The dedicated acceptance identity, API Key, two node gateways, synthetic real
|
|
|
|
|
canary images, and production snapshot transport are bootstrapped when their
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_* overrides are omitted.
|
|
|
|
|
|
|
|
|
|
Optional model overrides (otherwise selected from current production candidates):
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL
|
|
|
|
@@ -76,10 +75,10 @@ fi
|
|
|
|
|
load_cluster_env
|
|
|
|
|
require_commands curl git go jq node openssl sed
|
|
|
|
|
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:?}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:?}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_USER_ID:?}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_USER_ID:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS:=31}"
|
|
|
|
@@ -94,9 +93,17 @@ require_commands curl git go jq node openssl sed
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:?}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:?}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:?}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}"
|
|
|
|
|
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}"
|
|
|
|
|
: "${AI_GATEWAY_ONLINE_ACCOUNT:=}"
|
|
|
|
|
: "${AI_GATEWAY_ONLINE_PASSWORD:=}"
|
|
|
|
|
: "${AI_GATEWAY_ONLINE_BASE_URL:=}"
|
|
|
|
|
if [[ -z $AI_GATEWAY_ACCEPTANCE_GATEWAYS ]]; then
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_GATEWAYS="https://${CLUSTER_NINGBO_HOST#root@},https://${CLUSTER_HONGKONG_HOST#root@}"
|
|
|
|
|
: "${AI_GATEWAY_DEPLOY_DOMAIN:?}"
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:-$AI_GATEWAY_DEPLOY_DOMAIN}
|
|
|
|
|
fi
|
|
|
|
|
[[ $AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS =~ ^[1-9][0-9]*$ &&
|
|
|
|
|
$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS -le 256 ]] || {
|
|
|
|
|
echo 'AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS must be between 1 and 256' >&2
|
|
|
|
@@ -240,6 +247,42 @@ cleanup() {
|
|
|
|
|
trap cleanup EXIT
|
|
|
|
|
trap 'failure_gate_id=workflow_interrupted; failure_reason="production acceptance interrupted by signal"; exit 130' HUP INT TERM
|
|
|
|
|
|
|
|
|
|
bootstrap_acceptance_admin_token() {
|
|
|
|
|
if [[ -n $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN ]]; then
|
|
|
|
|
echo 'acceptance_admin_identity=PASS source=explicit_token'
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
[[ -n $AI_GATEWAY_ONLINE_ACCOUNT && -n $AI_GATEWAY_ONLINE_PASSWORD &&
|
|
|
|
|
$AI_GATEWAY_ONLINE_BASE_URL == https://* ]] || {
|
|
|
|
|
echo 'acceptance admin token or HTTPS online account credentials are required' >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
local response=$temporary_root/admin-login.json
|
|
|
|
|
local body status
|
|
|
|
|
body=$(jq -cn \
|
|
|
|
|
--arg account "$AI_GATEWAY_ONLINE_ACCOUNT" \
|
|
|
|
|
--arg password "$AI_GATEWAY_ONLINE_PASSWORD" \
|
|
|
|
|
'{account:$account,password:$password}')
|
|
|
|
|
status=$(curl -sS --max-time 15 -o "$response" -w '%{http_code}' \
|
|
|
|
|
-H 'Content-Type: application/json' --data-binary "$body" \
|
|
|
|
|
"${AI_GATEWAY_ONLINE_BASE_URL%/}/api/v1/auth/login")
|
|
|
|
|
chmod 0600 "$response"
|
|
|
|
|
[[ $status == 200 &&
|
|
|
|
|
$(jq '(.user.role // []) as $roles | if ($roles|type)=="array" then ($roles|index("manager")) != null else $roles == "manager" end' "$response") == true ]] || {
|
|
|
|
|
echo "acceptance Manager login failed: status=$status" >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN=$(jq -r '.accessToken // empty' "$response")
|
|
|
|
|
[[ -n $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN ]] || {
|
|
|
|
|
echo 'acceptance Manager login did not return an access token' >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
export AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN
|
|
|
|
|
echo 'acceptance_admin_identity=PASS source=short_lived_manager_login'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bootstrap_acceptance_admin_token
|
|
|
|
|
|
|
|
|
|
(
|
|
|
|
|
cd "$cluster_root/apps/api"
|
|
|
|
|
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
|
|
|
|
@@ -247,10 +290,41 @@ trap 'failure_gate_id=workflow_interrupted; failure_reason="production acceptanc
|
|
|
|
|
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
|
|
|
|
|
-trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
|
|
|
|
|
)
|
|
|
|
|
AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \
|
|
|
|
|
"$acceptance_snapshot_binary" export \
|
|
|
|
|
--release-sha "$release_sha" \
|
|
|
|
|
--output "$current_production_snapshot" >/dev/null
|
|
|
|
|
export_production_snapshot() {
|
|
|
|
|
local output=$1
|
|
|
|
|
if [[ -n $AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL ]]; then
|
|
|
|
|
AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \
|
|
|
|
|
"$acceptance_snapshot_binary" export \
|
|
|
|
|
--release-sha "$release_sha" \
|
|
|
|
|
--output "$output" >/dev/null
|
|
|
|
|
else
|
|
|
|
|
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$output" <<'REMOTE'
|
|
|
|
|
set -euo pipefail
|
|
|
|
|
release_sha=$1
|
|
|
|
|
namespace=easyai
|
|
|
|
|
pod=$(k3s kubectl get pods -n "$namespace" \
|
|
|
|
|
-l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
|
|
|
|
|
--field-selector status.phase=Running \
|
|
|
|
|
-o 'jsonpath={.items[0].metadata.name}')
|
|
|
|
|
[[ $pod == easyai-api-ningbo-* ]]
|
|
|
|
|
[[ $(k3s kubectl get pod "$pod" -n "$namespace" \
|
|
|
|
|
-o 'jsonpath={.status.containerStatuses[0].ready}') == true ]]
|
|
|
|
|
snapshot=/tmp/easyai-production-acceptance-snapshot-$release_sha.json
|
|
|
|
|
cleanup() {
|
|
|
|
|
k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
|
|
|
|
|
}
|
|
|
|
|
trap cleanup EXIT
|
|
|
|
|
k3s kubectl exec -n "$namespace" "$pod" -- \
|
|
|
|
|
/app/easyai-ai-gateway-acceptance-snapshot export \
|
|
|
|
|
--release-sha "$release_sha" --output "$snapshot" >/dev/null
|
|
|
|
|
k3s kubectl exec -n "$namespace" "$pod" -- cat "$snapshot"
|
|
|
|
|
REMOTE
|
|
|
|
|
fi
|
|
|
|
|
chmod 0600 "$output"
|
|
|
|
|
"$acceptance_snapshot_binary" validate --input "$output" >/dev/null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export_production_snapshot "$current_production_snapshot"
|
|
|
|
|
if [[ $skip_local_acceptance == true ]]; then
|
|
|
|
|
local_acceptance_report=$temporary_root/online-waiver-base.json
|
|
|
|
|
node "$cluster_root/scripts/acceptance/report.mjs" build-online-waiver \
|
|
|
|
@@ -283,6 +357,164 @@ database_query() {
|
|
|
|
|
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bootstrap_acceptance_primary_identity() {
|
|
|
|
|
if [[ -n $AI_GATEWAY_ACCEPTANCE_API_KEY || -n $AI_GATEWAY_ACCEPTANCE_API_KEY_ID ||
|
|
|
|
|
-n $AI_GATEWAY_ACCEPTANCE_USER_ID ]]; then
|
|
|
|
|
[[ -n $AI_GATEWAY_ACCEPTANCE_API_KEY &&
|
|
|
|
|
$AI_GATEWAY_ACCEPTANCE_API_KEY_ID =~ ^[0-9a-f-]{36}$ &&
|
|
|
|
|
$AI_GATEWAY_ACCEPTANCE_USER_ID =~ ^[0-9a-f-]{36}$ ]] || {
|
|
|
|
|
echo 'explicit acceptance primary identity variables must be provided together' >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
echo 'acceptance_primary_identity=PASS source=explicit'
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
local material
|
|
|
|
|
material=$(database_query "
|
|
|
|
|
WITH primary_user AS (
|
|
|
|
|
INSERT INTO gateway_users (
|
|
|
|
|
user_key, source, username, display_name, roles, auth_profile, metadata, status
|
|
|
|
|
)
|
|
|
|
|
VALUES (
|
|
|
|
|
'production-acceptance-primary', 'gateway', 'production-acceptance-primary',
|
|
|
|
|
'Production Acceptance Primary', '[\"user\"]'::jsonb, '{}'::jsonb,
|
|
|
|
|
'{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb, 'active'
|
|
|
|
|
)
|
|
|
|
|
ON CONFLICT (user_key) DO UPDATE
|
|
|
|
|
SET source=EXCLUDED.source,
|
|
|
|
|
username=EXCLUDED.username,
|
|
|
|
|
display_name=EXCLUDED.display_name,
|
|
|
|
|
roles=EXCLUDED.roles,
|
|
|
|
|
auth_profile=EXCLUDED.auth_profile,
|
|
|
|
|
metadata=EXCLUDED.metadata,
|
|
|
|
|
status='active',
|
|
|
|
|
deleted_at=NULL,
|
|
|
|
|
updated_at=now()
|
|
|
|
|
RETURNING id
|
|
|
|
|
), existing_key AS (
|
|
|
|
|
SELECT key.id, key.gateway_user_id, key.key_secret
|
|
|
|
|
FROM gateway_api_keys key
|
|
|
|
|
JOIN primary_user ON primary_user.id=key.gateway_user_id
|
|
|
|
|
WHERE key.name='Production Acceptance Primary'
|
|
|
|
|
AND key.status='active'
|
|
|
|
|
AND key.deleted_at IS NULL
|
|
|
|
|
AND COALESCE(key.key_secret, '') <> ''
|
|
|
|
|
ORDER BY key.created_at
|
|
|
|
|
LIMIT 1
|
|
|
|
|
), new_key_material AS (
|
|
|
|
|
SELECT primary_user.id AS gateway_user_id,
|
|
|
|
|
'sk-gw-' || encode(gen_random_bytes(32), 'hex') AS secret
|
|
|
|
|
FROM primary_user
|
|
|
|
|
WHERE NOT EXISTS (SELECT 1 FROM existing_key)
|
|
|
|
|
), inserted_key AS (
|
|
|
|
|
INSERT INTO gateway_api_keys (
|
|
|
|
|
gateway_user_id, user_id, key_prefix, key_secret, key_hash, name, scopes,
|
|
|
|
|
rate_limit_policy, quota_policy, metadata, status
|
|
|
|
|
)
|
|
|
|
|
SELECT material.gateway_user_id,
|
|
|
|
|
material.gateway_user_id::text,
|
|
|
|
|
left(material.secret, 18),
|
|
|
|
|
material.secret,
|
|
|
|
|
crypt(material.secret, gen_salt('bf', 10)),
|
|
|
|
|
'Production Acceptance Primary',
|
|
|
|
|
'[\"chat\",\"embedding\",\"rerank\",\"image\",\"image_vectorize\",\"video\",\"video_enhance\",\"music\",\"audio\",\"voice_clone\"]'::jsonb,
|
|
|
|
|
'{}'::jsonb,
|
|
|
|
|
'{}'::jsonb,
|
|
|
|
|
'{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb,
|
|
|
|
|
'active'
|
|
|
|
|
FROM new_key_material material
|
|
|
|
|
RETURNING id, gateway_user_id, key_secret
|
|
|
|
|
), selected_key AS (
|
|
|
|
|
SELECT * FROM existing_key
|
|
|
|
|
UNION ALL
|
|
|
|
|
SELECT * FROM inserted_key
|
|
|
|
|
), wallet AS (
|
|
|
|
|
INSERT INTO gateway_wallet_accounts (
|
|
|
|
|
gateway_user_id, user_id, currency, balance, total_recharged, metadata, status
|
|
|
|
|
)
|
|
|
|
|
SELECT primary_user.id,
|
|
|
|
|
primary_user.id::text,
|
|
|
|
|
'resource',
|
|
|
|
|
1000000000::numeric,
|
|
|
|
|
1000000000::numeric,
|
|
|
|
|
'{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb,
|
|
|
|
|
'active'
|
|
|
|
|
FROM primary_user
|
|
|
|
|
ON CONFLICT (gateway_user_id, currency) DO UPDATE
|
|
|
|
|
SET balance=GREATEST(gateway_wallet_accounts.balance, EXCLUDED.balance),
|
|
|
|
|
total_recharged=GREATEST(gateway_wallet_accounts.total_recharged, EXCLUDED.total_recharged),
|
|
|
|
|
metadata=EXCLUDED.metadata,
|
|
|
|
|
status='active',
|
|
|
|
|
updated_at=now()
|
|
|
|
|
RETURNING gateway_user_id
|
|
|
|
|
)
|
|
|
|
|
SELECT primary_user.id::text || E'\\t' || selected_key.id::text || E'\\t' || selected_key.key_secret
|
|
|
|
|
FROM primary_user
|
|
|
|
|
CROSS JOIN selected_key
|
|
|
|
|
CROSS JOIN (SELECT count(*) FROM wallet) wallet_write_barrier;")
|
|
|
|
|
[[ $material == *$'\t'* ]] || {
|
|
|
|
|
echo 'acceptance primary identity bootstrap returned an invalid payload' >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
IFS=$'\t' read -r AI_GATEWAY_ACCEPTANCE_USER_ID \
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_API_KEY_ID AI_GATEWAY_ACCEPTANCE_API_KEY <<<"$material"
|
|
|
|
|
[[ $AI_GATEWAY_ACCEPTANCE_USER_ID =~ ^[0-9a-f-]{36}$ &&
|
|
|
|
|
$AI_GATEWAY_ACCEPTANCE_API_KEY_ID =~ ^[0-9a-f-]{36}$ &&
|
|
|
|
|
$AI_GATEWAY_ACCEPTANCE_API_KEY =~ ^sk-gw-[0-9a-f]{64}$ ]] || {
|
|
|
|
|
echo 'acceptance primary identity bootstrap failed validation' >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
|
|
|
|
|
export AI_GATEWAY_ACCEPTANCE_USER_ID AI_GATEWAY_ACCEPTANCE_API_KEY_ID
|
|
|
|
|
export AI_GATEWAY_ACCEPTANCE_API_KEY AI_GATEWAY_ACCEPTANCE_API_KEYS
|
|
|
|
|
echo 'acceptance_primary_identity=PASS source=idempotent_database_bootstrap isolated=true'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ensure_acceptance_real_images() {
|
|
|
|
|
if [[ -n $AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS ]]; then
|
|
|
|
|
[[ $(awk -F',' '{print NF}' <<<"$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS") -ge 3 ]] || {
|
|
|
|
|
echo 'explicit acceptance real image URLs must contain at least three entries' >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
echo 'acceptance_real_images=PASS source=explicit count=3_or_more'
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
[[ $AI_GATEWAY_ONLINE_BASE_URL == https://* ]] || {
|
|
|
|
|
echo 'HTTPS online base URL is required to upload synthetic real canary images' >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
local fixture_root=$temporary_root/real-canary-images
|
|
|
|
|
local response url index
|
|
|
|
|
local -a urls=()
|
|
|
|
|
install -d -m 0700 "$fixture_root"
|
|
|
|
|
node "$cluster_root/scripts/acceptance/generate-reference-images.mjs" "$fixture_root" >/dev/null
|
|
|
|
|
for index in 1 2 3; do
|
|
|
|
|
response=$fixture_root/upload-$index.json
|
|
|
|
|
chmod 0600 "$fixture_root/production-acceptance-reference-$index.png"
|
|
|
|
|
{
|
|
|
|
|
printf 'silent\nshow-error\nfail-with-body\nmax-time = 120\n'
|
|
|
|
|
printf 'header = "Authorization: Bearer %s"\n' "$AI_GATEWAY_ACCEPTANCE_API_KEY"
|
|
|
|
|
printf 'form = "source=production-acceptance"\n'
|
|
|
|
|
printf 'form = "file=@%s;type=image/png"\n' \
|
|
|
|
|
"$fixture_root/production-acceptance-reference-$index.png"
|
|
|
|
|
printf 'url = "%s/api/v1/files/upload"\n' "${AI_GATEWAY_ONLINE_BASE_URL%/}"
|
|
|
|
|
} | curl --config - >"$response"
|
|
|
|
|
chmod 0600 "$response"
|
|
|
|
|
url=$(jq -r '.url // .data.url // empty' "$response")
|
|
|
|
|
if [[ $url == /* ]]; then
|
|
|
|
|
url="${AI_GATEWAY_ONLINE_BASE_URL%/}$url"
|
|
|
|
|
fi
|
|
|
|
|
[[ $url == https://* ]] || {
|
|
|
|
|
echo "synthetic acceptance image upload $index did not return an HTTPS URL" >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
urls+=("$url")
|
|
|
|
|
done
|
|
|
|
|
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(IFS=,; printf '%s' "${urls[*]}")
|
|
|
|
|
export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS
|
|
|
|
|
echo 'acceptance_real_images=PASS source=synthetic_gateway_upload count=3 dimensions=512x512'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
admin_request() {
|
|
|
|
|
local method=$1
|
|
|
|
|
local path=$2
|
|
|
|
@@ -2646,8 +2878,10 @@ WHERE task.acceptance_run_id='$run_id'::uuid
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
verify_release_cas
|
|
|
|
|
bootstrap_acceptance_primary_identity
|
|
|
|
|
ensure_acceptance_user_group
|
|
|
|
|
ensure_acceptance_identity_shards
|
|
|
|
|
ensure_acceptance_real_images
|
|
|
|
|
select_acceptance_models
|
|
|
|
|
ensure_acceptance_model_access
|
|
|
|
|
create_and_activate_run
|
|
|
|
|