BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(acceptance): 自举线上隔离验收身份

Browse Source 
线上验收此前依赖人工准备管理员 Token、专属用户、API Key、参考图片和数据库连接，导致跳过本地验收后仍无法安全、可重复执行。

本提交增加短期 Manager 登录、隔离用户/钱包/API Key 幂等创建、Pod 内脱敏快照导出和合成图片真实上传；敏感材料仅保留在进程或 0600 临时文件中。

验证：bash -n、ShellCheck、验收报告测试、参考图片生成测试、git diff --check。
This commit is contained in:
wangbo
2026-07-31 23:25:43 +08:00
parent 0b681275ed
commit 709434a256
4 changed files with 343 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cluster/run-production-acceptance.sh
+251 -17
View File
@@ -30,12 +30,11 @@ with a user-directed waiver. It never records them as passed and preserves all
online simulation, real canary, resource, consistency, and release CAS gates.
Required private environment values:
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN
AI_GATEWAY_ACCEPTANCE_API_KEY
AI_GATEWAY_ACCEPTANCE_API_KEY_ID
AI_GATEWAY_ACCEPTANCE_USER_ID
AI_GATEWAY_ACCEPTANCE_GATEWAYS
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN, or AI_GATEWAY_ONLINE_ACCOUNT/PASSWORD
The dedicated acceptance identity, API Key, two node gateways, synthetic real
canary images, and production snapshot transport are bootstrapped when their
AI_GATEWAY_ACCEPTANCE_* overrides are omitted.
Optional model overrides (otherwise selected from current production candidates):
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL
@@ -76,10 +75,10 @@ fi
load_cluster_env
require_commands curl git go jq node openssl sed
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:?}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:?}"
: "${AI_GATEWAY_ACCEPTANCE_USER_ID:?}"
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:=}"
: "${AI_GATEWAY_ACCEPTANCE_USER_ID:=}"
: "${AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL:=}"
: "${AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS:=31}"
@@ -94,9 +93,17 @@ require_commands curl git go jq node openssl sed
: "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:?}"
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:?}"
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:?}"
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}"
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}"
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}"
: "${AI_GATEWAY_ONLINE_ACCOUNT:=}"
: "${AI_GATEWAY_ONLINE_PASSWORD:=}"
: "${AI_GATEWAY_ONLINE_BASE_URL:=}"
if [[ -z $AI_GATEWAY_ACCEPTANCE_GATEWAYS ]]; then
AI_GATEWAY_ACCEPTANCE_GATEWAYS="https://${CLUSTER_NINGBO_HOST#root@},https://${CLUSTER_HONGKONG_HOST#root@}"
: "${AI_GATEWAY_DEPLOY_DOMAIN:?}"
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:-$AI_GATEWAY_DEPLOY_DOMAIN}
fi
[[ $AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS -le 256 ]] || {
echo 'AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS must be between 1 and 256' >&2
@@ -240,6 +247,42 @@ cleanup() {
trap cleanup EXIT
trap 'failure_gate_id=workflow_interrupted; failure_reason="production acceptance interrupted by signal"; exit 130' HUP INT TERM
bootstrap_acceptance_admin_token() {
if [[ -n $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN ]]; then
echo 'acceptance_admin_identity=PASS source=explicit_token'
return 0
fi
[[ -n $AI_GATEWAY_ONLINE_ACCOUNT && -n $AI_GATEWAY_ONLINE_PASSWORD &&
$AI_GATEWAY_ONLINE_BASE_URL == https://* ]] || {
echo 'acceptance admin token or HTTPS online account credentials are required' >&2
return 1
}
local response=$temporary_root/admin-login.json
local body status
body=$(jq -cn \
--arg account "$AI_GATEWAY_ONLINE_ACCOUNT" \
--arg password "$AI_GATEWAY_ONLINE_PASSWORD" \
'{account:$account,password:$password}')
status=$(curl -sS --max-time 15 -o "$response" -w '%{http_code}' \
-H 'Content-Type: application/json' --data-binary "$body" \
"${AI_GATEWAY_ONLINE_BASE_URL%/}/api/v1/auth/login")
chmod 0600 "$response"
[[ $status == 200 &&
$(jq '(.user.role // []) as $roles | if ($roles|type)=="array" then ($roles|index("manager")) != null else $roles == "manager" end' "$response") == true ]] || {
echo "acceptance Manager login failed: status=$status" >&2
return 1
}
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN=$(jq -r '.accessToken // empty' "$response")
[[ -n $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN ]] || {
echo 'acceptance Manager login did not return an access token' >&2
return 1
}
export AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN
echo 'acceptance_admin_identity=PASS source=short_lived_manager_login'
}
bootstrap_acceptance_admin_token
(
cd "$cluster_root/apps/api"
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
@@ -247,10 +290,41 @@ trap 'failure_gate_id=workflow_interrupted; failure_reason="production acceptanc
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
-trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
)
AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \
"$acceptance_snapshot_binary" export \
--release-sha "$release_sha" \
--output "$current_production_snapshot" >/dev/null
export_production_snapshot() {
local output=$1
if [[ -n $AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL ]]; then
AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \
"$acceptance_snapshot_binary" export \
--release-sha "$release_sha" \
--output "$output" >/dev/null
else
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$output" <<'REMOTE'
set -euo pipefail
release_sha=$1
namespace=easyai
pod=$(k3s kubectl get pods -n "$namespace" \
-l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
--field-selector status.phase=Running \
-o 'jsonpath={.items[0].metadata.name}')
[[ $pod == easyai-api-ningbo-* ]]
[[ $(k3s kubectl get pod "$pod" -n "$namespace" \
-o 'jsonpath={.status.containerStatuses[0].ready}') == true ]]
snapshot=/tmp/easyai-production-acceptance-snapshot-$release_sha.json
cleanup() {
k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true
}
trap cleanup EXIT
k3s kubectl exec -n "$namespace" "$pod" -- \
/app/easyai-ai-gateway-acceptance-snapshot export \
--release-sha "$release_sha" --output "$snapshot" >/dev/null
k3s kubectl exec -n "$namespace" "$pod" -- cat "$snapshot"
REMOTE
fi
chmod 0600 "$output"
"$acceptance_snapshot_binary" validate --input "$output" >/dev/null
}
export_production_snapshot "$current_production_snapshot"
if [[ $skip_local_acceptance == true ]]; then
local_acceptance_report=$temporary_root/online-waiver-base.json
node "$cluster_root/scripts/acceptance/report.mjs" build-online-waiver \
@@ -283,6 +357,164 @@ database_query() {
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql"
}
bootstrap_acceptance_primary_identity() {
if [[ -n $AI_GATEWAY_ACCEPTANCE_API_KEY || -n $AI_GATEWAY_ACCEPTANCE_API_KEY_ID ||
-n $AI_GATEWAY_ACCEPTANCE_USER_ID ]]; then
[[ -n $AI_GATEWAY_ACCEPTANCE_API_KEY &&
$AI_GATEWAY_ACCEPTANCE_API_KEY_ID =~ ^[0-9a-f-]{36}$ &&
$AI_GATEWAY_ACCEPTANCE_USER_ID =~ ^[0-9a-f-]{36}$ ]] || {
echo 'explicit acceptance primary identity variables must be provided together' >&2
return 1
}
echo 'acceptance_primary_identity=PASS source=explicit'
return 0
fi
local material
material=$(database_query "
WITH primary_user AS (
INSERT INTO gateway_users (
user_key, source, username, display_name, roles, auth_profile, metadata, status
)
VALUES (
'production-acceptance-primary', 'gateway', 'production-acceptance-primary',
'Production Acceptance Primary', '[\"user\"]'::jsonb, '{}'::jsonb,
'{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb, 'active'
)
ON CONFLICT (user_key) DO UPDATE
SET source=EXCLUDED.source,
username=EXCLUDED.username,
display_name=EXCLUDED.display_name,
roles=EXCLUDED.roles,
auth_profile=EXCLUDED.auth_profile,
metadata=EXCLUDED.metadata,
status='active',
deleted_at=NULL,
updated_at=now()
RETURNING id
), existing_key AS (
SELECT key.id, key.gateway_user_id, key.key_secret
FROM gateway_api_keys key
JOIN primary_user ON primary_user.id=key.gateway_user_id
WHERE key.name='Production Acceptance Primary'
AND key.status='active'
AND key.deleted_at IS NULL
AND COALESCE(key.key_secret, '') <> ''
ORDER BY key.created_at
LIMIT 1
), new_key_material AS (
SELECT primary_user.id AS gateway_user_id,
'sk-gw-' || encode(gen_random_bytes(32), 'hex') AS secret
FROM primary_user
WHERE NOT EXISTS (SELECT 1 FROM existing_key)
), inserted_key AS (
INSERT INTO gateway_api_keys (
gateway_user_id, user_id, key_prefix, key_secret, key_hash, name, scopes,
rate_limit_policy, quota_policy, metadata, status
)
SELECT material.gateway_user_id,
material.gateway_user_id::text,
left(material.secret, 18),
material.secret,
crypt(material.secret, gen_salt('bf', 10)),
'Production Acceptance Primary',
'[\"chat\",\"embedding\",\"rerank\",\"image\",\"image_vectorize\",\"video\",\"video_enhance\",\"music\",\"audio\",\"voice_clone\"]'::jsonb,
'{}'::jsonb,
'{}'::jsonb,
'{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb,
'active'
FROM new_key_material material
RETURNING id, gateway_user_id, key_secret
), selected_key AS (
SELECT * FROM existing_key
UNION ALL
SELECT * FROM inserted_key
), wallet AS (
INSERT INTO gateway_wallet_accounts (
gateway_user_id, user_id, currency, balance, total_recharged, metadata, status
)
SELECT primary_user.id,
primary_user.id::text,
'resource',
1000000000::numeric,
1000000000::numeric,
'{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb,
'active'
FROM primary_user
ON CONFLICT (gateway_user_id, currency) DO UPDATE
SET balance=GREATEST(gateway_wallet_accounts.balance, EXCLUDED.balance),
total_recharged=GREATEST(gateway_wallet_accounts.total_recharged, EXCLUDED.total_recharged),
metadata=EXCLUDED.metadata,
status='active',
updated_at=now()
RETURNING gateway_user_id
)
SELECT primary_user.id::text || E'\\t' || selected_key.id::text || E'\\t' || selected_key.key_secret
FROM primary_user
CROSS JOIN selected_key
CROSS JOIN (SELECT count(*) FROM wallet) wallet_write_barrier;")
[[ $material == *$'\t'* ]] || {
echo 'acceptance primary identity bootstrap returned an invalid payload' >&2
return 1
}
IFS=$'\t' read -r AI_GATEWAY_ACCEPTANCE_USER_ID \
AI_GATEWAY_ACCEPTANCE_API_KEY_ID AI_GATEWAY_ACCEPTANCE_API_KEY <<<"$material"
[[ $AI_GATEWAY_ACCEPTANCE_USER_ID =~ ^[0-9a-f-]{36}$ &&
$AI_GATEWAY_ACCEPTANCE_API_KEY_ID =~ ^[0-9a-f-]{36}$ &&
$AI_GATEWAY_ACCEPTANCE_API_KEY =~ ^sk-gw-[0-9a-f]{64}$ ]] || {
echo 'acceptance primary identity bootstrap failed validation' >&2
return 1
}
AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
export AI_GATEWAY_ACCEPTANCE_USER_ID AI_GATEWAY_ACCEPTANCE_API_KEY_ID
export AI_GATEWAY_ACCEPTANCE_API_KEY AI_GATEWAY_ACCEPTANCE_API_KEYS
echo 'acceptance_primary_identity=PASS source=idempotent_database_bootstrap isolated=true'
}
ensure_acceptance_real_images() {
if [[ -n $AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS ]]; then
[[ $(awk -F',' '{print NF}' <<<"$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS") -ge 3 ]] || {
echo 'explicit acceptance real image URLs must contain at least three entries' >&2
return 1
}
echo 'acceptance_real_images=PASS source=explicit count=3_or_more'
return 0
fi
[[ $AI_GATEWAY_ONLINE_BASE_URL == https://* ]] || {
echo 'HTTPS online base URL is required to upload synthetic real canary images' >&2
return 1
}
local fixture_root=$temporary_root/real-canary-images
local response url index
local -a urls=()
install -d -m 0700 "$fixture_root"
node "$cluster_root/scripts/acceptance/generate-reference-images.mjs" "$fixture_root" >/dev/null
for index in 1 2 3; do
response=$fixture_root/upload-$index.json
chmod 0600 "$fixture_root/production-acceptance-reference-$index.png"
{
printf 'silent\nshow-error\nfail-with-body\nmax-time = 120\n'
printf 'header = "Authorization: Bearer %s"\n' "$AI_GATEWAY_ACCEPTANCE_API_KEY"
printf 'form = "source=production-acceptance"\n'
printf 'form = "file=@%s;type=image/png"\n' \
"$fixture_root/production-acceptance-reference-$index.png"
printf 'url = "%s/api/v1/files/upload"\n' "${AI_GATEWAY_ONLINE_BASE_URL%/}"
} | curl --config - >"$response"
chmod 0600 "$response"
url=$(jq -r '.url // .data.url // empty' "$response")
if [[ $url == /* ]]; then
url="${AI_GATEWAY_ONLINE_BASE_URL%/}$url"
fi
[[ $url == https://* ]] || {
echo "synthetic acceptance image upload $index did not return an HTTPS URL" >&2
return 1
}
urls+=("$url")
done
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(IFS=,; printf '%s' "${urls[*]}")
export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS
echo 'acceptance_real_images=PASS source=synthetic_gateway_upload count=3 dimensions=512x512'
}
admin_request() {
local method=$1
local path=$2
@@ -2646,8 +2878,10 @@ WHERE task.acceptance_run_id='$run_id'::uuid
}
verify_release_cas
bootstrap_acceptance_primary_identity
ensure_acceptance_user_group
ensure_acceptance_identity_shards
ensure_acceptance_real_images
select_acceptance_models
ensure_acceptance_model_access
create_and_activate_run