diff --git a/docs/operations/production-acceptance.md b/docs/operations/production-acceptance.md index bbdd4ec..450bf77 100644 --- a/docs/operations/production-acceptance.md +++ b/docs/operations/production-acceptance.md @@ -116,10 +116,14 @@ easyai-ai-gateway-cluster-release capacity ## 执行 -先把主验收用户、钱包、API Key、管理员 Token、两个 Gateway 地址和三张真实小流量参考图 -写入本地私有环境。脚本会以主身份为模板幂等准备其余隔离身份；模型变量可省略，脚本会从 -当前启用候选中选择 Gemini 图片编辑模型，以及 `omni_video.max_images >= 9` 的视频模型 -并优先 Seedance 2.0/fast。 +主验收用户、钱包和 API Key 可以由脚本在隔离验收组中幂等创建；管理员 Token 未提供时， +脚本使用本地私有环境中的 `AI_GATEWAY_ONLINE_ACCOUNT/PASSWORD` 登录获取短期 Manager Token。 +两个 Gateway 地址默认从宁波、香港节点推导，三张真实小流量参考图默认生成 512×512 合成 +PNG 并通过验收 Key 上传到 Gateway。生产快照默认在已部署的 API Pod 内只读导出，无需把 +数据库连接串传到本机。所有自动生成的 Key、Token 和连接信息仅存在于进程或 `0600` 临时 +文件中，不写入报告或日志。上述值仍可用 `AI_GATEWAY_ACCEPTANCE_*` 显式覆盖。脚本会以主 +身份为模板幂等准备其余隔离身份；模型变量可省略，脚本会从当前启用候选中选择 Gemini 图片 +编辑模型，以及 `omni_video.max_images >= 9` 的视频模型并优先 Seedance 2.0/fast。 为保证请求严格各有一半命中宁波和香港，可以把两个 Gateway URL 配成两个节点的 `https://<节点IP>`，并设置 diff --git a/scripts/acceptance/generate-reference-images.mjs b/scripts/acceptance/generate-reference-images.mjs new file mode 100644 index 0000000..ee37e16 --- /dev/null +++ b/scripts/acceptance/generate-reference-images.mjs @@ -0,0 +1,62 @@ +#!/usr/bin/env node +import { lstatSync, writeFileSync } from 'node:fs'; +import { resolve, join } from 'node:path'; +import { deflateSync } from 'node:zlib'; + +function crc32(buffer) { + let value = 0xffffffff; + for (const byte of buffer) { + value ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + value = (value >>> 1) ^ (value & 1 ? 0xedb88320 : 0); + } + } + return (value ^ 0xffffffff) >>> 0; +} + +function chunk(type, data) { + const typeBuffer = Buffer.from(type); + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const checksum = Buffer.alloc(4); + checksum.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data]))); + return Buffer.concat([length, typeBuffer, data, checksum]); +} + +function fixture(seed, width = 512, height = 512) { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 2; + const rows = []; + for (let y = 0; y < height; y += 1) { + const row = Buffer.alloc(1 + width * 3); + for (let x = 0; x < width; x += 1) { + row[1 + x * 3] = (x + seed * 53) % 256; + row[2 + x * 3] = (y + seed * 97) % 256; + row[3 + x * 3] = (x + y + seed * 131) % 256; + } + rows.push(row); + } + return Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + chunk('IHDR', header), + chunk('IDAT', deflateSync(Buffer.concat(rows))), + chunk('IEND', Buffer.alloc(0)) + ]); +} + +const outputDirectory = resolve(process.argv[2] ?? ''); +const info = lstatSync(outputDirectory); +if (!info.isDirectory() || info.isSymbolicLink()) { + throw new Error('output directory must be a real directory'); +} +for (let index = 1; index <= 3; index += 1) { + writeFileSync( + join(outputDirectory, `production-acceptance-reference-${index}.png`), + fixture(index), + { mode: 0o600, flag: 'wx' } + ); +} +process.stdout.write('acceptance_reference_images=PASS count=3 dimensions=512x512\n'); diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 75f2e73..4429ec9 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -30,12 +30,11 @@ with a user-directed waiver. It never records them as passed and preserves all online simulation, real canary, resource, consistency, and release CAS gates. Required private environment values: - AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN - AI_GATEWAY_ACCEPTANCE_API_KEY - AI_GATEWAY_ACCEPTANCE_API_KEY_ID - AI_GATEWAY_ACCEPTANCE_USER_ID - AI_GATEWAY_ACCEPTANCE_GATEWAYS - AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS + AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN, or AI_GATEWAY_ONLINE_ACCOUNT/PASSWORD + +The dedicated acceptance identity, API Key, two node gateways, synthetic real +canary images, and production snapshot transport are bootstrapped when their +AI_GATEWAY_ACCEPTANCE_* overrides are omitted. Optional model overrides (otherwise selected from current production candidates): AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL @@ -76,10 +75,10 @@ fi load_cluster_env require_commands curl git go jq node openssl sed -: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}" -: "${AI_GATEWAY_ACCEPTANCE_API_KEY:?}" -: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:?}" -: "${AI_GATEWAY_ACCEPTANCE_USER_ID:?}" +: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}" +: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}" +: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:=}" +: "${AI_GATEWAY_ACCEPTANCE_USER_ID:=}" : "${AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL:=}" : "${AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL:=}" : "${AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS:=31}" @@ -94,9 +93,17 @@ require_commands curl git go jq node openssl sed : "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}" : "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}" : "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}" -: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:?}" -: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:?}" -: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:?}" +: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}" +: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}" +: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}" +: "${AI_GATEWAY_ONLINE_ACCOUNT:=}" +: "${AI_GATEWAY_ONLINE_PASSWORD:=}" +: "${AI_GATEWAY_ONLINE_BASE_URL:=}" +if [[ -z $AI_GATEWAY_ACCEPTANCE_GATEWAYS ]]; then + AI_GATEWAY_ACCEPTANCE_GATEWAYS="https://${CLUSTER_NINGBO_HOST#root@},https://${CLUSTER_HONGKONG_HOST#root@}" + : "${AI_GATEWAY_DEPLOY_DOMAIN:?}" + AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:-$AI_GATEWAY_DEPLOY_DOMAIN} +fi [[ $AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS =~ ^[1-9][0-9]*$ && $AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS -le 256 ]] || { echo 'AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS must be between 1 and 256' >&2 @@ -240,6 +247,42 @@ cleanup() { trap cleanup EXIT trap 'failure_gate_id=workflow_interrupted; failure_reason="production acceptance interrupted by signal"; exit 130' HUP INT TERM +bootstrap_acceptance_admin_token() { + if [[ -n $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN ]]; then + echo 'acceptance_admin_identity=PASS source=explicit_token' + return 0 + fi + [[ -n $AI_GATEWAY_ONLINE_ACCOUNT && -n $AI_GATEWAY_ONLINE_PASSWORD && + $AI_GATEWAY_ONLINE_BASE_URL == https://* ]] || { + echo 'acceptance admin token or HTTPS online account credentials are required' >&2 + return 1 + } + local response=$temporary_root/admin-login.json + local body status + body=$(jq -cn \ + --arg account "$AI_GATEWAY_ONLINE_ACCOUNT" \ + --arg password "$AI_GATEWAY_ONLINE_PASSWORD" \ + '{account:$account,password:$password}') + status=$(curl -sS --max-time 15 -o "$response" -w '%{http_code}' \ + -H 'Content-Type: application/json' --data-binary "$body" \ + "${AI_GATEWAY_ONLINE_BASE_URL%/}/api/v1/auth/login") + chmod 0600 "$response" + [[ $status == 200 && + $(jq '(.user.role // []) as $roles | if ($roles|type)=="array" then ($roles|index("manager")) != null else $roles == "manager" end' "$response") == true ]] || { + echo "acceptance Manager login failed: status=$status" >&2 + return 1 + } + AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN=$(jq -r '.accessToken // empty' "$response") + [[ -n $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN ]] || { + echo 'acceptance Manager login did not return an access token' >&2 + return 1 + } + export AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN + echo 'acceptance_admin_identity=PASS source=short_lived_manager_login' +} + +bootstrap_acceptance_admin_token + ( cd "$cluster_root/apps/api" env -u AI_GATEWAY_TEST_DATABASE_URL go build \ @@ -247,10 +290,41 @@ trap 'failure_gate_id=workflow_interrupted; failure_reason="production acceptanc env -u AI_GATEWAY_TEST_DATABASE_URL go build \ -trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot ) -AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \ - "$acceptance_snapshot_binary" export \ - --release-sha "$release_sha" \ - --output "$current_production_snapshot" >/dev/null +export_production_snapshot() { + local output=$1 + if [[ -n $AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL ]]; then + AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \ + "$acceptance_snapshot_binary" export \ + --release-sha "$release_sha" \ + --output "$output" >/dev/null + else + cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$release_sha" >"$output" <<'REMOTE' +set -euo pipefail +release_sha=$1 +namespace=easyai +pod=$(k3s kubectl get pods -n "$namespace" \ + -l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \ + --field-selector status.phase=Running \ + -o 'jsonpath={.items[0].metadata.name}') +[[ $pod == easyai-api-ningbo-* ]] +[[ $(k3s kubectl get pod "$pod" -n "$namespace" \ + -o 'jsonpath={.status.containerStatuses[0].ready}') == true ]] +snapshot=/tmp/easyai-production-acceptance-snapshot-$release_sha.json +cleanup() { + k3s kubectl exec -n "$namespace" "$pod" -- rm -f -- "$snapshot" >/dev/null 2>&1 || true +} +trap cleanup EXIT +k3s kubectl exec -n "$namespace" "$pod" -- \ + /app/easyai-ai-gateway-acceptance-snapshot export \ + --release-sha "$release_sha" --output "$snapshot" >/dev/null +k3s kubectl exec -n "$namespace" "$pod" -- cat "$snapshot" +REMOTE + fi + chmod 0600 "$output" + "$acceptance_snapshot_binary" validate --input "$output" >/dev/null +} + +export_production_snapshot "$current_production_snapshot" if [[ $skip_local_acceptance == true ]]; then local_acceptance_report=$temporary_root/online-waiver-base.json node "$cluster_root/scripts/acceptance/report.mjs" build-online-waiver \ @@ -283,6 +357,164 @@ database_query() { psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql" } +bootstrap_acceptance_primary_identity() { + if [[ -n $AI_GATEWAY_ACCEPTANCE_API_KEY || -n $AI_GATEWAY_ACCEPTANCE_API_KEY_ID || + -n $AI_GATEWAY_ACCEPTANCE_USER_ID ]]; then + [[ -n $AI_GATEWAY_ACCEPTANCE_API_KEY && + $AI_GATEWAY_ACCEPTANCE_API_KEY_ID =~ ^[0-9a-f-]{36}$ && + $AI_GATEWAY_ACCEPTANCE_USER_ID =~ ^[0-9a-f-]{36}$ ]] || { + echo 'explicit acceptance primary identity variables must be provided together' >&2 + return 1 + } + echo 'acceptance_primary_identity=PASS source=explicit' + return 0 + fi + local material + material=$(database_query " +WITH primary_user AS ( + INSERT INTO gateway_users ( + user_key, source, username, display_name, roles, auth_profile, metadata, status + ) + VALUES ( + 'production-acceptance-primary', 'gateway', 'production-acceptance-primary', + 'Production Acceptance Primary', '[\"user\"]'::jsonb, '{}'::jsonb, + '{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb, 'active' + ) + ON CONFLICT (user_key) DO UPDATE + SET source=EXCLUDED.source, + username=EXCLUDED.username, + display_name=EXCLUDED.display_name, + roles=EXCLUDED.roles, + auth_profile=EXCLUDED.auth_profile, + metadata=EXCLUDED.metadata, + status='active', + deleted_at=NULL, + updated_at=now() + RETURNING id +), existing_key AS ( + SELECT key.id, key.gateway_user_id, key.key_secret + FROM gateway_api_keys key + JOIN primary_user ON primary_user.id=key.gateway_user_id + WHERE key.name='Production Acceptance Primary' + AND key.status='active' + AND key.deleted_at IS NULL + AND COALESCE(key.key_secret, '') <> '' + ORDER BY key.created_at + LIMIT 1 +), new_key_material AS ( + SELECT primary_user.id AS gateway_user_id, + 'sk-gw-' || encode(gen_random_bytes(32), 'hex') AS secret + FROM primary_user + WHERE NOT EXISTS (SELECT 1 FROM existing_key) +), inserted_key AS ( + INSERT INTO gateway_api_keys ( + gateway_user_id, user_id, key_prefix, key_secret, key_hash, name, scopes, + rate_limit_policy, quota_policy, metadata, status + ) + SELECT material.gateway_user_id, + material.gateway_user_id::text, + left(material.secret, 18), + material.secret, + crypt(material.secret, gen_salt('bf', 10)), + 'Production Acceptance Primary', + '[\"chat\",\"embedding\",\"rerank\",\"image\",\"image_vectorize\",\"video\",\"video_enhance\",\"music\",\"audio\",\"voice_clone\"]'::jsonb, + '{}'::jsonb, + '{}'::jsonb, + '{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb, + 'active' + FROM new_key_material material + RETURNING id, gateway_user_id, key_secret +), selected_key AS ( + SELECT * FROM existing_key + UNION ALL + SELECT * FROM inserted_key +), wallet AS ( + INSERT INTO gateway_wallet_accounts ( + gateway_user_id, user_id, currency, balance, total_recharged, metadata, status + ) + SELECT primary_user.id, + primary_user.id::text, + 'resource', + 1000000000::numeric, + 1000000000::numeric, + '{\"purpose\":\"production_acceptance\",\"isolated\":true}'::jsonb, + 'active' + FROM primary_user + ON CONFLICT (gateway_user_id, currency) DO UPDATE + SET balance=GREATEST(gateway_wallet_accounts.balance, EXCLUDED.balance), + total_recharged=GREATEST(gateway_wallet_accounts.total_recharged, EXCLUDED.total_recharged), + metadata=EXCLUDED.metadata, + status='active', + updated_at=now() + RETURNING gateway_user_id +) +SELECT primary_user.id::text || E'\\t' || selected_key.id::text || E'\\t' || selected_key.key_secret +FROM primary_user +CROSS JOIN selected_key +CROSS JOIN (SELECT count(*) FROM wallet) wallet_write_barrier;") + [[ $material == *$'\t'* ]] || { + echo 'acceptance primary identity bootstrap returned an invalid payload' >&2 + return 1 + } + IFS=$'\t' read -r AI_GATEWAY_ACCEPTANCE_USER_ID \ + AI_GATEWAY_ACCEPTANCE_API_KEY_ID AI_GATEWAY_ACCEPTANCE_API_KEY <<<"$material" + [[ $AI_GATEWAY_ACCEPTANCE_USER_ID =~ ^[0-9a-f-]{36}$ && + $AI_GATEWAY_ACCEPTANCE_API_KEY_ID =~ ^[0-9a-f-]{36}$ && + $AI_GATEWAY_ACCEPTANCE_API_KEY =~ ^sk-gw-[0-9a-f]{64}$ ]] || { + echo 'acceptance primary identity bootstrap failed validation' >&2 + return 1 + } + AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY + export AI_GATEWAY_ACCEPTANCE_USER_ID AI_GATEWAY_ACCEPTANCE_API_KEY_ID + export AI_GATEWAY_ACCEPTANCE_API_KEY AI_GATEWAY_ACCEPTANCE_API_KEYS + echo 'acceptance_primary_identity=PASS source=idempotent_database_bootstrap isolated=true' +} + +ensure_acceptance_real_images() { + if [[ -n $AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS ]]; then + [[ $(awk -F',' '{print NF}' <<<"$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS") -ge 3 ]] || { + echo 'explicit acceptance real image URLs must contain at least three entries' >&2 + return 1 + } + echo 'acceptance_real_images=PASS source=explicit count=3_or_more' + return 0 + fi + [[ $AI_GATEWAY_ONLINE_BASE_URL == https://* ]] || { + echo 'HTTPS online base URL is required to upload synthetic real canary images' >&2 + return 1 + } + local fixture_root=$temporary_root/real-canary-images + local response url index + local -a urls=() + install -d -m 0700 "$fixture_root" + node "$cluster_root/scripts/acceptance/generate-reference-images.mjs" "$fixture_root" >/dev/null + for index in 1 2 3; do + response=$fixture_root/upload-$index.json + chmod 0600 "$fixture_root/production-acceptance-reference-$index.png" + { + printf 'silent\nshow-error\nfail-with-body\nmax-time = 120\n' + printf 'header = "Authorization: Bearer %s"\n' "$AI_GATEWAY_ACCEPTANCE_API_KEY" + printf 'form = "source=production-acceptance"\n' + printf 'form = "file=@%s;type=image/png"\n' \ + "$fixture_root/production-acceptance-reference-$index.png" + printf 'url = "%s/api/v1/files/upload"\n' "${AI_GATEWAY_ONLINE_BASE_URL%/}" + } | curl --config - >"$response" + chmod 0600 "$response" + url=$(jq -r '.url // .data.url // empty' "$response") + if [[ $url == /* ]]; then + url="${AI_GATEWAY_ONLINE_BASE_URL%/}$url" + fi + [[ $url == https://* ]] || { + echo "synthetic acceptance image upload $index did not return an HTTPS URL" >&2 + return 1 + } + urls+=("$url") + done + AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$(IFS=,; printf '%s' "${urls[*]}") + export AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS + echo 'acceptance_real_images=PASS source=synthetic_gateway_upload count=3 dimensions=512x512' +} + admin_request() { local method=$1 local path=$2 @@ -2646,8 +2878,10 @@ WHERE task.acceptance_run_id='$run_id'::uuid } verify_release_cas +bootstrap_acceptance_primary_identity ensure_acceptance_user_group ensure_acceptance_identity_shards +ensure_acceptance_real_images select_acceptance_models ensure_acceptance_model_access create_and_activate_run diff --git a/tests/acceptance/reference-images-test.mjs b/tests/acceptance/reference-images-test.mjs new file mode 100644 index 0000000..1e253be --- /dev/null +++ b/tests/acceptance/reference-images-test.mjs @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +const root = resolve(import.meta.dirname, '../..'); +const output = mkdtempSync(join(tmpdir(), 'easyai-reference-images-')); +execFileSync('node', [join(root, 'scripts/acceptance/generate-reference-images.mjs'), output]); + +const hashes = new Set(); +for (let index = 1; index <= 3; index += 1) { + const path = join(output, `production-acceptance-reference-${index}.png`); + const payload = readFileSync(path); + assert.deepEqual([...payload.subarray(0, 8)], [137, 80, 78, 71, 13, 10, 26, 10]); + assert.equal(payload.readUInt32BE(16), 512); + assert.equal(payload.readUInt32BE(20), 512); + assert.equal(statSync(path).mode & 0o777, 0o600); + hashes.add(createHash('sha256').update(payload).digest('hex')); +} +assert.equal(hashes.size, 3);