perf(acceptance): 使用多钱包身份分片模拟并发

单一验收钱包会把预留和结算串行化，掩盖双节点 Worker 的真实吞吐。验收现在幂等准备 32 个隔离身份和钱包，按请求轮询 API Key，并在 Run 配置中登记允许的 Key/User 身份对；密钥仅经 stdin 传给集群内压测进程。\n\n仍保留真实账务、候选权限、回调、重复扣费和强杀恢复校验。\n\n验证：Go 全量测试、临时 PostgreSQL 集成测试、go vet、OpenAPI 生成、迁移安全检查、bash -n、ShellCheck。

scripts/cluster/run-production-acceptance.sh

@@ -28,6 +28,7 @@ Optional model overrides (otherwise selected from current production candidates)
   AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL
   AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL
   AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME
+  AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS
 EOF
 }
 
@@ -52,6 +53,7 @@ require_commands curl git jq node openssl sed
 : "${AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL:=}"
 : "${AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL:=}"
 : "${AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS:=64}"
+: "${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:=32}"
 : "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
 : "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:?}"
 : "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:?}"
@@ -60,6 +62,11 @@ require_commands curl git jq node openssl sed
   echo 'AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS must be between 1 and 256' >&2
   exit 1
 }
+[[ $AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS =~ ^[1-9][0-9]*$ &&
+   $AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS -le 128 ]] || {
+  echo 'AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS must be between 1 and 128' >&2
+  exit 1
+}
 
 node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
 release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
@@ -102,6 +109,9 @@ active_profile=P24
 failure_reason=
 failure_recorded=false
 acceptance_admin_base=
+acceptance_group_id=
+acceptance_participants_json='[]'
+AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
 
 cleanup() {
   local status=$?
@@ -306,12 +316,15 @@ SELECT
   (SELECT count(*)
    FROM gateway_users
    WHERE default_user_group_id='$group_id'::uuid
-     AND id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid)
+     AND id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
+     AND COALESCE(metadata->>'purpose','') <> 'production_acceptance_shard')
   ||':'||
   (SELECT count(*)
-   FROM gateway_api_keys
-   WHERE user_group_id='$group_id'::uuid
-     AND gateway_user_id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid);")
+   FROM gateway_api_keys key
+   JOIN gateway_users user_record ON user_record.id=key.gateway_user_id
+   WHERE key.user_group_id='$group_id'::uuid
+     AND key.gateway_user_id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
+     AND COALESCE(user_record.metadata->>'purpose','') <> 'production_acceptance_shard');")
     [[ $isolation_state == 0:0 ]] || {
       echo 'acceptance user group is referenced by non-acceptance identities' >&2
       return 1
@@ -323,6 +336,7 @@ SELECT
     echo 'acceptance user group API returned an invalid group ID' >&2
     return 1
   }
+  acceptance_group_id=$group_id
 
   admin_request GET /api/admin/users '' "$users_response"
   user_body=$(jq -c \
@@ -392,12 +406,15 @@ SELECT
   (SELECT count(*)
    FROM gateway_users
    WHERE default_user_group_id='$group_id'::uuid
-     AND id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid)
+     AND id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
+     AND COALESCE(metadata->>'purpose','') <> 'production_acceptance_shard')
   ||':'||
   (SELECT count(*)
-   FROM gateway_api_keys
-   WHERE user_group_id='$group_id'::uuid
-     AND gateway_user_id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid);")
+   FROM gateway_api_keys key
+   JOIN gateway_users user_record ON user_record.id=key.gateway_user_id
+   WHERE key.user_group_id='$group_id'::uuid
+     AND key.gateway_user_id <> '$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
+     AND COALESCE(user_record.metadata->>'purpose','') <> 'production_acceptance_shard');")
   [[ $isolation_state == 1:1:1:1:0:0 ]] || {
     echo 'dedicated acceptance identity isolation verification failed' >&2
     return 1
@@ -405,6 +422,154 @@ SELECT
   echo 'acceptance_user_group=PASS rate_limit=production_candidates active_keys=1 isolated=true'
 }
 
+ensure_acceptance_identity_shards() {
+  local identity_material key_count participant_count
+  [[ $acceptance_group_id =~ ^[0-9a-f-]{36}$ ]] || {
+    echo 'acceptance user group must be prepared before identity shards' >&2
+    return 1
+  }
+  identity_material=$(database_query "
+WITH desired AS (
+  SELECT ordinal,
+         'production-acceptance-shard-' || lpad(ordinal::text, 3, '0') AS user_key
+  FROM generate_series(1, $((AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS - 1))) ordinal
+), shard_users AS (
+  INSERT INTO gateway_users (
+    user_key, source, username, display_name, default_user_group_id,
+    roles, auth_profile, metadata, status
+  )
+  SELECT desired.user_key,
+         'gateway',
+         desired.user_key,
+         'Production Acceptance Shard ' || lpad(desired.ordinal::text, 3, '0'),
+         '$acceptance_group_id'::uuid,
+         '[]'::jsonb,
+         '{}'::jsonb,
+         jsonb_build_object(
+           'purpose', 'production_acceptance_shard',
+           'ordinal', desired.ordinal,
+           'isolated', true
+         ),
+         'active'
+  FROM desired
+  ON CONFLICT (user_key) DO UPDATE
+  SET default_user_group_id=EXCLUDED.default_user_group_id,
+      metadata=EXCLUDED.metadata,
+      status='active',
+      deleted_at=NULL,
+      updated_at=now()
+  RETURNING id, user_key
+), primary_key AS (
+  SELECT id, gateway_user_id, key_secret, scopes, rate_limit_policy, quota_policy
+  FROM gateway_api_keys
+  WHERE id='$AI_GATEWAY_ACCEPTANCE_API_KEY_ID'::uuid
+    AND gateway_user_id='$AI_GATEWAY_ACCEPTANCE_USER_ID'::uuid
+    AND status='active'
+    AND deleted_at IS NULL
+    AND COALESCE(key_secret, '') <> ''
+), existing_keys AS (
+  SELECT selected.id, selected.gateway_user_id, selected.key_secret
+  FROM shard_users shard
+  JOIN LATERAL (
+    SELECT key.id, key.gateway_user_id, key.key_secret
+    FROM gateway_api_keys key
+    WHERE key.gateway_user_id=shard.id
+      AND key.name='Production Acceptance Shard'
+      AND key.status='active'
+      AND key.deleted_at IS NULL
+      AND COALESCE(key.key_secret, '') <> ''
+    ORDER BY key.created_at
+    LIMIT 1
+  ) selected ON true
+), new_key_material AS (
+  SELECT shard.id AS gateway_user_id,
+         'sk-gw-' || encode(gen_random_bytes(32), 'hex') AS secret
+  FROM shard_users shard
+  WHERE NOT EXISTS (
+    SELECT 1 FROM existing_keys existing
+    WHERE existing.gateway_user_id=shard.id
+  )
+), inserted_keys AS (
+  INSERT INTO gateway_api_keys (
+    gateway_user_id, user_id, key_prefix, key_secret, key_hash, name, scopes,
+    user_group_id, rate_limit_policy, quota_policy, status
+  )
+  SELECT material.gateway_user_id,
+         material.gateway_user_id::text,
+         left(material.secret, 18),
+         material.secret,
+         crypt(material.secret, gen_salt('bf', 10)),
+         'Production Acceptance Shard',
+         template.scopes,
+         '$acceptance_group_id'::uuid,
+         template.rate_limit_policy,
+         template.quota_policy,
+         'active'
+  FROM new_key_material material
+  CROSS JOIN primary_key template
+  RETURNING id, gateway_user_id, key_secret
+), shard_keys AS (
+  SELECT * FROM existing_keys
+  UNION ALL
+  SELECT * FROM inserted_keys
+), wallets AS (
+  INSERT INTO gateway_wallet_accounts (
+    gateway_user_id, user_id, currency, balance, total_recharged, metadata, status
+  )
+  SELECT shard.id,
+         shard.id::text,
+         'resource',
+         1000000000::numeric,
+         1000000000::numeric,
+         '{\"purpose\":\"production_acceptance_shard\",\"isolated\":true}'::jsonb,
+         'active'
+  FROM shard_users shard
+  ON CONFLICT (gateway_user_id, currency) DO UPDATE
+  SET balance=GREATEST(gateway_wallet_accounts.balance, EXCLUDED.balance),
+      total_recharged=GREATEST(gateway_wallet_accounts.total_recharged, EXCLUDED.total_recharged),
+      metadata=EXCLUDED.metadata,
+      status='active',
+      updated_at=now()
+  RETURNING gateway_user_id
+), participants AS (
+  SELECT 0 AS ordinal, key.id, key.gateway_user_id, key.key_secret
+  FROM primary_key key
+  UNION ALL
+  SELECT row_number() OVER (ORDER BY shard.user_key)::int,
+         key.id,
+         key.gateway_user_id,
+         key.key_secret
+  FROM shard_keys key
+  JOIN shard_users shard ON shard.id=key.gateway_user_id
+)
+SELECT jsonb_agg(
+         jsonb_build_object(
+           'apiKeyId', participant.id,
+           'userId', participant.gateway_user_id
+         )
+         ORDER BY participant.ordinal
+       )::text
+       || E'\\t' ||
+       string_agg(participant.key_secret, ',' ORDER BY participant.ordinal)
+FROM participants participant
+CROSS JOIN (SELECT count(*) FROM wallets) wallet_write_barrier;")
+  [[ $identity_material == *$'\t'* ]] || {
+    echo 'acceptance identity shard provisioning returned an invalid payload' >&2
+    return 1
+  }
+  acceptance_participants_json=${identity_material%%$'\t'*}
+  AI_GATEWAY_ACCEPTANCE_API_KEYS=${identity_material#*$'\t'}
+  participant_count=$(jq 'length' <<<"$acceptance_participants_json")
+  key_count=$(awk -F',' '{print NF}' <<<"$AI_GATEWAY_ACCEPTANCE_API_KEYS")
+  [[ $participant_count == "$AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS" &&
+     $key_count == "$AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS" ]] || {
+    echo 'acceptance identity shard count verification failed' >&2
+    return 1
+  }
+  export AI_GATEWAY_ACCEPTANCE_API_KEYS
+  echo "acceptance_identity_shards=PASS count=$participant_count isolated_wallets=true"
+}
+
 select_acceptance_models() {
   if [[ -z $AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL ]]; then
     AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(database_query "
@@ -510,10 +675,10 @@ FROM resources;")
     return 1
   }
   body=$(jq -cn \
-    --arg subjectID "$AI_GATEWAY_ACCEPTANCE_API_KEY_ID" \
+    --arg subjectID "$acceptance_group_id" \
     --argjson resources "$resources" \
     '{
-      subjectType:"api_key",
+      subjectType:"user_group",
       subjectId:$subjectID,
       effect:"allow",
       upsertResources:$resources,
@@ -549,15 +714,15 @@ FROM resources resource
 WHERE NOT EXISTS (
   SELECT 1
   FROM gateway_access_rules rule
-  WHERE rule.subject_type='api_key'
-    AND rule.subject_id='$AI_GATEWAY_ACCEPTANCE_API_KEY_ID'::uuid
+  WHERE rule.subject_type='user_group'
+    AND rule.subject_id='$acceptance_group_id'::uuid
     AND rule.effect='allow'
     AND rule.status='active'
     AND rule.resource_type=resource.resource_type
     AND rule.resource_id=resource.resource_id
 );")
   [[ $missing == 0 ]] || {
-    echo "acceptance API Key is missing $missing selected candidate access rules" >&2
+    echo "acceptance user group is missing $missing selected candidate access rules" >&2
     return 1
   }
   echo "acceptance_model_access=PASS resources=$expected"
@@ -587,6 +752,7 @@ create_and_activate_run() {
     --arg token "$run_token" \
     --arg emulatorURL 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' \
     --arg callbackURL 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090/callbacks' \
+    --argjson participants "$acceptance_participants_json" \
     '{
       releaseSha: $releaseSha,
       apiImageDigest: $apiDigest,
@@ -597,7 +763,10 @@ create_and_activate_run() {
       emulatorBaseUrl: $emulatorURL,
       callbackUrl: $callbackURL,
       capacityProfile: "P24",
-      config: {workloads: ["gemini_image_edit", "multi_reference_video"]}
+      config: {
+        workloads: ["gemini_image_edit", "multi_reference_video"],
+        participants: $participants
+      }
     }')
   admin_request POST /api/admin/system/acceptance/runs "$body" "$create_response"
   run_id=$(jq -r '.id // empty' "$create_response")
@@ -746,7 +915,7 @@ set -eu
 read -r gateways_b64
 read -r tls_name_b64
 read -r emulator_b64
-read -r api_key_b64
+read -r api_keys_b64
 read -r run_id_b64
 read -r run_token_b64
 read -r gemini_model_b64
@@ -758,7 +927,7 @@ decode() {
 export AI_GATEWAY_ACCEPTANCE_GATEWAYS=$(decode "$gateways_b64")
 export AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=$(decode "$tls_name_b64")
 export AI_GATEWAY_ACCEPTANCE_EMULATOR_URL=$(decode "$emulator_b64")
-export AI_GATEWAY_ACCEPTANCE_API_KEY=$(decode "$api_key_b64")
+export AI_GATEWAY_ACCEPTANCE_API_KEYS=$(decode "$api_keys_b64")
 export AI_GATEWAY_ACCEPTANCE_RUN_ID=$(decode "$run_id_b64")
 export AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=$(decode "$run_token_b64")
 export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(decode "$gemini_model_b64")
@@ -779,7 +948,7 @@ exit "$status"
     printf '%s' 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' |
       openssl base64 -A
     printf '\n'
-    printf '%s' "$AI_GATEWAY_ACCEPTANCE_API_KEY" | openssl base64 -A
+    printf '%s' "$AI_GATEWAY_ACCEPTANCE_API_KEYS" | openssl base64 -A
     printf '\n'
     printf '%s' "$run_id" | openssl base64 -A
     printf '\n'
@@ -1359,6 +1528,7 @@ finish_and_promote() {
 
 verify_release_cas
 ensure_acceptance_user_group
+ensure_acceptance_identity_shards
 select_acceptance_models
 ensure_acceptance_model_access
 deploy_protocol_emulator