fix(acceptance): 修复线上双站压测失真

将线上压力生成器拆分为宁波、香港宿主机上的 K3s 外进程，按全局序号分片并保守合并站点报告，避免操作者本机上行带宽成为容量瓶颈。\n\n为每次负载执行生成独立幂等键和 Run 级图片变体，阻止三轮验收重放旧任务或复用历史媒体缓存，并新增直接 OSS 物化一致性门禁。\n\n验证：Go 全量测试、go vet、迁移安全检查、bash -n、ShellCheck、manual-release-test 和双宿主机 linux/amd64 启动冒烟通过。
2026-08-01 01:02:07 +08:00
parent 9fd9c267d6
commit be6ce7f78a
5 changed files with 359 additions and 42 deletions
@@ -74,7 +74,7 @@ if [[ $skip_local_acceptance != true ]]; then
 fi

 load_cluster_env
-require_commands curl git go jq node openssl sed
+require_commands curl git go jq node openssl sed shasum

 : "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
 : "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
@@ -225,6 +225,7 @@ acceptance_group_id=
 acceptance_participants_json='[]'
 AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
 acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
+acceptance_load_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-load-linux-amd64
 acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
 current_production_snapshot=$temporary_root/current-production-snapshot.json
 local_snapshot_config_hash=
@@ -235,6 +236,7 @@ if [[ $skip_local_acceptance != true ]]; then
 fi
 active_load_pid=
 active_pressure_pid=
+remote_load_drivers_installed=false
 capacity_profiles_payload=
 image_stable_throughput=
 image_admitted_throughput=
@@ -254,6 +256,9 @@ cleanup() {
    kill "$active_pressure_pid" >/dev/null 2>&1 || true
    wait "$active_pressure_pid" >/dev/null 2>&1 || true
  fi
+  if [[ $remote_load_drivers_installed == true ]]; then
+    cleanup_remote_load_drivers >/dev/null 2>&1 || true
+  fi
  if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
    set +e
    apply_capacity_profile "$stable_profile" >/dev/null 2>&1
@@ -307,6 +312,8 @@ bootstrap_acceptance_admin_token
  cd "$cluster_root/apps/api"
  env -u AI_GATEWAY_TEST_DATABASE_URL go build \
    -trimpath -o "$acceptance_load_binary" ./cmd/acceptance-load
+  env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
+    -trimpath -o "$acceptance_load_linux_binary" ./cmd/acceptance-load
  env -u AI_GATEWAY_TEST_DATABASE_URL go build \
    -trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
 )
@@ -1750,24 +1757,14 @@ run_mixed_load_profile() {
  local video_rate=$4
  local report_path=$5
  local timeout_value=$6
-  local load_stdout=$temporary_root/load-$profile-stdout.json
-  AI_GATEWAY_ACCEPTANCE_GATEWAYS="$AI_GATEWAY_ACCEPTANCE_GATEWAYS" \
-  AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME="$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" \
-  AI_GATEWAY_ACCEPTANCE_EMULATOR_URL='http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' \
-  AI_GATEWAY_ACCEPTANCE_API_KEYS="$AI_GATEWAY_ACCEPTANCE_API_KEYS" \
-  AI_GATEWAY_ACCEPTANCE_RUN_ID="$run_id" \
-  AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$run_token" \
-  AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL" \
-  AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
-  AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS="$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" \
-    "$acceptance_load_binary" \
-      -profile "$profile" \
-      -duration "$duration" \
-      -image-rate "$image_rate" \
-      -video-rate "$video_rate" \
-      -timeout "$timeout_value" \
-      -report "$report_path" >"$load_stdout"
-  jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
+  local site_image_rate site_video_rate
+  site_image_rate=$(awk -v value="$image_rate" 'BEGIN {printf "%.9f", value/2}')
+  site_video_rate=$(awk -v value="$video_rate" 'BEGIN {printf "%.9f", value/2}')
+  run_distributed_load_profile "$profile" "$report_path" \
+    -duration "$duration" \
+    -image-rate "$site_image_rate" \
+    -video-rate "$site_video_rate" \
+    -timeout "$timeout_value"
 }

 run_mixed_soak_and_overload() {
@@ -1904,10 +1901,196 @@ activate_certified_autoscaling() {
  verify_runtime_gates
 }

+remote_load_binary_path() {
+  [[ $run_id =~ ^[0-9a-f-]{36}$ ]]
+  printf '/root/easyai-acceptance-load-%s' "$run_id"
+}
+
+remote_load_env_path() {
+  local site=$1
+  [[ $site == ningbo || $site == hongkong ]]
+  [[ $run_id =~ ^[0-9a-f-]{36}$ ]]
+  printf '/root/easyai-acceptance-load-%s-%s.env' "$run_id" "$site"
+}
+
+cleanup_remote_load_drivers() {
+  [[ $run_id =~ ^[0-9a-f-]{36}$ ]] || return 0
+  local binary_path ningbo_env hongkong_env
+  binary_path=$(remote_load_binary_path)
+  ningbo_env=$(remote_load_env_path ningbo)
+  hongkong_env=$(remote_load_env_path hongkong)
+  cluster_ssh "$CLUSTER_NINGBO_HOST" \
+    "pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$ningbo_env' ]] || unlink '$ningbo_env'" || true
+  cluster_ssh "$CLUSTER_HONGKONG_HOST" \
+    "pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$hongkong_env' ]] || unlink '$hongkong_env'" || true
+  remote_load_drivers_installed=false
+}
+
+write_remote_load_env() {
+  local output=$1
+  local gateway=$2
+  install -m 0600 /dev/null "$output"
+  {
+    printf 'AI_GATEWAY_ACCEPTANCE_GATEWAYS=%q\n' "$gateway"
+    printf 'AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=%q\n' "$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME"
+    printf 'AI_GATEWAY_ACCEPTANCE_EMULATOR_URL=%q\n' 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090'
+    printf 'AI_GATEWAY_ACCEPTANCE_API_KEYS=%q\n' "$AI_GATEWAY_ACCEPTANCE_API_KEYS"
+    printf 'AI_GATEWAY_ACCEPTANCE_RUN_ID=%q\n' "$run_id"
+    printf 'AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=%q\n' "$run_token"
+    printf 'AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=%q\n' "$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL"
+    printf 'AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=%q\n' "$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL"
+    printf 'AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=%q\n' "$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS"
+  } >>"$output"
+}
+
+install_remote_load_drivers() {
+  [[ $remote_load_drivers_installed == false ]] || return 0
+  local binary_path local_digest remote_digest site host gateway env_file remote_env
+  binary_path=$(remote_load_binary_path)
+  local_digest=$(shasum -a 256 "$acceptance_load_linux_binary" | awk '{print $1}')
+  local -a sites=(ningbo hongkong)
+  local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST")
+  local -a gateways=("https://${CLUSTER_NINGBO_HOST#root@}" "https://${CLUSTER_HONGKONG_HOST#root@}")
+  for site_index in 0 1; do
+    site=${sites[$site_index]}
+    host=${hosts[$site_index]}
+    gateway=${gateways[$site_index]}
+    env_file=$temporary_root/remote-load-$site.env
+    remote_env=$(remote_load_env_path "$site")
+    write_remote_load_env "$env_file" "$gateway"
+    cluster_scp "$acceptance_load_linux_binary" "$host:$binary_path" >/dev/null
+    cluster_scp "$env_file" "$host:$remote_env" >/dev/null
+    cluster_ssh "$host" "chmod 0755 '$binary_path'; chmod 0600 '$remote_env'; sha256sum '$binary_path'" >"$temporary_root/remote-load-$site.sha256"
+    remote_digest=$(awk '{print $1}' "$temporary_root/remote-load-$site.sha256")
+    [[ $remote_digest == "$local_digest" ]]
+  done
+  remote_load_drivers_installed=true
+  echo 'acceptance_load_drivers=PASS sites=2 placement=k3s_external_host_processes digest_verified=true'
+}
+
+run_remote_load_site() {
+  local site=$1
+  local host=$2
+  local profile=$3
+  local artifact=$4
+  local shard_index=$5
+  shift 5
+  [[ $site == ningbo || $site == hongkong ]]
+  [[ $profile =~ ^[a-z-]+$ && $artifact =~ ^[a-z0-9-]+\.json$ ]]
+  local binary_path remote_env remote_report local_report status=0
+  binary_path=$(remote_load_binary_path)
+  remote_env=$(remote_load_env_path "$site")
+  remote_report=/root/easyai-acceptance-load-"$run_id"-"$artifact"
+  local_report=$temporary_root/"$site"-"$artifact"
+  cluster_ssh "$host" bash -s -- \
+    "$binary_path" "$remote_env" "$profile" "$remote_report" "$shard_index" 2 "${artifact%.json}" "$@" \
+    >"$temporary_root/$site-$artifact.stdout" <<'REMOTE' || status=$?
+set -euo pipefail
+binary_path=$1
+env_file=$2
+profile=$3
+report=$4
+shard_index=$5
+shard_count=$6
+execution_id=$7
+shift 7
+set -a
+# shellcheck source=/dev/null
+source "$env_file"
+set +a
+"$binary_path" -profile "$profile" -report "$report" \
+  -shard-index "$shard_index" -shard-count "$shard_count" \
+  -execution-id "$execution_id" "$@"
+REMOTE
+  cluster_scp "$host:$remote_report" "$local_report" >/dev/null || return 1
+  cluster_ssh "$host" "[[ ! -e '$remote_report' ]] || unlink '$remote_report'" >/dev/null || true
+  jq -e --arg runId "$run_id" --arg profile "$profile" \
+    '.schemaVersion == "acceptance-load-report/v1" and .runId == $runId and .profile == $profile and .secretSafe == true' \
+    "$local_report" >/dev/null || return 1
+  return "$status"
+}
+
+merge_remote_load_reports() {
+  local profile=$1
+  local ningbo_report=$2
+  local hongkong_report=$3
+  local output=$4
+  jq -s --arg profile "$profile" '
+    def sum_field($name): [.[].phases[0][$name] // 0] | add;
+    def max_field($name): [.[].phases[0][$name] // 0] | max;
+    . as $reports
+    | {
+        schemaVersion:"acceptance-load-report/v1",
+        runId:$reports[0].runId,
+        profile:$profile,
+        startedAt:([$reports[].startedAt] | min),
+        finishedAt:([$reports[].finishedAt] | max),
+        passed:([$reports[].passed] | all),
+        phases:[{
+          name:$profile,
+          requests:sum_field("requests"),
+          completed:sum_field("completed"),
+          failed:sum_field("failed"),
+          durationMs:max_field("durationMs"),
+          submissionDurationMs:max_field("submissionDurationMs"),
+          p50Ms:max_field("p50Ms"),
+          p95Ms:max_field("p95Ms"),
+          p99Ms:max_field("p99Ms"),
+          decodedOutputBytes:sum_field("decodedOutputBytes"),
+          outputSha256:([$reports[].phases[0].outputSha256 // ""] | map(select(length>0)) | first // ""),
+          uniqueTaskIds:sum_field("uniqueTaskIds"),
+          uniqueImageCombinations:max_field("uniqueImageCombinations"),
+          forcedConversionTasks:sum_field("forcedConversionTasks"),
+          throttled:sum_field("throttled"),
+          unexpected5xx:sum_field("unexpected5xx"),
+          offeredRatePerSecond:sum_field("offeredRatePerSecond")
+        }],
+        failure:([$reports[] | select(.passed == false) | .failure] | first // null),
+        failureOperation:([$reports[] | select(.passed == false) | .failureOperation] | first // null),
+        aggregation:{
+          sites:["ningbo","hongkong"],
+          percentileMethod:"conservative_max_of_site_percentiles",
+          durationMethod:"max_site_duration",
+          placement:"k3s_external_host_processes"
+        },
+        secretSafe:true
+      }
+    | with_entries(select(.value != null))
+  ' "$ningbo_report" "$hongkong_report" >"$output"
+  chmod 0600 "$output"
+}
+
+run_distributed_load_profile() {
+  local profile=$1
+  local report_path=$2
+  shift 2
+  install_remote_load_drivers
+  local artifact base ningbo_report hongkong_report ningbo_pid hongkong_pid
+  local ningbo_status=0 hongkong_status=0
+  base=$(basename "$report_path" .json)
+  artifact=$base.json
+  run_remote_load_site ningbo "$CLUSTER_NINGBO_HOST" "$profile" "$artifact" 0 "$@" &
+  ningbo_pid=$!
+  run_remote_load_site hongkong "$CLUSTER_HONGKONG_HOST" "$profile" "$artifact" 1 "$@" &
+  hongkong_pid=$!
+  wait "$ningbo_pid" || ningbo_status=$?
+  wait "$hongkong_pid" || hongkong_status=$?
+  ningbo_report=$temporary_root/ningbo-$artifact
+  hongkong_report=$temporary_root/hongkong-$artifact
+  [[ -f $ningbo_report && -f $hongkong_report ]] || return 1
+  merge_remote_load_reports "$profile" "$ningbo_report" "$hongkong_report" "$report_path"
+  (( ningbo_status == 0 && hongkong_status == 0 )) || return 1
+  jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
+}
+
 run_load_profile() {
  local profile=$1
  local report_path=$2
  [[ $profile =~ ^[a-z-]+$ ]] || return 1
+  if [[ $profile != real-canary ]]; then
+    run_distributed_load_profile "$profile" "$report_path"
+    return
+  fi
  local load_stdout=$temporary_root/load-$profile-stdout.json
  AI_GATEWAY_ACCEPTANCE_GATEWAYS="$AI_GATEWAY_ACCEPTANCE_GATEWAYS" \
  AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME="$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" \
@@ -1919,6 +2102,7 @@ run_load_profile() {
  AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
  AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS="$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" \
    "$acceptance_load_binary" -profile "$profile" -report "$report_path" \
+      -execution-id "$(basename "$report_path" .json)" \
      >"$load_stdout"
  jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
 }
@@ -2617,6 +2801,7 @@ finish_and_report() {
  local gemini_invalid video_invalid reference_count_3 reference_count_6 reference_count_9
  local reference_role_count first_frame_role_count last_frame_role_count
  local unique_image_hashes unique_video_tasks
+  local unique_gemini_input_hashes direct_gemini_input_assets
  local duplicate_remote_tasks duplicate_terminal_events duplicate_settlements
  local duplicate_wallet_transactions duplicate_callback_rows pending_settlements pending_callbacks
  local missing_billings
@@ -2625,6 +2810,19 @@ finish_and_report() {
  gemini_task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND run_mode='acceptance';")
  forced_tasks=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND request::text LIKE '%acceptance-force-conversion%';")
  compact_payloads=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND (pg_column_size(request) >= 1048576 OR pg_column_size(result) >= 1048576);")
+  unique_gemini_input_hashes=$(database_query "
+SELECT count(DISTINCT request #>> '{image,assetRef,sha256}')
+FROM gateway_tasks
+WHERE acceptance_run_id='$run_id'::uuid
+  AND kind='images.edits'
+  AND run_mode='acceptance';")
+  direct_gemini_input_assets=$(database_query "
+SELECT count(*)
+FROM gateway_tasks
+WHERE acceptance_run_id='$run_id'::uuid
+  AND kind='images.edits'
+  AND run_mode='acceptance'
+  AND request #>> '{image,assetRef,storageProvider}' = 'aliyun_oss_direct';")
  submissions=$(jq -r '.videoSubmissions' "$emulator_report")
  duplicates=$(jq -r '.duplicates' "$callback_report")
  callbacks=$(jq -r '.deliveries' "$callback_report")
@@ -2653,6 +2851,10 @@ finish_and_report() {
     $missing_idempotency == 0 && $compact_payloads == 0 ]] ||
    fail_acceptance_gate idempotency_and_materialization \
      'duplicate submission, missing idempotency key, or inline database media detected'
+  [[ $unique_gemini_input_hashes == "$gemini_task_count" &&
+     $direct_gemini_input_assets == "$gemini_task_count" ]] ||
+    fail_acceptance_gate gemini_input_materialization \
+      'Gemini inputs were reused across executions or bypassed direct OSS materialization'
  [[ $gemini_invalid == 0 && $video_invalid == 0 ]] ||
    fail_acceptance_gate provider_protocol_validity 'Gemini or Volces protocol validation failed'
  [[ $reference_count_3 -gt 0 && $reference_count_6 -gt 0 && $reference_count_9 -gt 0 ]] ||
@@ -2796,6 +2998,8 @@ WHERE task.acceptance_run_id='$run_id'::uuid
    --argjson lastFrameRoleCount "$last_frame_role_count" \
    --argjson uniqueImageHashes "$unique_image_hashes" \
    --argjson uniqueVideoTasks "$unique_video_tasks" \
+    --argjson uniqueGeminiInputHashes "$unique_gemini_input_hashes" \
+    --argjson directGeminiInputAssets "$direct_gemini_input_assets" \
    --argjson duplicateRemoteTasks "$duplicate_remote_tasks" \
    --argjson duplicateTerminalEvents "$duplicate_terminal_events" \
    --argjson duplicateSettlements "$duplicate_settlements" \
@@ -2838,7 +3042,9 @@ WHERE task.acceptance_run_id='$run_id'::uuid
          last_frame:$lastFrameRoleCount
        },
        uniqueImageHashes:$uniqueImageHashes,
-        uniqueVideoTasks:$uniqueVideoTasks
+        uniqueVideoTasks:$uniqueVideoTasks,
+        uniqueGeminiInputHashes:$uniqueGeminiInputHashes,
+        directGeminiInputAssets:$directGeminiInputAssets
      },
      databaseConsistency:{
        duplicateRemoteTasks:$duplicateRemoteTasks,