fix(acceptance): 修复线上双站压测失真

将线上压力生成器拆分为宁波、香港宿主机上的 K3s 外进程,按全局序号分片并保守合并站点报告,避免操作者本机上行带宽成为容量瓶颈。\n\n为每次负载执行生成独立幂等键和 Run 级图片变体,阻止三轮验收重放旧任务或复用历史媒体缓存,并新增直接 OSS 物化一致性门禁。\n\n验证:Go 全量测试、go vet、迁移安全检查、bash -n、ShellCheck、manual-release-test 和双宿主机 linux/amd64 启动冒烟通过。
This commit is contained in:
2026-08-01 01:02:07 +08:00
parent 9fd9c267d6
commit be6ce7f78a
5 changed files with 359 additions and 42 deletions
+70 -16
View File
@@ -56,6 +56,9 @@ type options struct {
mixedDuration time.Duration
imageRate float64
videoRate float64
shardIndex int
shardCount int
executionID string
}
type report struct {
@@ -194,12 +197,18 @@ func parseOptions() (options, error) {
var mixedDuration time.Duration
var imageRate float64
var videoRate float64
var shardIndex int
var shardCount int
var executionID string
flag.StringVar(&profile, "profile", env("AI_GATEWAY_ACCEPTANCE_PROFILE", "simulated-all"), "simulated-all, Gemini profile, video profile, or real-canary")
flag.StringVar(&reportPath, "report", env("AI_GATEWAY_ACCEPTANCE_REPORT", ""), "secret-safe JSON report path")
flag.DurationVar(&timeout, "timeout", 45*time.Minute, "overall timeout")
flag.DurationVar(&mixedDuration, "duration", envDuration("AI_GATEWAY_ACCEPTANCE_MIXED_DURATION", 10*time.Minute), "mixed workload duration")
flag.Float64Var(&imageRate, "image-rate", envFloat("AI_GATEWAY_ACCEPTANCE_IMAGE_RATE", 0), "mixed image requests per second")
flag.Float64Var(&videoRate, "video-rate", envFloat("AI_GATEWAY_ACCEPTANCE_VIDEO_RATE", 0), "mixed video requests per second")
flag.IntVar(&shardIndex, "shard-index", 0, "zero-based distributed load shard index")
flag.IntVar(&shardCount, "shard-count", 1, "distributed load shard count")
flag.StringVar(&executionID, "execution-id", "", "unique workload execution identifier")
flag.Parse()
opts := options{
gateways: splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAYS")),
@@ -218,9 +227,21 @@ func parseOptions() (options, error) {
mixedDuration: mixedDuration,
imageRate: imageRate,
videoRate: videoRate,
shardIndex: shardIndex,
shardCount: shardCount,
executionID: strings.ToLower(strings.TrimSpace(executionID)),
}
if len(opts.gateways) != 2 {
return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAYS must contain exactly two API base URLs")
if opts.executionID == "" {
opts.executionID = opts.profile
}
if matched, _ := regexp.MatchString(`^[a-z0-9][a-z0-9-]{0,127}$`, opts.executionID); !matched {
return options{}, errors.New("execution-id must contain only lowercase letters, numbers, and hyphens")
}
if opts.shardCount < 1 || opts.shardCount > 32 || opts.shardIndex < 0 || opts.shardIndex >= opts.shardCount {
return options{}, fmt.Errorf("invalid load shard %d/%d", opts.shardIndex, opts.shardCount)
}
if len(opts.gateways) < 1 || len(opts.gateways) > 2 || (opts.shardCount == 1 && len(opts.gateways) != 2) {
return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAYS must contain two API base URLs, or one URL for a distributed shard")
}
for index := range opts.gateways {
opts.gateways[index] = strings.TrimRight(opts.gateways[index], "/")
@@ -262,6 +283,17 @@ func parseOptions() (options, error) {
return opts, nil
}
func (o options) shardRequestCount(total int) int {
if total <= o.shardIndex {
return 0
}
return (total-1-o.shardIndex)/o.shardCount + 1
}
func (o options) logicalRequestIndex(localIndex int) int {
return o.shardIndex + localIndex*o.shardCount
}
func run(ctx context.Context, opts options) ([]phaseResult, error) {
var tlsConfig *tls.Config
if opts.gatewayTLSName != "" || opts.gatewayCAFile != "" {
@@ -289,23 +321,26 @@ func run(ctx context.Context, opts options) ([]phaseResult, error) {
runPhase := func(name string) error {
var result phaseResult
if profile, ok := acceptanceworkload.GeminiProfileByName(name); ok {
result = runGemini(ctx, client, opts, name, profile.Requests, profile.InputBytes, profile.OutputBytes, false)
result = runGemini(
ctx, client, opts, name, opts.shardRequestCount(profile.Requests),
profile.InputBytes, profile.OutputBytes, false, opts.shardIndex, opts.shardCount,
)
} else {
switch name {
case "smoke-gemini":
result = runGemini(ctx, client, opts, name, 4, 256<<10, 256<<10, false)
result = runGemini(ctx, client, opts, name, opts.shardRequestCount(4), 256<<10, 256<<10, false, opts.shardIndex, opts.shardCount)
case "smoke-video":
result = runVideo(ctx, client, opts, name, 10, false, opts.emulatorFixtureURLs(), false, 0)
result = runVideo(ctx, client, opts, name, opts.shardRequestCount(10), false, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
case "video-throughput":
result = runVideo(ctx, client, opts, name, 1200, false, opts.emulatorFixtureURLs(), false, 0)
result = runVideo(ctx, client, opts, name, opts.shardRequestCount(1200), false, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
case "video-recovery":
result = runVideo(ctx, client, opts, name, 96, true, opts.emulatorFixtureURLs(), false, 0)
result = runVideo(ctx, client, opts, name, opts.shardRequestCount(96), true, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
case "mixed-soak", "mixed-overload":
result = runMixed(ctx, client, opts, name == "mixed-overload")
case "real-gemini-canary":
result = runGemini(ctx, client, opts, name, 1, 256<<10, 0, true)
result = runGemini(ctx, client, opts, name, 1, 256<<10, 0, true, 0, 1)
case "real-video-canary":
result = runVideo(ctx, client, opts, name, 1, false, opts.realImageURLs, true, 0)
result = runVideo(ctx, client, opts, name, 1, false, opts.realImageURLs, true, 0, 1)
default:
return fmt.Errorf("unknown phase %q", name)
}
@@ -338,7 +373,12 @@ func runGemini(
inputBytes int,
expectedOutputBytes int,
realUpstream bool,
requestOffset int,
requestStride int,
) phaseResult {
if requestStride < 1 {
requestStride = 1
}
startedAt := time.Now()
expectedHash := ""
if expectedOutputBytes > 0 {
@@ -358,6 +398,7 @@ func runGemini(
wg.Add(1)
go func() {
defer wg.Done()
logicalIndex := requestOffset + index*requestStride
select {
case slots <- struct{}{}:
case <-ctx.Done():
@@ -365,8 +406,8 @@ func runGemini(
}
defer func() { <-slots }()
requestStarted := time.Now()
endpoint := opts.gateways[index%len(opts.gateways)] + "/v1beta/models/" + url.PathEscape(opts.geminiModel) + ":generateContent"
requestBody := streamGeminiRequestBody(paddedPNGVariant(inputBytes, index+1))
endpoint := opts.gateways[logicalIndex%len(opts.gateways)] + "/v1beta/models/" + url.PathEscape(opts.geminiModel) + ":generateContent"
requestBody := streamGeminiRequestBody(paddedPNGVariant(inputBytes, geminiInputVariant(opts.runID+"/"+opts.executionID, logicalIndex)))
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
@@ -377,7 +418,7 @@ func runGemini(
_ = requestBody.Close()
}
if err == nil {
opts.setHeaders(req, index, realUpstream)
opts.setHeaders(req, logicalIndex, realUpstream)
req.Header.Set("Content-Type", "application/json")
var response *http.Response
response, err = client.Do(req)
@@ -472,7 +513,11 @@ func runVideo(
imageURLs []string,
realUpstream bool,
requestOffset int,
requestStride int,
) phaseResult {
if requestStride < 1 {
requestStride = 1
}
startedAt := time.Now()
combinationCount := 128
if realUpstream {
@@ -494,7 +539,7 @@ func runVideo(
wg.Add(1)
go func() {
defer wg.Done()
logicalIndex := index + requestOffset
logicalIndex := requestOffset + index*requestStride
requestStarted := time.Now()
startedTasks[index] = requestStarted
imageCount := acceptanceworkload.VideoImageCount(logicalIndex)
@@ -590,7 +635,7 @@ func runVideo(
continue
}
index, taskID := index, taskID
logicalIndex := index + requestOffset
logicalIndex := requestOffset + index*requestStride
wg.Add(1)
go func() {
defer wg.Done()
@@ -685,7 +730,7 @@ func runMixed(
generating = false
continue
}
requestIndex := requests
requestIndex := opts.logicalRequestIndex(requests)
requests++
imageAccumulator += opts.imageRate
isImage := imageAccumulator >= totalRate
@@ -698,7 +743,7 @@ func runMixed(
defer func() { <-slots }()
var result phaseResult
if isImage {
result = runGemini(runCtx, client, opts, "mixed-image", 1, 256<<10, 256<<10, false)
result = runGemini(runCtx, client, opts, "mixed-image", 1, 256<<10, 256<<10, false, requestIndex, 1)
} else {
result = runVideo(
runCtx,
@@ -710,6 +755,7 @@ func runMixed(
opts.emulatorFixtureURLs(),
false,
requestIndex,
1,
)
}
latency := time.Since(startedAt)
@@ -940,6 +986,9 @@ func (o options) setHeaders(request *http.Request, requestIndex int, realUpstrea
request.Header.Set("Authorization", "Bearer "+o.apiKeys[requestIndex%len(o.apiKeys)])
request.Header.Set(runHeader, o.runID)
request.Header.Set(tokenHeader, o.runToken)
if request.Method == http.MethodPost {
request.Header.Set("Idempotency-Key", fmt.Sprintf("acceptance-%s-%s-%d", o.runID, o.executionID, requestIndex))
}
if realUpstream {
request.Header.Set(upstreamHeader, "real")
}
@@ -1115,6 +1164,11 @@ func paddedPNG(size int) []byte {
return paddedPNGVariant(size, 0)
}
func geminiInputVariant(runID string, logicalIndex int) int {
digest := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", strings.TrimSpace(runID), logicalIndex)))
return int(binary.BigEndian.Uint64(digest[:8]) & uint64(^uint(0)>>1))
}
func paddedPNGVariant(size int, variant int) []byte {
base, _ := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
if size <= len(base) {
+48 -1
View File
@@ -146,7 +146,7 @@ func TestGeminiLoadIsSplitAcrossTwoGatewayAPIs(t *testing.T) {
gateways: []string{first.URL, second.URL}, apiKeys: []string{"key-1", "key-2"}, runID: "run-1",
runToken: "token-1", geminiModel: "gemini-image-test",
}
result := runGemini(t.Context(), http.DefaultClient, opts, "dual-api", 8, 256<<10, 256<<10, false)
result := runGemini(t.Context(), http.DefaultClient, opts, "dual-api", 8, 256<<10, 256<<10, false, 0, 1)
if result.err != nil || result.report.Completed != 8 {
t.Fatalf("result=%+v err=%v", result.report, result.err)
}
@@ -158,6 +158,53 @@ func TestGeminiLoadIsSplitAcrossTwoGatewayAPIs(t *testing.T) {
}
}
func TestDistributedShardIndexesCoverWorkloadWithoutOverlap(t *testing.T) {
first := options{shardIndex: 0, shardCount: 2}
second := options{shardIndex: 1, shardCount: 2}
if first.shardRequestCount(1001) != 501 || second.shardRequestCount(1001) != 500 {
t.Fatalf("shard counts=%d/%d", first.shardRequestCount(1001), second.shardRequestCount(1001))
}
seen := map[int]bool{}
for local := 0; local < first.shardRequestCount(1001); local++ {
seen[first.logicalRequestIndex(local)] = true
}
for local := 0; local < second.shardRequestCount(1001); local++ {
index := second.logicalRequestIndex(local)
if seen[index] {
t.Fatalf("duplicate logical index %d", index)
}
seen[index] = true
}
if len(seen) != 1001 {
t.Fatalf("covered indexes=%d", len(seen))
}
}
func TestGeminiInputVariantIsRunScoped(t *testing.T) {
if geminiInputVariant("run-a", 7) == geminiInputVariant("run-b", 7) {
t.Fatal("different Run IDs produced the same input variant")
}
if geminiInputVariant("run-a", 7) == geminiInputVariant("run-a", 8) {
t.Fatal("different logical indexes produced the same input variant")
}
}
func TestAcceptanceIdempotencyKeyIncludesExecutionAndLogicalIndex(t *testing.T) {
request := httptest.NewRequest(http.MethodPost, "https://gateway.example/v1", nil)
opts := options{
apiKeys: []string{"key-1"}, runID: "run-1", runToken: "token-1", executionID: "p24-2-gemini-baseline",
}
opts.setHeaders(request, 17, false)
if got := request.Header.Get("Idempotency-Key"); got != "acceptance-run-1-p24-2-gemini-baseline-17" {
t.Fatalf("idempotency key=%q", got)
}
poll := httptest.NewRequest(http.MethodGet, "https://gateway.example/result", nil)
opts.setHeaders(poll, 17, false)
if got := poll.Header.Get("Idempotency-Key"); got != "" {
t.Fatalf("GET request unexpectedly has idempotency key %q", got)
}
}
func TestValidateVideoAssetDownloadsFinalMedia(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if request.Header.Get("Authorization") != "" {
+13 -4
View File
@@ -8,7 +8,8 @@
验收脚本是显式生产写操作,会暂停新正式任务、调整 Worker 容量并在恢复场景删除一个
Worker Pod。它不会自动发布新版本;只能针对已经发布、完整 SHA 和 digest 固定的 manifest
执行。进入线上前必须先通过[本地三节点同构验收](local-isomorphic-acceptance.md)
执行。默认先通过[本地三节点同构验收](local-isomorphic-acceptance.md);用户明确授权在线验收时,
可以记录审计豁免并跳过本地阶段,但不能把跳过记录成通过。
## 隔离与流量门禁
@@ -61,10 +62,12 @@ WebP、合法 4K 图片和越过 Seedance 2.0 官方输入边界的 `6144x2160`
收集器。它校验参考图数量、角色、格式、尺寸与内容哈希,并统计重复上游提交和重复回调。
默认通过 `AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS=32` 幂等准备 32 个仅属于验收用户组的
身份、API Key 和独立钱包,并按请求序号轮询使用。这样可模拟真实多租户并发,避免把 1000
身份、API Key 和独立钱包,并按全局请求序号轮询使用。每次负载执行都有独立 execution ID
它同时进入幂等键和输入图片哈希,因此三次重复验收既不会重放旧任务,也不会命中前一轮的
输入资产缓存。这样可模拟真实多租户并发,避免把 1000
个任务全部压在同一钱包行锁上;每个任务仍执行真实预留、结算和重复扣费检查。该值可在
`1..128` 内通过环境变量调整,Run 只保存 Key/User ID 对,Key Secret 仅通过 stdin 传入
集群内压测进程,不进入 Pod spec、数据库 Run 配置、报告或日志。
双站压测进程,不进入 Pod spec、数据库 Run 配置、报告或日志。
## Worker 配置
@@ -96,7 +99,7 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG
重新编码,避免执行槽升档时把大图峰值内存同比放大。2 GiB Worker 的起始值为 `2`;它不占用
视频提交后的上游等待槽,只有实测单 Pod RSS 和节点预算允许时才单独升档。
API Pod 使用独立发布配置 `AI_GATEWAY_API_DATABASE_MAX_CONNS`,生产同构验收默认使用 `64`
API Pod 使用独立发布配置 `AI_GATEWAY_API_DATABASE_MAX_CONNS`,生产同构验收默认使用 `31`
也可通过 `AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS` 覆盖本次验收值,无需修改 Go 代码。
所有 API/Worker Pool 的 `AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 固定为 `4`,仅预热必要连接,
`AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS``30`,突发扩出的空闲连接会及时回落,
@@ -135,6 +138,12 @@ PNG 并通过验收 Key 上传到 Gateway。生产快照默认在已部署的 AP
`AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=ai.51easyai.com`。压测器会继续校验正式
证书,并同时使用该域名作为 HTTP Host;这不是跳过 TLS 校验。
线上高负载不再由操作者本机单进程跨公网发送。脚本构建同一源码的静态 linux/amd64
压测器,校验 SHA-256 后以 `0600` 临时环境分别运行在宁波、香港宿主机的 K3s 之外;两个
进程同时只访问本站 TLS 入口,并按奇偶全局序号分担请求。这样避免本机上行带宽成为容量
瓶颈,同时仍覆盖正式 TLS/Nginx 入口。站点报告按请求数相加、耗时取较慢站点、p50/p95/p99
取两站较差值进行保守合并。进程、私有环境和远端部分报告在 Run 退出时按精确路径清理。
```bash
scripts/cluster/run-production-acceptance.sh \
--execute dist/releases/<完整SHA>.json \
+2 -1
View File
@@ -161,7 +161,8 @@ run_load() {
AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$acceptance_run_token" \
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \
"$load_binary" -profile "$profile" -report "$report_path" "$@" \
"$load_binary" -profile "$profile" -report "$report_path" \
-execution-id "$(basename "$report_path" .json)" "$@" \
>"$stdout_temporary" &
load_pid=$!
active_load_pid=$load_pid
+226 -20
View File
@@ -74,7 +74,7 @@ if [[ $skip_local_acceptance != true ]]; then
fi
load_cluster_env
require_commands curl git go jq node openssl sed
require_commands curl git go jq node openssl sed shasum
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
@@ -225,6 +225,7 @@ acceptance_group_id=
acceptance_participants_json='[]'
AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
acceptance_load_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-load-linux-amd64
acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
current_production_snapshot=$temporary_root/current-production-snapshot.json
local_snapshot_config_hash=
@@ -235,6 +236,7 @@ if [[ $skip_local_acceptance != true ]]; then
fi
active_load_pid=
active_pressure_pid=
remote_load_drivers_installed=false
capacity_profiles_payload=
image_stable_throughput=
image_admitted_throughput=
@@ -254,6 +256,9 @@ cleanup() {
kill "$active_pressure_pid" >/dev/null 2>&1 || true
wait "$active_pressure_pid" >/dev/null 2>&1 || true
fi
if [[ $remote_load_drivers_installed == true ]]; then
cleanup_remote_load_drivers >/dev/null 2>&1 || true
fi
if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
set +e
apply_capacity_profile "$stable_profile" >/dev/null 2>&1
@@ -307,6 +312,8 @@ bootstrap_acceptance_admin_token
cd "$cluster_root/apps/api"
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
-trimpath -o "$acceptance_load_binary" ./cmd/acceptance-load
env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-trimpath -o "$acceptance_load_linux_binary" ./cmd/acceptance-load
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
-trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
)
@@ -1750,24 +1757,14 @@ run_mixed_load_profile() {
local video_rate=$4
local report_path=$5
local timeout_value=$6
local load_stdout=$temporary_root/load-$profile-stdout.json
AI_GATEWAY_ACCEPTANCE_GATEWAYS="$AI_GATEWAY_ACCEPTANCE_GATEWAYS" \
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME="$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" \
AI_GATEWAY_ACCEPTANCE_EMULATOR_URL='http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' \
AI_GATEWAY_ACCEPTANCE_API_KEYS="$AI_GATEWAY_ACCEPTANCE_API_KEYS" \
AI_GATEWAY_ACCEPTANCE_RUN_ID="$run_id" \
AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$run_token" \
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL" \
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS="$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" \
"$acceptance_load_binary" \
-profile "$profile" \
-duration "$duration" \
-image-rate "$image_rate" \
-video-rate "$video_rate" \
-timeout "$timeout_value" \
-report "$report_path" >"$load_stdout"
jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
local site_image_rate site_video_rate
site_image_rate=$(awk -v value="$image_rate" 'BEGIN {printf "%.9f", value/2}')
site_video_rate=$(awk -v value="$video_rate" 'BEGIN {printf "%.9f", value/2}')
run_distributed_load_profile "$profile" "$report_path" \
-duration "$duration" \
-image-rate "$site_image_rate" \
-video-rate "$site_video_rate" \
-timeout "$timeout_value"
}
run_mixed_soak_and_overload() {
@@ -1904,10 +1901,196 @@ activate_certified_autoscaling() {
verify_runtime_gates
}
remote_load_binary_path() {
[[ $run_id =~ ^[0-9a-f-]{36}$ ]]
printf '/root/easyai-acceptance-load-%s' "$run_id"
}
remote_load_env_path() {
local site=$1
[[ $site == ningbo || $site == hongkong ]]
[[ $run_id =~ ^[0-9a-f-]{36}$ ]]
printf '/root/easyai-acceptance-load-%s-%s.env' "$run_id" "$site"
}
cleanup_remote_load_drivers() {
[[ $run_id =~ ^[0-9a-f-]{36}$ ]] || return 0
local binary_path ningbo_env hongkong_env
binary_path=$(remote_load_binary_path)
ningbo_env=$(remote_load_env_path ningbo)
hongkong_env=$(remote_load_env_path hongkong)
cluster_ssh "$CLUSTER_NINGBO_HOST" \
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$ningbo_env' ]] || unlink '$ningbo_env'" || true
cluster_ssh "$CLUSTER_HONGKONG_HOST" \
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$hongkong_env' ]] || unlink '$hongkong_env'" || true
remote_load_drivers_installed=false
}
write_remote_load_env() {
local output=$1
local gateway=$2
install -m 0600 /dev/null "$output"
{
printf 'AI_GATEWAY_ACCEPTANCE_GATEWAYS=%q\n' "$gateway"
printf 'AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=%q\n' "$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME"
printf 'AI_GATEWAY_ACCEPTANCE_EMULATOR_URL=%q\n' 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090'
printf 'AI_GATEWAY_ACCEPTANCE_API_KEYS=%q\n' "$AI_GATEWAY_ACCEPTANCE_API_KEYS"
printf 'AI_GATEWAY_ACCEPTANCE_RUN_ID=%q\n' "$run_id"
printf 'AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=%q\n' "$run_token"
printf 'AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=%q\n' "$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL"
printf 'AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=%q\n' "$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL"
printf 'AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=%q\n' "$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS"
} >>"$output"
}
install_remote_load_drivers() {
[[ $remote_load_drivers_installed == false ]] || return 0
local binary_path local_digest remote_digest site host gateway env_file remote_env
binary_path=$(remote_load_binary_path)
local_digest=$(shasum -a 256 "$acceptance_load_linux_binary" | awk '{print $1}')
local -a sites=(ningbo hongkong)
local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST")
local -a gateways=("https://${CLUSTER_NINGBO_HOST#root@}" "https://${CLUSTER_HONGKONG_HOST#root@}")
for site_index in 0 1; do
site=${sites[$site_index]}
host=${hosts[$site_index]}
gateway=${gateways[$site_index]}
env_file=$temporary_root/remote-load-$site.env
remote_env=$(remote_load_env_path "$site")
write_remote_load_env "$env_file" "$gateway"
cluster_scp "$acceptance_load_linux_binary" "$host:$binary_path" >/dev/null
cluster_scp "$env_file" "$host:$remote_env" >/dev/null
cluster_ssh "$host" "chmod 0755 '$binary_path'; chmod 0600 '$remote_env'; sha256sum '$binary_path'" >"$temporary_root/remote-load-$site.sha256"
remote_digest=$(awk '{print $1}' "$temporary_root/remote-load-$site.sha256")
[[ $remote_digest == "$local_digest" ]]
done
remote_load_drivers_installed=true
echo 'acceptance_load_drivers=PASS sites=2 placement=k3s_external_host_processes digest_verified=true'
}
run_remote_load_site() {
local site=$1
local host=$2
local profile=$3
local artifact=$4
local shard_index=$5
shift 5
[[ $site == ningbo || $site == hongkong ]]
[[ $profile =~ ^[a-z-]+$ && $artifact =~ ^[a-z0-9-]+\.json$ ]]
local binary_path remote_env remote_report local_report status=0
binary_path=$(remote_load_binary_path)
remote_env=$(remote_load_env_path "$site")
remote_report=/root/easyai-acceptance-load-"$run_id"-"$artifact"
local_report=$temporary_root/"$site"-"$artifact"
cluster_ssh "$host" bash -s -- \
"$binary_path" "$remote_env" "$profile" "$remote_report" "$shard_index" 2 "${artifact%.json}" "$@" \
>"$temporary_root/$site-$artifact.stdout" <<'REMOTE' || status=$?
set -euo pipefail
binary_path=$1
env_file=$2
profile=$3
report=$4
shard_index=$5
shard_count=$6
execution_id=$7
shift 7
set -a
# shellcheck source=/dev/null
source "$env_file"
set +a
"$binary_path" -profile "$profile" -report "$report" \
-shard-index "$shard_index" -shard-count "$shard_count" \
-execution-id "$execution_id" "$@"
REMOTE
cluster_scp "$host:$remote_report" "$local_report" >/dev/null || return 1
cluster_ssh "$host" "[[ ! -e '$remote_report' ]] || unlink '$remote_report'" >/dev/null || true
jq -e --arg runId "$run_id" --arg profile "$profile" \
'.schemaVersion == "acceptance-load-report/v1" and .runId == $runId and .profile == $profile and .secretSafe == true' \
"$local_report" >/dev/null || return 1
return "$status"
}
merge_remote_load_reports() {
local profile=$1
local ningbo_report=$2
local hongkong_report=$3
local output=$4
jq -s --arg profile "$profile" '
def sum_field($name): [.[].phases[0][$name] // 0] | add;
def max_field($name): [.[].phases[0][$name] // 0] | max;
. as $reports
| {
schemaVersion:"acceptance-load-report/v1",
runId:$reports[0].runId,
profile:$profile,
startedAt:([$reports[].startedAt] | min),
finishedAt:([$reports[].finishedAt] | max),
passed:([$reports[].passed] | all),
phases:[{
name:$profile,
requests:sum_field("requests"),
completed:sum_field("completed"),
failed:sum_field("failed"),
durationMs:max_field("durationMs"),
submissionDurationMs:max_field("submissionDurationMs"),
p50Ms:max_field("p50Ms"),
p95Ms:max_field("p95Ms"),
p99Ms:max_field("p99Ms"),
decodedOutputBytes:sum_field("decodedOutputBytes"),
outputSha256:([$reports[].phases[0].outputSha256 // ""] | map(select(length>0)) | first // ""),
uniqueTaskIds:sum_field("uniqueTaskIds"),
uniqueImageCombinations:max_field("uniqueImageCombinations"),
forcedConversionTasks:sum_field("forcedConversionTasks"),
throttled:sum_field("throttled"),
unexpected5xx:sum_field("unexpected5xx"),
offeredRatePerSecond:sum_field("offeredRatePerSecond")
}],
failure:([$reports[] | select(.passed == false) | .failure] | first // null),
failureOperation:([$reports[] | select(.passed == false) | .failureOperation] | first // null),
aggregation:{
sites:["ningbo","hongkong"],
percentileMethod:"conservative_max_of_site_percentiles",
durationMethod:"max_site_duration",
placement:"k3s_external_host_processes"
},
secretSafe:true
}
| with_entries(select(.value != null))
' "$ningbo_report" "$hongkong_report" >"$output"
chmod 0600 "$output"
}
run_distributed_load_profile() {
local profile=$1
local report_path=$2
shift 2
install_remote_load_drivers
local artifact base ningbo_report hongkong_report ningbo_pid hongkong_pid
local ningbo_status=0 hongkong_status=0
base=$(basename "$report_path" .json)
artifact=$base.json
run_remote_load_site ningbo "$CLUSTER_NINGBO_HOST" "$profile" "$artifact" 0 "$@" &
ningbo_pid=$!
run_remote_load_site hongkong "$CLUSTER_HONGKONG_HOST" "$profile" "$artifact" 1 "$@" &
hongkong_pid=$!
wait "$ningbo_pid" || ningbo_status=$?
wait "$hongkong_pid" || hongkong_status=$?
ningbo_report=$temporary_root/ningbo-$artifact
hongkong_report=$temporary_root/hongkong-$artifact
[[ -f $ningbo_report && -f $hongkong_report ]] || return 1
merge_remote_load_reports "$profile" "$ningbo_report" "$hongkong_report" "$report_path"
(( ningbo_status == 0 && hongkong_status == 0 )) || return 1
jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
}
run_load_profile() {
local profile=$1
local report_path=$2
[[ $profile =~ ^[a-z-]+$ ]] || return 1
if [[ $profile != real-canary ]]; then
run_distributed_load_profile "$profile" "$report_path"
return
fi
local load_stdout=$temporary_root/load-$profile-stdout.json
AI_GATEWAY_ACCEPTANCE_GATEWAYS="$AI_GATEWAY_ACCEPTANCE_GATEWAYS" \
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME="$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" \
@@ -1919,6 +2102,7 @@ run_load_profile() {
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS="$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" \
"$acceptance_load_binary" -profile "$profile" -report "$report_path" \
-execution-id "$(basename "$report_path" .json)" \
>"$load_stdout"
jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
}
@@ -2617,6 +2801,7 @@ finish_and_report() {
local gemini_invalid video_invalid reference_count_3 reference_count_6 reference_count_9
local reference_role_count first_frame_role_count last_frame_role_count
local unique_image_hashes unique_video_tasks
local unique_gemini_input_hashes direct_gemini_input_assets
local duplicate_remote_tasks duplicate_terminal_events duplicate_settlements
local duplicate_wallet_transactions duplicate_callback_rows pending_settlements pending_callbacks
local missing_billings
@@ -2625,6 +2810,19 @@ finish_and_report() {
gemini_task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND run_mode='acceptance';")
forced_tasks=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND request::text LIKE '%acceptance-force-conversion%';")
compact_payloads=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND (pg_column_size(request) >= 1048576 OR pg_column_size(result) >= 1048576);")
unique_gemini_input_hashes=$(database_query "
SELECT count(DISTINCT request #>> '{image,assetRef,sha256}')
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid
AND kind='images.edits'
AND run_mode='acceptance';")
direct_gemini_input_assets=$(database_query "
SELECT count(*)
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid
AND kind='images.edits'
AND run_mode='acceptance'
AND request #>> '{image,assetRef,storageProvider}' = 'aliyun_oss_direct';")
submissions=$(jq -r '.videoSubmissions' "$emulator_report")
duplicates=$(jq -r '.duplicates' "$callback_report")
callbacks=$(jq -r '.deliveries' "$callback_report")
@@ -2653,6 +2851,10 @@ finish_and_report() {
$missing_idempotency == 0 && $compact_payloads == 0 ]] ||
fail_acceptance_gate idempotency_and_materialization \
'duplicate submission, missing idempotency key, or inline database media detected'
[[ $unique_gemini_input_hashes == "$gemini_task_count" &&
$direct_gemini_input_assets == "$gemini_task_count" ]] ||
fail_acceptance_gate gemini_input_materialization \
'Gemini inputs were reused across executions or bypassed direct OSS materialization'
[[ $gemini_invalid == 0 && $video_invalid == 0 ]] ||
fail_acceptance_gate provider_protocol_validity 'Gemini or Volces protocol validation failed'
[[ $reference_count_3 -gt 0 && $reference_count_6 -gt 0 && $reference_count_9 -gt 0 ]] ||
@@ -2796,6 +2998,8 @@ WHERE task.acceptance_run_id='$run_id'::uuid
--argjson lastFrameRoleCount "$last_frame_role_count" \
--argjson uniqueImageHashes "$unique_image_hashes" \
--argjson uniqueVideoTasks "$unique_video_tasks" \
--argjson uniqueGeminiInputHashes "$unique_gemini_input_hashes" \
--argjson directGeminiInputAssets "$direct_gemini_input_assets" \
--argjson duplicateRemoteTasks "$duplicate_remote_tasks" \
--argjson duplicateTerminalEvents "$duplicate_terminal_events" \
--argjson duplicateSettlements "$duplicate_settlements" \
@@ -2838,7 +3042,9 @@ WHERE task.acceptance_run_id='$run_id'::uuid
last_frame:$lastFrameRoleCount
},
uniqueImageHashes:$uniqueImageHashes,
uniqueVideoTasks:$uniqueVideoTasks
uniqueVideoTasks:$uniqueVideoTasks,
uniqueGeminiInputHashes:$uniqueGeminiInputHashes,
directGeminiInputAssets:$directGeminiInputAssets
},
databaseConsistency:{
duplicateRemoteTasks:$duplicateRemoteTasks,