fix(acceptance): 修复线上双站压测失真
将线上压力生成器拆分为宁波、香港宿主机上的 K3s 外进程,按全局序号分片并保守合并站点报告,避免操作者本机上行带宽成为容量瓶颈。\n\n为每次负载执行生成独立幂等键和 Run 级图片变体,阻止三轮验收重放旧任务或复用历史媒体缓存,并新增直接 OSS 物化一致性门禁。\n\n验证:Go 全量测试、go vet、迁移安全检查、bash -n、ShellCheck、manual-release-test 和双宿主机 linux/amd64 启动冒烟通过。
This commit is contained in:
@@ -56,6 +56,9 @@ type options struct {
|
|||||||
mixedDuration time.Duration
|
mixedDuration time.Duration
|
||||||
imageRate float64
|
imageRate float64
|
||||||
videoRate float64
|
videoRate float64
|
||||||
|
shardIndex int
|
||||||
|
shardCount int
|
||||||
|
executionID string
|
||||||
}
|
}
|
||||||
|
|
||||||
type report struct {
|
type report struct {
|
||||||
@@ -194,12 +197,18 @@ func parseOptions() (options, error) {
|
|||||||
var mixedDuration time.Duration
|
var mixedDuration time.Duration
|
||||||
var imageRate float64
|
var imageRate float64
|
||||||
var videoRate float64
|
var videoRate float64
|
||||||
|
var shardIndex int
|
||||||
|
var shardCount int
|
||||||
|
var executionID string
|
||||||
flag.StringVar(&profile, "profile", env("AI_GATEWAY_ACCEPTANCE_PROFILE", "simulated-all"), "simulated-all, Gemini profile, video profile, or real-canary")
|
flag.StringVar(&profile, "profile", env("AI_GATEWAY_ACCEPTANCE_PROFILE", "simulated-all"), "simulated-all, Gemini profile, video profile, or real-canary")
|
||||||
flag.StringVar(&reportPath, "report", env("AI_GATEWAY_ACCEPTANCE_REPORT", ""), "secret-safe JSON report path")
|
flag.StringVar(&reportPath, "report", env("AI_GATEWAY_ACCEPTANCE_REPORT", ""), "secret-safe JSON report path")
|
||||||
flag.DurationVar(&timeout, "timeout", 45*time.Minute, "overall timeout")
|
flag.DurationVar(&timeout, "timeout", 45*time.Minute, "overall timeout")
|
||||||
flag.DurationVar(&mixedDuration, "duration", envDuration("AI_GATEWAY_ACCEPTANCE_MIXED_DURATION", 10*time.Minute), "mixed workload duration")
|
flag.DurationVar(&mixedDuration, "duration", envDuration("AI_GATEWAY_ACCEPTANCE_MIXED_DURATION", 10*time.Minute), "mixed workload duration")
|
||||||
flag.Float64Var(&imageRate, "image-rate", envFloat("AI_GATEWAY_ACCEPTANCE_IMAGE_RATE", 0), "mixed image requests per second")
|
flag.Float64Var(&imageRate, "image-rate", envFloat("AI_GATEWAY_ACCEPTANCE_IMAGE_RATE", 0), "mixed image requests per second")
|
||||||
flag.Float64Var(&videoRate, "video-rate", envFloat("AI_GATEWAY_ACCEPTANCE_VIDEO_RATE", 0), "mixed video requests per second")
|
flag.Float64Var(&videoRate, "video-rate", envFloat("AI_GATEWAY_ACCEPTANCE_VIDEO_RATE", 0), "mixed video requests per second")
|
||||||
|
flag.IntVar(&shardIndex, "shard-index", 0, "zero-based distributed load shard index")
|
||||||
|
flag.IntVar(&shardCount, "shard-count", 1, "distributed load shard count")
|
||||||
|
flag.StringVar(&executionID, "execution-id", "", "unique workload execution identifier")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
opts := options{
|
opts := options{
|
||||||
gateways: splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAYS")),
|
gateways: splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAYS")),
|
||||||
@@ -218,9 +227,21 @@ func parseOptions() (options, error) {
|
|||||||
mixedDuration: mixedDuration,
|
mixedDuration: mixedDuration,
|
||||||
imageRate: imageRate,
|
imageRate: imageRate,
|
||||||
videoRate: videoRate,
|
videoRate: videoRate,
|
||||||
|
shardIndex: shardIndex,
|
||||||
|
shardCount: shardCount,
|
||||||
|
executionID: strings.ToLower(strings.TrimSpace(executionID)),
|
||||||
}
|
}
|
||||||
if len(opts.gateways) != 2 {
|
if opts.executionID == "" {
|
||||||
return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAYS must contain exactly two API base URLs")
|
opts.executionID = opts.profile
|
||||||
|
}
|
||||||
|
if matched, _ := regexp.MatchString(`^[a-z0-9][a-z0-9-]{0,127}$`, opts.executionID); !matched {
|
||||||
|
return options{}, errors.New("execution-id must contain only lowercase letters, numbers, and hyphens")
|
||||||
|
}
|
||||||
|
if opts.shardCount < 1 || opts.shardCount > 32 || opts.shardIndex < 0 || opts.shardIndex >= opts.shardCount {
|
||||||
|
return options{}, fmt.Errorf("invalid load shard %d/%d", opts.shardIndex, opts.shardCount)
|
||||||
|
}
|
||||||
|
if len(opts.gateways) < 1 || len(opts.gateways) > 2 || (opts.shardCount == 1 && len(opts.gateways) != 2) {
|
||||||
|
return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAYS must contain two API base URLs, or one URL for a distributed shard")
|
||||||
}
|
}
|
||||||
for index := range opts.gateways {
|
for index := range opts.gateways {
|
||||||
opts.gateways[index] = strings.TrimRight(opts.gateways[index], "/")
|
opts.gateways[index] = strings.TrimRight(opts.gateways[index], "/")
|
||||||
@@ -262,6 +283,17 @@ func parseOptions() (options, error) {
|
|||||||
return opts, nil
|
return opts, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (o options) shardRequestCount(total int) int {
|
||||||
|
if total <= o.shardIndex {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return (total-1-o.shardIndex)/o.shardCount + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o options) logicalRequestIndex(localIndex int) int {
|
||||||
|
return o.shardIndex + localIndex*o.shardCount
|
||||||
|
}
|
||||||
|
|
||||||
func run(ctx context.Context, opts options) ([]phaseResult, error) {
|
func run(ctx context.Context, opts options) ([]phaseResult, error) {
|
||||||
var tlsConfig *tls.Config
|
var tlsConfig *tls.Config
|
||||||
if opts.gatewayTLSName != "" || opts.gatewayCAFile != "" {
|
if opts.gatewayTLSName != "" || opts.gatewayCAFile != "" {
|
||||||
@@ -289,23 +321,26 @@ func run(ctx context.Context, opts options) ([]phaseResult, error) {
|
|||||||
runPhase := func(name string) error {
|
runPhase := func(name string) error {
|
||||||
var result phaseResult
|
var result phaseResult
|
||||||
if profile, ok := acceptanceworkload.GeminiProfileByName(name); ok {
|
if profile, ok := acceptanceworkload.GeminiProfileByName(name); ok {
|
||||||
result = runGemini(ctx, client, opts, name, profile.Requests, profile.InputBytes, profile.OutputBytes, false)
|
result = runGemini(
|
||||||
|
ctx, client, opts, name, opts.shardRequestCount(profile.Requests),
|
||||||
|
profile.InputBytes, profile.OutputBytes, false, opts.shardIndex, opts.shardCount,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
switch name {
|
switch name {
|
||||||
case "smoke-gemini":
|
case "smoke-gemini":
|
||||||
result = runGemini(ctx, client, opts, name, 4, 256<<10, 256<<10, false)
|
result = runGemini(ctx, client, opts, name, opts.shardRequestCount(4), 256<<10, 256<<10, false, opts.shardIndex, opts.shardCount)
|
||||||
case "smoke-video":
|
case "smoke-video":
|
||||||
result = runVideo(ctx, client, opts, name, 10, false, opts.emulatorFixtureURLs(), false, 0)
|
result = runVideo(ctx, client, opts, name, opts.shardRequestCount(10), false, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
|
||||||
case "video-throughput":
|
case "video-throughput":
|
||||||
result = runVideo(ctx, client, opts, name, 1200, false, opts.emulatorFixtureURLs(), false, 0)
|
result = runVideo(ctx, client, opts, name, opts.shardRequestCount(1200), false, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
|
||||||
case "video-recovery":
|
case "video-recovery":
|
||||||
result = runVideo(ctx, client, opts, name, 96, true, opts.emulatorFixtureURLs(), false, 0)
|
result = runVideo(ctx, client, opts, name, opts.shardRequestCount(96), true, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
|
||||||
case "mixed-soak", "mixed-overload":
|
case "mixed-soak", "mixed-overload":
|
||||||
result = runMixed(ctx, client, opts, name == "mixed-overload")
|
result = runMixed(ctx, client, opts, name == "mixed-overload")
|
||||||
case "real-gemini-canary":
|
case "real-gemini-canary":
|
||||||
result = runGemini(ctx, client, opts, name, 1, 256<<10, 0, true)
|
result = runGemini(ctx, client, opts, name, 1, 256<<10, 0, true, 0, 1)
|
||||||
case "real-video-canary":
|
case "real-video-canary":
|
||||||
result = runVideo(ctx, client, opts, name, 1, false, opts.realImageURLs, true, 0)
|
result = runVideo(ctx, client, opts, name, 1, false, opts.realImageURLs, true, 0, 1)
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unknown phase %q", name)
|
return fmt.Errorf("unknown phase %q", name)
|
||||||
}
|
}
|
||||||
@@ -338,7 +373,12 @@ func runGemini(
|
|||||||
inputBytes int,
|
inputBytes int,
|
||||||
expectedOutputBytes int,
|
expectedOutputBytes int,
|
||||||
realUpstream bool,
|
realUpstream bool,
|
||||||
|
requestOffset int,
|
||||||
|
requestStride int,
|
||||||
) phaseResult {
|
) phaseResult {
|
||||||
|
if requestStride < 1 {
|
||||||
|
requestStride = 1
|
||||||
|
}
|
||||||
startedAt := time.Now()
|
startedAt := time.Now()
|
||||||
expectedHash := ""
|
expectedHash := ""
|
||||||
if expectedOutputBytes > 0 {
|
if expectedOutputBytes > 0 {
|
||||||
@@ -358,6 +398,7 @@ func runGemini(
|
|||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
logicalIndex := requestOffset + index*requestStride
|
||||||
select {
|
select {
|
||||||
case slots <- struct{}{}:
|
case slots <- struct{}{}:
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@@ -365,8 +406,8 @@ func runGemini(
|
|||||||
}
|
}
|
||||||
defer func() { <-slots }()
|
defer func() { <-slots }()
|
||||||
requestStarted := time.Now()
|
requestStarted := time.Now()
|
||||||
endpoint := opts.gateways[index%len(opts.gateways)] + "/v1beta/models/" + url.PathEscape(opts.geminiModel) + ":generateContent"
|
endpoint := opts.gateways[logicalIndex%len(opts.gateways)] + "/v1beta/models/" + url.PathEscape(opts.geminiModel) + ":generateContent"
|
||||||
requestBody := streamGeminiRequestBody(paddedPNGVariant(inputBytes, index+1))
|
requestBody := streamGeminiRequestBody(paddedPNGVariant(inputBytes, geminiInputVariant(opts.runID+"/"+opts.executionID, logicalIndex)))
|
||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodPost,
|
http.MethodPost,
|
||||||
@@ -377,7 +418,7 @@ func runGemini(
|
|||||||
_ = requestBody.Close()
|
_ = requestBody.Close()
|
||||||
}
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
opts.setHeaders(req, index, realUpstream)
|
opts.setHeaders(req, logicalIndex, realUpstream)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
var response *http.Response
|
var response *http.Response
|
||||||
response, err = client.Do(req)
|
response, err = client.Do(req)
|
||||||
@@ -472,7 +513,11 @@ func runVideo(
|
|||||||
imageURLs []string,
|
imageURLs []string,
|
||||||
realUpstream bool,
|
realUpstream bool,
|
||||||
requestOffset int,
|
requestOffset int,
|
||||||
|
requestStride int,
|
||||||
) phaseResult {
|
) phaseResult {
|
||||||
|
if requestStride < 1 {
|
||||||
|
requestStride = 1
|
||||||
|
}
|
||||||
startedAt := time.Now()
|
startedAt := time.Now()
|
||||||
combinationCount := 128
|
combinationCount := 128
|
||||||
if realUpstream {
|
if realUpstream {
|
||||||
@@ -494,7 +539,7 @@ func runVideo(
|
|||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
logicalIndex := index + requestOffset
|
logicalIndex := requestOffset + index*requestStride
|
||||||
requestStarted := time.Now()
|
requestStarted := time.Now()
|
||||||
startedTasks[index] = requestStarted
|
startedTasks[index] = requestStarted
|
||||||
imageCount := acceptanceworkload.VideoImageCount(logicalIndex)
|
imageCount := acceptanceworkload.VideoImageCount(logicalIndex)
|
||||||
@@ -590,7 +635,7 @@ func runVideo(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
index, taskID := index, taskID
|
index, taskID := index, taskID
|
||||||
logicalIndex := index + requestOffset
|
logicalIndex := requestOffset + index*requestStride
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
@@ -685,7 +730,7 @@ func runMixed(
|
|||||||
generating = false
|
generating = false
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
requestIndex := requests
|
requestIndex := opts.logicalRequestIndex(requests)
|
||||||
requests++
|
requests++
|
||||||
imageAccumulator += opts.imageRate
|
imageAccumulator += opts.imageRate
|
||||||
isImage := imageAccumulator >= totalRate
|
isImage := imageAccumulator >= totalRate
|
||||||
@@ -698,7 +743,7 @@ func runMixed(
|
|||||||
defer func() { <-slots }()
|
defer func() { <-slots }()
|
||||||
var result phaseResult
|
var result phaseResult
|
||||||
if isImage {
|
if isImage {
|
||||||
result = runGemini(runCtx, client, opts, "mixed-image", 1, 256<<10, 256<<10, false)
|
result = runGemini(runCtx, client, opts, "mixed-image", 1, 256<<10, 256<<10, false, requestIndex, 1)
|
||||||
} else {
|
} else {
|
||||||
result = runVideo(
|
result = runVideo(
|
||||||
runCtx,
|
runCtx,
|
||||||
@@ -710,6 +755,7 @@ func runMixed(
|
|||||||
opts.emulatorFixtureURLs(),
|
opts.emulatorFixtureURLs(),
|
||||||
false,
|
false,
|
||||||
requestIndex,
|
requestIndex,
|
||||||
|
1,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
latency := time.Since(startedAt)
|
latency := time.Since(startedAt)
|
||||||
@@ -940,6 +986,9 @@ func (o options) setHeaders(request *http.Request, requestIndex int, realUpstrea
|
|||||||
request.Header.Set("Authorization", "Bearer "+o.apiKeys[requestIndex%len(o.apiKeys)])
|
request.Header.Set("Authorization", "Bearer "+o.apiKeys[requestIndex%len(o.apiKeys)])
|
||||||
request.Header.Set(runHeader, o.runID)
|
request.Header.Set(runHeader, o.runID)
|
||||||
request.Header.Set(tokenHeader, o.runToken)
|
request.Header.Set(tokenHeader, o.runToken)
|
||||||
|
if request.Method == http.MethodPost {
|
||||||
|
request.Header.Set("Idempotency-Key", fmt.Sprintf("acceptance-%s-%s-%d", o.runID, o.executionID, requestIndex))
|
||||||
|
}
|
||||||
if realUpstream {
|
if realUpstream {
|
||||||
request.Header.Set(upstreamHeader, "real")
|
request.Header.Set(upstreamHeader, "real")
|
||||||
}
|
}
|
||||||
@@ -1115,6 +1164,11 @@ func paddedPNG(size int) []byte {
|
|||||||
return paddedPNGVariant(size, 0)
|
return paddedPNGVariant(size, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func geminiInputVariant(runID string, logicalIndex int) int {
|
||||||
|
digest := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", strings.TrimSpace(runID), logicalIndex)))
|
||||||
|
return int(binary.BigEndian.Uint64(digest[:8]) & uint64(^uint(0)>>1))
|
||||||
|
}
|
||||||
|
|
||||||
func paddedPNGVariant(size int, variant int) []byte {
|
func paddedPNGVariant(size int, variant int) []byte {
|
||||||
base, _ := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
|
base, _ := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
|
||||||
if size <= len(base) {
|
if size <= len(base) {
|
||||||
|
|||||||
@@ -146,7 +146,7 @@ func TestGeminiLoadIsSplitAcrossTwoGatewayAPIs(t *testing.T) {
|
|||||||
gateways: []string{first.URL, second.URL}, apiKeys: []string{"key-1", "key-2"}, runID: "run-1",
|
gateways: []string{first.URL, second.URL}, apiKeys: []string{"key-1", "key-2"}, runID: "run-1",
|
||||||
runToken: "token-1", geminiModel: "gemini-image-test",
|
runToken: "token-1", geminiModel: "gemini-image-test",
|
||||||
}
|
}
|
||||||
result := runGemini(t.Context(), http.DefaultClient, opts, "dual-api", 8, 256<<10, 256<<10, false)
|
result := runGemini(t.Context(), http.DefaultClient, opts, "dual-api", 8, 256<<10, 256<<10, false, 0, 1)
|
||||||
if result.err != nil || result.report.Completed != 8 {
|
if result.err != nil || result.report.Completed != 8 {
|
||||||
t.Fatalf("result=%+v err=%v", result.report, result.err)
|
t.Fatalf("result=%+v err=%v", result.report, result.err)
|
||||||
}
|
}
|
||||||
@@ -158,6 +158,53 @@ func TestGeminiLoadIsSplitAcrossTwoGatewayAPIs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDistributedShardIndexesCoverWorkloadWithoutOverlap(t *testing.T) {
|
||||||
|
first := options{shardIndex: 0, shardCount: 2}
|
||||||
|
second := options{shardIndex: 1, shardCount: 2}
|
||||||
|
if first.shardRequestCount(1001) != 501 || second.shardRequestCount(1001) != 500 {
|
||||||
|
t.Fatalf("shard counts=%d/%d", first.shardRequestCount(1001), second.shardRequestCount(1001))
|
||||||
|
}
|
||||||
|
seen := map[int]bool{}
|
||||||
|
for local := 0; local < first.shardRequestCount(1001); local++ {
|
||||||
|
seen[first.logicalRequestIndex(local)] = true
|
||||||
|
}
|
||||||
|
for local := 0; local < second.shardRequestCount(1001); local++ {
|
||||||
|
index := second.logicalRequestIndex(local)
|
||||||
|
if seen[index] {
|
||||||
|
t.Fatalf("duplicate logical index %d", index)
|
||||||
|
}
|
||||||
|
seen[index] = true
|
||||||
|
}
|
||||||
|
if len(seen) != 1001 {
|
||||||
|
t.Fatalf("covered indexes=%d", len(seen))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGeminiInputVariantIsRunScoped(t *testing.T) {
|
||||||
|
if geminiInputVariant("run-a", 7) == geminiInputVariant("run-b", 7) {
|
||||||
|
t.Fatal("different Run IDs produced the same input variant")
|
||||||
|
}
|
||||||
|
if geminiInputVariant("run-a", 7) == geminiInputVariant("run-a", 8) {
|
||||||
|
t.Fatal("different logical indexes produced the same input variant")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAcceptanceIdempotencyKeyIncludesExecutionAndLogicalIndex(t *testing.T) {
|
||||||
|
request := httptest.NewRequest(http.MethodPost, "https://gateway.example/v1", nil)
|
||||||
|
opts := options{
|
||||||
|
apiKeys: []string{"key-1"}, runID: "run-1", runToken: "token-1", executionID: "p24-2-gemini-baseline",
|
||||||
|
}
|
||||||
|
opts.setHeaders(request, 17, false)
|
||||||
|
if got := request.Header.Get("Idempotency-Key"); got != "acceptance-run-1-p24-2-gemini-baseline-17" {
|
||||||
|
t.Fatalf("idempotency key=%q", got)
|
||||||
|
}
|
||||||
|
poll := httptest.NewRequest(http.MethodGet, "https://gateway.example/result", nil)
|
||||||
|
opts.setHeaders(poll, 17, false)
|
||||||
|
if got := poll.Header.Get("Idempotency-Key"); got != "" {
|
||||||
|
t.Fatalf("GET request unexpectedly has idempotency key %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateVideoAssetDownloadsFinalMedia(t *testing.T) {
|
func TestValidateVideoAssetDownloadsFinalMedia(t *testing.T) {
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||||
if request.Header.Get("Authorization") != "" {
|
if request.Header.Get("Authorization") != "" {
|
||||||
|
|||||||
@@ -8,7 +8,8 @@
|
|||||||
|
|
||||||
验收脚本是显式生产写操作,会暂停新正式任务、调整 Worker 容量并在恢复场景删除一个
|
验收脚本是显式生产写操作,会暂停新正式任务、调整 Worker 容量并在恢复场景删除一个
|
||||||
Worker Pod。它不会自动发布新版本;只能针对已经发布、完整 SHA 和 digest 固定的 manifest
|
Worker Pod。它不会自动发布新版本;只能针对已经发布、完整 SHA 和 digest 固定的 manifest
|
||||||
执行。进入线上前必须先通过[本地三节点同构验收](local-isomorphic-acceptance.md)。
|
执行。默认先通过[本地三节点同构验收](local-isomorphic-acceptance.md);用户明确授权在线验收时,
|
||||||
|
可以记录审计豁免并跳过本地阶段,但不能把跳过记录成通过。
|
||||||
|
|
||||||
## 隔离与流量门禁
|
## 隔离与流量门禁
|
||||||
|
|
||||||
@@ -61,10 +62,12 @@ WebP、合法 4K 图片和越过 Seedance 2.0 官方输入边界的 `6144x2160`
|
|||||||
收集器。它校验参考图数量、角色、格式、尺寸与内容哈希,并统计重复上游提交和重复回调。
|
收集器。它校验参考图数量、角色、格式、尺寸与内容哈希,并统计重复上游提交和重复回调。
|
||||||
|
|
||||||
默认通过 `AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS=32` 幂等准备 32 个仅属于验收用户组的
|
默认通过 `AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS=32` 幂等准备 32 个仅属于验收用户组的
|
||||||
身份、API Key 和独立钱包,并按请求序号轮询使用。这样可模拟真实多租户并发,避免把 1000
|
身份、API Key 和独立钱包,并按全局请求序号轮询使用。每次负载执行都有独立 execution ID,
|
||||||
|
它同时进入幂等键和输入图片哈希,因此三次重复验收既不会重放旧任务,也不会命中前一轮的
|
||||||
|
输入资产缓存。这样可模拟真实多租户并发,避免把 1000
|
||||||
个任务全部压在同一钱包行锁上;每个任务仍执行真实预留、结算和重复扣费检查。该值可在
|
个任务全部压在同一钱包行锁上;每个任务仍执行真实预留、结算和重复扣费检查。该值可在
|
||||||
`1..128` 内通过环境变量调整,Run 只保存 Key/User ID 对,Key Secret 仅通过 stdin 传入
|
`1..128` 内通过环境变量调整,Run 只保存 Key/User ID 对,Key Secret 仅通过 stdin 传入
|
||||||
集群内压测进程,不进入 Pod spec、数据库 Run 配置、报告或日志。
|
双站压测进程,不进入 Pod spec、数据库 Run 配置、报告或日志。
|
||||||
|
|
||||||
## Worker 配置
|
## Worker 配置
|
||||||
|
|
||||||
@@ -96,7 +99,7 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG
|
|||||||
重新编码,避免执行槽升档时把大图峰值内存同比放大。2 GiB Worker 的起始值为 `2`;它不占用
|
重新编码,避免执行槽升档时把大图峰值内存同比放大。2 GiB Worker 的起始值为 `2`;它不占用
|
||||||
视频提交后的上游等待槽,只有实测单 Pod RSS 和节点预算允许时才单独升档。
|
视频提交后的上游等待槽,只有实测单 Pod RSS 和节点预算允许时才单独升档。
|
||||||
|
|
||||||
API Pod 使用独立发布配置 `AI_GATEWAY_API_DATABASE_MAX_CONNS`,生产同构验收默认使用 `64`;
|
API Pod 使用独立发布配置 `AI_GATEWAY_API_DATABASE_MAX_CONNS`,生产同构验收默认使用 `31`;
|
||||||
也可通过 `AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS` 覆盖本次验收值,无需修改 Go 代码。
|
也可通过 `AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS` 覆盖本次验收值,无需修改 Go 代码。
|
||||||
所有 API/Worker Pool 的 `AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 固定为 `4`,仅预热必要连接,
|
所有 API/Worker Pool 的 `AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 固定为 `4`,仅预热必要连接,
|
||||||
`AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS` 为 `30`,突发扩出的空闲连接会及时回落,
|
`AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS` 为 `30`,突发扩出的空闲连接会及时回落,
|
||||||
@@ -135,6 +138,12 @@ PNG 并通过验收 Key 上传到 Gateway。生产快照默认在已部署的 AP
|
|||||||
`AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=ai.51easyai.com`。压测器会继续校验正式
|
`AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=ai.51easyai.com`。压测器会继续校验正式
|
||||||
证书,并同时使用该域名作为 HTTP Host;这不是跳过 TLS 校验。
|
证书,并同时使用该域名作为 HTTP Host;这不是跳过 TLS 校验。
|
||||||
|
|
||||||
|
线上高负载不再由操作者本机单进程跨公网发送。脚本构建同一源码的静态 linux/amd64
|
||||||
|
压测器,校验 SHA-256 后以 `0600` 临时环境分别运行在宁波、香港宿主机的 K3s 之外;两个
|
||||||
|
进程同时只访问本站 TLS 入口,并按奇偶全局序号分担请求。这样避免本机上行带宽成为容量
|
||||||
|
瓶颈,同时仍覆盖正式 TLS/Nginx 入口。站点报告按请求数相加、耗时取较慢站点、p50/p95/p99
|
||||||
|
取两站较差值进行保守合并。进程、私有环境和远端部分报告在 Run 退出时按精确路径清理。
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
scripts/cluster/run-production-acceptance.sh \
|
scripts/cluster/run-production-acceptance.sh \
|
||||||
--execute dist/releases/<完整SHA>.json \
|
--execute dist/releases/<完整SHA>.json \
|
||||||
|
|||||||
@@ -161,7 +161,8 @@ run_load() {
|
|||||||
AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$acceptance_run_token" \
|
AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$acceptance_run_token" \
|
||||||
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \
|
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \
|
||||||
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \
|
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \
|
||||||
"$load_binary" -profile "$profile" -report "$report_path" "$@" \
|
"$load_binary" -profile "$profile" -report "$report_path" \
|
||||||
|
-execution-id "$(basename "$report_path" .json)" "$@" \
|
||||||
>"$stdout_temporary" &
|
>"$stdout_temporary" &
|
||||||
load_pid=$!
|
load_pid=$!
|
||||||
active_load_pid=$load_pid
|
active_load_pid=$load_pid
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ if [[ $skip_local_acceptance != true ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
load_cluster_env
|
load_cluster_env
|
||||||
require_commands curl git go jq node openssl sed
|
require_commands curl git go jq node openssl sed shasum
|
||||||
|
|
||||||
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
|
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
|
||||||
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
|
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
|
||||||
@@ -225,6 +225,7 @@ acceptance_group_id=
|
|||||||
acceptance_participants_json='[]'
|
acceptance_participants_json='[]'
|
||||||
AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
|
AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
|
||||||
acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
|
acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
|
||||||
|
acceptance_load_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-load-linux-amd64
|
||||||
acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
|
acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
|
||||||
current_production_snapshot=$temporary_root/current-production-snapshot.json
|
current_production_snapshot=$temporary_root/current-production-snapshot.json
|
||||||
local_snapshot_config_hash=
|
local_snapshot_config_hash=
|
||||||
@@ -235,6 +236,7 @@ if [[ $skip_local_acceptance != true ]]; then
|
|||||||
fi
|
fi
|
||||||
active_load_pid=
|
active_load_pid=
|
||||||
active_pressure_pid=
|
active_pressure_pid=
|
||||||
|
remote_load_drivers_installed=false
|
||||||
capacity_profiles_payload=
|
capacity_profiles_payload=
|
||||||
image_stable_throughput=
|
image_stable_throughput=
|
||||||
image_admitted_throughput=
|
image_admitted_throughput=
|
||||||
@@ -254,6 +256,9 @@ cleanup() {
|
|||||||
kill "$active_pressure_pid" >/dev/null 2>&1 || true
|
kill "$active_pressure_pid" >/dev/null 2>&1 || true
|
||||||
wait "$active_pressure_pid" >/dev/null 2>&1 || true
|
wait "$active_pressure_pid" >/dev/null 2>&1 || true
|
||||||
fi
|
fi
|
||||||
|
if [[ $remote_load_drivers_installed == true ]]; then
|
||||||
|
cleanup_remote_load_drivers >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
|
if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
|
||||||
set +e
|
set +e
|
||||||
apply_capacity_profile "$stable_profile" >/dev/null 2>&1
|
apply_capacity_profile "$stable_profile" >/dev/null 2>&1
|
||||||
@@ -307,6 +312,8 @@ bootstrap_acceptance_admin_token
|
|||||||
cd "$cluster_root/apps/api"
|
cd "$cluster_root/apps/api"
|
||||||
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
|
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
|
||||||
-trimpath -o "$acceptance_load_binary" ./cmd/acceptance-load
|
-trimpath -o "$acceptance_load_binary" ./cmd/acceptance-load
|
||||||
|
env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||||
|
-trimpath -o "$acceptance_load_linux_binary" ./cmd/acceptance-load
|
||||||
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
|
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
|
||||||
-trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
|
-trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
|
||||||
)
|
)
|
||||||
@@ -1750,24 +1757,14 @@ run_mixed_load_profile() {
|
|||||||
local video_rate=$4
|
local video_rate=$4
|
||||||
local report_path=$5
|
local report_path=$5
|
||||||
local timeout_value=$6
|
local timeout_value=$6
|
||||||
local load_stdout=$temporary_root/load-$profile-stdout.json
|
local site_image_rate site_video_rate
|
||||||
AI_GATEWAY_ACCEPTANCE_GATEWAYS="$AI_GATEWAY_ACCEPTANCE_GATEWAYS" \
|
site_image_rate=$(awk -v value="$image_rate" 'BEGIN {printf "%.9f", value/2}')
|
||||||
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME="$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" \
|
site_video_rate=$(awk -v value="$video_rate" 'BEGIN {printf "%.9f", value/2}')
|
||||||
AI_GATEWAY_ACCEPTANCE_EMULATOR_URL='http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' \
|
run_distributed_load_profile "$profile" "$report_path" \
|
||||||
AI_GATEWAY_ACCEPTANCE_API_KEYS="$AI_GATEWAY_ACCEPTANCE_API_KEYS" \
|
-duration "$duration" \
|
||||||
AI_GATEWAY_ACCEPTANCE_RUN_ID="$run_id" \
|
-image-rate "$site_image_rate" \
|
||||||
AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$run_token" \
|
-video-rate "$site_video_rate" \
|
||||||
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL" \
|
-timeout "$timeout_value"
|
||||||
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
|
|
||||||
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS="$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" \
|
|
||||||
"$acceptance_load_binary" \
|
|
||||||
-profile "$profile" \
|
|
||||||
-duration "$duration" \
|
|
||||||
-image-rate "$image_rate" \
|
|
||||||
-video-rate "$video_rate" \
|
|
||||||
-timeout "$timeout_value" \
|
|
||||||
-report "$report_path" >"$load_stdout"
|
|
||||||
jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
|
|
||||||
}
|
}
|
||||||
|
|
||||||
run_mixed_soak_and_overload() {
|
run_mixed_soak_and_overload() {
|
||||||
@@ -1904,10 +1901,196 @@ activate_certified_autoscaling() {
|
|||||||
verify_runtime_gates
|
verify_runtime_gates
|
||||||
}
|
}
|
||||||
|
|
||||||
|
remote_load_binary_path() {
|
||||||
|
[[ $run_id =~ ^[0-9a-f-]{36}$ ]]
|
||||||
|
printf '/root/easyai-acceptance-load-%s' "$run_id"
|
||||||
|
}
|
||||||
|
|
||||||
|
remote_load_env_path() {
|
||||||
|
local site=$1
|
||||||
|
[[ $site == ningbo || $site == hongkong ]]
|
||||||
|
[[ $run_id =~ ^[0-9a-f-]{36}$ ]]
|
||||||
|
printf '/root/easyai-acceptance-load-%s-%s.env' "$run_id" "$site"
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup_remote_load_drivers() {
|
||||||
|
[[ $run_id =~ ^[0-9a-f-]{36}$ ]] || return 0
|
||||||
|
local binary_path ningbo_env hongkong_env
|
||||||
|
binary_path=$(remote_load_binary_path)
|
||||||
|
ningbo_env=$(remote_load_env_path ningbo)
|
||||||
|
hongkong_env=$(remote_load_env_path hongkong)
|
||||||
|
cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||||
|
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$ningbo_env' ]] || unlink '$ningbo_env'" || true
|
||||||
|
cluster_ssh "$CLUSTER_HONGKONG_HOST" \
|
||||||
|
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$hongkong_env' ]] || unlink '$hongkong_env'" || true
|
||||||
|
remote_load_drivers_installed=false
|
||||||
|
}
|
||||||
|
|
||||||
|
write_remote_load_env() {
|
||||||
|
local output=$1
|
||||||
|
local gateway=$2
|
||||||
|
install -m 0600 /dev/null "$output"
|
||||||
|
{
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_GATEWAYS=%q\n' "$gateway"
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=%q\n' "$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME"
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_EMULATOR_URL=%q\n' 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090'
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_API_KEYS=%q\n' "$AI_GATEWAY_ACCEPTANCE_API_KEYS"
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_RUN_ID=%q\n' "$run_id"
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=%q\n' "$run_token"
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=%q\n' "$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL"
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=%q\n' "$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL"
|
||||||
|
printf 'AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=%q\n' "$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS"
|
||||||
|
} >>"$output"
|
||||||
|
}
|
||||||
|
|
||||||
|
install_remote_load_drivers() {
|
||||||
|
[[ $remote_load_drivers_installed == false ]] || return 0
|
||||||
|
local binary_path local_digest remote_digest site host gateway env_file remote_env
|
||||||
|
binary_path=$(remote_load_binary_path)
|
||||||
|
local_digest=$(shasum -a 256 "$acceptance_load_linux_binary" | awk '{print $1}')
|
||||||
|
local -a sites=(ningbo hongkong)
|
||||||
|
local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST")
|
||||||
|
local -a gateways=("https://${CLUSTER_NINGBO_HOST#root@}" "https://${CLUSTER_HONGKONG_HOST#root@}")
|
||||||
|
for site_index in 0 1; do
|
||||||
|
site=${sites[$site_index]}
|
||||||
|
host=${hosts[$site_index]}
|
||||||
|
gateway=${gateways[$site_index]}
|
||||||
|
env_file=$temporary_root/remote-load-$site.env
|
||||||
|
remote_env=$(remote_load_env_path "$site")
|
||||||
|
write_remote_load_env "$env_file" "$gateway"
|
||||||
|
cluster_scp "$acceptance_load_linux_binary" "$host:$binary_path" >/dev/null
|
||||||
|
cluster_scp "$env_file" "$host:$remote_env" >/dev/null
|
||||||
|
cluster_ssh "$host" "chmod 0755 '$binary_path'; chmod 0600 '$remote_env'; sha256sum '$binary_path'" >"$temporary_root/remote-load-$site.sha256"
|
||||||
|
remote_digest=$(awk '{print $1}' "$temporary_root/remote-load-$site.sha256")
|
||||||
|
[[ $remote_digest == "$local_digest" ]]
|
||||||
|
done
|
||||||
|
remote_load_drivers_installed=true
|
||||||
|
echo 'acceptance_load_drivers=PASS sites=2 placement=k3s_external_host_processes digest_verified=true'
|
||||||
|
}
|
||||||
|
|
||||||
|
run_remote_load_site() {
|
||||||
|
local site=$1
|
||||||
|
local host=$2
|
||||||
|
local profile=$3
|
||||||
|
local artifact=$4
|
||||||
|
local shard_index=$5
|
||||||
|
shift 5
|
||||||
|
[[ $site == ningbo || $site == hongkong ]]
|
||||||
|
[[ $profile =~ ^[a-z-]+$ && $artifact =~ ^[a-z0-9-]+\.json$ ]]
|
||||||
|
local binary_path remote_env remote_report local_report status=0
|
||||||
|
binary_path=$(remote_load_binary_path)
|
||||||
|
remote_env=$(remote_load_env_path "$site")
|
||||||
|
remote_report=/root/easyai-acceptance-load-"$run_id"-"$artifact"
|
||||||
|
local_report=$temporary_root/"$site"-"$artifact"
|
||||||
|
cluster_ssh "$host" bash -s -- \
|
||||||
|
"$binary_path" "$remote_env" "$profile" "$remote_report" "$shard_index" 2 "${artifact%.json}" "$@" \
|
||||||
|
>"$temporary_root/$site-$artifact.stdout" <<'REMOTE' || status=$?
|
||||||
|
set -euo pipefail
|
||||||
|
binary_path=$1
|
||||||
|
env_file=$2
|
||||||
|
profile=$3
|
||||||
|
report=$4
|
||||||
|
shard_index=$5
|
||||||
|
shard_count=$6
|
||||||
|
execution_id=$7
|
||||||
|
shift 7
|
||||||
|
set -a
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "$env_file"
|
||||||
|
set +a
|
||||||
|
"$binary_path" -profile "$profile" -report "$report" \
|
||||||
|
-shard-index "$shard_index" -shard-count "$shard_count" \
|
||||||
|
-execution-id "$execution_id" "$@"
|
||||||
|
REMOTE
|
||||||
|
cluster_scp "$host:$remote_report" "$local_report" >/dev/null || return 1
|
||||||
|
cluster_ssh "$host" "[[ ! -e '$remote_report' ]] || unlink '$remote_report'" >/dev/null || true
|
||||||
|
jq -e --arg runId "$run_id" --arg profile "$profile" \
|
||||||
|
'.schemaVersion == "acceptance-load-report/v1" and .runId == $runId and .profile == $profile and .secretSafe == true' \
|
||||||
|
"$local_report" >/dev/null || return 1
|
||||||
|
return "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
merge_remote_load_reports() {
|
||||||
|
local profile=$1
|
||||||
|
local ningbo_report=$2
|
||||||
|
local hongkong_report=$3
|
||||||
|
local output=$4
|
||||||
|
jq -s --arg profile "$profile" '
|
||||||
|
def sum_field($name): [.[].phases[0][$name] // 0] | add;
|
||||||
|
def max_field($name): [.[].phases[0][$name] // 0] | max;
|
||||||
|
. as $reports
|
||||||
|
| {
|
||||||
|
schemaVersion:"acceptance-load-report/v1",
|
||||||
|
runId:$reports[0].runId,
|
||||||
|
profile:$profile,
|
||||||
|
startedAt:([$reports[].startedAt] | min),
|
||||||
|
finishedAt:([$reports[].finishedAt] | max),
|
||||||
|
passed:([$reports[].passed] | all),
|
||||||
|
phases:[{
|
||||||
|
name:$profile,
|
||||||
|
requests:sum_field("requests"),
|
||||||
|
completed:sum_field("completed"),
|
||||||
|
failed:sum_field("failed"),
|
||||||
|
durationMs:max_field("durationMs"),
|
||||||
|
submissionDurationMs:max_field("submissionDurationMs"),
|
||||||
|
p50Ms:max_field("p50Ms"),
|
||||||
|
p95Ms:max_field("p95Ms"),
|
||||||
|
p99Ms:max_field("p99Ms"),
|
||||||
|
decodedOutputBytes:sum_field("decodedOutputBytes"),
|
||||||
|
outputSha256:([$reports[].phases[0].outputSha256 // ""] | map(select(length>0)) | first // ""),
|
||||||
|
uniqueTaskIds:sum_field("uniqueTaskIds"),
|
||||||
|
uniqueImageCombinations:max_field("uniqueImageCombinations"),
|
||||||
|
forcedConversionTasks:sum_field("forcedConversionTasks"),
|
||||||
|
throttled:sum_field("throttled"),
|
||||||
|
unexpected5xx:sum_field("unexpected5xx"),
|
||||||
|
offeredRatePerSecond:sum_field("offeredRatePerSecond")
|
||||||
|
}],
|
||||||
|
failure:([$reports[] | select(.passed == false) | .failure] | first // null),
|
||||||
|
failureOperation:([$reports[] | select(.passed == false) | .failureOperation] | first // null),
|
||||||
|
aggregation:{
|
||||||
|
sites:["ningbo","hongkong"],
|
||||||
|
percentileMethod:"conservative_max_of_site_percentiles",
|
||||||
|
durationMethod:"max_site_duration",
|
||||||
|
placement:"k3s_external_host_processes"
|
||||||
|
},
|
||||||
|
secretSafe:true
|
||||||
|
}
|
||||||
|
| with_entries(select(.value != null))
|
||||||
|
' "$ningbo_report" "$hongkong_report" >"$output"
|
||||||
|
chmod 0600 "$output"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_distributed_load_profile() {
|
||||||
|
local profile=$1
|
||||||
|
local report_path=$2
|
||||||
|
shift 2
|
||||||
|
install_remote_load_drivers
|
||||||
|
local artifact base ningbo_report hongkong_report ningbo_pid hongkong_pid
|
||||||
|
local ningbo_status=0 hongkong_status=0
|
||||||
|
base=$(basename "$report_path" .json)
|
||||||
|
artifact=$base.json
|
||||||
|
run_remote_load_site ningbo "$CLUSTER_NINGBO_HOST" "$profile" "$artifact" 0 "$@" &
|
||||||
|
ningbo_pid=$!
|
||||||
|
run_remote_load_site hongkong "$CLUSTER_HONGKONG_HOST" "$profile" "$artifact" 1 "$@" &
|
||||||
|
hongkong_pid=$!
|
||||||
|
wait "$ningbo_pid" || ningbo_status=$?
|
||||||
|
wait "$hongkong_pid" || hongkong_status=$?
|
||||||
|
ningbo_report=$temporary_root/ningbo-$artifact
|
||||||
|
hongkong_report=$temporary_root/hongkong-$artifact
|
||||||
|
[[ -f $ningbo_report && -f $hongkong_report ]] || return 1
|
||||||
|
merge_remote_load_reports "$profile" "$ningbo_report" "$hongkong_report" "$report_path"
|
||||||
|
(( ningbo_status == 0 && hongkong_status == 0 )) || return 1
|
||||||
|
jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
run_load_profile() {
|
run_load_profile() {
|
||||||
local profile=$1
|
local profile=$1
|
||||||
local report_path=$2
|
local report_path=$2
|
||||||
[[ $profile =~ ^[a-z-]+$ ]] || return 1
|
[[ $profile =~ ^[a-z-]+$ ]] || return 1
|
||||||
|
if [[ $profile != real-canary ]]; then
|
||||||
|
run_distributed_load_profile "$profile" "$report_path"
|
||||||
|
return
|
||||||
|
fi
|
||||||
local load_stdout=$temporary_root/load-$profile-stdout.json
|
local load_stdout=$temporary_root/load-$profile-stdout.json
|
||||||
AI_GATEWAY_ACCEPTANCE_GATEWAYS="$AI_GATEWAY_ACCEPTANCE_GATEWAYS" \
|
AI_GATEWAY_ACCEPTANCE_GATEWAYS="$AI_GATEWAY_ACCEPTANCE_GATEWAYS" \
|
||||||
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME="$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" \
|
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME="$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME" \
|
||||||
@@ -1919,6 +2102,7 @@ run_load_profile() {
|
|||||||
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
|
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL" \
|
||||||
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS="$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" \
|
AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS="$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS" \
|
||||||
"$acceptance_load_binary" -profile "$profile" -report "$report_path" \
|
"$acceptance_load_binary" -profile "$profile" -report "$report_path" \
|
||||||
|
-execution-id "$(basename "$report_path" .json)" \
|
||||||
>"$load_stdout"
|
>"$load_stdout"
|
||||||
jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
|
jq -e '.secretSafe == true and .passed == true' "$report_path" >/dev/null
|
||||||
}
|
}
|
||||||
@@ -2617,6 +2801,7 @@ finish_and_report() {
|
|||||||
local gemini_invalid video_invalid reference_count_3 reference_count_6 reference_count_9
|
local gemini_invalid video_invalid reference_count_3 reference_count_6 reference_count_9
|
||||||
local reference_role_count first_frame_role_count last_frame_role_count
|
local reference_role_count first_frame_role_count last_frame_role_count
|
||||||
local unique_image_hashes unique_video_tasks
|
local unique_image_hashes unique_video_tasks
|
||||||
|
local unique_gemini_input_hashes direct_gemini_input_assets
|
||||||
local duplicate_remote_tasks duplicate_terminal_events duplicate_settlements
|
local duplicate_remote_tasks duplicate_terminal_events duplicate_settlements
|
||||||
local duplicate_wallet_transactions duplicate_callback_rows pending_settlements pending_callbacks
|
local duplicate_wallet_transactions duplicate_callback_rows pending_settlements pending_callbacks
|
||||||
local missing_billings
|
local missing_billings
|
||||||
@@ -2625,6 +2810,19 @@ finish_and_report() {
|
|||||||
gemini_task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND run_mode='acceptance';")
|
gemini_task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND run_mode='acceptance';")
|
||||||
forced_tasks=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND request::text LIKE '%acceptance-force-conversion%';")
|
forced_tasks=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND request::text LIKE '%acceptance-force-conversion%';")
|
||||||
compact_payloads=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND (pg_column_size(request) >= 1048576 OR pg_column_size(result) >= 1048576);")
|
compact_payloads=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND (pg_column_size(request) >= 1048576 OR pg_column_size(result) >= 1048576);")
|
||||||
|
unique_gemini_input_hashes=$(database_query "
|
||||||
|
SELECT count(DISTINCT request #>> '{image,assetRef,sha256}')
|
||||||
|
FROM gateway_tasks
|
||||||
|
WHERE acceptance_run_id='$run_id'::uuid
|
||||||
|
AND kind='images.edits'
|
||||||
|
AND run_mode='acceptance';")
|
||||||
|
direct_gemini_input_assets=$(database_query "
|
||||||
|
SELECT count(*)
|
||||||
|
FROM gateway_tasks
|
||||||
|
WHERE acceptance_run_id='$run_id'::uuid
|
||||||
|
AND kind='images.edits'
|
||||||
|
AND run_mode='acceptance'
|
||||||
|
AND request #>> '{image,assetRef,storageProvider}' = 'aliyun_oss_direct';")
|
||||||
submissions=$(jq -r '.videoSubmissions' "$emulator_report")
|
submissions=$(jq -r '.videoSubmissions' "$emulator_report")
|
||||||
duplicates=$(jq -r '.duplicates' "$callback_report")
|
duplicates=$(jq -r '.duplicates' "$callback_report")
|
||||||
callbacks=$(jq -r '.deliveries' "$callback_report")
|
callbacks=$(jq -r '.deliveries' "$callback_report")
|
||||||
@@ -2653,6 +2851,10 @@ finish_and_report() {
|
|||||||
$missing_idempotency == 0 && $compact_payloads == 0 ]] ||
|
$missing_idempotency == 0 && $compact_payloads == 0 ]] ||
|
||||||
fail_acceptance_gate idempotency_and_materialization \
|
fail_acceptance_gate idempotency_and_materialization \
|
||||||
'duplicate submission, missing idempotency key, or inline database media detected'
|
'duplicate submission, missing idempotency key, or inline database media detected'
|
||||||
|
[[ $unique_gemini_input_hashes == "$gemini_task_count" &&
|
||||||
|
$direct_gemini_input_assets == "$gemini_task_count" ]] ||
|
||||||
|
fail_acceptance_gate gemini_input_materialization \
|
||||||
|
'Gemini inputs were reused across executions or bypassed direct OSS materialization'
|
||||||
[[ $gemini_invalid == 0 && $video_invalid == 0 ]] ||
|
[[ $gemini_invalid == 0 && $video_invalid == 0 ]] ||
|
||||||
fail_acceptance_gate provider_protocol_validity 'Gemini or Volces protocol validation failed'
|
fail_acceptance_gate provider_protocol_validity 'Gemini or Volces protocol validation failed'
|
||||||
[[ $reference_count_3 -gt 0 && $reference_count_6 -gt 0 && $reference_count_9 -gt 0 ]] ||
|
[[ $reference_count_3 -gt 0 && $reference_count_6 -gt 0 && $reference_count_9 -gt 0 ]] ||
|
||||||
@@ -2796,6 +2998,8 @@ WHERE task.acceptance_run_id='$run_id'::uuid
|
|||||||
--argjson lastFrameRoleCount "$last_frame_role_count" \
|
--argjson lastFrameRoleCount "$last_frame_role_count" \
|
||||||
--argjson uniqueImageHashes "$unique_image_hashes" \
|
--argjson uniqueImageHashes "$unique_image_hashes" \
|
||||||
--argjson uniqueVideoTasks "$unique_video_tasks" \
|
--argjson uniqueVideoTasks "$unique_video_tasks" \
|
||||||
|
--argjson uniqueGeminiInputHashes "$unique_gemini_input_hashes" \
|
||||||
|
--argjson directGeminiInputAssets "$direct_gemini_input_assets" \
|
||||||
--argjson duplicateRemoteTasks "$duplicate_remote_tasks" \
|
--argjson duplicateRemoteTasks "$duplicate_remote_tasks" \
|
||||||
--argjson duplicateTerminalEvents "$duplicate_terminal_events" \
|
--argjson duplicateTerminalEvents "$duplicate_terminal_events" \
|
||||||
--argjson duplicateSettlements "$duplicate_settlements" \
|
--argjson duplicateSettlements "$duplicate_settlements" \
|
||||||
@@ -2838,7 +3042,9 @@ WHERE task.acceptance_run_id='$run_id'::uuid
|
|||||||
last_frame:$lastFrameRoleCount
|
last_frame:$lastFrameRoleCount
|
||||||
},
|
},
|
||||||
uniqueImageHashes:$uniqueImageHashes,
|
uniqueImageHashes:$uniqueImageHashes,
|
||||||
uniqueVideoTasks:$uniqueVideoTasks
|
uniqueVideoTasks:$uniqueVideoTasks,
|
||||||
|
uniqueGeminiInputHashes:$uniqueGeminiInputHashes,
|
||||||
|
directGeminiInputAssets:$directGeminiInputAssets
|
||||||
},
|
},
|
||||||
databaseConsistency:{
|
databaseConsistency:{
|
||||||
duplicateRemoteTasks:$duplicateRemoteTasks,
|
duplicateRemoteTasks:$duplicateRemoteTasks,
|
||||||
|
|||||||
Reference in New Issue
Block a user