From 3053ba4925f589acd8d39e465b53881cdf0c4558 Mon Sep 17 00:00:00 2001 From: wangbo Date: Thu, 30 Jul 2026 23:06:19 +0800 Subject: [PATCH] =?UTF-8?q?feat(acceptance):=20=E5=A2=9E=E5=8A=A0=E7=94=9F?= =?UTF-8?q?=E4=BA=A7=E5=90=8C=E6=9E=84=E5=AA=92=E4=BD=93=E5=8E=8B=E5=8A=9B?= =?UTF-8?q?=E9=AA=8C=E6=94=B6=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 引入动态流量门禁、隔离验收身份与协议级 Gemini/Volces 模拟器，覆盖双站点 API、Worker、PostgreSQL、River、账务、回调和媒体物化链路。 新增 P24/P28/P32 容量阶梯、Worker 强杀恢复、真实小流量 canary、CAS 放量和失败保持 validation 的生产编排；Worker 执行槽、连接池、媒体并发和双站点副本数改为环境配置。 验证：Go 全量测试、真实 PostgreSQL 迁移集成测试、迁移安全检查、OpenAPI 生成、ShellCheck、Kustomize、gofmt 和 git diff --check。 --- Dockerfile | 6 +- README.md | 4 +- apps/api/cmd/acceptance-emulator/main.go | 28 + apps/api/cmd/acceptance-load/main.go | 825 ++++++++++++++++ apps/api/cmd/acceptance-load/main_test.go | 127 +++ apps/api/docs/swagger.json | 457 +++++++++ apps/api/docs/swagger.yaml | 288 ++++++ .../api/internal/acceptanceemulator/server.go | 658 +++++++++++++ .../acceptanceemulator/server_test.go | 186 ++++ .../internal/acceptanceworkload/profiles.go | 52 + .../acceptanceworkload/profiles_test.go | 23 + apps/api/internal/clients/clients_test.go | 2 + apps/api/internal/clients/volces.go | 2 + .../internal/httpapi/acceptance_handlers.go | 300 ++++++ .../httpapi/acceptance_handlers_test.go | 27 + .../internal/httpapi/gateway_task_creation.go | 11 +- .../gemini_base64_stress_integration_test.go | 10 +- apps/api/internal/httpapi/gemini_compat.go | 13 +- apps/api/internal/httpapi/handlers.go | 13 +- apps/api/internal/httpapi/kling_compat.go | 31 +- apps/api/internal/httpapi/server.go | 8 + .../httpapi/volces_compat_handlers.go | 5 + .../httpapi/volces_compat_handlers_test.go | 25 + apps/api/internal/runner/billing_mode_test.go | 11 + apps/api/internal/runner/service.go | 63 +- apps/api/internal/securityevents/metrics.go | 12 + .../internal/securityevents/metrics_test.go | 22 +- apps/api/internal/store/acceptance.go | 533 ++++++++++ .../store/acceptance_integration_test.go | 136 +++ apps/api/internal/store/acceptance_test.go | 37 + apps/api/internal/store/postgres.go | 22 +- .../internal/store/postgres_pool_metrics.go | 25 + apps/api/internal/store/tasks_runtime.go | 22 +- .../0094_production_acceptance_runs.sql | 50 + .../kubernetes/acceptance/kustomization.yaml | 4 + .../acceptance/protocol-emulator.yaml | 67 ++ .../easyai-ai-gateway-cluster-release | 78 +- ...ai-ai-gateway-cluster-release.conf.example | 10 + .../production/application-config.yaml | 2 +- deploy/kubernetes/production/application.yaml | 4 + docs/design.md | 4 +- docs/operations/k3s-ha-runbook.md | 3 + docs/operations/production-acceptance.md | 131 +++ scripts/cluster/install-release-helper.sh | 9 +- scripts/cluster/run-production-acceptance.sh | 913 ++++++++++++++++++ 45 files changed, 5214 insertions(+), 45 deletions(-) create mode 100644 apps/api/cmd/acceptance-emulator/main.go create mode 100644 apps/api/cmd/acceptance-load/main.go create mode 100644 apps/api/cmd/acceptance-load/main_test.go create mode 100644 apps/api/internal/acceptanceemulator/server.go create mode 100644 apps/api/internal/acceptanceemulator/server_test.go create mode 100644 apps/api/internal/acceptanceworkload/profiles.go create mode 100644 apps/api/internal/acceptanceworkload/profiles_test.go create mode 100644 apps/api/internal/httpapi/acceptance_handlers.go create mode 100644 apps/api/internal/httpapi/acceptance_handlers_test.go create mode 100644 apps/api/internal/store/acceptance.go create mode 100644 apps/api/internal/store/acceptance_integration_test.go create mode 100644 apps/api/internal/store/acceptance_test.go create mode 100644 apps/api/internal/store/postgres_pool_metrics.go create mode 100644 apps/api/migrations/0094_production_acceptance_runs.sql create mode 100644 deploy/kubernetes/acceptance/kustomization.yaml create mode 100644 deploy/kubernetes/acceptance/protocol-emulator.yaml create mode 100644 docs/operations/production-acceptance.md create mode 100755 scripts/cluster/run-production-acceptance.sh diff --git a/Dockerfile b/Dockerfile index 032afa9..b6218ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,9 @@ RUN --mount=type=cache,target=/go/pkg/mod \ cd apps/api && \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway ./cmd/gateway && \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway-migrate ./cmd/migrate && \ - CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway-backfill-binary-results ./cmd/backfill-binary-results + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway-backfill-binary-results ./cmd/backfill-binary-results && \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway-acceptance-emulator ./cmd/acceptance-emulator && \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway-acceptance-load ./cmd/acceptance-load FROM ${API_RUNTIME_IMAGE} AS api @@ -44,6 +46,8 @@ WORKDIR /app COPY --from=api-builder /out/easyai-ai-gateway /app/easyai-ai-gateway COPY --from=api-builder /out/easyai-ai-gateway-migrate /app/easyai-ai-gateway-migrate COPY --from=api-builder /out/easyai-ai-gateway-backfill-binary-results /app/easyai-ai-gateway-backfill-binary-results +COPY --from=api-builder /out/easyai-ai-gateway-acceptance-emulator /app/easyai-ai-gateway-acceptance-emulator +COPY --from=api-builder /out/easyai-ai-gateway-acceptance-load /app/easyai-ai-gateway-acceptance-load COPY apps/api/migrations /app/migrations RUN mkdir -p /app/data/static/generated /app/data/static/uploaded && \ diff --git a/README.md b/README.md index 441661f..3ae9409 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,9 @@ docker login --username= registry.cn-shanghai.aliyuncs.com 完整安装、验证、失败处理和停用旧自动化步骤见[人工生产发布运行手册](docs/runbooks/production-ci-cd.md)； 三节点 K3s、CloudNativePG、双 NGINX、备份恢复和跨节点文件验收见 -[K3s 高可用运行手册](docs/operations/k3s-ha-runbook.md)。决策背景见 +[K3s 高可用运行手册](docs/operations/k3s-ha-runbook.md)；正式流量接入前的 Gemini +带图、多参考图视频、Worker 强杀和容量阶梯验收见 +[生产同构验收手册](docs/operations/production-acceptance.md)。决策背景见 [ADR-003](docs/decisions/003-manual-agent-release.md)。 Compose 默认使用独立容器数据库 `postgres:18-alpine`，数据卷会保留在 `postgres_data` 和 `api_data`。为避免本地开发 `.env` 中的 `localhost` 数据库地址污染容器部署，compose 使用 `AI_GATEWAY_COMPOSE_*` 变量作为容器部署专用覆盖，例如： diff --git a/apps/api/cmd/acceptance-emulator/main.go b/apps/api/cmd/acceptance-emulator/main.go new file mode 100644 index 0000000..bf6a523 --- /dev/null +++ b/apps/api/cmd/acceptance-emulator/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "log/slog" + "net/http" + "os" + "strings" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceemulator" +) + +func main() { + address := strings.TrimSpace(os.Getenv("HTTP_ADDR")) + if address == "" { + address = ":8090" + } + server := &http.Server{ + Addr: address, + Handler: acceptanceemulator.New(acceptanceemulator.Config{}).Handler(), + ReadHeaderTimeout: 10 * time.Second, + } + slog.Info("acceptance protocol emulator started", "address", address) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + slog.Error("acceptance protocol emulator stopped", "error", err) + os.Exit(1) + } +} diff --git a/apps/api/cmd/acceptance-load/main.go b/apps/api/cmd/acceptance-load/main.go new file mode 100644 index 0000000..b61753f --- /dev/null +++ b/apps/api/cmd/acceptance-load/main.go @@ -0,0 +1,825 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "crypto/tls" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "hash/crc32" + "io" + "net/http" + "net/url" + "os" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceworkload" +) + +const ( + runHeader = "X-EasyAI-Acceptance-Run" + tokenHeader = "X-EasyAI-Acceptance-Token" + upstreamHeader = "X-EasyAI-Acceptance-Upstream" +) + +var reportURLPattern = regexp.MustCompile(`(?i)(?:https?|postgres(?:ql)?):\/\/[^\s"'<>]+`) + +type options struct { + gateways []string + gatewayTLSName string + emulatorURL string + apiKey string + runID string + runToken string + geminiModel string + videoModel string + profile string + reportPath string + realImageURLs []string + timeout time.Duration +} + +type report struct { + RunID string `json:"runId"` + Profile string `json:"profile"` + StartedAt time.Time `json:"startedAt"` + FinishedAt time.Time `json:"finishedAt"` + Passed bool `json:"passed"` + Phases []phaseReport `json:"phases"` + Failure string `json:"failure,omitempty"` + SecretSafe bool `json:"secretSafe"` +} + +type phaseReport struct { + Name string `json:"name"` + Requests int `json:"requests"` + Completed int `json:"completed"` + Failed int `json:"failed"` + DurationMS int64 `json:"durationMs"` + SubmissionDurationMS int64 `json:"submissionDurationMs,omitempty"` + P50MS float64 `json:"p50Ms"` + P95MS float64 `json:"p95Ms"` + P99MS float64 `json:"p99Ms"` + DecodedOutputBytes int64 `json:"decodedOutputBytes,omitempty"` + OutputSHA256 string `json:"outputSha256,omitempty"` + UniqueTaskIDs int `json:"uniqueTaskIds,omitempty"` + UniqueImageCombos int `json:"uniqueImageCombinations,omitempty"` + ForcedConversionTasks int `json:"forcedConversionTasks,omitempty"` +} + +type phaseResult struct { + report phaseReport + latencies []time.Duration + err error +} + +func main() { + opts, err := parseOptions() + if err != nil { + fmt.Fprintln(os.Stderr, "acceptance load configuration error:", err) + os.Exit(2) + } + ctx, cancel := context.WithTimeout(context.Background(), opts.timeout) + defer cancel() + result := report{ + RunID: opts.runID, Profile: opts.profile, StartedAt: time.Now().UTC(), SecretSafe: true, + } + phaseResults, runErr := run(ctx, opts) + for _, phase := range phaseResults { + phase.report.P50MS = percentileMilliseconds(phase.latencies, 0.50) + phase.report.P95MS = percentileMilliseconds(phase.latencies, 0.95) + phase.report.P99MS = percentileMilliseconds(phase.latencies, 0.99) + result.Phases = append(result.Phases, phase.report) + if runErr == nil && phase.err != nil { + runErr = phase.err + } + } + result.FinishedAt = time.Now().UTC() + result.Passed = runErr == nil + if runErr != nil { + result.Failure = redactError(runErr.Error(), opts) + } + payload, _ := json.MarshalIndent(result, "", " ") + fmt.Println(string(payload)) + if opts.reportPath != "" { + if err := os.WriteFile(opts.reportPath, append(payload, '\n'), 0o600); err != nil { + fmt.Fprintln(os.Stderr, "write acceptance report failed:", err) + os.Exit(1) + } + } + if runErr != nil { + os.Exit(1) + } +} + +func parseOptions() (options, error) { + var profile string + var reportPath string + var timeout time.Duration + flag.StringVar(&profile, "profile", env("AI_GATEWAY_ACCEPTANCE_PROFILE", "simulated-all"), "simulated-all, Gemini profile, video profile, or real-canary") + flag.StringVar(&reportPath, "report", env("AI_GATEWAY_ACCEPTANCE_REPORT", ""), "secret-safe JSON report path") + flag.DurationVar(&timeout, "timeout", 45*time.Minute, "overall timeout") + flag.Parse() + opts := options{ + gateways: splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAYS")), + gatewayTLSName: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME")), + emulatorURL: strings.TrimRight(strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_EMULATOR_URL")), "/"), + apiKey: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_API_KEY")), + runID: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_RUN_ID")), + runToken: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_RUN_TOKEN")), + geminiModel: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL")), + videoModel: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL")), + profile: strings.ToLower(strings.TrimSpace(profile)), + reportPath: strings.TrimSpace(reportPath), + realImageURLs: splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS")), + timeout: timeout, + } + if len(opts.gateways) != 2 { + return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAYS must contain exactly two API base URLs") + } + for index := range opts.gateways { + opts.gateways[index] = strings.TrimRight(opts.gateways[index], "/") + if _, err := url.ParseRequestURI(opts.gateways[index]); err != nil { + return options{}, fmt.Errorf("gateway %d URL is invalid", index+1) + } + } + if opts.gatewayTLSName != "" && + (strings.ContainsAny(opts.gatewayTLSName, "/:@") || !strings.Contains(opts.gatewayTLSName, ".")) { + return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME must be a DNS hostname") + } + if opts.apiKey == "" || opts.runID == "" || opts.runToken == "" { + return options{}, errors.New("acceptance API key, Run ID, and Run Token environment variables are required") + } + if opts.geminiModel == "" || opts.videoModel == "" { + return options{}, errors.New("Gemini and video model environment variables are required") + } + if opts.timeout <= 0 { + return options{}, errors.New("timeout must be positive") + } + switch opts.profile { + case "simulated-all", "gemini-baseline", "gemini-large", "gemini-peak", "video-throughput", "video-recovery": + if opts.emulatorURL == "" { + return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_EMULATOR_URL is required for simulated profiles") + } + case "real-canary": + if len(opts.realImageURLs) < 3 { + return options{}, errors.New("real-canary requires at least three public AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS") + } + default: + return options{}, fmt.Errorf("unsupported profile %q", opts.profile) + } + return opts, nil +} + +func run(ctx context.Context, opts options) ([]phaseResult, error) { + var tlsConfig *tls.Config + if opts.gatewayTLSName != "" { + tlsConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: opts.gatewayTLSName, + } + } + client := &http.Client{ + Timeout: opts.timeout, + Transport: &http.Transport{ + MaxIdleConns: 2048, + MaxIdleConnsPerHost: 1024, + MaxConnsPerHost: 1024, + TLSClientConfig: tlsConfig, + }, + } + results := make([]phaseResult, 0, 5) + runPhase := func(name string) error { + var result phaseResult + if profile, ok := acceptanceworkload.GeminiProfileByName(name); ok { + result = runGemini(ctx, client, opts, name, profile.Requests, profile.InputBytes, profile.OutputBytes, false) + } else { + switch name { + case "video-throughput": + result = runVideo(ctx, client, opts, name, 1200, false, opts.emulatorFixtureURLs(), false) + case "video-recovery": + result = runVideo(ctx, client, opts, name, 96, true, opts.emulatorFixtureURLs(), false) + case "real-gemini-canary": + result = runGemini(ctx, client, opts, name, 1, 256<<10, 0, true) + case "real-video-canary": + result = runVideo(ctx, client, opts, name, 1, false, opts.realImageURLs, true) + default: + return fmt.Errorf("unknown phase %q", name) + } + } + results = append(results, result) + return result.err + } + phases := []string{opts.profile} + if opts.profile == "simulated-all" { + phases = []string{"gemini-baseline", "gemini-large", "gemini-peak", "video-throughput", "video-recovery"} + } else if opts.profile == "real-canary" { + phases = []string{"real-gemini-canary", "real-video-canary"} + } + for _, phase := range phases { + if err := runPhase(phase); err != nil { + return results, err + } + } + return results, nil +} + +func runGemini( + ctx context.Context, + client *http.Client, + opts options, + name string, + requestCount int, + inputBytes int, + expectedOutputBytes int, + realUpstream bool, +) phaseResult { + startedAt := time.Now() + input := paddedPNG(inputBytes) + requestBody, _ := json.Marshal(map[string]any{ + "contents": []any{map[string]any{ + "role": "user", + "parts": []any{ + map[string]any{"text": "将参考图编辑为蓝色赛博朋克风格，保留主体结构"}, + map[string]any{"inlineData": map[string]any{ + "mimeType": "image/png", + "data": base64.StdEncoding.EncodeToString(input), + }}, + }, + }}, + "generationConfig": map[string]any{"responseModalities": []any{"IMAGE"}}, + }) + expectedHash := "" + if expectedOutputBytes > 0 { + sum := sha256.Sum256(paddedPNG(expectedOutputBytes)) + expectedHash = hex.EncodeToString(sum[:]) + } + latencies := make([]time.Duration, requestCount) + var decodedBytes int64 + var completed int + var failures int + var firstErr error + var mu sync.Mutex + slots := make(chan struct{}, requestCount) + var wg sync.WaitGroup + for index := 0; index < requestCount; index++ { + index := index + wg.Add(1) + go func() { + defer wg.Done() + select { + case slots <- struct{}{}: + case <-ctx.Done(): + return + } + defer func() { <-slots }() + requestStarted := time.Now() + endpoint := opts.gateways[index%len(opts.gateways)] + "/v1beta/models/" + url.PathEscape(opts.geminiModel) + ":generateContent" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(requestBody)) + if err == nil { + opts.setHeaders(req, realUpstream) + req.Header.Set("Content-Type", "application/json") + var response *http.Response + response, err = client.Do(req) + if err == nil { + if response.StatusCode != http.StatusOK { + err = responseStatusError(response) + } else { + var size int64 + var outputHash string + size, outputHash, err = streamGeminiImageHash(response.Body) + _ = response.Body.Close() + if err == nil && expectedOutputBytes > 0 && (size != int64(expectedOutputBytes) || outputHash != expectedHash) { + err = fmt.Errorf("Gemini output mismatch: bytes=%d hash=%s", size, outputHash) + } + if err == nil { + mu.Lock() + decodedBytes += size + if expectedHash == "" { + expectedHash = outputHash + } + mu.Unlock() + } + } + } + } + latency := time.Since(requestStarted) + mu.Lock() + latencies[index] = latency + if err != nil { + failures++ + if firstErr == nil { + firstErr = err + } + } else { + completed++ + } + mu.Unlock() + }() + } + wg.Wait() + elapsed := time.Since(startedAt) + result := phaseReport{ + Name: name, Requests: requestCount, Completed: completed, Failed: failures, + DurationMS: elapsed.Milliseconds(), DecodedOutputBytes: decodedBytes, OutputSHA256: expectedHash, + } + if firstErr == nil && name == "gemini-baseline" && elapsed > 8*time.Minute { + firstErr = fmt.Errorf("Gemini baseline exceeded 8 minutes: %s", elapsed) + } + if firstErr == nil && completed != requestCount { + firstErr = fmt.Errorf("Gemini completed %d of %d requests", completed, requestCount) + } + return phaseResult{report: result, latencies: latencies, err: firstErr} +} + +func runVideo( + ctx context.Context, + client *http.Client, + opts options, + name string, + requestCount int, + longRun bool, + imageURLs []string, + realUpstream bool, +) phaseResult { + startedAt := time.Now() + combinationCount := 128 + if realUpstream { + combinationCount = 1 + } + combinations := videoCombinations(imageURLs, combinationCount) + var taskIDs = make([]string, requestCount) + latencies := make([]time.Duration, requestCount) + startedTasks := make([]time.Time, requestCount) + var completed int + var failures int + var forcedConversions int + var firstErr error + var mu sync.Mutex + var wg sync.WaitGroup + submitStartedAt := time.Now() + for index := 0; index < requestCount; index++ { + index := index + wg.Add(1) + go func() { + defer wg.Done() + requestStarted := time.Now() + startedTasks[index] = requestStarted + imageCount := acceptanceworkload.VideoImageCount(index) + combo := append([]string(nil), combinations[index%len(combinations)]...) + if len(combo) > imageCount { + combo = combo[:imageCount] + } + if !realUpstream && index%4 == 0 { + combo[0] = imageURLs[12+(index/4)%4] + } + content := make([]any, 0, len(combo)+1) + prompt := "多参考图生成连续运镜视频，保持人物、服装和场景一致" + if longRun { + prompt += " acceptance-long-recovery" + } + if !realUpstream && index%4 == 0 { + prompt += " acceptance-force-conversion" + } + content = append(content, map[string]any{"type": "text", "text": prompt}) + for imageIndex, imageURL := range combo { + role := "reference_image" + if index%5 == 0 && imageIndex == 0 { + role = "first_frame" + } + if index%5 == 0 && len(combo) > 3 && imageIndex == len(combo)-1 { + role = "last_frame" + } + content = append(content, map[string]any{ + "type": "image_url", "role": role, + "image_url": map[string]any{"url": imageURL}, + }) + } + body, _ := json.Marshal(map[string]any{ + "model": opts.videoModel, "modelType": "omni_video", "content": content, "seed": index + 1, + "duration": 5, "ratio": "16:9", "resolution": "720p", + }) + endpoint := opts.gateways[index%len(opts.gateways)] + "/api/v1/videos/generations" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err == nil { + opts.setHeaders(req, realUpstream) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Async", "true") + var response *http.Response + response, err = client.Do(req) + if err == nil { + if response.StatusCode != http.StatusAccepted { + err = responseStatusError(response) + } else { + var accepted struct { + TaskID string `json:"taskId"` + } + err = json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&accepted) + _ = response.Body.Close() + taskIDs[index] = strings.TrimSpace(accepted.TaskID) + if err == nil && taskIDs[index] == "" { + err = errors.New("video acceptance response did not include taskId") + } + } + } + } + mu.Lock() + latencies[index] = time.Since(requestStarted) + if !realUpstream && index%4 == 0 { + forcedConversions++ + } + if err != nil { + failures++ + if firstErr == nil { + firstErr = err + } + } + mu.Unlock() + }() + } + wg.Wait() + submissionDuration := time.Since(submitStartedAt) + if firstErr == nil && name == "video-throughput" && submissionDuration > 10*time.Second { + firstErr = fmt.Errorf("1200 video submissions exceeded 10 seconds: %s", submissionDuration) + } + uniqueTaskIDs := map[string]struct{}{} + for _, taskID := range taskIDs { + if taskID != "" { + uniqueTaskIDs[taskID] = struct{}{} + } + } + if firstErr == nil && len(uniqueTaskIDs) != requestCount { + firstErr = fmt.Errorf("video task IDs are not unique: unique=%d requests=%d", len(uniqueTaskIDs), requestCount) + } + pollSlots := make(chan struct{}, 256) + for index, taskID := range taskIDs { + if taskID == "" { + continue + } + index, taskID := index, taskID + wg.Add(1) + go func() { + defer wg.Done() + select { + case pollSlots <- struct{}{}: + case <-ctx.Done(): + return + } + defer func() { <-pollSlots }() + err := pollVideoTask(ctx, client, opts, taskID, index, realUpstream) + mu.Lock() + latencies[index] = time.Since(startedTasks[index]) + if err != nil { + failures++ + if firstErr == nil { + firstErr = err + } + } else { + completed++ + } + mu.Unlock() + }() + } + wg.Wait() + comboSet := map[string]struct{}{} + for _, combo := range combinations { + comboSet[strings.Join(combo, "\x00")] = struct{}{} + } + elapsed := time.Since(startedAt) + result := phaseReport{ + Name: name, Requests: requestCount, Completed: completed, Failed: failures, + DurationMS: elapsed.Milliseconds(), SubmissionDurationMS: submissionDuration.Milliseconds(), + UniqueTaskIDs: len(uniqueTaskIDs), UniqueImageCombos: len(comboSet), + ForcedConversionTasks: forcedConversions, + } + if firstErr == nil && completed != requestCount { + firstErr = fmt.Errorf("video completed %d of %d tasks", completed, requestCount) + } + return phaseResult{report: result, latencies: latencies, err: firstErr} +} + +func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskID string, index int, realUpstream bool) error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + endpoint := opts.gateways[index%len(opts.gateways)] + "/api/v1/ai/result/" + url.PathEscape(taskID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return err + } + opts.setHeaders(req, realUpstream) + response, err := client.Do(req) + if err == nil { + if response.StatusCode != http.StatusOK { + err = responseStatusError(response) + } else { + var payload map[string]any + err = json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&payload) + _ = response.Body.Close() + status := strings.ToLower(strings.TrimSpace(fmt.Sprint(payload["status"]))) + switch status { + case "succeeded", "success": + mediaURL := findMediaURL(payload) + if mediaURL == "" { + return fmt.Errorf("video task %s succeeded without a media URL", taskID) + } + return validateVideoAsset(ctx, http.DefaultClient, mediaURL) + case "failed", "cancelled", "canceled": + return fmt.Errorf("video task %s finished with status %s", taskID, status) + } + } + } + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +func findMediaURL(value any) string { + switch typed := value.(type) { + case map[string]any: + for key, item := range typed { + normalized := strings.ToLower(strings.TrimSpace(key)) + if (normalized == "url" || normalized == "video_url") && strings.HasPrefix(strings.TrimSpace(fmt.Sprint(item)), "http") { + return strings.TrimSpace(fmt.Sprint(item)) + } + if mediaURL := findMediaURL(item); mediaURL != "" { + return mediaURL + } + } + case []any: + for _, item := range typed { + if mediaURL := findMediaURL(item); mediaURL != "" { + return mediaURL + } + } + } + return "" +} + +func validateVideoAsset(ctx context.Context, client *http.Client, mediaURL string) error { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, mediaURL, nil) + if err != nil { + return err + } + response, err := client.Do(request) + if err != nil { + return fmt.Errorf("download final video: %w", err) + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return fmt.Errorf("download final video: HTTP %d", response.StatusCode) + } + payload, err := io.ReadAll(io.LimitReader(response.Body, 64<<10)) + if err != nil { + return fmt.Errorf("read final video: %w", err) + } + contentType := strings.ToLower(strings.TrimSpace(response.Header.Get("Content-Type"))) + hasMP4Header := len(payload) >= 12 && string(payload[4:8]) == "ftyp" + if len(payload) < 12 || (!strings.HasPrefix(contentType, "video/") && !hasMP4Header) { + return fmt.Errorf("final video asset is invalid: bytes=%d content_type=%q", len(payload), contentType) + } + return nil +} + +func (o options) setHeaders(request *http.Request, realUpstream bool) { + if o.gatewayTLSName != "" { + request.Host = o.gatewayTLSName + } + request.Header.Set("Authorization", "Bearer "+o.apiKey) + request.Header.Set(runHeader, o.runID) + request.Header.Set(tokenHeader, o.runToken) + if realUpstream { + request.Header.Set(upstreamHeader, "real") + } +} + +func (o options) emulatorFixtureURLs() []string { + out := make([]string, 0, 16) + for index := 0; index < 4; index++ { + out = append(out, fmt.Sprintf("%s/fixtures/image-%02d.png", o.emulatorURL, index)) + } + for index := 4; index < 8; index++ { + out = append(out, fmt.Sprintf("%s/fixtures/image-%02d.jpg", o.emulatorURL, index)) + } + for index := 8; index < 12; index++ { + out = append(out, fmt.Sprintf("%s/fixtures/image-%02d.webp", o.emulatorURL, index)) + } + for index := 12; index < 16; index++ { + out = append(out, fmt.Sprintf("%s/fixtures/image-%02d-4k.jpg", o.emulatorURL, index)) + } + return out +} + +func videoCombinations(images []string, count int) [][]string { + if len(images) < 3 { + return nil + } + if len(images) < 9 { + return [][]string{append([]string(nil), images[:3]...)} + } + out := make([][]string, 0, count) + seen := map[string]struct{}{} + for seed := 0; len(out) < count; seed++ { + imageCount := acceptanceworkload.VideoImageCount(seed) + combo := make([]string, 0, imageCount) + used := map[int]struct{}{} + state := uint64(seed + 1) + for len(combo) < imageCount { + state = state*6364136223846793005 + 1442695040888963407 + index := int((state >> 32) % uint64(len(images))) + if _, exists := used[index]; exists { + continue + } + used[index] = struct{}{} + combo = append(combo, images[index]) + } + signature := strings.Join(combo, "\x00") + if _, exists := seen[signature]; exists { + continue + } + seen[signature] = struct{}{} + out = append(out, combo) + } + return out +} + +func streamGeminiImageHash(reader io.Reader) (int64, string, error) { + buffered := bufio.NewReaderSize(reader, 64<<10) + if err := scanUntil(buffered, []byte(`"inlineData"`), 256<<20); err != nil { + return 0, "", err + } + if err := scanUntil(buffered, []byte(`"data"`), 1<<20); err != nil { + return 0, "", err + } + for { + item, err := buffered.ReadByte() + if err != nil { + return 0, "", err + } + if item == ':' { + break + } + } + for { + item, err := buffered.ReadByte() + if err != nil { + return 0, "", err + } + if item == '"' { + break + } + if item != ' ' && item != '\t' && item != '\r' && item != '\n' { + return 0, "", errors.New("Gemini inlineData.data is not a JSON string") + } + } + jsonString := &base64JSONStringReader{reader: buffered} + digest := sha256.New() + decodedBytes, err := io.Copy(digest, base64.NewDecoder(base64.StdEncoding, jsonString)) + if err != nil { + return 0, "", fmt.Errorf("decode Gemini Base64 output: %w", err) + } + if decodedBytes == 0 { + return 0, "", errors.New("Gemini Base64 output is empty") + } + return decodedBytes, hex.EncodeToString(digest.Sum(nil)), nil +} + +type base64JSONStringReader struct { + reader *bufio.Reader + done bool +} + +func (r *base64JSONStringReader) Read(payload []byte) (int, error) { + if r.done { + return 0, io.EOF + } + written := 0 + for written < len(payload) { + item, err := r.reader.ReadByte() + if err != nil { + return written, err + } + if item == '"' { + r.done = true + if written == 0 { + return 0, io.EOF + } + return written, nil + } + if item == '\\' { + return written, errors.New("escaped Base64 JSON strings are not supported") + } + payload[written] = item + written++ + } + return written, nil +} + +func scanUntil(reader *bufio.Reader, target []byte, limit int64) error { + matched := 0 + for read := int64(0); read < limit; read++ { + item, err := reader.ReadByte() + if err != nil { + return err + } + if item == target[matched] { + matched++ + if matched == len(target) { + return nil + } + continue + } + if item == target[0] { + matched = 1 + } else { + matched = 0 + } + } + return errors.New("Gemini inlineData output was not found") +} + +func responseStatusError(response *http.Response) error { + defer response.Body.Close() + payload, _ := io.ReadAll(io.LimitReader(response.Body, 512)) + return fmt.Errorf("HTTP %d: %s", response.StatusCode, strings.TrimSpace(string(payload))) +} + +func percentileMilliseconds(values []time.Duration, quantile float64) float64 { + filtered := make([]time.Duration, 0, len(values)) + for _, value := range values { + if value > 0 { + filtered = append(filtered, value) + } + } + if len(filtered) == 0 { + return 0 + } + sort.Slice(filtered, func(i, j int) bool { return filtered[i] < filtered[j] }) + index := int(float64(len(filtered)-1) * quantile) + return float64(filtered[index].Microseconds()) / 1000 +} + +func paddedPNG(size int) []byte { + base, _ := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=") + if size <= len(base) { + return base + } + iendOffset := len(base) - 12 + paddingLength := size - len(base) - 12 + if paddingLength < 0 { + paddingLength = 0 + } + chunk := make([]byte, 12+paddingLength) + binary.BigEndian.PutUint32(chunk[:4], uint32(paddingLength)) + copy(chunk[4:8], []byte("teST")) + binary.BigEndian.PutUint32(chunk[8+paddingLength:], crc32.ChecksumIEEE(chunk[4:8+paddingLength])) + out := make([]byte, 0, size) + out = append(out, base[:iendOffset]...) + out = append(out, chunk...) + out = append(out, base[iendOffset:]...) + return out +} + +func redactError(message string, opts options) string { + for _, secret := range []string{opts.apiKey, opts.runToken} { + if secret != "" { + message = strings.ReplaceAll(message, secret, "[REDACTED]") + } + } + return reportURLPattern.ReplaceAllString(message, "[REDACTED_URL]") +} + +func splitCSV(value string) []string { + items := strings.Split(value, ",") + out := make([]string, 0, len(items)) + for _, item := range items { + if item = strings.TrimSpace(item); item != "" { + out = append(out, item) + } + } + return out +} + +func env(name string, fallback string) string { + if value := strings.TrimSpace(os.Getenv(name)); value != "" { + return value + } + return fallback +} diff --git a/apps/api/cmd/acceptance-load/main_test.go b/apps/api/cmd/acceptance-load/main_test.go new file mode 100644 index 0000000..90edcb9 --- /dev/null +++ b/apps/api/cmd/acceptance-load/main_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +func TestStreamGeminiImageHashDoesNotNeedWholeResponse(t *testing.T) { + payload := paddedPNG(4 << 20) + response := fmt.Sprintf(`{"candidates":[{"content":{"parts":[{"inlineData":{"mimeType":"image/png","data":"%s"}}]}}]}`, + base64.StdEncoding.EncodeToString(payload)) + size, digest, err := streamGeminiImageHash(bytes.NewBufferString(response)) + if err != nil { + t.Fatalf("stream Gemini output: %v", err) + } + expected := sha256.Sum256(payload) + if size != int64(len(payload)) || digest != hex.EncodeToString(expected[:]) { + t.Fatalf("size=%d digest=%s", size, digest) + } +} + +func TestVideoCombinationsProvide128UniqueInputs(t *testing.T) { + images := make([]string, 16) + for index := range images { + images[index] = fmt.Sprintf("https://fixtures.example/image-%02d", index) + } + combinations := videoCombinations(images, 128) + if len(combinations) != 128 { + t.Fatalf("combinations=%d", len(combinations)) + } + seen := map[string]struct{}{} + for _, combination := range combinations { + seen[fmt.Sprint(combination)] = struct{}{} + if len(combination) != 3 && len(combination) != 6 && len(combination) != 9 { + t.Fatalf("invalid combination size=%d", len(combination)) + } + } + if len(seen) != 128 { + t.Fatalf("unique combinations=%d", len(seen)) + } +} + +func TestGeminiLoadIsSplitAcrossTwoGatewayAPIs(t *testing.T) { + output := paddedPNG(256 << 10) + encoded := base64.StdEncoding.EncodeToString(output) + var firstCalls atomic.Int64 + var secondCalls atomic.Int64 + newGateway := func(calls *atomic.Int64) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.Header.Get(runHeader) != "run-1" || r.Header.Get(tokenHeader) != "token-1" { + t.Errorf("missing acceptance headers") + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode Gemini body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "candidates": []any{map[string]any{"content": map[string]any{ + "parts": []any{map[string]any{"inlineData": map[string]any{ + "mimeType": "image/png", "data": encoded, + }}}, + }}}, + }) + })) + } + first := newGateway(&firstCalls) + defer first.Close() + second := newGateway(&secondCalls) + defer second.Close() + + opts := options{ + gateways: []string{first.URL, second.URL}, apiKey: "key-1", runID: "run-1", + runToken: "token-1", geminiModel: "gemini-image-test", + } + result := runGemini(t.Context(), http.DefaultClient, opts, "dual-api", 8, 256<<10, 256<<10, false) + if result.err != nil || result.report.Completed != 8 { + t.Fatalf("result=%+v err=%v", result.report, result.err) + } + if firstCalls.Load() != 4 || secondCalls.Load() != 4 { + t.Fatalf("gateway calls=%d/%d", firstCalls.Load(), secondCalls.Load()) + } +} + +func TestValidateVideoAssetDownloadsFinalMedia(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write([]byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm'}) + })) + defer server.Close() + if err := validateVideoAsset(t.Context(), server.Client(), server.URL+"/result.mp4"); err != nil { + t.Fatalf("validate video asset: %v", err) + } + if got := findMediaURL(map[string]any{"content": map[string]any{"video_url": server.URL}}); got != server.URL { + t.Fatalf("media URL=%q", got) + } +} + +func TestAcceptanceReportErrorRedactsSecretsAndURLs(t *testing.T) { + got := redactError( + `token-1 failed at https://example.invalid/video.mp4?token=signed`, + options{runToken: "token-1"}, + ) + if got != `[REDACTED] failed at [REDACTED_URL]` { + t.Fatalf("redacted error=%q", got) + } +} + +func TestAcceptanceGatewayTLSNameSetsHostHeader(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "https://127.0.0.1/api/v1/healthz", nil) + opts := options{ + apiKey: "key-1", runID: "run-1", runToken: "token-1", + gatewayTLSName: "ai.example.com", + } + opts.setHeaders(request, false) + if request.Host != "ai.example.com" { + t.Fatalf("Host=%q", request.Host) + } +} diff --git a/apps/api/docs/swagger.json b/apps/api/docs/swagger.json index e4ac558..ab42e51 100644 --- a/apps/api/docs/swagger.json +++ b/apps/api/docs/swagger.json @@ -2575,6 +2575,302 @@ } } }, + "/api/admin/system/acceptance/runs": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "acceptance" + ], + "summary": "创建生产同构验收 Run", + "parameters": [ + { + "description": "验收 Run", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/store.CreateAcceptanceRunInput" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/store.AcceptanceRun" + } + } + } + } + }, + "/api/admin/system/acceptance/runs/{runID}": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "acceptance" + ], + "summary": "获取生产同构验收 Run", + "parameters": [ + { + "type": "string", + "description": "验收 Run ID", + "name": "runID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/store.AcceptanceRun" + } + } + } + } + }, + "/api/admin/system/acceptance/runs/{runID}/abort": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "acceptance" + ], + "summary": "人工 CAS 中止验收并切回 live", + "parameters": [ + { + "type": "string", + "description": "验收 Run ID", + "name": "runID", + "in": "path", + "required": true + }, + { + "description": "线上 release 与 digest CAS", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/store.PromoteAcceptanceRunInput" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/store.GatewayTrafficMode" + } + } + } + } + }, + "/api/admin/system/acceptance/runs/{runID}/activate": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "acceptance" + ], + "summary": "切换到 validation 并激活 Run", + "parameters": [ + { + "type": "string", + "description": "验收 Run ID", + "name": "runID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/store.GatewayTrafficMode" + } + } + } + } + }, + "/api/admin/system/acceptance/runs/{runID}/finish": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "acceptance" + ], + "summary": "记录验收门禁结果", + "parameters": [ + { + "type": "string", + "description": "验收 Run ID", + "name": "runID", + "in": "path", + "required": true + }, + { + "description": "门禁结果", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/store.FinishAcceptanceRunInput" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/store.AcceptanceRun" + } + } + } + } + }, + "/api/admin/system/acceptance/runs/{runID}/promote": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "acceptance" + ], + "summary": "通过 CAS 切回 live", + "parameters": [ + { + "type": "string", + "description": "验收 Run ID", + "name": "runID", + "in": "path", + "required": true + }, + { + "description": "线上 release 与 digest CAS", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/store.PromoteAcceptanceRunInput" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/store.GatewayTrafficMode" + } + } + } + } + }, + "/api/admin/system/acceptance/runs/{runID}/retry": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "acceptance" + ], + "summary": "在 validation 关闭正式流量的状态下重试失败 Run", + "parameters": [ + { + "type": "string", + "description": "验收 Run ID", + "name": "runID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/store.AcceptanceRun" + } + } + } + } + }, + "/api/admin/system/acceptance/traffic-mode": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "acceptance" + ], + "summary": "获取 Gateway 流量模式", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/store.GatewayTrafficMode" + } + } + } + } + }, "/api/admin/system/client-customization/settings": { "get": { "security": [ @@ -12618,6 +12914,67 @@ } } }, + "store.AcceptanceRun": { + "type": "object", + "properties": { + "apiImageDigest": { + "type": "string" + }, + "apiKeyId": { + "type": "string" + }, + "callbackUrl": { + "type": "string" + }, + "capacityProfile": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "createdAt": { + "type": "string" + }, + "emulatorBaseUrl": { + "type": "string" + }, + "failureReason": { + "type": "string" + }, + "finishedAt": { + "type": "string" + }, + "id": { + "type": "string" + }, + "promotedAt": { + "type": "string" + }, + "releaseSha": { + "type": "string" + }, + "report": { + "type": "object", + "additionalProperties": {} + }, + "startedAt": { + "type": "string" + }, + "status": { + "type": "string" + }, + "updatedAt": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "workerImageDigest": { + "type": "string" + } + } + }, "store.AccessRule": { "type": "object", "properties": { @@ -12758,6 +13115,9 @@ "store.AdminGatewayTask": { "type": "object", "properties": { + "acceptanceRunId": { + "type": "string" + }, "adminContext": { "$ref": "#/definitions/store.AdminTaskContext" }, @@ -13461,6 +13821,42 @@ } } }, + "store.CreateAcceptanceRunInput": { + "type": "object", + "properties": { + "apiImageDigest": { + "type": "string" + }, + "apiKeyId": { + "type": "string" + }, + "callbackUrl": { + "type": "string" + }, + "capacityProfile": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "emulatorBaseUrl": { + "type": "string" + }, + "releaseSha": { + "type": "string" + }, + "token": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "workerImageDigest": { + "type": "string" + } + } + }, "store.CreatePlatformInput": { "type": "object", "properties": { @@ -13753,9 +14149,27 @@ } } }, + "store.FinishAcceptanceRunInput": { + "type": "object", + "properties": { + "failureReason": { + "type": "string" + }, + "passed": { + "type": "boolean" + }, + "report": { + "type": "object", + "additionalProperties": {} + } + } + }, "store.GatewayTask": { "type": "object", "properties": { + "acceptanceRunId": { + "type": "string" + }, "apiKeyId": { "type": "string" }, @@ -14061,6 +14475,32 @@ } } }, + "store.GatewayTrafficMode": { + "type": "object", + "properties": { + "apiImageDigest": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "releaseSha": { + "type": "string" + }, + "revision": { + "type": "integer" + }, + "runId": { + "type": "string" + }, + "updatedAt": { + "type": "string" + }, + "workerImageDigest": { + "type": "string" + } + } + }, "store.GatewayUser": { "type": "object", "properties": { @@ -15058,6 +15498,23 @@ } } }, + "store.PromoteAcceptanceRunInput": { + "type": "object", + "properties": { + "apiImageDigest": { + "type": "string" + }, + "releaseSha": { + "type": "string" + }, + "revision": { + "type": "integer" + }, + "workerImageDigest": { + "type": "string" + } + } + }, "store.RateLimitMetricStatus": { "type": "object", "properties": { diff --git a/apps/api/docs/swagger.yaml b/apps/api/docs/swagger.yaml index 04c7dea..c564793 100644 --- a/apps/api/docs/swagger.yaml +++ b/apps/api/docs/swagger.yaml @@ -1993,6 +1993,47 @@ definitions: userId: type: string type: object + store.AcceptanceRun: + properties: + apiImageDigest: + type: string + apiKeyId: + type: string + callbackUrl: + type: string + capacityProfile: + type: string + config: + additionalProperties: {} + type: object + createdAt: + type: string + emulatorBaseUrl: + type: string + failureReason: + type: string + finishedAt: + type: string + id: + type: string + promotedAt: + type: string + releaseSha: + type: string + report: + additionalProperties: {} + type: object + startedAt: + type: string + status: + type: string + updatedAt: + type: string + userId: + type: string + workerImageDigest: + type: string + type: object store.AccessRule: properties: conditions: @@ -2087,6 +2128,8 @@ definitions: type: object store.AdminGatewayTask: properties: + acceptanceRunId: + type: string adminContext: $ref: '#/definitions/store.AdminTaskContext' apiKeyId: @@ -2562,6 +2605,30 @@ definitions: type: string type: array type: object + store.CreateAcceptanceRunInput: + properties: + apiImageDigest: + type: string + apiKeyId: + type: string + callbackUrl: + type: string + capacityProfile: + type: string + config: + additionalProperties: {} + type: object + emulatorBaseUrl: + type: string + releaseSha: + type: string + token: + type: string + userId: + type: string + workerImageDigest: + type: string + type: object store.CreatePlatformInput: properties: authType: @@ -2760,8 +2827,20 @@ definitions: resultUploadPolicy: type: string type: object + store.FinishAcceptanceRunInput: + properties: + failureReason: + type: string + passed: + type: boolean + report: + additionalProperties: {} + type: object + type: object store.GatewayTask: properties: + acceptanceRunId: + type: string apiKeyId: type: string apiKeyName: @@ -2970,6 +3049,23 @@ definitions: tenantKey: type: string type: object + store.GatewayTrafficMode: + properties: + apiImageDigest: + type: string + mode: + type: string + releaseSha: + type: string + revision: + type: integer + runId: + type: string + updatedAt: + type: string + workerImageDigest: + type: string + type: object store.GatewayUser: properties: authProfile: @@ -3642,6 +3738,17 @@ definitions: taskId: type: string type: object + store.PromoteAcceptanceRunInput: + properties: + apiImageDigest: + type: string + releaseSha: + type: string + revision: + type: integer + workerImageDigest: + type: string + type: object store.RateLimitMetricStatus: properties: currentValue: @@ -5645,6 +5752,187 @@ paths: summary: 更新 Runner 策略 tags: - runtime + /api/admin/system/acceptance/runs: + post: + consumes: + - application/json + parameters: + - description: 验收 Run + in: body + name: body + required: true + schema: + $ref: '#/definitions/store.CreateAcceptanceRunInput' + produces: + - application/json + responses: + "201": + description: Created + schema: + $ref: '#/definitions/store.AcceptanceRun' + security: + - BearerAuth: [] + summary: 创建生产同构验收 Run + tags: + - acceptance + /api/admin/system/acceptance/runs/{runID}: + get: + parameters: + - description: 验收 Run ID + in: path + name: runID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.AcceptanceRun' + security: + - BearerAuth: [] + summary: 获取生产同构验收 Run + tags: + - acceptance + /api/admin/system/acceptance/runs/{runID}/abort: + post: + consumes: + - application/json + parameters: + - description: 验收 Run ID + in: path + name: runID + required: true + type: string + - description: 线上 release 与 digest CAS + in: body + name: body + required: true + schema: + $ref: '#/definitions/store.PromoteAcceptanceRunInput' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.GatewayTrafficMode' + security: + - BearerAuth: [] + summary: 人工 CAS 中止验收并切回 live + tags: + - acceptance + /api/admin/system/acceptance/runs/{runID}/activate: + post: + parameters: + - description: 验收 Run ID + in: path + name: runID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.GatewayTrafficMode' + security: + - BearerAuth: [] + summary: 切换到 validation 并激活 Run + tags: + - acceptance + /api/admin/system/acceptance/runs/{runID}/finish: + post: + consumes: + - application/json + parameters: + - description: 验收 Run ID + in: path + name: runID + required: true + type: string + - description: 门禁结果 + in: body + name: body + required: true + schema: + $ref: '#/definitions/store.FinishAcceptanceRunInput' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.AcceptanceRun' + security: + - BearerAuth: [] + summary: 记录验收门禁结果 + tags: + - acceptance + /api/admin/system/acceptance/runs/{runID}/promote: + post: + consumes: + - application/json + parameters: + - description: 验收 Run ID + in: path + name: runID + required: true + type: string + - description: 线上 release 与 digest CAS + in: body + name: body + required: true + schema: + $ref: '#/definitions/store.PromoteAcceptanceRunInput' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.GatewayTrafficMode' + security: + - BearerAuth: [] + summary: 通过 CAS 切回 live + tags: + - acceptance + /api/admin/system/acceptance/runs/{runID}/retry: + post: + parameters: + - description: 验收 Run ID + in: path + name: runID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.AcceptanceRun' + security: + - BearerAuth: [] + summary: 在 validation 关闭正式流量的状态下重试失败 Run + tags: + - acceptance + /api/admin/system/acceptance/traffic-mode: + get: + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.GatewayTrafficMode' + security: + - BearerAuth: [] + summary: 获取 Gateway 流量模式 + tags: + - acceptance /api/admin/system/client-customization/settings: get: description: 返回客户端名称、英文名称、平台名和图标路径；数据库对象尚未创建时返回默认设置。 diff --git a/apps/api/internal/acceptanceemulator/server.go b/apps/api/internal/acceptanceemulator/server.go new file mode 100644 index 0000000..aa11bac --- /dev/null +++ b/apps/api/internal/acceptanceemulator/server.go @@ -0,0 +1,658 @@ +package acceptanceemulator + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "image" + "image/color" + "image/jpeg" + "image/png" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceworkload" + _ "golang.org/x/image/webp" +) + +const maxProtocolBodyBytes = 128 << 20 + +type Config struct { + Now func() time.Time + HTTPClient *http.Client + Wait func(context.Context, time.Duration) error +} + +type Server struct { + now func() time.Time + httpClient *http.Client + wait func(context.Context, time.Duration) error + mu sync.RWMutex + tasks map[string]videoTask + fixtures map[string]fixture + nextID atomic.Uint64 + report Report + callbacks map[string]map[int64]int + pngSmall string + pngLarge string + pngPeak string +} + +type Report struct { + GeminiRequests int64 `json:"geminiRequests"` + GeminiInvalid int64 `json:"geminiInvalid"` + GeminiInputBytes int64 `json:"geminiInputBytes"` + GeminiOutputBytes int64 `json:"geminiOutputBytes"` + VideoSubmissions int64 `json:"videoSubmissions"` + VideoPolls int64 `json:"videoPolls"` + VideoInvalid int64 `json:"videoInvalid"` + VideoReferenceCounts map[string]int64 `json:"videoReferenceCounts"` + VideoRoleCounts map[string]int64 `json:"videoRoleCounts"` + UniqueImageHashes int `json:"uniqueImageHashes"` + UniqueVideoTasks int `json:"uniqueVideoTasks"` + ForcedConversions int64 `json:"forcedConversionRequests"` + VerifiedConversions int64 `json:"verifiedConversions"` + CallbackEvents int64 `json:"callbackEvents"` + DuplicateCallbacks int64 `json:"duplicateCallbacks"` +} + +type videoTask struct { + ID string + Model string + CreatedAt time.Time + ReadyAt time.Time + ImageRefs []imageReference +} + +type imageReference struct { + Role string + SHA256 string + ContentType string + Width int + Height int +} + +type fixture struct { + ContentType string + Payload []byte +} + +func New(config Config) *Server { + now := config.Now + if now == nil { + now = time.Now + } + client := config.HTTPClient + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + waiter := config.Wait + if waiter == nil { + waiter = wait + } + return &Server{ + now: now, httpClient: client, wait: waiter, tasks: map[string]videoTask{}, fixtures: buildFixtures(), + callbacks: map[string]map[int64]int{}, + report: Report{ + VideoReferenceCounts: map[string]int64{}, + VideoRoleCounts: map[string]int64{}, + }, + pngSmall: base64.StdEncoding.EncodeToString(paddedPNG(256 << 10)), + pngLarge: base64.StdEncoding.EncodeToString(paddedPNG(4 << 20)), + pngPeak: base64.StdEncoding.EncodeToString(paddedPNG(8 << 20)), + } +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", s.health) + mux.HandleFunc("GET /report", s.getReport) + mux.HandleFunc("POST /v1beta/models/", s.geminiGenerateContent) + mux.HandleFunc("POST /v1/models/", s.geminiGenerateContent) + mux.HandleFunc("POST /contents/generations/tasks", s.submitVideo) + mux.HandleFunc("GET /contents/generations/tasks/{taskID}", s.getVideo) + mux.HandleFunc("DELETE /contents/generations/tasks/{taskID}", s.deleteVideo) + mux.HandleFunc("GET /media/{asset}", s.getMedia) + mux.HandleFunc("GET /fixtures/{asset}", s.getFixture) + mux.HandleFunc("POST /callbacks", s.collectCallback) + return mux +} + +func (s *Server) health(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) +} + +func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, ":generateContent") { + http.NotFound(w, r) + return + } + var body map[string]any + if err := decodeJSON(r, &body); err != nil { + s.recordGeminiInvalid() + writeProtocolError(w, http.StatusBadRequest, err.Error()) + return + } + inputBytes, err := validateGeminiImageRequest(body) + if err != nil { + s.recordGeminiInvalid() + writeProtocolError(w, http.StatusBadRequest, err.Error()) + return + } + outputBytes, outputBase64, delay := s.geminiProfile(inputBytes) + if err := s.wait(r.Context(), delay); err != nil { + return + } + s.mu.Lock() + s.report.GeminiRequests++ + s.report.GeminiInputBytes += int64(inputBytes) + s.report.GeminiOutputBytes += int64(outputBytes) + s.mu.Unlock() + w.Header().Set("X-Request-ID", "acceptance-gemini-"+strconv.FormatUint(s.nextID.Add(1), 10)) + writeJSON(w, http.StatusOK, map[string]any{ + "candidates": []any{map[string]any{ + "index": 0, + "content": map[string]any{ + "role": "model", + "parts": []any{map[string]any{"inlineData": map[string]any{ + "mimeType": "image/png", + "data": outputBase64, + }}}, + }, + "finishReason": "STOP", + }}, + "usageMetadata": map[string]any{ + "promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2, + }, + }) +} + +func (s *Server) geminiProfile(inputBytes int) (int, string, time.Duration) { + switch { + case inputBytes > acceptanceworkload.GeminiLarge.InputBytes: + profile := acceptanceworkload.GeminiPeak + return profile.OutputBytes, s.pngPeak, scaledDelay(profile.DelayMin, profile.DelayMax, inputBytes) + case inputBytes > acceptanceworkload.GeminiBaseline.InputBytes: + profile := acceptanceworkload.GeminiLarge + return profile.OutputBytes, s.pngLarge, scaledDelay(profile.DelayMin, profile.DelayMax, inputBytes) + default: + profile := acceptanceworkload.GeminiBaseline + return profile.OutputBytes, s.pngSmall, profile.DelayMin + } +} + +func (s *Server) submitVideo(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if err := decodeJSON(r, &body); err != nil { + s.recordVideoInvalid() + writeProtocolError(w, http.StatusBadRequest, err.Error()) + return + } + refs, longRun, forceConversion, err := s.validateVideoRequest(r.Context(), body) + if err != nil { + s.recordVideoInvalid() + writeProtocolError(w, http.StatusBadRequest, err.Error()) + return + } + delay := videoDelay(body, longRun) + id := "acceptance-video-" + strconv.FormatUint(s.nextID.Add(1), 10) + task := videoTask{ + ID: id, Model: strings.TrimSpace(stringValue(body["model"])), + CreatedAt: s.now(), ReadyAt: s.now().Add(delay), ImageRefs: refs, + } + s.mu.Lock() + s.tasks[id] = task + s.report.VideoSubmissions++ + s.report.VideoReferenceCounts[strconv.Itoa(len(refs))]++ + if forceConversion { + s.report.ForcedConversions++ + s.report.VerifiedConversions++ + } + for _, ref := range refs { + s.report.VideoRoleCounts[ref.Role]++ + } + s.mu.Unlock() + w.Header().Set("X-Request-ID", id) + writeJSON(w, http.StatusOK, map[string]any{ + "id": id, "model": task.Model, "status": "queued", + "created_at": task.CreatedAt.Unix(), + }) +} + +func (s *Server) getVideo(w http.ResponseWriter, r *http.Request) { + id := strings.TrimSpace(r.PathValue("taskID")) + s.mu.Lock() + task, ok := s.tasks[id] + if ok { + s.report.VideoPolls++ + } + s.mu.Unlock() + if !ok { + writeProtocolError(w, http.StatusNotFound, "video task not found") + return + } + status := "running" + content := map[string]any{} + if !s.now().Before(task.ReadyAt) { + status = "succeeded" + content["video_url"] = requestOrigin(r) + "/media/" + url.PathEscape(id) + ".mp4" + } + w.Header().Set("X-Request-ID", id) + writeJSON(w, http.StatusOK, map[string]any{ + "id": id, "model": task.Model, "status": status, "content": content, + "created_at": task.CreatedAt.Unix(), + "usage": map[string]any{"completion_tokens": 1, "total_tokens": 1}, + }) +} + +func (s *Server) deleteVideo(w http.ResponseWriter, r *http.Request) { + id := strings.TrimSpace(r.PathValue("taskID")) + s.mu.Lock() + _, ok := s.tasks[id] + delete(s.tasks, id) + s.mu.Unlock() + if !ok { + writeProtocolError(w, http.StatusNotFound, "video task not found") + return + } + writeJSON(w, http.StatusOK, map[string]any{"id": id, "status": "cancelled"}) +} + +func (s *Server) getMedia(w http.ResponseWriter, r *http.Request) { + asset := strings.TrimSuffix(strings.TrimSpace(r.PathValue("asset")), ".mp4") + s.mu.RLock() + _, ok := s.tasks[asset] + s.mu.RUnlock() + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "video/mp4") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(minimalMP4) +} + +func (s *Server) getFixture(w http.ResponseWriter, r *http.Request) { + fixture, ok := s.fixtures[strings.TrimSpace(r.PathValue("asset"))] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", fixture.ContentType) + w.Header().Set("Cache-Control", "public, max-age=3600") + w.Header().Set("X-Content-SHA256", payloadSHA256(fixture.Payload)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(fixture.Payload) +} + +func (s *Server) getReport(w http.ResponseWriter, _ *http.Request) { + s.mu.RLock() + report := s.report + tasks := make([]videoTask, 0, len(s.tasks)) + for _, task := range s.tasks { + tasks = append(tasks, task) + } + s.mu.RUnlock() + hashes := map[string]struct{}{} + for _, task := range tasks { + for _, ref := range task.ImageRefs { + hashes[ref.SHA256] = struct{}{} + } + } + report.UniqueImageHashes = len(hashes) + report.UniqueVideoTasks = len(tasks) + writeJSON(w, http.StatusOK, report) +} + +func (s *Server) collectCallback(w http.ResponseWriter, r *http.Request) { + var payload map[string]any + if err := decodeJSON(r, &payload); err != nil { + writeProtocolError(w, http.StatusBadRequest, err.Error()) + return + } + taskID := strings.TrimSpace(stringValue(payload["taskId"])) + seq := int64(0) + switch value := payload["seq"].(type) { + case json.Number: + seq, _ = value.Int64() + case float64: + seq = int64(value) + } + if taskID == "" || seq <= 0 { + writeProtocolError(w, http.StatusBadRequest, "callback taskId and seq are required") + return + } + s.mu.Lock() + seen := s.callbacks[taskID] + if seen == nil { + seen = map[int64]int{} + s.callbacks[taskID] = seen + } + seen[seq]++ + s.report.CallbackEvents++ + if seen[seq] > 1 { + s.report.DuplicateCallbacks++ + } + s.mu.Unlock() + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) +} + +func (s *Server) validateVideoRequest(ctx context.Context, body map[string]any) ([]imageReference, bool, bool, error) { + content, _ := body["content"].([]any) + refs := make([]imageReference, 0, 9) + firstFrames := 0 + lastFrames := 0 + longRun := false + forceConversion := false + for _, raw := range content { + item, _ := raw.(map[string]any) + switch strings.TrimSpace(stringValue(item["type"])) { + case "text": + if strings.Contains(strings.ToLower(stringValue(item["text"])), "acceptance-long-recovery") { + longRun = true + } + if strings.Contains(strings.ToLower(stringValue(item["text"])), "acceptance-force-conversion") { + forceConversion = true + } + case "image_url": + role := strings.TrimSpace(stringValue(item["role"])) + switch role { + case "reference_image": + case "first_frame": + firstFrames++ + case "last_frame": + lastFrames++ + default: + return nil, false, false, fmt.Errorf("unsupported image role %q", role) + } + nested, _ := item["image_url"].(map[string]any) + rawURL := strings.TrimSpace(stringValue(nested["url"])) + ref, err := s.fetchImageReference(ctx, role, rawURL) + if err != nil { + return nil, false, false, err + } + refs = append(refs, ref) + } + } + if len(refs) != 3 && len(refs) != 6 && len(refs) != 9 { + return nil, false, false, fmt.Errorf("expected 3, 6, or 9 image references, got %d", len(refs)) + } + if firstFrames > 1 || lastFrames > 1 { + return nil, false, false, errors.New("first_frame and last_frame must be unique") + } + if forceConversion { + for _, ref := range refs { + if ref.Width >= 3840 || ref.Height >= 2160 { + return nil, false, false, errors.New("forced 4K image did not pass through automatic normalization") + } + } + } + return refs, longRun, forceConversion, nil +} + +func (s *Server) fetchImageReference(ctx context.Context, role string, rawURL string) (imageReference, error) { + if rawURL == "" { + return imageReference{}, errors.New("image reference URL is required") + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return imageReference{}, err + } + response, err := s.httpClient.Do(request) + if err != nil { + return imageReference{}, fmt.Errorf("fetch image reference: %w", err) + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return imageReference{}, fmt.Errorf("fetch image reference: status %d", response.StatusCode) + } + payload, err := io.ReadAll(io.LimitReader(response.Body, 32<<20)) + if err != nil { + return imageReference{}, err + } + if len(payload) == 0 { + return imageReference{}, errors.New("image reference is empty") + } + config, format, err := image.DecodeConfig(bytes.NewReader(payload)) + if err != nil { + return imageReference{}, fmt.Errorf("decode image reference: %w", err) + } + sum := sha256.Sum256(payload) + return imageReference{ + Role: role, SHA256: hex.EncodeToString(sum[:]), ContentType: "image/" + format, + Width: config.Width, Height: config.Height, + }, nil +} + +func validateGeminiImageRequest(body map[string]any) (int, error) { + generationConfig, _ := body["generationConfig"].(map[string]any) + modalities, _ := generationConfig["responseModalities"].([]any) + hasImageModality := false + for _, modality := range modalities { + if strings.EqualFold(strings.TrimSpace(stringValue(modality)), "IMAGE") { + hasImageModality = true + } + } + if !hasImageModality { + return 0, errors.New("generationConfig.responseModalities must contain IMAGE") + } + total := 0 + contents, _ := body["contents"].([]any) + for _, rawContent := range contents { + content, _ := rawContent.(map[string]any) + parts, _ := content["parts"].([]any) + for _, rawPart := range parts { + part, _ := rawPart.(map[string]any) + inline, _ := part["inlineData"].(map[string]any) + if inline == nil { + inline, _ = part["inline_data"].(map[string]any) + } + encoded := strings.TrimSpace(stringValue(inline["data"])) + if encoded == "" { + continue + } + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return 0, fmt.Errorf("Gemini inlineData is not valid Base64: %w", err) + } + if len(decoded) == 0 { + return 0, errors.New("Gemini inlineData is empty") + } + total += len(decoded) + } + } + if total == 0 { + return 0, errors.New("Gemini request has no inlineData image") + } + return total, nil +} + +func (s *Server) recordGeminiInvalid() { + s.mu.Lock() + s.report.GeminiInvalid++ + s.mu.Unlock() +} + +func (s *Server) recordVideoInvalid() { + s.mu.Lock() + s.report.VideoInvalid++ + s.mu.Unlock() +} + +func decodeJSON(r *http.Request, target any) error { + reader := io.LimitReader(r.Body, maxProtocolBodyBytes+1) + payload, err := io.ReadAll(reader) + if err != nil { + return err + } + if len(payload) > maxProtocolBodyBytes { + return errors.New("protocol request body is too large") + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.UseNumber() + return decoder.Decode(target) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} + +func writeProtocolError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]any{"error": map[string]any{ + "code": "acceptance_protocol_invalid", "message": message, + }}) +} + +func wait(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func scaledDelay(minimum time.Duration, maximum time.Duration, seed int) time.Duration { + if maximum <= minimum { + return minimum + } + windowSeconds := int64((maximum - minimum) / time.Second) + if windowSeconds <= 0 { + return minimum + } + return minimum + time.Duration(int64(seed)%(windowSeconds+1))*time.Second +} + +func videoDelay(body map[string]any, longRun bool) time.Duration { + seed := int64(0) + switch value := body["seed"].(type) { + case json.Number: + seed, _ = value.Int64() + case float64: + seed = int64(value) + case int: + seed = int64(value) + } + if seed < 0 { + seed = -seed + } + if longRun { + return 2*time.Minute + time.Duration(seed%61)*time.Second + } + return 5*time.Second + time.Duration(seed%11)*time.Second +} + +func requestOrigin(r *http.Request) string { + scheme := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto")) + if scheme == "" { + scheme = "http" + } + host := strings.TrimSpace(r.Host) + return scheme + "://" + host +} + +func stringValue(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case json.Number: + return typed.String() + default: + return fmt.Sprint(value) + } +} + +func paddedPNG(size int) []byte { + base, _ := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=") + if size <= len(base) { + return base + } + iendOffset := len(base) - 12 + paddingLength := size - len(base) - 12 + if paddingLength < 0 { + paddingLength = 0 + } + chunk := make([]byte, 12+paddingLength) + binary.BigEndian.PutUint32(chunk[:4], uint32(paddingLength)) + copy(chunk[4:8], []byte("teST")) + binary.BigEndian.PutUint32(chunk[8+paddingLength:], crc32.ChecksumIEEE(chunk[4:8+paddingLength])) + out := make([]byte, 0, len(base)+len(chunk)) + out = append(out, base[:iendOffset]...) + out = append(out, chunk...) + out = append(out, base[iendOffset:]...) + return out +} + +func buildFixtures() map[string]fixture { + fixtures := map[string]fixture{} + for index := 0; index < 4; index++ { + imageValue := patternedImage(768, 512, index+1) + var encoded bytes.Buffer + _ = png.Encode(&encoded, imageValue) + fixtures[fmt.Sprintf("image-%02d.png", index)] = fixture{ContentType: "image/png", Payload: encoded.Bytes()} + } + for index := 0; index < 4; index++ { + imageValue := patternedImage(1024, 768, index+11) + var encoded bytes.Buffer + _ = jpeg.Encode(&encoded, imageValue, &jpeg.Options{Quality: 88}) + fixtures[fmt.Sprintf("image-%02d.jpg", index+4)] = fixture{ContentType: "image/jpeg", Payload: encoded.Bytes()} + } + webpPayload, _ := base64.StdEncoding.DecodeString("UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEALmk0mk0iIiIiIgBoSygABc6zbAAA") + for index := 0; index < 4; index++ { + payload := append([]byte(nil), webpPayload...) + payload = append(payload, byte(index)) + fixtures[fmt.Sprintf("image-%02d.webp", index+8)] = fixture{ContentType: "image/webp", Payload: payload} + } + for index := 0; index < 4; index++ { + imageValue := patternedImage(4096, 2160, index+21) + var encoded bytes.Buffer + _ = jpeg.Encode(&encoded, imageValue, &jpeg.Options{Quality: 90}) + fixtures[fmt.Sprintf("image-%02d-4k.jpg", index+12)] = fixture{ContentType: "image/jpeg", Payload: encoded.Bytes()} + } + return fixtures +} + +func patternedImage(width int, height int, seed int) image.Image { + out := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + out.SetNRGBA(x, y, color.NRGBA{ + R: uint8((x + seed*31) % 256), + G: uint8((y + seed*47) % 256), + B: uint8((x/8 + y/8 + seed*59) % 256), + A: 255, + }) + } + } + return out +} + +func payloadSHA256(payload []byte) string { + sum := sha256.Sum256(payload) + return hex.EncodeToString(sum[:]) +} + +var minimalMP4 = []byte{ + 0, 0, 0, 24, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm', + 0, 0, 0, 0, 'i', 's', 'o', 'm', 'm', 'p', '4', '2', + 0, 0, 0, 8, 'm', 'd', 'a', 't', +} diff --git a/apps/api/internal/acceptanceemulator/server_test.go b/apps/api/internal/acceptanceemulator/server_test.go new file mode 100644 index 0000000..a34f024 --- /dev/null +++ b/apps/api/internal/acceptanceemulator/server_test.go @@ -0,0 +1,186 @@ +package acceptanceemulator + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "image" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +func TestGeminiAndVolcesProtocolEmulation(t *testing.T) { + var mu sync.Mutex + now := time.Unix(1_800_000_000, 0) + server := New(Config{ + Now: func() time.Time { + mu.Lock() + defer mu.Unlock() + return now + }, + Wait: func(context.Context, time.Duration) error { return nil }, + }) + httpServer := httptest.NewServer(server.Handler()) + defer httpServer.Close() + + input := paddedPNG(256 << 10) + geminiBody, _ := json.Marshal(map[string]any{ + "contents": []any{map[string]any{"parts": []any{ + map[string]any{"text": "edit image"}, + map[string]any{"inlineData": map[string]any{ + "mimeType": "image/png", "data": base64.StdEncoding.EncodeToString(input), + }}, + }}}, + "generationConfig": map[string]any{"responseModalities": []any{"IMAGE"}}, + }) + response, err := http.Post(httpServer.URL+"/v1beta/models/gemini-test:generateContent", "application/json", bytes.NewReader(geminiBody)) + if err != nil { + t.Fatalf("Gemini request: %v", err) + } + if response.StatusCode != http.StatusOK { + payload, _ := io.ReadAll(response.Body) + t.Fatalf("Gemini status=%d body=%s", response.StatusCode, payload) + } + var geminiResult map[string]any + if err := json.NewDecoder(response.Body).Decode(&geminiResult); err != nil { + t.Fatalf("decode Gemini response: %v", err) + } + _ = response.Body.Close() + + fixtureResponse, err := http.Get(httpServer.URL + "/fixtures/image-12-4k.jpg") + if err != nil { + t.Fatalf("get 4K fixture: %v", err) + } + config, format, err := image.DecodeConfig(fixtureResponse.Body) + _ = fixtureResponse.Body.Close() + if err != nil || format != "jpeg" || config.Width != 4096 || config.Height != 2160 { + t.Fatalf("4K fixture format=%s config=%+v err=%v", format, config, err) + } + + content := []any{map[string]any{"type": "text", "text": "video"}} + for index, name := range []string{"image-00.png", "image-04.jpg", "image-08.webp"} { + role := "reference_image" + if index == 0 { + role = "first_frame" + } + content = append(content, map[string]any{ + "type": "image_url", "role": role, + "image_url": map[string]any{"url": httpServer.URL + "/fixtures/" + name}, + }) + } + videoBody, _ := json.Marshal(map[string]any{"model": "seedance-test", "content": content, "seed": 1}) + response, err = http.Post(httpServer.URL+"/contents/generations/tasks", "application/json", bytes.NewReader(videoBody)) + if err != nil { + t.Fatalf("submit video: %v", err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("submit video status=%d", response.StatusCode) + } + var submitted map[string]any + _ = json.NewDecoder(response.Body).Decode(&submitted) + _ = response.Body.Close() + taskID := strings.TrimSpace(stringValue(submitted["id"])) + if taskID == "" { + t.Fatal("video emulator returned no task ID") + } + mu.Lock() + now = now.Add(20 * time.Second) + mu.Unlock() + response, err = http.Get(httpServer.URL + "/contents/generations/tasks/" + taskID) + if err != nil { + t.Fatalf("poll video: %v", err) + } + var polled map[string]any + _ = json.NewDecoder(response.Body).Decode(&polled) + _ = response.Body.Close() + if polled["status"] != "succeeded" { + t.Fatalf("video status=%v", polled["status"]) + } + + response, err = http.Get(httpServer.URL + "/report") + if err != nil { + t.Fatalf("get report: %v", err) + } + var report Report + _ = json.NewDecoder(response.Body).Decode(&report) + _ = response.Body.Close() + if report.GeminiRequests != 1 || report.VideoSubmissions != 1 || report.VideoInvalid != 0 || + report.VideoReferenceCounts["3"] != 1 || report.UniqueImageHashes != 3 { + t.Fatalf("unexpected emulator report: %+v", report) + } +} + +func TestPaddedPNGIsValidAndExact(t *testing.T) { + for _, size := range []int{256 << 10, 4 << 20, 8 << 20} { + payload := paddedPNG(size) + if len(payload) != size { + t.Fatalf("PNG bytes=%d, want=%d", len(payload), size) + } + if _, _, err := image.DecodeConfig(bytes.NewReader(payload)); err != nil { + t.Fatalf("decode padded PNG: %v", err) + } + } +} + +func TestScaledDelayStaysOnWholeSecondsWithinProfile(t *testing.T) { + delay := scaledDelay(8*time.Second, 15*time.Second, 2<<20) + if delay < 8*time.Second || delay > 15*time.Second || delay%time.Second != 0 { + t.Fatalf("scaled delay=%s", delay) + } +} + +func TestVolcesProtocolAcceptsThreeSixAndNineReferenceImages(t *testing.T) { + server := New(Config{Wait: func(context.Context, time.Duration) error { return nil }}) + httpServer := httptest.NewServer(server.Handler()) + defer httpServer.Close() + + for _, imageCount := range []int{3, 6, 9} { + content := []any{map[string]any{"type": "text", "text": "multi-reference video"}} + for index := 0; index < imageCount; index++ { + role := "reference_image" + if index == 0 { + role = "first_frame" + } + if imageCount > 3 && index == imageCount-1 { + role = "last_frame" + } + content = append(content, map[string]any{ + "type": "image_url", "role": role, + "image_url": map[string]any{ + "url": fmt.Sprintf("%s/fixtures/image-%02d.png", httpServer.URL, index%4), + }, + }) + } + body, _ := json.Marshal(map[string]any{"model": "seedance-test", "content": content, "seed": imageCount}) + response, err := http.Post(httpServer.URL+"/contents/generations/tasks", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("%d-image submit: %v", imageCount, err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("%d-image status=%d", imageCount, response.StatusCode) + } + } + + response, err := http.Get(httpServer.URL + "/report") + if err != nil { + t.Fatalf("get report: %v", err) + } + defer response.Body.Close() + var report Report + if err := json.NewDecoder(response.Body).Decode(&report); err != nil { + t.Fatalf("decode report: %v", err) + } + for _, imageCount := range []string{"3", "6", "9"} { + if report.VideoReferenceCounts[imageCount] != 1 { + t.Fatalf("reference counts=%v", report.VideoReferenceCounts) + } + } +} diff --git a/apps/api/internal/acceptanceworkload/profiles.go b/apps/api/internal/acceptanceworkload/profiles.go new file mode 100644 index 0000000..519f2b5 --- /dev/null +++ b/apps/api/internal/acceptanceworkload/profiles.go @@ -0,0 +1,52 @@ +package acceptanceworkload + +import "time" + +type GeminiProfile struct { + Name string + Requests int + InputBytes int + OutputBytes int + DelayMin time.Duration + DelayMax time.Duration +} + +var ( + GeminiBaseline = GeminiProfile{ + Name: "gemini-baseline", Requests: 1000, InputBytes: 256 << 10, OutputBytes: 256 << 10, + DelayMin: 4 * time.Second, DelayMax: 4 * time.Second, + } + GeminiLarge = GeminiProfile{ + Name: "gemini-large", Requests: 128, InputBytes: 2 << 20, OutputBytes: 4 << 20, + DelayMin: 8 * time.Second, DelayMax: 15 * time.Second, + } + GeminiPeak = GeminiProfile{ + Name: "gemini-peak", Requests: 32, InputBytes: 8 << 20, OutputBytes: 8 << 20, + DelayMin: 15 * time.Second, DelayMax: 30 * time.Second, + } +) + +func GeminiProfileByName(name string) (GeminiProfile, bool) { + switch name { + case GeminiBaseline.Name: + return GeminiBaseline, true + case GeminiLarge.Name: + return GeminiLarge, true + case GeminiPeak.Name: + return GeminiPeak, true + default: + return GeminiProfile{}, false + } +} + +// VideoImageCount provides a deterministic 60/30/10 split for 3/6/9-image tasks. +func VideoImageCount(index int) int { + switch index % 10 { + case 0, 1, 2, 3, 4, 5: + return 3 + case 6, 7, 8: + return 6 + default: + return 9 + } +} diff --git a/apps/api/internal/acceptanceworkload/profiles_test.go b/apps/api/internal/acceptanceworkload/profiles_test.go new file mode 100644 index 0000000..434898c --- /dev/null +++ b/apps/api/internal/acceptanceworkload/profiles_test.go @@ -0,0 +1,23 @@ +package acceptanceworkload + +import "testing" + +func TestGeminiProfilesAndVideoDistribution(t *testing.T) { + for _, profile := range []GeminiProfile{GeminiBaseline, GeminiLarge, GeminiPeak} { + resolved, ok := GeminiProfileByName(profile.Name) + if !ok || resolved != profile { + t.Fatalf("profile %q resolved to %+v, ok=%v", profile.Name, resolved, ok) + } + if profile.Requests <= 0 || profile.InputBytes <= 0 || profile.OutputBytes <= 0 || + profile.DelayMin <= 0 || profile.DelayMax < profile.DelayMin { + t.Fatalf("invalid Gemini profile: %+v", profile) + } + } + counts := map[int]int{} + for index := 0; index < 100; index++ { + counts[VideoImageCount(index)]++ + } + if counts[3] != 60 || counts[6] != 30 || counts[9] != 10 { + t.Fatalf("video image distribution=%v", counts) + } +} diff --git a/apps/api/internal/clients/clients_test.go b/apps/api/internal/clients/clients_test.go index 2029c71..7396291 100644 --- a/apps/api/internal/clients/clients_test.go +++ b/apps/api/internal/clients/clients_test.go @@ -2435,6 +2435,8 @@ func TestVolcesVideoBodyAllowsOnlyTaskPayloadFields(t *testing.T) { "tools": []any{map[string]any{"type": "web_search"}}, "task_id": "local-task-id", "runMode": "simulation", + "modelType": "omni_video", + "model_type": "omni_video", "fps": 24, "content": []any{ map[string]any{"type": "text", "text": "Use <<>> in a product reveal"}, diff --git a/apps/api/internal/clients/volces.go b/apps/api/internal/clients/volces.go index 4b770d8..3f8bdf8 100644 --- a/apps/api/internal/clients/volces.go +++ b/apps/api/internal/clients/volces.go @@ -418,6 +418,8 @@ func cleanProviderBody(body map[string]any) map[string]any { "platformId", "platform_model_id", "platformModelId", + "modelType", + "model_type", } { delete(out, key) } diff --git a/apps/api/internal/httpapi/acceptance_handlers.go b/apps/api/internal/httpapi/acceptance_handlers.go new file mode 100644 index 0000000..eeb4267 --- /dev/null +++ b/apps/api/internal/httpapi/acceptance_handlers.go @@ -0,0 +1,300 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "net/http" + "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +const ( + acceptanceRunHeader = "X-EasyAI-Acceptance-Run" + acceptanceTokenHeader = "X-EasyAI-Acceptance-Token" + acceptanceUpstreamHeader = "X-EasyAI-Acceptance-Upstream" +) + +type taskTrafficAdmission struct { + RunMode string + AcceptanceRunID string +} + +type taskTrafficError struct { + Status int + Code string + Message string + Err error +} + +func (e *taskTrafficError) Error() string { + return e.Message +} + +func (e *taskTrafficError) ErrorCode() string { + return e.Code +} + +func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTrafficAdmission, error) { + runID, err := s.store.AuthorizeAcceptanceTask( + r.Context(), + r.Header.Get(acceptanceRunHeader), + r.Header.Get(acceptanceTokenHeader), + user, + ) + if err != nil { + switch { + case errors.Is(err, store.ErrProductionTrafficPaused): + return taskTrafficAdmission{}, &taskTrafficError{ + Status: http.StatusServiceUnavailable, Code: "validation_in_progress", + Message: "new production tasks are paused while validation is running", Err: err, + } + case errors.Is(err, store.ErrAcceptanceNotAuthorized): + return taskTrafficAdmission{}, &taskTrafficError{ + Status: http.StatusForbidden, Code: "acceptance_not_authorized", + Message: "acceptance credentials do not match the active run", Err: err, + } + case errors.Is(err, store.ErrAcceptanceRunNotActive): + return taskTrafficAdmission{}, &taskTrafficError{ + Status: http.StatusConflict, Code: "acceptance_run_not_active", + Message: "acceptance headers are not valid while live traffic is enabled", Err: err, + } + default: + return taskTrafficAdmission{}, &taskTrafficError{ + Status: http.StatusServiceUnavailable, Code: "traffic_gate_unavailable", + Message: "task traffic gate is unavailable", Err: err, + } + } + } + if runID != "" { + runMode := "acceptance" + if strings.EqualFold(strings.TrimSpace(r.Header.Get(acceptanceUpstreamHeader)), "real") { + runMode = "acceptance_canary" + } + return taskTrafficAdmission{RunMode: runMode, AcceptanceRunID: runID}, nil + } + return taskTrafficAdmission{RunMode: "production"}, nil +} + +func (s *Server) admittedTaskRunMode(admission taskTrafficAdmission, body map[string]any) (string, error) { + if admission.RunMode == "acceptance" || admission.RunMode == "acceptance_canary" { + return admission.RunMode, nil + } + requested := runModeFromRequest(body) + requestedMode := strings.ToLower(strings.TrimSpace(requested)) + if requestedMode == "acceptance" || requestedMode == "acceptance_canary" { + return "", &taskTrafficError{ + Status: http.StatusForbidden, Code: "acceptance_not_authorized", + Message: "acceptance mode is available only through an active acceptance run", + } + } + if strings.EqualFold(strings.TrimSpace(s.cfg.AppEnv), "production") && requestedMode == "simulation" { + return "", &taskTrafficError{ + Status: http.StatusForbidden, Code: "simulation_not_authorized", + Message: "simulation mode is available only through an active acceptance run", + } + } + return requested, nil +} + +func writeTaskTrafficError(w http.ResponseWriter, err error, protocol string) { + var trafficErr *taskTrafficError + if !errors.As(err, &trafficErr) { + trafficErr = &taskTrafficError{ + Status: http.StatusServiceUnavailable, Code: "traffic_gate_unavailable", + Message: "task traffic gate is unavailable", Err: err, + } + } + if protocol != "" { + writeProtocolError(w, protocol, trafficErr.Status, trafficErr.Message, nil, trafficErr.Code) + return + } + writeErrorWithDetails(w, trafficErr.Status, trafficErr.Message, nil, trafficErr.Code) +} + +// getGatewayTrafficMode godoc +// @Summary 获取 Gateway 流量模式 +// @Tags acceptance +// @Produce json +// @Security BearerAuth +// @Success 200 {object} store.GatewayTrafficMode +// @Router /api/admin/system/acceptance/traffic-mode [get] +func (s *Server) getGatewayTrafficMode(w http.ResponseWriter, r *http.Request) { + mode, err := s.store.GetGatewayTrafficMode(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, "get gateway traffic mode failed") + return + } + writeJSON(w, http.StatusOK, mode) +} + +// createAcceptanceRun godoc +// @Summary 创建生产同构验收 Run +// @Tags acceptance +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param body body store.CreateAcceptanceRunInput true "验收 Run" +// @Success 201 {object} store.AcceptanceRun +// @Router /api/admin/system/acceptance/runs [post] +func (s *Server) createAcceptanceRun(w http.ResponseWriter, r *http.Request) { + var input store.CreateAcceptanceRunInput + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, http.StatusBadRequest, "invalid json body") + return + } + run, err := s.store.CreateAcceptanceRun(r.Context(), input) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusCreated, run) +} + +// getAcceptanceRun godoc +// @Summary 获取生产同构验收 Run +// @Tags acceptance +// @Produce json +// @Security BearerAuth +// @Param runID path string true "验收 Run ID" +// @Success 200 {object} store.AcceptanceRun +// @Router /api/admin/system/acceptance/runs/{runID} [get] +func (s *Server) getAcceptanceRun(w http.ResponseWriter, r *http.Request) { + run, err := s.store.GetAcceptanceRun(r.Context(), r.PathValue("runID")) + if err != nil { + if store.IsNotFound(err) { + writeError(w, http.StatusNotFound, "acceptance run not found") + return + } + writeError(w, http.StatusInternalServerError, "get acceptance run failed") + return + } + writeJSON(w, http.StatusOK, run) +} + +// activateAcceptanceRun godoc +// @Summary 切换到 validation 并激活 Run +// @Tags acceptance +// @Produce json +// @Security BearerAuth +// @Param runID path string true "验收 Run ID" +// @Success 200 {object} store.GatewayTrafficMode +// @Router /api/admin/system/acceptance/runs/{runID}/activate [post] +func (s *Server) activateAcceptanceRun(w http.ResponseWriter, r *http.Request) { + mode, err := s.store.ActivateAcceptanceRun(r.Context(), r.PathValue("runID")) + if err != nil { + writeAcceptanceMutationError(w, err) + return + } + writeJSON(w, http.StatusOK, mode) +} + +// finishAcceptanceRun godoc +// @Summary 记录验收门禁结果 +// @Tags acceptance +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param runID path string true "验收 Run ID" +// @Param body body store.FinishAcceptanceRunInput true "门禁结果" +// @Success 200 {object} store.AcceptanceRun +// @Router /api/admin/system/acceptance/runs/{runID}/finish [post] +func (s *Server) finishAcceptanceRun(w http.ResponseWriter, r *http.Request) { + var input store.FinishAcceptanceRunInput + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, http.StatusBadRequest, "invalid json body") + return + } + input.RunID = r.PathValue("runID") + run, err := s.store.FinishAcceptanceRun(r.Context(), input) + if err != nil { + if store.IsNotFound(err) { + writeError(w, http.StatusConflict, "acceptance run is not running") + return + } + writeError(w, http.StatusInternalServerError, "finish acceptance run failed") + return + } + writeJSON(w, http.StatusOK, run) +} + +// retryAcceptanceRun godoc +// @Summary 在 validation 关闭正式流量的状态下重试失败 Run +// @Tags acceptance +// @Produce json +// @Security BearerAuth +// @Param runID path string true "验收 Run ID" +// @Success 200 {object} store.AcceptanceRun +// @Router /api/admin/system/acceptance/runs/{runID}/retry [post] +func (s *Server) retryAcceptanceRun(w http.ResponseWriter, r *http.Request) { + run, err := s.store.RetryAcceptanceRun(r.Context(), r.PathValue("runID")) + if err != nil { + writeAcceptanceMutationError(w, err) + return + } + writeJSON(w, http.StatusOK, run) +} + +// promoteAcceptanceRun godoc +// @Summary 通过 CAS 切回 live +// @Tags acceptance +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param runID path string true "验收 Run ID" +// @Param body body store.PromoteAcceptanceRunInput true "线上 release 与 digest CAS" +// @Success 200 {object} store.GatewayTrafficMode +// @Router /api/admin/system/acceptance/runs/{runID}/promote [post] +func (s *Server) promoteAcceptanceRun(w http.ResponseWriter, r *http.Request) { + var input store.PromoteAcceptanceRunInput + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, http.StatusBadRequest, "invalid json body") + return + } + input.RunID = r.PathValue("runID") + mode, err := s.store.PromoteAcceptanceRun(r.Context(), input) + if err != nil { + writeAcceptanceMutationError(w, err) + return + } + writeJSON(w, http.StatusOK, mode) +} + +// abortAcceptanceRun godoc +// @Summary 人工 CAS 中止验收并切回 live +// @Tags acceptance +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param runID path string true "验收 Run ID" +// @Param body body store.PromoteAcceptanceRunInput true "线上 release 与 digest CAS" +// @Success 200 {object} store.GatewayTrafficMode +// @Router /api/admin/system/acceptance/runs/{runID}/abort [post] +func (s *Server) abortAcceptanceRun(w http.ResponseWriter, r *http.Request) { + var input store.PromoteAcceptanceRunInput + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, http.StatusBadRequest, "invalid json body") + return + } + input.RunID = r.PathValue("runID") + mode, err := s.store.AbortAcceptanceRun(r.Context(), input) + if err != nil { + writeAcceptanceMutationError(w, err) + return + } + writeJSON(w, http.StatusOK, mode) +} + +func writeAcceptanceMutationError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, store.ErrAcceptanceStateConflict): + writeError(w, http.StatusConflict, err.Error()) + case errors.Is(err, store.ErrAcceptancePromotionGates): + writeError(w, http.StatusPreconditionFailed, err.Error()) + case store.IsNotFound(err): + writeError(w, http.StatusNotFound, "acceptance run not found") + default: + writeError(w, http.StatusInternalServerError, "acceptance state update failed") + } +} diff --git a/apps/api/internal/httpapi/acceptance_handlers_test.go b/apps/api/internal/httpapi/acceptance_handlers_test.go new file mode 100644 index 0000000..0d53d6a --- /dev/null +++ b/apps/api/internal/httpapi/acceptance_handlers_test.go @@ -0,0 +1,27 @@ +package httpapi + +import ( + "errors" + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" +) + +func TestAdmittedTaskRunModeReservesAcceptanceAndProductionSimulation(t *testing.T) { + server := &Server{cfg: config.Config{AppEnv: "production"}} + for _, runMode := range []string{"simulation", "SIMULATION", "acceptance", "acceptance_canary"} { + _, err := server.admittedTaskRunMode(taskTrafficAdmission{}, map[string]any{"runMode": runMode}) + var trafficErr *taskTrafficError + if !errors.As(err, &trafficErr) || trafficErr.Status != 403 { + t.Fatalf("run mode %q error=%v", runMode, err) + } + } + for _, runMode := range []string{"acceptance", "acceptance_canary"} { + got, err := server.admittedTaskRunMode(taskTrafficAdmission{RunMode: runMode}, map[string]any{ + "runMode": "production", + }) + if err != nil || got != runMode { + t.Fatalf("admitted mode=%q got=%q err=%v", runMode, got, err) + } + } +} diff --git a/apps/api/internal/httpapi/gateway_task_creation.go b/apps/api/internal/httpapi/gateway_task_creation.go index 76b3c7e..8abc22b 100644 --- a/apps/api/internal/httpapi/gateway_task_creation.go +++ b/apps/api/internal/httpapi/gateway_task_creation.go @@ -44,14 +44,23 @@ func (s *Server) prepareAndCreateGatewayTask( body map[string]any, async bool, ) (store.GatewayTask, error) { + admission, err := s.authorizeTaskTraffic(r, user) + if err != nil { + return store.GatewayTask{}, &gatewayTaskCreationError{Stage: gatewayTaskCreationPrepare, Err: err} + } prepared, err := s.prepareTaskRequest(ctx, r, user, body) if err != nil { return store.GatewayTask{}, &gatewayTaskCreationError{Stage: gatewayTaskCreationPrepare, Err: err} } + runMode, err := s.admittedTaskRunMode(admission, prepared.Body) + if err != nil { + return store.GatewayTask{}, &gatewayTaskCreationError{Stage: gatewayTaskCreationPrepare, Err: err} + } task, err := s.store.CreateTask(ctx, store.CreateTaskInput{ Kind: kind, Model: model, - RunMode: runModeFromRequest(prepared.Body), + RunMode: runMode, + AcceptanceRunID: admission.AcceptanceRunID, Async: async, Request: prepared.Body, ConversationID: prepared.ConversationID, diff --git a/apps/api/internal/httpapi/gemini_base64_stress_integration_test.go b/apps/api/internal/httpapi/gemini_base64_stress_integration_test.go index 17c9063..13919d7 100644 --- a/apps/api/internal/httpapi/gemini_base64_stress_integration_test.go +++ b/apps/api/internal/httpapi/gemini_base64_stress_integration_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceworkload" "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" "github.com/jackc/pgx/v5/pgxpool" @@ -37,13 +38,14 @@ func TestGeminiBase64ImageEditThousandSynchronousRequests(t *testing.T) { } const ( - taskCount = 1000 upstreamConcurrent = 64 inputVariants = 16 mediaConcurrent = 8 ) - imageBytes := stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_IMAGE_BYTES", 256<<10) - upstreamDelay := time.Duration(stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_UPSTREAM_DELAY_MS", 4000)) * time.Millisecond + baselineProfile := acceptanceworkload.GeminiBaseline + taskCount := baselineProfile.Requests + imageBytes := stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_IMAGE_BYTES", baselineProfile.InputBytes) + upstreamDelay := time.Duration(stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_UPSTREAM_DELAY_MS", int(baselineProfile.DelayMin/time.Millisecond))) * time.Millisecond maxHeapGrowth := uint64(stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_MAX_HEAP_BYTES", 1<<30)) clientConnections := stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_CLIENT_CONNECTIONS", 256) testTimeout := time.Duration(stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_TIMEOUT_SECONDS", 1200)) * time.Second @@ -387,7 +389,7 @@ WHERE task.model = $1 if tasks != taskCount || succeeded != taskCount || attempts != taskCount || duplicateAttemptTasks != 0 { t.Fatalf("task results tasks=%d succeeded=%d attempts=%d duplicate_attempt_tasks=%d", tasks, succeeded, attempts, duplicateAttemptTasks) } - if upstreamCalls.Load() != taskCount || invalidInputs.Load() != 0 { + if upstreamCalls.Load() != int64(taskCount) || invalidInputs.Load() != 0 { t.Fatalf("upstream calls=%d invalid_inputs=%d, want %d/0", upstreamCalls.Load(), invalidInputs.Load(), taskCount) } if upstreamPeak.Load() > mediaConcurrent || upstreamPeak.Load() < mediaConcurrent/2 { diff --git a/apps/api/internal/httpapi/gemini_compat.go b/apps/api/internal/httpapi/gemini_compat.go index 11063cd..f21d79a 100644 --- a/apps/api/internal/httpapi/gemini_compat.go +++ b/apps/api/internal/httpapi/gemini_compat.go @@ -117,6 +117,11 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) { writeGeminiTaskError(http.StatusUnauthorized, "unauthorized", nil, "unauthorized") return } + admission, err := s.authorizeTaskTraffic(r, user) + if err != nil { + writeTaskTrafficError(w, err, clients.ProtocolGeminiGenerateContent) + return + } releaseRequestBody, err := s.acquireMediaRequestBodySlot(r.Context()) if err != nil { return @@ -166,10 +171,16 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) { mapping.Body = nil releaseRequestBody() releaseRequestBody = nil + runMode, err := s.admittedTaskRunMode(admission, prepared.Body) + if err != nil { + writeTaskTrafficError(w, err, clients.ProtocolGeminiGenerateContent) + return + } createInput := store.CreateTaskInput{ Kind: mapping.Kind, Model: mapping.Model, - RunMode: runModeFromRequest(prepared.Body), + RunMode: runMode, + AcceptanceRunID: admission.AcceptanceRunID, Async: false, Request: prepared.Body, ConversationID: prepared.ConversationID, diff --git a/apps/api/internal/httpapi/handlers.go b/apps/api/internal/httpapi/handlers.go index e55c671..fc0696d 100644 --- a/apps/api/internal/httpapi/handlers.go +++ b/apps/api/internal/httpapi/handlers.go @@ -1095,6 +1095,11 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler { writeTaskError(http.StatusUnauthorized, "unauthorized", nil) return } + admission, err := s.authorizeTaskTraffic(r, user) + if err != nil { + writeTaskTrafficError(w, err, targetProtocol) + return + } body, err := s.decodeTaskRequestBody(r.Context(), w, r, kind) if err != nil { @@ -1147,10 +1152,16 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler { return } + runMode, err := s.admittedTaskRunMode(admission, prepared.Body) + if err != nil { + writeTaskTrafficError(w, err, targetProtocol) + return + } createInput := store.CreateTaskInput{ Kind: kind, Model: model, - RunMode: runModeFromRequest(prepared.Body), + RunMode: runMode, + AcceptanceRunID: admission.AcceptanceRunID, Async: responsePlan.asyncMode, Request: prepared.Body, ConversationID: prepared.ConversationID, diff --git a/apps/api/internal/httpapi/kling_compat.go b/apps/api/internal/httpapi/kling_compat.go index e9b6303..19e9ec7 100644 --- a/apps/api/internal/httpapi/kling_compat.go +++ b/apps/api/internal/httpapi/kling_compat.go @@ -112,6 +112,15 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v writeKlingCompatError(w, http.StatusForbidden, "api key scope does not allow video generation", "permission_denied") return } + admission, err := s.authorizeTaskTraffic(r, user) + if err != nil { + targetProtocol := clients.ProtocolKlingV1Omni + if version == "v2" { + targetProtocol = clients.ProtocolKlingV2Omni + } + writeTaskTrafficError(w, err, targetProtocol) + return + } body, externalTaskID, err := klingCompatTaskBody(version, model, native) if err != nil { writeKlingCompatError(w, http.StatusBadRequest, err.Error(), clients.ErrorCode(err)) @@ -122,13 +131,23 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v writeKlingCompatError(w, http.StatusBadRequest, err.Error(), clients.ErrorCode(err)) return } + runMode, err := s.admittedTaskRunMode(admission, prepared.Body) + if err != nil { + targetProtocol := clients.ProtocolKlingV1Omni + if version == "v2" { + targetProtocol = clients.ProtocolKlingV2Omni + } + writeTaskTrafficError(w, err, targetProtocol) + return + } createInput := store.CreateTaskInput{ - Kind: "videos.generations", - Model: model, - ExternalTaskID: externalTaskID, - RunMode: runModeFromRequest(prepared.Body), - Async: true, - Request: prepared.Body, + Kind: "videos.generations", + Model: model, + ExternalTaskID: externalTaskID, + RunMode: runMode, + AcceptanceRunID: admission.AcceptanceRunID, + Async: true, + Request: prepared.Body, } if idempotencyKey, hasKey, keyErr := optionalTaskIdempotencyKey(r); keyErr != nil { writeKlingCompatError(w, http.StatusBadRequest, "invalid Idempotency-Key", "invalid_idempotency_key") diff --git a/apps/api/internal/httpapi/server.go b/apps/api/internal/httpapi/server.go index ae17df6..e69a392 100644 --- a/apps/api/internal/httpapi/server.go +++ b/apps/api/internal/httpapi/server.go @@ -217,6 +217,14 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor mux.Handle("PATCH /api/admin/system/file-storage/settings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageSettings))) mux.Handle("GET /api/admin/system/client-customization/settings", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getClientCustomizationSettings))) mux.Handle("PATCH /api/admin/system/client-customization/settings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateClientCustomizationSettings))) + mux.Handle("GET /api/admin/system/acceptance/traffic-mode", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getGatewayTrafficMode))) + mux.Handle("POST /api/admin/system/acceptance/runs", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createAcceptanceRun))) + mux.Handle("GET /api/admin/system/acceptance/runs/{runID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getAcceptanceRun))) + mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/activate", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.activateAcceptanceRun))) + mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/finish", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.finishAcceptanceRun))) + mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/retry", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.retryAcceptanceRun))) + mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/promote", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.promoteAcceptanceRun))) + mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/abort", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.abortAcceptanceRun))) mux.Handle("GET /api/admin/system/identity/configuration", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityConfiguration))) mux.Handle("POST /api/admin/system/identity/pairings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.startIdentityPairing))) mux.Handle("GET /api/admin/system/identity/pairings/{pairingID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityPairing))) diff --git a/apps/api/internal/httpapi/volces_compat_handlers.go b/apps/api/internal/httpapi/volces_compat_handlers.go index afa0ed9..46d3b47 100644 --- a/apps/api/internal/httpapi/volces_compat_handlers.go +++ b/apps/api/internal/httpapi/volces_compat_handlers.go @@ -416,6 +416,11 @@ func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) { if errors.As(err, &staged) { err = staged.Err } + var trafficErr *taskTrafficError + if errors.As(err, &trafficErr) { + writeVolcesError(w, trafficErr.Status, trafficErr.Message, trafficErr.Code) + return + } var clientErr *clients.ClientError if errors.As(err, &clientErr) && clientErr.StatusCode > 0 { status = clientErr.StatusCode diff --git a/apps/api/internal/httpapi/volces_compat_handlers_test.go b/apps/api/internal/httpapi/volces_compat_handlers_test.go index 0900aab..87adcd4 100644 --- a/apps/api/internal/httpapi/volces_compat_handlers_test.go +++ b/apps/api/internal/httpapi/volces_compat_handlers_test.go @@ -1,6 +1,9 @@ package httpapi import ( + "encoding/json" + "net/http" + "net/http/httptest" "testing" "time" @@ -35,3 +38,25 @@ func TestVolcesCompatibleTaskBuildsProtocolResponseFromCanonicalResult(t *testin } } } + +func TestVolcesCompatibilityPreservesValidationGateStatus(t *testing.T) { + recorder := httptest.NewRecorder() + writeVolcesCompatibleTaskError(recorder, &gatewayTaskCreationError{ + Stage: gatewayTaskCreationPrepare, + Err: &taskTrafficError{ + Status: http.StatusServiceUnavailable, + Code: "validation_in_progress", + Message: "new production tasks are paused while validation is running", + }, + }) + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + var payload VolcesErrorEnvelope + if err := json.NewDecoder(recorder.Body).Decode(&payload); err != nil { + t.Fatalf("decode error: %v", err) + } + if payload.Error.Code != "validation_in_progress" { + t.Fatalf("error=%+v", payload.Error) + } +} diff --git a/apps/api/internal/runner/billing_mode_test.go b/apps/api/internal/runner/billing_mode_test.go index 06cfd44..157fed3 100644 --- a/apps/api/internal/runner/billing_mode_test.go +++ b/apps/api/internal/runner/billing_mode_test.go @@ -25,3 +25,14 @@ func TestBillingItemsFixedTotalKeepsNineDecimalPlaces(t *testing.T) { t.Fatalf("total = %s", got) } } + +func TestAcceptanceRunModesUseRealBilling(t *testing.T) { + for _, runMode := range []string{"production", "acceptance", "acceptance_canary"} { + if !isBillableRunMode(runMode) { + t.Fatalf("run mode %q must be billable", runMode) + } + } + if isBillableRunMode("simulation") { + t.Fatal("simulation must remain non-billable") + } +} diff --git a/apps/api/internal/runner/service.go b/apps/api/internal/runner/service.go index 0c3102b..c49f639 100644 --- a/apps/api/internal/runner/service.go +++ b/apps/api/internal/runner/service.go @@ -418,7 +418,7 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, preprocessingByCandidate := map[string]parameterPreprocessResult{} reservationBillings := []any(nil) reservationPricingSnapshot := map[string]any(nil) - if task.RunMode == "production" { + if isBillableRunMode(task.RunMode) { billingMode := normalizedBillingEngineMode(s.cfg.BillingEngineMode) if billingMode == "hold" { holdErr := &clients.ClientError{Code: "billing_hold", Message: "production billing is temporarily on hold", Retryable: true} @@ -809,7 +809,7 @@ candidatesLoop: finalAmount := fixedAmount(0) finalAmountText := "" pricingSnapshot := map[string]any{"pricingVersion": pricingVersionV2, "source": "simulation"} - if task.RunMode == "production" { + if isBillableRunMode(task.RunMode) { billingMode := normalizedBillingEngineMode(s.cfg.BillingEngineMode) var billingErr error if billingMode == "observe" { @@ -1171,9 +1171,16 @@ func (s *Service) runCandidate( admittedLeases []store.ConcurrencyLease, ) (clients.Response, error) { var err error - candidate, err = candidateWithEnvironmentCredentials(candidate) - if err != nil { - return clients.Response{}, err + if task.RunMode == "acceptance" { + candidate, err = s.acceptanceCandidate(ctx, task, candidate) + if err != nil { + return clients.Response{}, err + } + } else { + candidate, err = candidateWithEnvironmentCredentials(candidate) + if err != nil { + return clients.Response{}, err + } } simulated := isSimulation(task, candidate) baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing)) @@ -2085,7 +2092,14 @@ func (s *Service) emit(ctx context.Context, taskID string, eventType string, sta if event.ID == "" { s.observeTaskEventSkip(event.SkippedReason) } - if s.cfg.TaskProgressCallbackEnabled { + acceptanceURL, acceptance, callbackErr := s.store.AcceptanceCallbackURL(ctx, taskID) + if callbackErr != nil { + return callbackErr + } + if acceptance && acceptanceURL != "" { + return s.store.QueueTaskCallback(ctx, event, acceptanceURL) + } + if s.cfg.TaskProgressCallbackEnabled && s.cfg.TaskProgressCallbackURL != "" { return s.store.QueueTaskCallback(ctx, event, s.cfg.TaskProgressCallbackURL) } return nil @@ -2230,6 +2244,43 @@ func isSimulation(task store.GatewayTask, candidate store.RuntimeModelCandidate) return stringFromMap(candidate.Credentials, "mode") == "simulation" || boolFromMap(candidate.PlatformConfig, "testMode") } +func isBillableRunMode(runMode string) bool { + return runMode == "production" || runMode == "acceptance" || runMode == "acceptance_canary" +} + +func (s *Service) acceptanceCandidate( + ctx context.Context, + task store.GatewayTask, + candidate store.RuntimeModelCandidate, +) (store.RuntimeModelCandidate, error) { + if strings.TrimSpace(task.AcceptanceRunID) == "" { + return candidate, &clients.ClientError{ + Code: "acceptance_run_missing", + Message: "acceptance task is missing its run identifier", + Retryable: false, + } + } + baseURL, credential, err := s.store.AcceptanceCandidateOverride(ctx, task.AcceptanceRunID) + if err != nil { + return candidate, &clients.ClientError{ + Code: "acceptance_run_inactive", + Message: err.Error(), + Retryable: false, + } + } + candidate.BaseURL = baseURL + candidate.Credentials = map[string]any{"apiKey": credential} + platformConfig := cloneMap(candidate.PlatformConfig) + delete(platformConfig, "credentialEnv") + delete(platformConfig, "credential_env") + delete(platformConfig, "proxyMode") + delete(platformConfig, "httpProxy") + platformConfig["networkProxy"] = map[string]any{"mode": "none"} + platformConfig["acceptanceRunId"] = task.AcceptanceRunID + candidate.PlatformConfig = platformConfig + return candidate, nil +} + func retryEnabled(candidate store.RuntimeModelCandidate) bool { policy := effectiveRetryPolicy(candidate) if enabled, ok := policy["enabled"].(bool); ok { diff --git a/apps/api/internal/securityevents/metrics.go b/apps/api/internal/securityevents/metrics.go index 7fc9c3f..51288a3 100644 --- a/apps/api/internal/securityevents/metrics.go +++ b/apps/api/internal/securityevents/metrics.go @@ -279,6 +279,12 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str return } } + postgresPool := store.PostgresPoolMetricsSnapshot{} + if poolProvider, ok := provider.(interface { + PostgresPoolMetrics() store.PostgresPoolMetricsSnapshot + }); ok { + postgresPool = poolProvider.PostgresPoolMetrics() + } w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") outcomeCounters(w, "easyai_gateway_ssf_receipts_total", "Received SETs by bounded outcome.", []outcomeValue{ {"accepted", m.accepted.Load()}, {"rejected", m.rejected.Load()}, {"duplicate", m.duplicate.Load()}, @@ -350,6 +356,12 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str plainGauge(w, "easyai_gateway_worker_active_instances", "Active distributed worker instances with a fresh heartbeat.", m.asyncWorkerActiveInstances.Load()) plainGauge(w, "easyai_gateway_worker_global_capacity", "Global asynchronous execution capacity before instance allocation.", m.asyncWorkerGlobalCapacity.Load()) plainGauge(w, "easyai_gateway_worker_allocated_capacity", "Asynchronous execution capacity allocated to this worker instance.", m.asyncWorkerAllocated.Load()) + plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections)) + plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections)) + plainGauge(w, "easyai_gateway_postgres_pool_acquired_connections", "Currently acquired PostgreSQL connections in this process pool.", int64(postgresPool.AcquiredConnections)) + plainGauge(w, "easyai_gateway_postgres_pool_idle_connections", "Currently idle PostgreSQL connections in this process pool.", int64(postgresPool.IdleConnections)) + plainCounter(w, "easyai_gateway_postgres_pool_empty_acquire_total", "Pool acquires that had to wait for a PostgreSQL connection.", uint64(max(postgresPool.EmptyAcquireCount, 0))) + plainCounter(w, "easyai_gateway_postgres_pool_canceled_acquire_total", "Canceled PostgreSQL pool acquires.", uint64(max(postgresPool.CanceledAcquireCount, 0))) outcomeCounters(w, "easyai_gateway_async_worker_resizes_total", "Asynchronous worker resize attempts by bounded outcome.", []outcomeValue{ {"success", m.asyncWorkerResizeSuccess.Load()}, {"refresh_failed", m.asyncWorkerRefreshFailed.Load()}, diff --git a/apps/api/internal/securityevents/metrics_test.go b/apps/api/internal/securityevents/metrics_test.go index ff76210..fac43a1 100644 --- a/apps/api/internal/securityevents/metrics_test.go +++ b/apps/api/internal/securityevents/metrics_test.go @@ -7,17 +7,24 @@ import ( "strings" "testing" "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) type metricsSnapshot struct { mode string last time.Time + pool store.PostgresPoolMetricsSnapshot } func (m metricsSnapshot) SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error) { return m.mode, &m.last, nil } +func (m metricsSnapshot) PostgresPoolMetrics() store.PostgresPoolMetricsSnapshot { + return m.pool +} + func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) { metrics := &Metrics{} metrics.ObserveReceipt("accepted", 20*time.Millisecond, 2) @@ -34,7 +41,14 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) { metrics.ObserveTaskEventSkip("budget_exceeded") recorder := httptest.NewRecorder() - metrics.Handler(metricsSnapshot{mode: "push_healthy", last: time.Now().Add(-time.Minute)}, "issuer", "audience", true). + metrics.Handler(metricsSnapshot{ + mode: "push_healthy", + last: time.Now().Add(-time.Minute), + pool: store.PostgresPoolMetricsSnapshot{ + MaxConnections: 32, TotalConnections: 18, AcquiredConnections: 12, + IdleConnections: 6, EmptyAcquireCount: 3, CanceledAcquireCount: 1, + }, + }, "issuer", "audience", true). ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) if recorder.Code != http.StatusOK { t.Fatalf("metrics status=%d", recorder.Code) @@ -56,6 +70,12 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) { `easyai_gateway_concurrency_lease_renewals_total{outcome="lost"} 1`, `easyai_gateway_task_events_skipped_total{outcome="duplicate"} 1`, `easyai_gateway_task_events_skipped_total{outcome="budget_exceeded"} 1`, + `easyai_gateway_postgres_pool_max_connections 32`, + `easyai_gateway_postgres_pool_total_connections 18`, + `easyai_gateway_postgres_pool_acquired_connections 12`, + `easyai_gateway_postgres_pool_idle_connections 6`, + `easyai_gateway_postgres_pool_empty_acquire_total 3`, + `easyai_gateway_postgres_pool_canceled_acquire_total 1`, } { if !strings.Contains(recorder.Body.String(), expected) { t.Fatalf("missing metric %q in:\n%s", expected, recorder.Body.String()) diff --git a/apps/api/internal/store/acceptance.go b/apps/api/internal/store/acceptance.go new file mode 100644 index 0000000..d5834a4 --- /dev/null +++ b/apps/api/internal/store/acceptance.go @@ -0,0 +1,533 @@ +package store + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/jackc/pgx/v5" +) + +const SystemSettingGatewayTrafficMode = "gateway_traffic_mode" + +var ( + acceptanceReleaseSHAPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + acceptanceDigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) +) + +var ( + ErrProductionTrafficPaused = errors.New("production traffic is paused for validation") + ErrAcceptanceNotAuthorized = errors.New("acceptance request is not authorized") + ErrAcceptanceRunNotActive = errors.New("acceptance run is not active") + ErrAcceptanceStateConflict = errors.New("acceptance state changed") + ErrAcceptancePromotionGates = errors.New("acceptance promotion gates have not passed") +) + +type GatewayTrafficMode struct { + Mode string `json:"mode"` + RunID string `json:"runId,omitempty"` + Revision int64 `json:"revision"` + ReleaseSHA string `json:"releaseSha,omitempty"` + APIImageDigest string `json:"apiImageDigest,omitempty"` + WorkerImageDigest string `json:"workerImageDigest,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type AcceptanceRun struct { + ID string `json:"id"` + Status string `json:"status"` + ReleaseSHA string `json:"releaseSha"` + APIImageDigest string `json:"apiImageDigest"` + WorkerImageDigest string `json:"workerImageDigest"` + APIKeyID string `json:"apiKeyId"` + UserID string `json:"userId"` + EmulatorBaseURL string `json:"emulatorBaseUrl"` + CallbackURL string `json:"callbackUrl,omitempty"` + CapacityProfile string `json:"capacityProfile"` + Config map[string]any `json:"config"` + Report map[string]any `json:"report"` + FailureReason string `json:"failureReason,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` + PromotedAt string `json:"promotedAt,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type CreateAcceptanceRunInput struct { + ReleaseSHA string `json:"releaseSha"` + APIImageDigest string `json:"apiImageDigest"` + WorkerImageDigest string `json:"workerImageDigest"` + APIKeyID string `json:"apiKeyId"` + UserID string `json:"userId"` + Token string `json:"token"` + EmulatorBaseURL string `json:"emulatorBaseUrl"` + CallbackURL string `json:"callbackUrl,omitempty"` + CapacityProfile string `json:"capacityProfile"` + Config map[string]any `json:"config,omitempty"` +} + +type FinishAcceptanceRunInput struct { + RunID string `json:"-"` + Passed bool `json:"passed"` + Report map[string]any `json:"report"` + FailureReason string `json:"failureReason,omitempty"` +} + +type PromoteAcceptanceRunInput struct { + RunID string `json:"-"` + Revision int64 `json:"revision"` + ReleaseSHA string `json:"releaseSha"` + APIImageDigest string `json:"apiImageDigest"` + WorkerImageDigest string `json:"workerImageDigest"` +} + +func DefaultGatewayTrafficMode() GatewayTrafficMode { + return GatewayTrafficMode{Mode: "live"} +} + +func (s *Store) GetGatewayTrafficMode(ctx context.Context) (GatewayTrafficMode, error) { + var value []byte + var updatedAt time.Time + err := s.pool.QueryRow(ctx, ` +SELECT value, updated_at +FROM system_settings +WHERE setting_key = $1`, SystemSettingGatewayTrafficMode).Scan(&value, &updatedAt) + if err != nil { + if IsNotFound(err) || IsUndefinedDatabaseObject(err) { + return DefaultGatewayTrafficMode(), nil + } + return GatewayTrafficMode{}, err + } + var mode GatewayTrafficMode + if err := json.Unmarshal(value, &mode); err != nil { + return GatewayTrafficMode{}, fmt.Errorf("decode gateway traffic mode: %w", err) + } + mode.Mode = normalizeTrafficMode(mode.Mode) + mode.UpdatedAt = updatedAt + return mode, nil +} + +func (s *Store) CreateAcceptanceRun(ctx context.Context, input CreateAcceptanceRunInput) (AcceptanceRun, error) { + input.ReleaseSHA = strings.TrimSpace(input.ReleaseSHA) + input.APIImageDigest = strings.TrimSpace(input.APIImageDigest) + input.WorkerImageDigest = strings.TrimSpace(input.WorkerImageDigest) + input.APIKeyID = strings.TrimSpace(input.APIKeyID) + input.UserID = strings.TrimSpace(input.UserID) + input.Token = strings.TrimSpace(input.Token) + input.EmulatorBaseURL = strings.TrimRight(strings.TrimSpace(input.EmulatorBaseURL), "/") + input.CallbackURL = strings.TrimSpace(input.CallbackURL) + input.CapacityProfile = strings.ToUpper(strings.TrimSpace(input.CapacityProfile)) + if input.CapacityProfile == "" { + input.CapacityProfile = "P24" + } + if input.ReleaseSHA == "" || input.APIImageDigest == "" || input.WorkerImageDigest == "" || + input.APIKeyID == "" || input.UserID == "" || input.Token == "" || input.EmulatorBaseURL == "" || + input.CallbackURL == "" { + return AcceptanceRun{}, errors.New("release SHA, image digests, API key, user, token, emulator URL, and callback URL are required") + } + if !acceptanceReleaseSHAPattern.MatchString(input.ReleaseSHA) || + !acceptanceDigestPattern.MatchString(input.APIImageDigest) || + !acceptanceDigestPattern.MatchString(input.WorkerImageDigest) { + return AcceptanceRun{}, errors.New("release SHA and image digests must use full immutable identifiers") + } + if len(input.Token) < 32 { + return AcceptanceRun{}, errors.New("acceptance token must contain at least 32 characters") + } + if !validAcceptanceURL(input.EmulatorBaseURL) || !validAcceptanceURL(input.CallbackURL) { + return AcceptanceRun{}, errors.New("emulator and callback URLs must be absolute HTTP URLs without user information") + } + switch input.CapacityProfile { + case "P24", "P28", "P32": + default: + return AcceptanceRun{}, errors.New("capacity profile must be P24, P28, or P32") + } + config, _ := json.Marshal(sanitizeAcceptanceMetadata(sanitizeJSONForStorage(input.Config))) + tokenHash := acceptanceTokenSHA256(input.Token) + return scanAcceptanceRun(s.pool.QueryRow(ctx, ` +INSERT INTO gateway_acceptance_runs ( + release_sha, api_image_digest, worker_image_digest, api_key_id, user_id, + token_sha256, emulator_base_url, callback_url, capacity_profile, config +) +VALUES ($1, $2, $3, $4, $5, $6, $7, NULLIF($8, ''), $9, $10::jsonb) +RETURNING `+acceptanceRunColumns, + input.ReleaseSHA, input.APIImageDigest, input.WorkerImageDigest, input.APIKeyID, input.UserID, + tokenHash, input.EmulatorBaseURL, input.CallbackURL, input.CapacityProfile, config, + )) +} + +func (s *Store) GetAcceptanceRun(ctx context.Context, runID string) (AcceptanceRun, error) { + return scanAcceptanceRun(s.pool.QueryRow(ctx, ` +SELECT `+acceptanceRunColumns+` +FROM gateway_acceptance_runs +WHERE id = $1::uuid`, strings.TrimSpace(runID))) +} + +func (s *Store) ActivateAcceptanceRun(ctx context.Context, runID string) (GatewayTrafficMode, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return GatewayTrafficMode{}, err + } + defer rollbackTransaction(tx) + + var currentValue []byte + if err := tx.QueryRow(ctx, ` +SELECT value +FROM system_settings +WHERE setting_key = $1 +FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(¤tValue); err != nil { + return GatewayTrafficMode{}, err + } + var current GatewayTrafficMode + if err := json.Unmarshal(currentValue, ¤t); err != nil { + return GatewayTrafficMode{}, err + } + if normalizeTrafficMode(current.Mode) != "live" { + return GatewayTrafficMode{}, ErrAcceptanceStateConflict + } + run, err := scanAcceptanceRun(tx.QueryRow(ctx, ` +SELECT `+acceptanceRunColumns+` +FROM gateway_acceptance_runs +WHERE id = $1::uuid +FOR UPDATE`, strings.TrimSpace(runID))) + if err != nil { + return GatewayTrafficMode{}, err + } + if run.Status != "pending" && run.Status != "failed" { + return GatewayTrafficMode{}, ErrAcceptanceStateConflict + } + next := GatewayTrafficMode{ + Mode: "validation", + RunID: run.ID, + Revision: current.Revision + 1, + ReleaseSHA: run.ReleaseSHA, + APIImageDigest: run.APIImageDigest, + WorkerImageDigest: run.WorkerImageDigest, + } + value, _ := json.Marshal(next) + if _, err := tx.Exec(ctx, ` +UPDATE system_settings +SET value = $2::jsonb, updated_at = now() +WHERE setting_key = $1`, SystemSettingGatewayTrafficMode, value); err != nil { + return GatewayTrafficMode{}, err + } + if _, err := tx.Exec(ctx, ` +UPDATE gateway_acceptance_runs +SET status = 'running', started_at = COALESCE(started_at, now()), + finished_at = NULL, failure_reason = NULL, updated_at = now() +WHERE id = $1::uuid`, run.ID); err != nil { + return GatewayTrafficMode{}, err + } + if err := tx.Commit(ctx); err != nil { + return GatewayTrafficMode{}, err + } + next.UpdatedAt = time.Now() + return next, nil +} + +func (s *Store) FinishAcceptanceRun(ctx context.Context, input FinishAcceptanceRunInput) (AcceptanceRun, error) { + status := "failed" + if input.Passed { + status = "passed" + } + report, _ := json.Marshal(sanitizeAcceptanceMetadata(sanitizeJSONForStorage(input.Report))) + return scanAcceptanceRun(s.pool.QueryRow(ctx, ` +UPDATE gateway_acceptance_runs +SET status = $2, report = $3::jsonb, failure_reason = NULLIF($4, ''), + finished_at = now(), updated_at = now() +WHERE id = $1::uuid AND status = 'running' +RETURNING `+acceptanceRunColumns, + strings.TrimSpace(input.RunID), status, report, strings.TrimSpace(input.FailureReason), + )) +} + +func (s *Store) RetryAcceptanceRun(ctx context.Context, runID string) (AcceptanceRun, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return AcceptanceRun{}, err + } + defer rollbackTransaction(tx) + var value []byte + if err := tx.QueryRow(ctx, ` +SELECT value +FROM system_settings +WHERE setting_key = $1 +FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&value); err != nil { + return AcceptanceRun{}, err + } + var mode GatewayTrafficMode + if err := json.Unmarshal(value, &mode); err != nil { + return AcceptanceRun{}, err + } + if normalizeTrafficMode(mode.Mode) != "validation" || mode.RunID != strings.TrimSpace(runID) { + return AcceptanceRun{}, ErrAcceptanceStateConflict + } + run, err := scanAcceptanceRun(tx.QueryRow(ctx, ` +UPDATE gateway_acceptance_runs +SET status = 'running', report = '{}'::jsonb, failure_reason = NULL, + finished_at = NULL, updated_at = now() +WHERE id = $1::uuid AND status = 'failed' +RETURNING `+acceptanceRunColumns, strings.TrimSpace(runID))) + if err != nil { + return AcceptanceRun{}, err + } + if err := tx.Commit(ctx); err != nil { + return AcceptanceRun{}, err + } + return run, nil +} + +func (s *Store) PromoteAcceptanceRun(ctx context.Context, input PromoteAcceptanceRunInput) (GatewayTrafficMode, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return GatewayTrafficMode{}, err + } + defer rollbackTransaction(tx) + var value []byte + if err := tx.QueryRow(ctx, ` +SELECT value +FROM system_settings +WHERE setting_key = $1 +FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&value); err != nil { + return GatewayTrafficMode{}, err + } + var current GatewayTrafficMode + if err := json.Unmarshal(value, ¤t); err != nil { + return GatewayTrafficMode{}, err + } + if normalizeTrafficMode(current.Mode) != "validation" || + current.RunID != strings.TrimSpace(input.RunID) || + current.Revision != input.Revision || + current.ReleaseSHA != strings.TrimSpace(input.ReleaseSHA) || + current.APIImageDigest != strings.TrimSpace(input.APIImageDigest) || + current.WorkerImageDigest != strings.TrimSpace(input.WorkerImageDigest) { + return GatewayTrafficMode{}, ErrAcceptanceStateConflict + } + var status string + if err := tx.QueryRow(ctx, ` +SELECT status +FROM gateway_acceptance_runs +WHERE id = $1::uuid +FOR UPDATE`, current.RunID).Scan(&status); err != nil { + return GatewayTrafficMode{}, err + } + if status != "passed" { + return GatewayTrafficMode{}, ErrAcceptancePromotionGates + } + next := GatewayTrafficMode{Mode: "live", Revision: current.Revision + 1} + nextValue, _ := json.Marshal(next) + if _, err := tx.Exec(ctx, ` +UPDATE system_settings +SET value = $2::jsonb, updated_at = now() +WHERE setting_key = $1`, SystemSettingGatewayTrafficMode, nextValue); err != nil { + return GatewayTrafficMode{}, err + } + if _, err := tx.Exec(ctx, ` +UPDATE gateway_acceptance_runs +SET status = 'promoted', promoted_at = now(), updated_at = now() +WHERE id = $1::uuid`, current.RunID); err != nil { + return GatewayTrafficMode{}, err + } + if err := tx.Commit(ctx); err != nil { + return GatewayTrafficMode{}, err + } + next.UpdatedAt = time.Now() + return next, nil +} + +func (s *Store) AbortAcceptanceRun(ctx context.Context, input PromoteAcceptanceRunInput) (GatewayTrafficMode, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return GatewayTrafficMode{}, err + } + defer rollbackTransaction(tx) + var value []byte + if err := tx.QueryRow(ctx, ` +SELECT value +FROM system_settings +WHERE setting_key = $1 +FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&value); err != nil { + return GatewayTrafficMode{}, err + } + var current GatewayTrafficMode + if err := json.Unmarshal(value, ¤t); err != nil { + return GatewayTrafficMode{}, err + } + if normalizeTrafficMode(current.Mode) != "validation" || + current.RunID != strings.TrimSpace(input.RunID) || + current.Revision != input.Revision || + current.ReleaseSHA != strings.TrimSpace(input.ReleaseSHA) || + current.APIImageDigest != strings.TrimSpace(input.APIImageDigest) || + current.WorkerImageDigest != strings.TrimSpace(input.WorkerImageDigest) { + return GatewayTrafficMode{}, ErrAcceptanceStateConflict + } + next := GatewayTrafficMode{Mode: "live", Revision: current.Revision + 1} + nextValue, _ := json.Marshal(next) + if _, err := tx.Exec(ctx, ` +UPDATE system_settings +SET value = $2::jsonb, updated_at = now() +WHERE setting_key = $1`, SystemSettingGatewayTrafficMode, nextValue); err != nil { + return GatewayTrafficMode{}, err + } + if _, err := tx.Exec(ctx, ` +UPDATE gateway_acceptance_runs +SET status = 'aborted', finished_at = COALESCE(finished_at, now()), updated_at = now() +WHERE id = $1::uuid AND status IN ('running', 'failed', 'passed')`, current.RunID); err != nil { + return GatewayTrafficMode{}, err + } + if err := tx.Commit(ctx); err != nil { + return GatewayTrafficMode{}, err + } + next.UpdatedAt = time.Now() + return next, nil +} + +func (s *Store) AuthorizeAcceptanceTask(ctx context.Context, runID string, token string, user *auth.User) (string, error) { + mode, err := s.GetGatewayTrafficMode(ctx) + if err != nil { + return "", err + } + runID = strings.TrimSpace(runID) + token = strings.TrimSpace(token) + if mode.Mode == "live" { + if runID != "" || token != "" { + return "", ErrAcceptanceRunNotActive + } + return "", nil + } + if runID == "" || token == "" || runID != mode.RunID { + return "", ErrProductionTrafficPaused + } + if user == nil || strings.TrimSpace(user.APIKeyID) == "" { + return "", ErrAcceptanceNotAuthorized + } + var activeRunID, apiKeyID, userID, tokenHash, status string + err = s.pool.QueryRow(ctx, ` +SELECT id::text, api_key_id, user_id, token_sha256, status +FROM gateway_acceptance_runs +WHERE id = $1::uuid`, runID).Scan(&activeRunID, &apiKeyID, &userID, &tokenHash, &status) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", ErrAcceptanceNotAuthorized + } + return "", err + } + if status != "running" || apiKeyID != strings.TrimSpace(user.APIKeyID) || userID != strings.TrimSpace(user.ID) || + subtle.ConstantTimeCompare([]byte(tokenHash), []byte(acceptanceTokenSHA256(token))) != 1 { + return "", ErrAcceptanceNotAuthorized + } + return activeRunID, nil +} + +func (s *Store) AcceptanceCandidateOverride(ctx context.Context, runID string) (string, string, error) { + var baseURL, status string + err := s.pool.QueryRow(ctx, ` +SELECT emulator_base_url, status +FROM gateway_acceptance_runs +WHERE id = $1::uuid`, strings.TrimSpace(runID)).Scan(&baseURL, &status) + if err != nil { + return "", "", err + } + if status != "running" && status != "passed" { + return "", "", ErrAcceptanceRunNotActive + } + return strings.TrimRight(strings.TrimSpace(baseURL), "/"), "acceptance-protocol-emulator", nil +} + +func (s *Store) AcceptanceCallbackURL(ctx context.Context, taskID string) (string, bool, error) { + var callbackURL string + err := s.pool.QueryRow(ctx, ` +SELECT COALESCE(run.callback_url, '') +FROM gateway_tasks task +JOIN gateway_acceptance_runs run ON run.id = task.acceptance_run_id +WHERE task.id = $1::uuid`, strings.TrimSpace(taskID)).Scan(&callbackURL) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", false, nil + } + return "", false, err + } + return strings.TrimSpace(callbackURL), true, nil +} + +func normalizeTrafficMode(value string) string { + if strings.EqualFold(strings.TrimSpace(value), "validation") { + return "validation" + } + return "live" +} + +func acceptanceTokenSHA256(token string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(token))) + return hex.EncodeToString(sum[:]) +} + +func validAcceptanceURL(raw string) bool { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed.User != nil || parsed.Host == "" { + return false + } + return parsed.Scheme == "http" || parsed.Scheme == "https" +} + +func sanitizeAcceptanceMetadata(value any) any { + switch typed := value.(type) { + case map[string]any: + next := make(map[string]any, len(typed)) + for key, item := range typed { + normalized := strings.ToLower(strings.NewReplacer("_", "", "-", "", ".", "").Replace(key)) + if strings.Contains(normalized, "password") || + strings.Contains(normalized, "secret") || + strings.Contains(normalized, "credential") || + strings.Contains(normalized, "connectionstring") || + strings.Contains(normalized, "databaseurl") || + (strings.Contains(normalized, "token") && !strings.HasSuffix(normalized, "count")) { + next[key] = "[REDACTED]" + continue + } + next[key] = sanitizeAcceptanceMetadata(item) + } + return next + case []any: + next := make([]any, len(typed)) + for index, item := range typed { + next[index] = sanitizeAcceptanceMetadata(item) + } + return next + default: + return value + } +} + +const acceptanceRunColumns = ` +id::text, status, release_sha, api_image_digest, worker_image_digest, +api_key_id, user_id, emulator_base_url, COALESCE(callback_url, ''), +capacity_profile, config, report, COALESCE(failure_reason, ''), +COALESCE(started_at::text, ''), COALESCE(finished_at::text, ''), +COALESCE(promoted_at::text, ''), created_at, updated_at` + +func scanAcceptanceRun(scanner taskScanner) (AcceptanceRun, error) { + var run AcceptanceRun + var config, report []byte + err := scanner.Scan( + &run.ID, &run.Status, &run.ReleaseSHA, &run.APIImageDigest, &run.WorkerImageDigest, + &run.APIKeyID, &run.UserID, &run.EmulatorBaseURL, &run.CallbackURL, + &run.CapacityProfile, &config, &report, &run.FailureReason, + &run.StartedAt, &run.FinishedAt, &run.PromotedAt, &run.CreatedAt, &run.UpdatedAt, + ) + if err != nil { + return AcceptanceRun{}, err + } + run.Config = decodeObject(config) + run.Report = decodeObject(report) + return run, nil +} diff --git a/apps/api/internal/store/acceptance_integration_test.go b/apps/api/internal/store/acceptance_integration_test.go new file mode 100644 index 0000000..405dce6 --- /dev/null +++ b/apps/api/internal/store/acceptance_integration_test.go @@ -0,0 +1,136 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" +) + +func TestAcceptanceTrafficGateAndCASPromotion(t *testing.T) { + databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run acceptance traffic integration test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + applyOIDCJITTestMigrations(t, ctx, databaseURL) + db, err := Connect(ctx, databaseURL) + if err != nil { + t.Fatalf("connect store: %v", err) + } + defer db.Close() + resetTrafficMode := func() { + _, _ = db.pool.Exec(context.Background(), ` +UPDATE system_settings +SET value = '{"mode":"live","revision":0}'::jsonb, updated_at = now() +WHERE setting_key = $1`, SystemSettingGatewayTrafficMode) + } + resetTrafficMode() + t.Cleanup(resetTrafficMode) + + token := "acceptance-token-" + time.Now().Format("150405.000000000") + run, err := db.CreateAcceptanceRun(ctx, CreateAcceptanceRunInput{ + ReleaseSHA: strings.Repeat("a", 40), + APIImageDigest: "sha256:" + strings.Repeat("b", 64), + WorkerImageDigest: "sha256:" + strings.Repeat("c", 64), + APIKeyID: "acceptance-api-key", + UserID: "acceptance-user", + Token: token, + EmulatorBaseURL: "http://acceptance-emulator:8090", + CallbackURL: "http://acceptance-emulator:8090/callbacks", + CapacityProfile: "P24", + }) + if err != nil { + t.Fatalf("create acceptance run: %v", err) + } + encoded, _ := json.Marshal(run) + if strings.Contains(string(encoded), token) { + t.Fatal("acceptance run response exposed the raw token") + } + mode, err := db.ActivateAcceptanceRun(ctx, run.ID) + if err != nil { + t.Fatalf("activate acceptance run: %v", err) + } + if mode.Mode != "validation" || mode.RunID != run.ID || mode.Revision != 1 { + t.Fatalf("unexpected validation mode: %+v", mode) + } + user := &auth.User{ID: "acceptance-user", APIKeyID: "acceptance-api-key"} + if _, err := db.AuthorizeAcceptanceTask(ctx, "", "", user); !errors.Is(err, ErrProductionTrafficPaused) { + t.Fatalf("production request error=%v, want traffic paused", err) + } + if _, err := db.AuthorizeAcceptanceTask(ctx, run.ID, "wrong-token", user); !errors.Is(err, ErrAcceptanceNotAuthorized) { + t.Fatalf("wrong acceptance token error=%v", err) + } + if authorizedRunID, err := db.AuthorizeAcceptanceTask(ctx, run.ID, token, user); err != nil || authorizedRunID != run.ID { + t.Fatalf("authorize acceptance task run=%q err=%v", authorizedRunID, err) + } + if baseURL, credential, err := db.AcceptanceCandidateOverride(ctx, run.ID); err != nil || + baseURL != "http://acceptance-emulator:8090" || credential == "" || credential == token { + t.Fatalf("candidate override base=%q credential=%q err=%v", baseURL, credential, err) + } + if _, err := db.FinishAcceptanceRun(ctx, FinishAcceptanceRunInput{ + RunID: run.ID, Passed: true, Report: map[string]any{"queueFinal": 0}, + }); err != nil { + t.Fatalf("finish acceptance run: %v", err) + } + promotion := PromoteAcceptanceRunInput{ + RunID: run.ID, Revision: mode.Revision, + ReleaseSHA: mode.ReleaseSHA, APIImageDigest: mode.APIImageDigest, WorkerImageDigest: mode.WorkerImageDigest, + } + stale := promotion + stale.Revision++ + if _, err := db.PromoteAcceptanceRun(ctx, stale); !errors.Is(err, ErrAcceptanceStateConflict) { + t.Fatalf("stale promotion error=%v, want state conflict", err) + } + live, err := db.PromoteAcceptanceRun(ctx, promotion) + if err != nil { + t.Fatalf("promote acceptance run: %v", err) + } + if live.Mode != "live" || live.Revision != 2 { + t.Fatalf("unexpected live mode: %+v", live) + } + + failedRun, err := db.CreateAcceptanceRun(ctx, CreateAcceptanceRunInput{ + ReleaseSHA: strings.Repeat("d", 40), + APIImageDigest: "sha256:" + strings.Repeat("e", 64), + WorkerImageDigest: "sha256:" + strings.Repeat("f", 64), + APIKeyID: "acceptance-api-key", + UserID: "acceptance-user", + Token: token + "-retry", + EmulatorBaseURL: "http://acceptance-emulator:8090", + CallbackURL: "http://acceptance-emulator:8090/callbacks", + CapacityProfile: "P24", + }) + if err != nil { + t.Fatalf("create retry acceptance run: %v", err) + } + failedMode, err := db.ActivateAcceptanceRun(ctx, failedRun.ID) + if err != nil { + t.Fatalf("activate retry acceptance run: %v", err) + } + if _, err := db.FinishAcceptanceRun(ctx, FinishAcceptanceRunInput{ + RunID: failedRun.ID, Passed: false, FailureReason: "load failed", + }); err != nil { + t.Fatalf("fail acceptance run: %v", err) + } + if retried, err := db.RetryAcceptanceRun(ctx, failedRun.ID); err != nil || retried.Status != "running" { + t.Fatalf("retry acceptance run=%+v err=%v", retried, err) + } + aborted, err := db.AbortAcceptanceRun(ctx, PromoteAcceptanceRunInput{ + RunID: failedRun.ID, Revision: failedMode.Revision, + ReleaseSHA: failedMode.ReleaseSHA, APIImageDigest: failedMode.APIImageDigest, + WorkerImageDigest: failedMode.WorkerImageDigest, + }) + if err != nil { + t.Fatalf("abort acceptance run: %v", err) + } + if aborted.Mode != "live" || aborted.Revision != failedMode.Revision+1 { + t.Fatalf("unexpected aborted mode: %+v", aborted) + } +} diff --git a/apps/api/internal/store/acceptance_test.go b/apps/api/internal/store/acceptance_test.go new file mode 100644 index 0000000..6120778 --- /dev/null +++ b/apps/api/internal/store/acceptance_test.go @@ -0,0 +1,37 @@ +package store + +import "testing" + +func TestAcceptanceMetadataRedactsSecrets(t *testing.T) { + value := sanitizeAcceptanceMetadata(map[string]any{ + "token": "raw-token", + "nested": map[string]any{ + "databaseUrl": "postgresql://example.invalid/database", + "taskCount": 1200, + }, + }) + next := value.(map[string]any) + if next["token"] != "[REDACTED]" { + t.Fatalf("token=%v", next["token"]) + } + nested := next["nested"].(map[string]any) + if nested["databaseUrl"] != "[REDACTED]" || nested["taskCount"] != 1200 { + t.Fatalf("nested=%v", nested) + } +} + +func TestAcceptanceURLsMustBeAbsoluteAndCredentialFree(t *testing.T) { + for _, raw := range []string{ + "http://acceptance-emulator.easyai.svc.cluster.local:8090", + "https://acceptance.example.invalid/callbacks", + } { + if !validAcceptanceURL(raw) { + t.Fatalf("valid URL rejected: %s", raw) + } + } + for _, raw := range []string{"relative/path", "ftp://example.invalid/file", "https://user:pass@example.invalid"} { + if validAcceptanceURL(raw) { + t.Fatalf("invalid URL accepted: %s", raw) + } + } +} diff --git a/apps/api/internal/store/postgres.go b/apps/api/internal/store/postgres.go index 07b9e96..f6844b4 100644 --- a/apps/api/internal/store/postgres.go +++ b/apps/api/internal/store/postgres.go @@ -545,6 +545,7 @@ type CreateTaskInput struct { Model string `json:"model"` ExternalTaskID string `json:"externalTaskId,omitempty"` RunMode string `json:"runMode"` + AcceptanceRunID string `json:"acceptanceRunId,omitempty"` Async bool `json:"async"` Request map[string]any `json:"request"` ConversationID string `json:"conversationId"` @@ -564,6 +565,7 @@ type GatewayTask struct { ExternalTaskID string `json:"externalTaskId,omitempty"` Kind string `json:"kind"` RunMode string `json:"runMode"` + AcceptanceRunID string `json:"acceptanceRunId,omitempty"` UserID string `json:"userId"` GatewayUserID string `json:"gatewayUserId,omitempty"` UserSource string `json:"userSource,omitempty"` @@ -629,7 +631,8 @@ type GatewayTask struct { } const gatewayTaskColumns = ` -id::text, COALESCE(external_task_id, ''), kind, run_mode, user_id, COALESCE(gateway_user_id::text, ''), user_source, +id::text, COALESCE(external_task_id, ''), kind, run_mode, COALESCE(acceptance_run_id::text, ''), +user_id, COALESCE(gateway_user_id::text, ''), user_source, COALESCE(gateway_tenant_id::text, ''), COALESCE(tenant_id, ''), COALESCE(tenant_key, ''), COALESCE(api_key_id, ''), COALESCE(api_key_name, ''), COALESCE(api_key_prefix, ''), COALESCE(user_group_id::text, ''), COALESCE(user_group_key, ''), model, @@ -2103,15 +2106,19 @@ func (s *Store) CreateTaskIdempotent(ctx context.Context, input CreateTaskInput, task, err := scanGatewayTask(tx.QueryRow(ctx, ` INSERT INTO gateway_tasks ( - external_task_id, kind, run_mode, user_id, gateway_user_id, user_source, gateway_tenant_id, tenant_id, tenant_key, + external_task_id, kind, run_mode, acceptance_run_id, user_id, gateway_user_id, user_source, gateway_tenant_id, tenant_id, tenant_key, api_key_id, api_key_name, api_key_prefix, user_group_id, user_group_key, model, requested_model, request, async_mode, status, result, billings, conversation_id, new_message_count, idempotency_key_hash, idempotency_request_hash, finished_at ) - VALUES (NULLIF($1, ''), $2, $3, $4, NULLIF($5, '')::uuid, COALESCE(NULLIF($6, ''), 'gateway'), NULLIF($7, '')::uuid, NULLIF($8, ''), NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, ''), NULLIF($13, '')::uuid, NULLIF($14, ''), $15, $15, $16, $17, $18, $19::jsonb, $20::jsonb, NULLIF($21, '')::uuid, $22, NULLIF($23, ''), NULLIF($24, ''), NULL) + VALUES (NULLIF($1, ''), $2, $3, NULLIF($4, '')::uuid, $5, NULLIF($6, '')::uuid, COALESCE(NULLIF($7, ''), 'gateway'), NULLIF($8, '')::uuid, NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, ''), NULLIF($13, ''), NULLIF($14, '')::uuid, NULLIF($15, ''), $16, $16, $17, $18, $19, $20::jsonb, $21::jsonb, NULLIF($22, '')::uuid, $23, NULLIF($24, ''), NULLIF($25, ''), NULL) ON CONFLICT (user_id, idempotency_key_hash) WHERE idempotency_key_hash IS NOT NULL DO NOTHING RETURNING `+gatewayTaskColumns, - strings.TrimSpace(input.ExternalTaskID), input.Kind, runMode, user.ID, user.GatewayUserID, user.Source, user.GatewayTenantID, user.TenantID, user.TenantKey, user.APIKeyID, user.APIKeyName, user.APIKeyPrefix, user.UserGroupID, user.UserGroupKey, input.Model, requestBody, input.Async, status, resultBody, billingsBody, input.ConversationID, input.NewMessageCount, strings.TrimSpace(input.IdempotencyKeyHash), strings.TrimSpace(input.IdempotencyRequestHash), + strings.TrimSpace(input.ExternalTaskID), input.Kind, runMode, strings.TrimSpace(input.AcceptanceRunID), + user.ID, user.GatewayUserID, user.Source, user.GatewayTenantID, user.TenantID, user.TenantKey, + user.APIKeyID, user.APIKeyName, user.APIKeyPrefix, user.UserGroupID, user.UserGroupKey, + input.Model, requestBody, input.Async, status, resultBody, billingsBody, input.ConversationID, + input.NewMessageCount, strings.TrimSpace(input.IdempotencyKeyHash), strings.TrimSpace(input.IdempotencyRequestHash), )) replayed := false if errors.Is(err, pgx.ErrNoRows) && strings.TrimSpace(input.IdempotencyKeyHash) != "" { @@ -2204,6 +2211,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) { &task.ExternalTaskID, &task.Kind, &task.RunMode, + &task.AcceptanceRunID, &task.UserID, &task.GatewayUserID, &task.UserSource, @@ -2461,6 +2469,12 @@ func normalizeRunMode(input string, request map[string]any) string { if mode == "simulation" { return "simulation" } + if mode == "acceptance" { + return "acceptance" + } + if mode == "acceptance_canary" { + return "acceptance_canary" + } return "production" } diff --git a/apps/api/internal/store/postgres_pool_metrics.go b/apps/api/internal/store/postgres_pool_metrics.go new file mode 100644 index 0000000..ed8a921 --- /dev/null +++ b/apps/api/internal/store/postgres_pool_metrics.go @@ -0,0 +1,25 @@ +package store + +type PostgresPoolMetricsSnapshot struct { + MaxConnections int32 + TotalConnections int32 + AcquiredConnections int32 + IdleConnections int32 + EmptyAcquireCount int64 + CanceledAcquireCount int64 +} + +func (s *Store) PostgresPoolMetrics() PostgresPoolMetricsSnapshot { + if s == nil || s.pool == nil { + return PostgresPoolMetricsSnapshot{} + } + statistics := s.pool.Stat() + return PostgresPoolMetricsSnapshot{ + MaxConnections: statistics.MaxConns(), + TotalConnections: statistics.TotalConns(), + AcquiredConnections: statistics.AcquiredConns(), + IdleConnections: statistics.IdleConns(), + EmptyAcquireCount: statistics.EmptyAcquireCount(), + CanceledAcquireCount: statistics.CanceledAcquireCount(), + } +} diff --git a/apps/api/internal/store/tasks_runtime.go b/apps/api/internal/store/tasks_runtime.go index 0b2edcb..ae2940c 100644 --- a/apps/api/internal/store/tasks_runtime.go +++ b/apps/api/internal/store/tasks_runtime.go @@ -294,7 +294,7 @@ func (s *Store) ClaimTaskExecution(ctx context.Context, taskID string, execution if err := tx.QueryRow(ctx, ` SELECT status = 'queued' AND next_run_at <= now(), status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now()), - run_mode = 'production', + run_mode IN ('production', 'acceptance', 'acceptance_canary'), gateway_user_id IS NOT NULL FROM gateway_tasks WHERE id = $1::uuid @@ -431,7 +431,7 @@ SELECT status = 'queued' OR execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now() ), - run_mode = 'production', + run_mode IN ('production', 'acceptance', 'acceptance_canary'), gateway_user_id IS NOT NULL FROM gateway_tasks WHERE id = $1::uuid @@ -513,7 +513,7 @@ SET status = 'failed', error_code = NULLIF($2, ''), error_message = NULLIF($3, ''), billing_status = CASE - WHEN run_mode <> 'production' OR gateway_user_id IS NULL THEN 'not_required' + WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required' ELSE 'released' END, billing_updated_at = now(), @@ -910,7 +910,7 @@ SET status = 'cancelled', error_code = 'client_disconnected', error_message = NULLIF($3::text, ''), billing_status = CASE - WHEN run_mode <> 'production' OR gateway_user_id IS NULL THEN 'not_required' + WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required' ELSE 'released' END, billing_updated_at = now(), @@ -991,7 +991,7 @@ SET status = 'cancelled', error_code = 'task_cancelled', error_message = NULLIF($2, ''), billing_status = CASE - WHEN run_mode <> 'production' OR gateway_user_id IS NULL THEN 'not_required' + WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required' WHEN reservation_amount > 0 THEN 'pending' ELSE 'released' END, @@ -1027,7 +1027,7 @@ SELECT id, 'task.billing.release', 'release', reservation_amount, billing_curren pricing_snapshot, $2::jsonb, 'pending', now() FROM gateway_tasks WHERE id = $1::uuid - AND run_mode = 'production' + AND run_mode IN ('production', 'acceptance', 'acceptance_canary') AND gateway_user_id IS NOT NULL AND reservation_amount > 0 ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)) @@ -1569,7 +1569,7 @@ SET status = 'succeeded', final_charge_amount = $9::numeric, billing_version = 'effective-pricing-v2', billing_status = CASE - WHEN run_mode = 'production' AND gateway_user_id IS NOT NULL THEN 'pending' + WHEN run_mode IN ('production', 'acceptance', 'acceptance_canary') AND gateway_user_id IS NOT NULL THEN 'pending' ELSE 'not_required' END, billing_currency = $10, @@ -1624,7 +1624,7 @@ INSERT INTO settlement_outbox ( SELECT id, 'task.billing.settle', 'settle', $2::numeric, $3, $4::jsonb, $5::jsonb, 'pending', now() FROM gateway_tasks WHERE id = $1::uuid - AND run_mode = 'production' + AND run_mode IN ('production', 'acceptance', 'acceptance_canary') AND gateway_user_id IS NOT NULL ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, finalChargeAmount, currency, string(pricingSnapshotJSON), string(payloadJSON)) @@ -1725,7 +1725,7 @@ SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currenc pricing_snapshot, $2::jsonb, 'manual_review', now(), $3 FROM gateway_tasks WHERE id = $1::uuid - AND run_mode = 'production' + AND run_mode IN ('production', 'acceptance', 'acceptance_canary') AND gateway_user_id IS NOT NULL ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON), input.Code) return err @@ -1902,7 +1902,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn response_duration_ms = $8, result = $9::jsonb, billing_status = CASE - WHEN run_mode <> 'production' OR gateway_user_id IS NULL THEN 'not_required' + WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required' WHEN reservation_amount > 0 THEN 'pending' ELSE 'released' END, @@ -1944,7 +1944,7 @@ SELECT id, 'task.billing.release', 'release', reservation_amount, billing_curren pricing_snapshot, $2::jsonb, 'pending', now() FROM gateway_tasks WHERE id = $1::uuid - AND run_mode = 'production' + AND run_mode IN ('production', 'acceptance', 'acceptance_canary') AND gateway_user_id IS NOT NULL AND reservation_amount > 0 ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON)) diff --git a/apps/api/migrations/0094_production_acceptance_runs.sql b/apps/api/migrations/0094_production_acceptance_runs.sql new file mode 100644 index 0000000..fd5a00f --- /dev/null +++ b/apps/api/migrations/0094_production_acceptance_runs.sql @@ -0,0 +1,50 @@ +CREATE TABLE gateway_acceptance_runs ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + status text NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'running', 'passed', 'failed', 'promoted', 'aborted')), + release_sha text NOT NULL, + api_image_digest text NOT NULL, + worker_image_digest text NOT NULL, + api_key_id text NOT NULL, + user_id text NOT NULL, + token_sha256 text NOT NULL, + emulator_base_url text NOT NULL, + callback_url text, + capacity_profile text NOT NULL DEFAULT 'P24' + CHECK (capacity_profile IN ('P24', 'P28', 'P32')), + config jsonb NOT NULL DEFAULT '{}'::jsonb, + report jsonb NOT NULL DEFAULT '{}'::jsonb, + failure_reason text, + started_at timestamptz, + finished_at timestamptz, + promoted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX idx_gateway_acceptance_runs_status_created + ON gateway_acceptance_runs (status, created_at DESC); + +ALTER TABLE gateway_tasks + ADD COLUMN acceptance_run_id uuid REFERENCES gateway_acceptance_runs(id), + ADD CONSTRAINT gateway_tasks_acceptance_run_mode_check CHECK ( + ( + run_mode IN ('acceptance', 'acceptance_canary') + AND acceptance_run_id IS DISTINCT FROM NULL + ) + OR ( + run_mode NOT IN ('acceptance', 'acceptance_canary') + AND acceptance_run_id IS NULL + ) + ) NOT VALID; + +CREATE INDEX idx_gateway_tasks_acceptance_run + ON gateway_tasks (acceptance_run_id, status, created_at) + WHERE acceptance_run_id IS NOT NULL; + +INSERT INTO system_settings (setting_key, value) +VALUES ( + 'gateway_traffic_mode', + '{"mode":"live","revision":0}'::jsonb +) +ON CONFLICT (setting_key) DO NOTHING; diff --git a/deploy/kubernetes/acceptance/kustomization.yaml b/deploy/kubernetes/acceptance/kustomization.yaml new file mode 100644 index 0000000..1552678 --- /dev/null +++ b/deploy/kubernetes/acceptance/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - protocol-emulator.yaml diff --git a/deploy/kubernetes/acceptance/protocol-emulator.yaml b/deploy/kubernetes/acceptance/protocol-emulator.yaml new file mode 100644 index 0000000..8880c12 --- /dev/null +++ b/deploy/kubernetes/acceptance/protocol-emulator.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: easyai-acceptance-emulator + namespace: easyai + labels: + app.kubernetes.io/name: easyai-acceptance-emulator + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: easyai-acceptance-emulator + template: + metadata: + labels: + app.kubernetes.io/name: easyai-acceptance-emulator + app.kubernetes.io/part-of: easyai-ai-gateway + spec: + nodeSelector: + easyai.io/site: ningbo + easyai.io/workload: "true" + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: emulator + image: easyai-api + command: ["/app/easyai-ai-gateway-acceptance-emulator"] + env: + - name: HTTP_ADDR + value: ":8090" + ports: + - name: http + containerPort: 8090 + readinessProbe: + httpGet: + path: /healthz + port: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "2" + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true +--- +apiVersion: v1 +kind: Service +metadata: + name: easyai-acceptance-emulator + namespace: easyai +spec: + selector: + app.kubernetes.io/name: easyai-acceptance-emulator + ports: + - name: http + port: 8090 + targetPort: http diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index 0784a48..95922e8 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -14,10 +14,40 @@ source "$config_file" : "${NAMESPACE:=easyai}" : "${PUBLIC_BASE_URL:=https://ai.51easyai.com}" : "${K3S_BIN:=/usr/local/bin/k3s}" +: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}" +: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}" +: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}" +: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}" +: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}" +: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}" +: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}" [[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || { echo 'invalid cluster release configuration' >&2 exit 1 } +for capacity_value in \ + "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ + "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ + "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ + "$AI_GATEWAY_DATABASE_MAX_CONNS" \ + "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \ + "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do + [[ $capacity_value =~ ^[1-9][0-9]*$ ]] || { + echo 'worker capacity configuration must contain positive integers' >&2 + exit 1 + } +done +(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 && + AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 && + AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 && + AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 && + AI_GATEWAY_DATABASE_MAX_CONNS <= 256 && + AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 && + AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || { + echo 'worker capacity configuration exceeds the release safety bounds' >&2 + exit 1 +} [[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { echo 'invalid public base URL' >&2 exit 1 @@ -87,11 +117,47 @@ verify_site() { rollout_worker_site() { local site=$1 local api_image=$2 - "${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \ - "worker=$api_image" + local replicas + case $site in + ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;; + hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;; + *) return 1 ;; + esac + "${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \ + "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ + "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ + "AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \ + "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \ + "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" + "${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \ + --replicas="$replicas" + if [[ -n $api_image ]]; then + "${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \ + "worker=$api_image" + fi "${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \ -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]] + [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \ + -o jsonpath='{.status.readyReplicas}') == "$replicas" ]] +} + +rollout_api_capacity_site() { + local site=$1 + "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \ + "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" + "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s +} + +apply_capacity_config() { + rollout_worker_site hongkong "" + rollout_worker_site ningbo "" + rollout_api_capacity_site hongkong + rollout_api_capacity_site ningbo + verify_site hongkong + verify_site ningbo + wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true' + echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" } rollout_site() { @@ -101,6 +167,8 @@ rollout_site() { local api_changed=$4 local web_changed=$5 if [[ $api_changed == true ]]; then + "${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \ + "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \ "api=$api_image" "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s @@ -305,8 +373,12 @@ case ${1:-} in [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; } activate_manifest "$target" rollback ;; + capacity) + [[ $# -eq 1 ]] || exit 64 + apply_capacity_config + ;; *) - echo 'usage: easyai-ai-gateway-cluster-release {status|adopt |deploy |rollback }' >&2 + echo 'usage: easyai-ai-gateway-cluster-release {status|adopt |deploy |rollback |capacity}' >&2 exit 64 ;; esac diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example index d25fc35..1f2e399 100644 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example @@ -2,3 +2,13 @@ RELEASES_DIR=/var/lib/easyai-ai-gateway-cluster-release/releases NAMESPACE=easyai PUBLIC_BASE_URL=https://ai.51easyai.com K3S_BIN=/usr/local/bin/k3s + +# Worker 容量只需修改本发布配置并执行获授权的 `capacity`，不再修改 Go 代码。 +# 副本数属于 Deployment 配置，不能由容器内部环境变量改变；发布工具读取这两个变量并写入 replicas。 +AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 +AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 +AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24 +AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 +AI_GATEWAY_DATABASE_MAX_CONNS=32 +AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24 +AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 diff --git a/deploy/kubernetes/production/application-config.yaml b/deploy/kubernetes/production/application-config.yaml index 750aca5..0d1760e 100644 --- a/deploy/kubernetes/production/application-config.yaml +++ b/deploy/kubernetes/production/application-config.yaml @@ -37,5 +37,5 @@ data: AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: "268435456" AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: "536870912" AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true" - AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "2048" + AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "48" AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5" diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index 09eb9c4..eddd98e 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -316,6 +316,8 @@ spec: value: "24" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY value: "24" + - name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY + value: "24" - name: POD_NAMESPACE valueFrom: fieldRef: @@ -445,6 +447,8 @@ spec: value: "24" - name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY value: "24" + - name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY + value: "24" - name: POD_NAMESPACE valueFrom: fieldRef: diff --git a/docs/design.md b/docs/design.md index e512773..f6901cb 100644 --- a/docs/design.md +++ b/docs/design.md @@ -461,7 +461,7 @@ Gateway Web Console 默认使用 HttpOnly OIDC Cookie 会话解决标签页隔 - `idempotency_key` - `remote_task_id` - `remote_task_payload`：仅保存 provider 显式白名单内、跨进程恢复不可重算的 checkpoint；禁止保存提交响应、最终响应和完整请求副本。 -- `run_mode`：`production`、`simulation` +- `run_mode`：`production`、`simulation`、`acceptance`、`acceptance_canary` - `simulation_profile` - `simulation_seed` - `locked_by` @@ -990,7 +990,7 @@ Provider 原始响应只在内存中参与标准结果转换，不进入 `result | `id` | `uuid` | PK | Gateway 任务 ID | | `external_task_id` | `text` | nullable | 与 `server-main` / 前端兼容的外部任务 ID | | `kind` | `text` | not null | `chat.completions`、`images.generations` 等 | -| `run_mode` | `text` | not null | `production`、`simulation` | +| `run_mode` | `text` | not null | `production`、`simulation`、`acceptance`、`acceptance_canary` | | `user_id` | `text` | not null | 请求 claim 中的用户 ID | | `gateway_user_id` | `uuid` | nullable | Gateway 用户表 ID | | `user_source` | `text` | not null, default `gateway` | `gateway`、`server-main`、`sync` | diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md index 74418ed..882fa31 100644 --- a/docs/operations/k3s-ha-runbook.md +++ b/docs/operations/k3s-ha-runbook.md @@ -6,6 +6,9 @@ CloudNativePG；洛杉矶只作为 K3s server/etcd 仲裁节点，并带 `easyai.io/witness=true:NoSchedule` 污点。 +生产上线前的双节点高媒体负载、Worker 强杀和 P24/P28/P32 容量搜索见 +[生产同构验收模式与高媒体压力测试](production-acceptance.md)。 + - 健康双库：同步复制，RPO=0。 - 单库降级：`dataDurability: preferred` 保持写入，`archive_timeout=60s`，灾难 RPO 目标不超过 5 分钟。 - 数据库故障 RTO：不超过 5 分钟。 diff --git a/docs/operations/production-acceptance.md b/docs/operations/production-acceptance.md new file mode 100644 index 0000000..4342910 --- /dev/null +++ b/docs/operations/production-acceptance.md @@ -0,0 +1,131 @@ +# 生产同构验收模式与高媒体压力测试 + +## 目标与边界 + +生产同构验收使用当前生产 API、双站点 Worker、PostgreSQL、River、候选路由、限流、 +账务、回调和文件物化链路。只有候选确定后的上游连接会被克隆并指向集群内的协议模拟器， +因此不会绕过真实 Gemini 与 Volces 客户端实现。 + +验收脚本是显式生产写操作，会暂停新正式任务、调整 Worker 容量并在恢复场景删除一个 +Worker Pod。它不会自动发布新版本；只能针对已经发布、完整 SHA 和 digest 固定的 manifest +执行。 + +## 隔离与流量门禁 + +Gateway 流量模式保存在数据库中，支持 `live` 和 `validation`： + +- `live`：正常接收正式任务，带验收 Header 的请求被拒绝。 +- `validation`：普通新任务返回 `503 validation_in_progress`；已有正式任务继续排空。 +- 验收请求必须同时匹配当前 Run ID、随机 Run Token、专属 API Key ID 和专属用户 ID。 +- 普通生产请求体不能打开 `simulation`。 +- `X-EasyAI-Acceptance-Upstream: real` 只对已授权验收身份有效，用于最后两条真实小流量请求。 + +验收请求使用以下 Header，Token 只存在于私有环境和进程内，不写入数据库、报告或日志： + +```text +X-EasyAI-Acceptance-Run: +X-EasyAI-Acceptance-Token: +X-EasyAI-Acceptance-Upstream: real +``` + +Run 记录保存 release SHA、API/Worker image digest、容量档位和脱敏报告。切回 `live` 必须 +同时匹配 Run ID、流量模式 revision、release SHA 与两个 digest，并且 Run 已标记为 +`passed`。CAS 任一项变化都会拒绝切换。 + +## 验收负载 + +Gemini 原生图片编辑调用 `generateContent`，输入 `contents.parts` 同时包含文本和 +`inlineData`，并设置 `responseModalities=["IMAGE"]`。压测端边读 Base64 边解码和计算 +SHA-256，不集中保存响应： + +| Profile | 请求数 | 输入图片 | 输出图片 | 模拟延迟 | +|---|---:|---:|---:|---:| +| `gemini-baseline` | 1000 | 256 KiB | 256 KiB | 4 秒 | +| `gemini-large` | 128 | 2 MiB | 4 MiB | 8–15 秒 | +| `gemini-peak` | 32 | 8 MiB | 8 MiB | 15–30 秒 | + +多参考图视频使用正式 `content` 结构，按 60%/30%/10% 分配 3/6/9 图，包含 JPEG、PNG、 +WebP 和 4K 图片。128 组输入组合避免只命中相同缓存；每四个任务至少一个携带 4K 图片并 +要求在到达上游前完成缩放或重编码。 + +- `video-throughput`：1200 个任务并发提交，提交窗口不超过 10 秒。 +- `video-recovery`：96 个 2–3 分钟任务，运行中删除香港 Worker Pod，验证远程任务恢复、 + 租约续期和重复提交保护。 +- `real-canary`：一条 Gemini 带图编辑和一条多参考图视频，使用真实候选与真实上游。 + +模拟器实现 Gemini `generateContent` 与 Volces +`/contents/generations/tasks` 提交、查询、取消协议，还提供格式混合的图片池和独立回调 +收集器。它校验参考图数量、角色、格式、尺寸与内容哈希，并统计重复上游提交和重复回调。 + +## Worker 配置 + +执行槽、连接池和媒体并发进入 Pod 环境变量，副本数由发布工具更新 Deployment： + +```text +AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT +AI_GATEWAY_DATABASE_MAX_CONNS +AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY +AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY +AI_GATEWAY_WORKER_REPLICAS_NINGBO +AI_GATEWAY_WORKER_REPLICAS_HONGKONG +``` + +`AI_GATEWAY_WORKER_REPLICAS_*` 是发布工具配置，不会传入容器。当前验收固定宁波、香港各 +一个 2 GiB Worker，不创建额外 Pod： + +| 档位 | 单实例执行槽 | 数据库池 | 媒体并发 | 全局执行槽 | +|---|---:|---:|---:|---:| +| P24 | 24 | 32 | 24 | 48 | +| P28 | 28 | 36 | 28 | 56 | +| P32 | 32 | 40 | 32 | 64 | + +每档依次运行全部五个模拟 Profile 三次。失败时恢复上一个已完整通过的档位，流量保持 +`validation`，不会自动放开正式请求。 + +常规容量调整只需修改服务器上的私有发布配置并执行获授权的滚动命令，不要求源码改动或 +新镜像： + +```bash +easyai-ai-gateway-cluster-release capacity +``` + +## 执行 + +先把专属验收用户、钱包、API Key、管理员 Token、两个 Gateway 地址和三张真实小流量 +参考图写入本地私有环境。模型变量可省略，脚本会从当前启用候选中选择 Gemini 图片编辑 +模型，以及 `omni_video.max_images >= 9` 的视频模型并优先 Seedance 2.0/fast。 + +为保证请求严格各有一半命中宁波和香港，可以把两个 Gateway URL 配成两个节点的 +`https://<节点IP>`，并设置 +`AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=ai.51easyai.com`。压测器会继续校验正式 +证书，并同时使用该域名作为 HTTP Host；这不是跳过 TLS 校验。 + +```bash +scripts/cluster/run-production-acceptance.sh \ + --execute dist/releases/<完整SHA>.json +``` + +脚本依次执行： + +1. 校验工作区、manifest、线上 release SHA、digest、双站点镜像和副本数。 +2. 部署 digest 固定且仅集群内可访问的协议模拟器。 +3. 创建 Run、切换 `validation`、等待已有正式任务排空。 +4. 按 P24、P28、P32 执行三轮负载和 Worker 强杀。 +5. 执行两条真实小流量请求。 +6. 校验任务、账务、回调、队列、连接池、RSS、节点内存、数据库同步、六向 WireGuard 和 + 公网健康。 +7. 重新校验 SHA/digest/revision 后切回 `live`。 + +报告保存在 `dist/acceptance//`，包含每阶段吞吐、p50/p95/p99、队列和资源 CSV、 +协议模拟器统计及最终摘要。报告不包含图片原文、Base64、Token、密码或连接串。 + +## 通过和失败处理 + +通过要求包括：任务全部成功、Gemini 输出可解码且哈希/字节数正确、视频参考图校验全部 +通过、无重复任务/上游提交/结算/回调、队列归零、Worker 心跳小于 30 秒、单实例分配不 +超过当前档位、无 OOM/重启、Pod RSS 小于 1.5 GiB、节点内存低于 80%、PostgreSQL 连接 +低于 75%、连接池不连续满载、双副本同步、六向链路和公网健康通过。 + +失败 Run 会记录原因、恢复上一稳定容量并保持 `validation`。排除问题后调用 Run 的 +`retry` 接口重新执行。只有人工决定放弃验收时，才使用带相同 CAS 字段的 `abort` 接口 +恢复 `live`；该操作不会把失败验收标记为通过。 diff --git a/scripts/cluster/install-release-helper.sh b/scripts/cluster/install-release-helper.sh index 572e3c3..86f7430 100755 --- a/scripts/cluster/install-release-helper.sh +++ b/scripts/cluster/install-release-helper.sh @@ -7,15 +7,20 @@ source "$script_dir/common.sh" load_cluster_env cluster_ssh "$CLUSTER_NINGBO_HOST" \ - 'install -d -m 0755 /usr/local/lib/easyai-ai-gateway-release' + 'install -d -m 0755 /usr/local/lib/easyai-ai-gateway-release /usr/local/share/easyai-ai-gateway-release' cluster_scp "$cluster_root/deploy/kubernetes/easyai-ai-gateway-cluster-release" \ "$CLUSTER_NINGBO_HOST:/usr/local/sbin/easyai-ai-gateway-cluster-release" cluster_scp "$cluster_root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example" \ - "$CLUSTER_NINGBO_HOST:/etc/easyai-ai-gateway-cluster-release.conf" + "$CLUSTER_NINGBO_HOST:/usr/local/share/easyai-ai-gateway-release/easyai-ai-gateway-cluster-release.conf.example" cluster_scp "$cluster_root/scripts/release-manifest.mjs" \ "$CLUSTER_NINGBO_HOST:/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs" cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' set -euo pipefail +if [[ ! -e /etc/easyai-ai-gateway-cluster-release.conf ]]; then + install -m 0640 \ + /usr/local/share/easyai-ai-gateway-release/easyai-ai-gateway-cluster-release.conf.example \ + /etc/easyai-ai-gateway-cluster-release.conf +fi chmod 0750 /usr/local/sbin/easyai-ai-gateway-cluster-release chmod 0640 /etc/easyai-ai-gateway-cluster-release.conf chmod 0755 /usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh new file mode 100755 index 0000000..065a52d --- /dev/null +++ b/scripts/cluster/run-production-acceptance.sh @@ -0,0 +1,913 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: + scripts/cluster/run-production-acceptance.sh --execute dist/releases/.json + +This is a destructive production acceptance workflow. It pauses new live tasks, +changes Worker capacity, creates paid real canaries, and deletes one Worker Pod +during each recovery phase. A failed run deliberately keeps traffic in +validation mode and restores only the last stable capacity profile. + +Required private environment values: + AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN + AI_GATEWAY_ACCEPTANCE_API_KEY + AI_GATEWAY_ACCEPTANCE_API_KEY_ID + AI_GATEWAY_ACCEPTANCE_USER_ID + AI_GATEWAY_ACCEPTANCE_GATEWAYS + AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS + +Optional model overrides (otherwise selected from current production candidates): + AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL + AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL + AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME +EOF +} + +[[ ${1:-} == --execute && $# -eq 2 ]] || { + usage >&2 + exit 64 +} + +release_manifest=$2 +[[ -f $release_manifest && ! -L $release_manifest ]] || { + echo 'release manifest must be a regular file' >&2 + exit 1 +} + +load_cluster_env +require_commands curl git go jq node openssl sed + +: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}" +: "${AI_GATEWAY_ACCEPTANCE_API_KEY:?}" +: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:?}" +: "${AI_GATEWAY_ACCEPTANCE_USER_ID:?}" +: "${AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL:=}" +: "${AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL:=}" +: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}" +: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:?}" +: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:?}" + +node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null +release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha) +api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api) +[[ $release_sha =~ ^[0-9a-f]{40}$ && $api_image =~ @sha256:[0-9a-f]{64}$ ]] || { + echo 'release manifest is not full-SHA and digest pinned' >&2 + exit 1 +} +api_digest=${api_image##*@} +worker_digest=$api_digest +[[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" ]] || { + echo 'working copy HEAD must equal the release SHA' >&2 + exit 1 +} +[[ -z $(git -C "$cluster_root" status --short) ]] || { + echo 'production acceptance requires a clean release working copy' >&2 + exit 1 +} + +namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai} +remote_release_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-cluster-release} +[[ $namespace =~ ^[a-z0-9-]+$ && $remote_release_helper =~ ^/[A-Za-z0-9._/-]+$ ]] || { + echo 'invalid namespace or remote release helper' >&2 + exit 1 +} +[[ $namespace == easyai ]] || { + echo 'the production acceptance manifests currently require namespace easyai' >&2 + exit 1 +} + +temporary_root=$(mktemp -d) +chmod 0700 "$temporary_root" +current_manifest=$temporary_root/current.json +load_binary=$temporary_root/easyai-ai-gateway-acceptance-load +run_token=$(openssl rand -hex 32) +run_id= +report_root= +stable_profile=P24 +active_profile=P24 +failure_reason= +failure_recorded=false + +cleanup() { + local status=$? + trap - EXIT + if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then + set +e + apply_capacity_profile "$stable_profile" >/dev/null 2>&1 + mark_run_failed 'acceptance workflow exited unexpectedly' + set -e + fi + rm -rf -- "$temporary_root" + exit "$status" +} +trap cleanup EXIT + +remote_kubectl() { + local command_text + printf -v command_text '%q ' "$@" + cluster_ssh "$CLUSTER_NINGBO_HOST" "k3s kubectl $command_text" +} + +database_query() { + local sql=$1 + local primary + primary=$(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.currentPrimary}') + [[ $primary =~ ^easyai-postgres-[0-9]+$ ]] + remote_kubectl exec -n "$namespace" "$primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql" +} + +admin_request() { + local method=$1 + local path=$2 + local body=${3:-} + local output=$4 + local admin_base=${AI_GATEWAY_ACCEPTANCE_GATEWAYS%%,*} + local status + if [[ -n $body ]]; then + status=$(curl -sS --max-time 30 -o "$output" -w '%{http_code}' \ + -X "$method" \ + -H "Authorization: Bearer $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" \ + -H 'Content-Type: application/json' \ + --data-binary "$body" \ + "$admin_base$path") + else + status=$(curl -sS --max-time 30 -o "$output" -w '%{http_code}' \ + -X "$method" \ + -H "Authorization: Bearer $AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" \ + "$admin_base$path") + fi + [[ $status =~ ^2[0-9][0-9]$ ]] || { + echo "acceptance control API failed: method=$method path=$path status=$status" >&2 + return 1 + } +} + +verify_release_cas() { + cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_release_helper status" >"$current_manifest" + node "$cluster_root/scripts/release-manifest.mjs" validate "$current_manifest" >/dev/null + local current_sha current_api + current_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" sourceSha) + current_api=$(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" images.api) + [[ $current_sha == "$release_sha" && $current_api == "$api_image" ]] || { + echo "production release CAS mismatch: expected=$release_sha current=$current_sha" >&2 + return 1 + } + for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do + [[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \ + -o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]] + done + [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \ + -o 'jsonpath={.spec.replicas}') == 1 ]] + [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \ + -o 'jsonpath={.spec.replicas}') == 1 ]] +} + +wait_for_existing_tasks_to_drain() { + local deadline=$((SECONDS + 900)) + local active + while (( SECONDS < deadline )); do + active=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id IS NULL AND status IN ('queued','running');") + if [[ $active == 0 ]]; then + return 0 + fi + echo "waiting_for_existing_tasks=$active" + sleep 5 + done + echo 'existing production tasks did not drain within 15 minutes' >&2 + return 1 +} + +select_acceptance_models() { + if [[ -z $AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL ]]; then + AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$(database_query " +SELECT b.invocation_name +FROM platform_models model +JOIN integration_platforms platform ON platform.id=model.platform_id +JOIN base_model_catalog b ON b.id=model.base_model_id +WHERE platform.status='enabled' AND platform.deleted_at IS NULL + AND model.enabled=true + AND lower(platform.provider) IN ('gemini','google-gemini','gemini-openai') + AND model.model_type @> '[\"image_edit\"]'::jsonb +ORDER BY + CASE WHEN lower(b.invocation_name) LIKE '%gemini%image%' THEN 0 ELSE 1 END, + platform.priority, model.created_at +LIMIT 1;") + fi + if [[ -z $AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL ]]; then + AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$(database_query " +SELECT b.invocation_name +FROM platform_models model +JOIN integration_platforms platform ON platform.id=model.platform_id +JOIN base_model_catalog b ON b.id=model.base_model_id +WHERE platform.status='enabled' AND platform.deleted_at IS NULL + AND model.enabled=true + AND model.model_type @> '[\"omni_video\"]'::jsonb + AND COALESCE( + NULLIF(model.capability_override #>> '{omni_video,max_images}','')::int, + NULLIF(model.capabilities #>> '{omni_video,max_images}','')::int, + NULLIF(b.capabilities #>> '{omni_video,max_images}','')::int, + 0 + ) >= 9 +ORDER BY + CASE + WHEN lower(b.invocation_name) LIKE '%seedance%2%fast%' THEN 0 + WHEN lower(b.invocation_name) LIKE '%seedance%2%' THEN 1 + ELSE 2 + END, + platform.priority, model.created_at +LIMIT 1;") + fi + [[ -n $AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL && -n $AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL ]] || { + echo 'no enabled Gemini image-edit or omni-video max_images>=9 model was found' >&2 + return 1 + } + export AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL +} + +deploy_protocol_emulator() { + sed "s|image: easyai-api|image: $api_image|" \ + "$cluster_root/deploy/kubernetes/acceptance/protocol-emulator.yaml" | + cluster_ssh "$CLUSTER_NINGBO_HOST" 'k3s kubectl apply -f -' >/dev/null + remote_kubectl rollout status deployment/easyai-acceptance-emulator \ + -n "$namespace" --timeout=300s +} + +create_and_activate_run() { + local create_response=$temporary_root/create-run.json + local activate_response=$temporary_root/activate-run.json + local body + body=$(jq -cn \ + --arg releaseSha "$release_sha" \ + --arg apiDigest "$api_digest" \ + --arg workerDigest "$worker_digest" \ + --arg apiKeyID "$AI_GATEWAY_ACCEPTANCE_API_KEY_ID" \ + --arg userID "$AI_GATEWAY_ACCEPTANCE_USER_ID" \ + --arg token "$run_token" \ + --arg emulatorURL 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' \ + --arg callbackURL 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090/callbacks' \ + '{ + releaseSha: $releaseSha, + apiImageDigest: $apiDigest, + workerImageDigest: $workerDigest, + apiKeyId: $apiKeyID, + userId: $userID, + token: $token, + emulatorBaseUrl: $emulatorURL, + callbackUrl: $callbackURL, + capacityProfile: "P24", + config: {workloads: ["gemini_image_edit", "multi_reference_video"]} + }') + admin_request POST /api/admin/system/acceptance/runs "$body" "$create_response" + run_id=$(jq -r '.id // empty' "$create_response") + [[ $run_id =~ ^[0-9a-f-]{36}$ ]] || { + echo 'acceptance control API returned an invalid Run ID' >&2 + return 1 + } + report_root=${AI_GATEWAY_ACCEPTANCE_REPORT_DIR:-"$cluster_root/dist/acceptance/$run_id"} + mkdir -p "$report_root" + chmod 0700 "$report_root" + admin_request POST "/api/admin/system/acceptance/runs/$run_id/activate" '' "$activate_response" + [[ $(jq -r '.mode' "$activate_response") == validation ]] +} + +mark_run_failed() { + local reason=$1 + local response=$temporary_root/failed-run.json + local body + [[ -n $run_id ]] || return 0 + body=$(jq -cn --arg reason "$reason" \ + '{passed:false,failureReason:$reason,report:{queueFinal:null}}') + if admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$body" "$response"; then + failure_recorded=true + fi +} + +apply_capacity_profile() { + local profile=$1 + local slots pool media global + case $profile in + P24) slots=24; pool=32; media=24; global=48 ;; + P28) slots=28; pool=36; media=28; global=56 ;; + P32) slots=32; pool=40; media=32; global=64 ;; + *) return 1 ;; + esac + active_profile=$profile + for site in ningbo hongkong; do + remote_kubectl set env "deployment/easyai-worker-$site" -n "$namespace" \ + "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$slots" \ + "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \ + "AI_GATEWAY_DATABASE_MAX_CONNS=$pool" \ + "AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$media" \ + "AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$media" >/dev/null + remote_kubectl rollout status "deployment/easyai-worker-$site" \ + -n "$namespace" --timeout=300s + remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \ + "AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" >/dev/null + remote_kubectl rollout status "deployment/easyai-api-$site" \ + -n "$namespace" --timeout=300s + done +} + +run_load_profile() { + local profile=$1 + local report_path=$2 + ( + cd "$cluster_root/apps/api" + AI_GATEWAY_ACCEPTANCE_PROFILE=$profile \ + AI_GATEWAY_ACCEPTANCE_GATEWAYS=$AI_GATEWAY_ACCEPTANCE_GATEWAYS \ + AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=$AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME \ + AI_GATEWAY_ACCEPTANCE_EMULATOR_URL=http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090 \ + AI_GATEWAY_ACCEPTANCE_API_KEY=$AI_GATEWAY_ACCEPTANCE_API_KEY \ + AI_GATEWAY_ACCEPTANCE_RUN_ID=$run_id \ + AI_GATEWAY_ACCEPTANCE_RUN_TOKEN=$run_token \ + AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL=$AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL \ + AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL=$AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL \ + AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS=$AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS \ + "$load_binary" -profile "$profile" -report "$report_path" >/dev/null + ) +} + +kill_one_worker_during_recovery() { + local deadline=$((SECONDS + 120)) + local running pod + while (( SECONDS < deadline )); do + running=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND status='running' AND request::text LIKE '%acceptance-long-recovery%';") + if (( running > 0 )); then + break + fi + sleep 1 + done + (( running > 0 )) || { + echo 'recovery workload never reached a running Worker' >&2 + return 1 + } + pod=$(remote_kubectl get pods -n "$namespace" \ + -l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \ + -o 'jsonpath={.items[0].metadata.name}') + [[ $pod =~ ^easyai-worker-hongkong-[a-z0-9-]+$ ]] + remote_kubectl delete pod "$pod" -n "$namespace" --wait=false >/dev/null + remote_kubectl rollout status deployment/easyai-worker-hongkong \ + -n "$namespace" --timeout=300s +} + +run_recovery_profile() { + local report_path=$1 + run_load_profile video-recovery "$report_path" & + local load_pid=$! + if ! kill_one_worker_during_recovery; then + kill "$load_pid" >/dev/null 2>&1 || true + wait "$load_pid" >/dev/null 2>&1 || true + return 1 + fi + wait "$load_pid" +} + +verify_wireguard() { + local source_host=$1 + local target_ip=$2 + local max_rtt=$3 + local label=$4 + local output loss average handshakes now latest age + output=$(cluster_ssh "$source_host" "ping -q -c 10 -W 2 $target_ip") + loss=$(awk -F',' '/packet loss/ {gsub(/[^0-9.]/, "", $3); print $3}' <<<"$output") + average=$(awk -F'=' '/min\\/avg\\/max/ {split($2, values, \"/\"); gsub(/ /, "", values[2]); print values[2]}' <<<"$output") + [[ -n $loss && -n $average ]] + awk -v loss="$loss" -v average="$average" -v max="$max_rtt" \ + 'BEGIN { exit !(loss < 1 && average < max) }' + now=$(date +%s) + latest=$(cluster_ssh "$source_host" "wg show all latest-handshakes | awk '{print \\$3}' | sort -nr | head -n 2") + handshakes=$(wc -l <<<"$latest" | tr -d ' ') + (( handshakes >= 2 )) + while IFS= read -r timestamp; do + [[ $timestamp =~ ^[0-9]+$ && $timestamp -gt 0 ]] + age=$((now - timestamp)) + (( age < 180 )) + done <<<"$latest" + echo "wireguard=$label loss_percent=$loss average_rtt_ms=$average recent_handshakes=$handshakes" +} + +verify_cluster_links() { + verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.2 80 'ningbo_to_hongkong' + verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.1 80 'hongkong_to_ningbo' + verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.3 300 'ningbo_to_los_angeles' + verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo' + verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles' + verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong' +} + +verify_control_plane_and_recent_logs() { + local host ready_output pod error_count + for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do + ready_output=$(cluster_ssh "$host" "k3s kubectl get --raw='/readyz?verbose'") + grep -Fq '[+]etcd ok' <<<"$ready_output" + done + while read -r pod; do + error_count=$(remote_kubectl logs "$pod" -n "$namespace" --since=10m | + awk 'BEGIN {IGNORECASE=1} + /postgres_unavailable/ || + /postgres readiness.*(fail|unavailable|error)/ || + /leadership elector.*(error|fail)/ || + /concurrency lease.*(renew.*(error|fail)|lost)/ {count++} + END {print count+0}') + [[ $error_count == 0 ]] + done < <(remote_kubectl get pods -n "$namespace" \ + -l 'app.kubernetes.io/name=easyai-worker' \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + while read -r pod; do + error_count=$(remote_kubectl logs "$pod" -n "$namespace" -c postgres --since=10m | + awk 'BEGIN {IGNORECASE=1} + /apply request took too long/ || + /dial tcp.*6443/ || + /synchronous commit.*cancel/ || + /canceling statement due to user request/ || + /(liveness|readiness).*fail/ {count++} + END {print count+0}') + [[ $error_count == 0 ]] + done < <(remote_kubectl get pods -n "$namespace" \ + -l 'cnpg.io/cluster=easyai-postgres' \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') +} + +current_max_pod_memory_mib() { + remote_kubectl top pods -n "$namespace" \ + -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers | + awk '{ + value=$3 + if (value ~ /Gi$/) {sub(/Gi$/, "", value); value*=1024} + else if (value ~ /Mi$/) {sub(/Mi$/, "", value)} + else if (value ~ /Ki$/) {sub(/Ki$/, "", value); value/=1024} + if (value>max) max=value + } END {printf "%.0f", max+0}' +} + +wait_for_memory_recovery() { + local baseline_mib=$1 + local deadline=$((SECONDS + 120)) + local current_mib + while (( SECONDS < deadline )); do + current_mib=$(current_max_pod_memory_mib) + if (( current_mib <= baseline_mib + 256 )); then + return 0 + fi + sleep 5 + done + echo "Pod memory did not return near baseline: baseline_mib=$baseline_mib current_mib=$current_mib" >&2 + return 1 +} + +verify_runtime_gates() { + local expected_slots expected_global expected_pool + case $active_profile in + P24) expected_slots=24; expected_pool=32; expected_global=48 ;; + P28) expected_slots=28; expected_pool=36; expected_global=56 ;; + P32) expected_slots=32; expected_pool=40; expected_global=64 ;; + esac + [[ $(remote_kubectl get nodes -o json | + jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') == 3 ]] + [[ $(remote_kubectl get nodes -o json | + jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] + [[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.readyInstances}') == 2 ]] + local sync_state + sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;") + [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] + verify_control_plane_and_recent_logs + + local unhealthy_containers pod_memory + unhealthy_containers=$(remote_kubectl get pods -n "$namespace" -o json | + jq '[.items[] + | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or + .metadata.labels["app.kubernetes.io/name"] == "easyai-worker") + | .status.containerStatuses[]? + | select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or + .state.terminated.reason == "OOMKilled")] | length') + [[ $unhealthy_containers == 0 ]] + while read -r _node _cpu _cpu_percent _memory memory_percent; do + memory_percent=${memory_percent%\%} + [[ $memory_percent =~ ^[0-9]+$ ]] + (( memory_percent < 80 )) + done < <(remote_kubectl top nodes --no-headers) + while read -r _pod _cpu pod_memory; do + case $pod_memory in + *Gi) pod_memory=$(awk -v value="${pod_memory%Gi}" 'BEGIN { printf "%.0f", value * 1024 }') ;; + *Mi) pod_memory=${pod_memory%Mi} ;; + *Ki) pod_memory=$(awk -v value="${pod_memory%Ki}" 'BEGIN { printf "%.0f", value / 1024 }') ;; + *) return 1 ;; + esac + [[ $pod_memory =~ ^[0-9]+$ ]] + (( pod_memory < 1536 )) + done < <(remote_kubectl top pods -n "$namespace" \ + -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers) + local pod_name pod_anon_mib + while read -r pod_name; do + pod_anon_mib=$(remote_kubectl exec -n "$namespace" "$pod_name" -- \ + cat /sys/fs/cgroup/memory.stat | + awk '$1=="anon" {printf "%.0f", $2/1048576}') + [[ $pod_anon_mib =~ ^[0-9]+$ ]] + (( pod_anon_mib < 1536 )) + done < <(remote_kubectl get pods -n "$namespace" -o json | + jq -r '.items[] + | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or + .metadata.labels["app.kubernetes.io/name"] == "easyai-worker") + | .metadata.name') + + local completion_deadline=$((SECONDS + 180)) + local incomplete side_effects + while (( SECONDS < completion_deadline )); do + incomplete=$(database_query " +SELECT count(*) +FROM gateway_tasks +WHERE acceptance_run_id='$run_id'::uuid + AND ( + status <> 'succeeded' + OR billing_status <> 'settled' + );") + side_effects=$(database_query " +SELECT + (SELECT count(*) + FROM settlement_outbox settlement + WHERE settlement.task_id IN ( + SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid + ) + AND settlement.status <> 'completed') + + + (SELECT count(*) + FROM gateway_tasks task + WHERE task.acceptance_run_id='$run_id'::uuid + AND NOT EXISTS ( + SELECT 1 + FROM gateway_task_callback_outbox callback + WHERE callback.task_id=task.id + AND callback.status='delivered' + ));") + if [[ $incomplete == 0 && $side_effects == 0 ]]; then + break + fi + sleep 2 + done + [[ $incomplete == 0 && $side_effects == 0 ]] + + local queue_state duplicates settlements wallet_duplicates missing_billings callbacks connections max_connections + local active_workers stale_workers + queue_state=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;") + [[ $queue_state == 0:0 ]] + duplicates=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) duplicate;") + [[ $duplicates == 0 ]] + settlements=$(database_query "SELECT count(*) FROM (SELECT task_id,action FROM settlement_outbox WHERE task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY task_id,action HAVING count(*)>1) duplicate;") + [[ $settlements == 0 ]] + wallet_duplicates=$(database_query " +SELECT count(*) +FROM ( + SELECT reference_id,transaction_type + FROM gateway_wallet_transactions + WHERE reference_type='gateway_task' + AND reference_id IN ( + SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid + ) + GROUP BY reference_id,transaction_type + HAVING count(*)>1 +) duplicate;") + missing_billings=$(database_query " +SELECT count(*) +FROM gateway_tasks task +WHERE task.acceptance_run_id='$run_id'::uuid + AND NOT EXISTS ( + SELECT 1 + FROM gateway_wallet_transactions transaction + WHERE transaction.reference_type='gateway_task' + AND transaction.reference_id=task.id::text + AND transaction.transaction_type='task_billing' + );") + [[ $wallet_duplicates == 0 && $missing_billings == 0 ]] + callbacks=$(database_query " +SELECT count(*) +FROM gateway_task_callback_outbox +WHERE task_id IN ( + SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid +) + AND status <> 'delivered';") + [[ $callbacks == 0 ]] + active_workers=$(database_query " +SELECT count(*) +FROM gateway_worker_instances +WHERE status='active' + AND heartbeat_at > now()-interval '30 seconds';") + stale_workers=$(database_query " +SELECT count(*) +FROM gateway_worker_instances +WHERE status='active' + AND ( + heartbeat_at <= now()-interval '30 seconds' + OR desired_capacity <> $expected_slots + OR capacity_limit <> $expected_slots + OR allocated_capacity > $expected_slots + );") + [[ $active_workers == 2 && $stale_workers == 0 ]] + connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';") + max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';") + (( connections * 4 < max_connections * 3 )) + + local pod metrics desired current global allocated active_instances pool_max acquired idle + local canceled_acquires lease_failures lease_lost + for _ in 1 2 3 4 5 6; do + for site in ningbo hongkong; do + pod=$(remote_kubectl get pods -n "$namespace" \ + -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" \ + -o 'jsonpath={.items[0].metadata.name}') + metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- wget -qO- http://127.0.0.1:8088/metrics) + desired=$(awk '$1=="easyai_gateway_async_worker_desired_capacity" {print $2}' <<<"$metrics") + current=$(awk '$1=="easyai_gateway_async_worker_capacity" {print $2}' <<<"$metrics") + global=$(awk '$1=="easyai_gateway_worker_global_capacity" {print $2}' <<<"$metrics") + allocated=$(awk '$1=="easyai_gateway_worker_allocated_capacity" {print $2}' <<<"$metrics") + active_instances=$(awk '$1=="easyai_gateway_worker_active_instances" {print $2}' <<<"$metrics") + pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics") + acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics") + idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics") + canceled_acquires=$(awk '$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {print $2}' <<<"$metrics") + lease_failures=$(awk 'index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {print $2}' <<<"$metrics") + lease_lost=$(awk 'index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {print $2}' <<<"$metrics") + [[ ${desired%.*} == "$expected_slots" && ${current%.*} == "$expected_slots" ]] + [[ ${global%.*} == "$expected_global" && ${allocated%.*} -le "$expected_slots" ]] + [[ ${active_instances%.*} == 2 ]] + [[ ${pool_max%.*} == "$expected_pool" ]] + [[ ${canceled_acquires%.*} == 0 && ${lease_failures%.*} == 0 && ${lease_lost%.*} == 0 ]] + if [[ ${acquired%.*} == "$expected_pool" && ${idle%.*} == 0 ]]; then + echo "Worker connection pool saturated for site $site" >&2 + return 1 + fi + done + sleep 5 + done + verify_cluster_links + curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/healthz >/dev/null + curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz >/dev/null +} + +sample_pressure() { + local output=$1 + local stop_file=$2 + local database_state node_memory_percent pod_memory_mib pool_state pod metrics + echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max' >"$output" + while [[ ! -f $stop_file ]]; do + database_state=$(database_query " +SELECT + count(*) FILTER (WHERE status='queued')||','|| + count(*) FILTER (WHERE status='running')||','|| + COALESCE(EXTRACT(EPOCH FROM (now()-(min(created_at) FILTER (WHERE status='queued')))),0)::bigint||','|| + (SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','|| + (SELECT setting FROM pg_settings WHERE name='max_connections') +FROM gateway_tasks +WHERE acceptance_run_id='$run_id'::uuid;") || break + node_memory_percent=$(remote_kubectl top nodes --no-headers | + awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break + pod_memory_mib=$(current_max_pod_memory_mib) || break + pool_state=$( + for site in ningbo hongkong; do + pod=$(remote_kubectl get pods -n "$namespace" \ + -l "app.kubernetes.io/name=easyai-worker,easyai.io/site=$site" -o json | + jq -r '[.items[] + | select(any(.status.conditions[]?; .type=="Ready" and .status=="True")) + | .metadata.name][0] // empty') || exit + [[ -n $pod ]] || continue + metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \ + wget -qO- http://127.0.0.1:8088/metrics) || exit + awk ' + $1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2} + $1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2} + $1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2} + $1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2} + $1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2} + index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2} + index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2} + $1=="easyai_gateway_worker_active_instances" {active=$2} + $1=="easyai_gateway_worker_allocated_capacity" {allocated=$2} + END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated} + ' <<<"$metrics" + done | + awk -F',' ' + NR==1 { + acquired=$1; pool_max=$2; idle=$3; empty=$4; canceled=$5 + failure=$6; lost=$7; active=$8; allocated=$9 + next + } + { + if ($1>acquired) acquired=$1 + if ($2>pool_max) pool_max=$2 + if ($3allocated) allocated=$9 + } + END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated} + ' + ) || break + printf '%s,%s,%s,%s,%s\n' \ + "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$database_state" "$node_memory_percent" "$pod_memory_mib" "$pool_state" >>"$output" + sleep 5 + done +} + +verify_pressure_report() { + local report_path=$1 + local expected_slots=$2 + awk -F',' ' + NR == 1 { next } + { + samples++ + if (samples == 1) { + first_empty=$12 + first_canceled=$13 + first_lease_failure=$14 + first_lease_lost=$15 + } + last_empty=$12 + last_canceled=$13 + last_lease_failure=$14 + last_lease_lost=$15 + if ($13 > first_canceled || $14 > first_lease_failure || $15 > first_lease_lost) { + failed=1 + } + if ($9 >= $10 && $11 == 0) { + saturated++ + } else { + saturated=0 + } + if ($4 >= 900 || $6 <= 0 || $5 * 4 >= $6 * 3 || $7 >= 80 || $8 >= 1536 || + $10 <= 0 || saturated >= 6 || $16 < 1 || $16 > 2 || $17 > slots) { + failed=1 + } + } + END { + if (last_canceled > first_canceled || + last_lease_failure > first_lease_failure || + last_lease_lost > first_lease_lost || + (last_empty > first_empty && saturated >= 6)) { + failed=1 + } + exit !(samples > 0 && failed == 0) + } + ' slots="$expected_slots" "$report_path" +} + +run_capacity_round() { + local profile=$1 + local repetition=$2 + local prefix=$report_root/${profile,,}-$repetition + local pressure_report=$prefix-pressure.csv + local pressure_stop=$temporary_root/pressure-stop + local pressure_pid status=0 + local baseline_memory_mib + local expected_slots + case $profile in + P24) expected_slots=24 ;; + P28) expected_slots=28 ;; + P32) expected_slots=32 ;; + esac + baseline_memory_mib=$(current_max_pod_memory_mib) + rm -f -- "$pressure_stop" + sample_pressure "$pressure_report" "$pressure_stop" & + pressure_pid=$! + run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$? + if (( status == 0 )); then run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?; fi + if (( status == 0 )); then run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?; fi + if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi + if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi + touch "$pressure_stop" + wait "$pressure_pid" || true + (( status == 0 )) || return "$status" + verify_pressure_report "$pressure_report" "$expected_slots" + verify_runtime_gates + wait_for_memory_recovery "$baseline_memory_mib" +} + +finish_and_promote() { + local emulator_report=$report_root/emulator-report.json + local finish_response=$temporary_root/finish.json + local mode_response=$temporary_root/mode.json + remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-emulator -- \ + wget -qO- http://127.0.0.1:8090/report >"$emulator_report" + local task_count video_task_count gemini_task_count submissions duplicates callbacks + local gemini_requests forced_tasks forced_requests verified_conversions compact_payloads + task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;") + video_task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='videos.generations' AND run_mode='acceptance';") + gemini_task_count=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND run_mode='acceptance';") + forced_tasks=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND request::text LIKE '%acceptance-force-conversion%';") + compact_payloads=$(database_query "SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND kind='images.edits' AND (pg_column_size(request) >= 1048576 OR pg_column_size(result) >= 1048576);") + submissions=$(jq -r '.videoSubmissions' "$emulator_report") + duplicates=$(jq -r '.duplicateCallbacks' "$emulator_report") + callbacks=$(jq -r '.callbackEvents' "$emulator_report") + gemini_requests=$(jq -r '.geminiRequests' "$emulator_report") + forced_requests=$(jq -r '.forcedConversionRequests' "$emulator_report") + verified_conversions=$(jq -r '.verifiedConversions' "$emulator_report") + [[ $submissions == "$video_task_count" && $gemini_requests == "$gemini_task_count" ]] + [[ $forced_requests == "$forced_tasks" && $verified_conversions == "$forced_tasks" ]] + [[ $duplicates == 0 && $compact_payloads == 0 ]] + [[ $callbacks -gt 0 ]] + admin_request GET /api/admin/system/acceptance/traffic-mode '' "$mode_response" + verify_release_cas + local promotion_body + promotion_body=$(jq -cn \ + --argjson revision "$(jq -r '.revision' "$mode_response")" \ + --arg releaseSha "$release_sha" \ + --arg apiDigest "$api_digest" \ + --arg workerDigest "$worker_digest" \ + '{revision:$revision,releaseSha:$releaseSha,apiImageDigest:$apiDigest,workerImageDigest:$workerDigest}') + jq -n \ + --arg runId "$run_id" \ + --arg releaseSha "$release_sha" \ + --arg apiImageDigest "$api_digest" \ + --arg workerImageDigest "$worker_digest" \ + --arg stableCapacityProfile "$stable_profile" \ + --argjson tasks "$task_count" \ + --argjson geminiTasks "$gemini_task_count" \ + --argjson videoTasks "$video_task_count" \ + --argjson upstreamSubmissions "$submissions" \ + --argjson callbackDeliveries "$callbacks" \ + '{ + runId:$runId, + releaseSha:$releaseSha, + apiImageDigest:$apiImageDigest, + workerImageDigest:$workerImageDigest, + stableCapacityProfile:$stableCapacityProfile, + tasks:$tasks, + geminiTasks:$geminiTasks, + videoTasks:$videoTasks, + upstreamSubmissions:$upstreamSubmissions, + callbackDeliveries:$callbackDeliveries, + promotionRequested:true, + passed:true + }' >"$report_root/summary.json" + local finish_body + finish_body=$(jq -cn \ + --argjson tasks "$task_count" \ + --argjson videos "$video_task_count" \ + --argjson submissions "$submissions" \ + --arg profile "$stable_profile" \ + '{passed:true,report:{tasks:$tasks,videos:$videos,upstreamSubmissions:$submissions,stableCapacityProfile:$profile,queueFinal:0}}') + admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response" + admin_request POST "/api/admin/system/acceptance/runs/$run_id/promote" "$promotion_body" "$finish_response" + [[ $(jq -r '.mode' "$finish_response") == live ]] +} + +verify_release_cas +select_acceptance_models +deploy_protocol_emulator +create_and_activate_run +wait_for_existing_tasks_to_drain +( + cd "$cluster_root/apps/api" + go build -trimpath -o "$load_binary" ./cmd/acceptance-load +) + +for profile in P24 P28 P32; do + if ! apply_capacity_profile "$profile"; then + failure_reason="failed to apply $profile" + break + fi + profile_passed=true + for repetition in 1 2 3; do + if ! run_capacity_round "$profile" "$repetition"; then + profile_passed=false + failure_reason="$profile repetition $repetition failed" + break + fi + done + if [[ $profile_passed != true ]]; then + apply_capacity_profile "$stable_profile" || true + break + fi + stable_profile=$profile +done + +if [[ -n $failure_reason ]]; then + mark_run_failed "$failure_reason" + echo "production_acceptance=FAIL run_id=$run_id reason=$failure_reason traffic_mode=validation restored_profile=$stable_profile" >&2 + exit 1 +fi + +if ! run_load_profile real-canary "$report_root/real-canary.json"; then + failure_reason='real canary failed' +elif ! verify_runtime_gates; then + failure_reason='post-canary runtime gates failed' +fi +if [[ -n $failure_reason ]]; then + mark_run_failed "$failure_reason" + echo "production_acceptance=FAIL run_id=$run_id reason=$failure_reason traffic_mode=validation restored_profile=$stable_profile" >&2 + exit 1 +fi +finish_and_promote +remote_kubectl delete deployment easyai-acceptance-emulator -n "$namespace" --ignore-not-found >/dev/null || + echo 'warning: acceptance emulator Deployment cleanup failed' >&2 +remote_kubectl delete service easyai-acceptance-emulator -n "$namespace" --ignore-not-found >/dev/null || + echo 'warning: acceptance emulator Service cleanup failed' >&2 +echo "production_acceptance=PASS run_id=$run_id release=$release_sha stable_profile=$stable_profile traffic_mode=live"