BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(acceptance): 建立同构验收与弹性容量体系

Browse Source 
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
wangbo
2026-07-31 18:02:24 +08:00
parent 93d36d0e55
commit e05922b0f4
94 changed files with 12379 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/cmd/acceptance-load/main.go
+281 -30
View File
@@ -6,6 +6,7 @@ import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/binary"
"encoding/hex"
@@ -18,10 +19,13 @@ import (
"net/http"
"net/url"
"os"
"os/signal"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceworkload"
@@ -38,6 +42,7 @@ var reportURLPattern = regexp.MustCompile(`(?i)(?:https?|postgres(?:ql)?):\/\/[^
type options struct {
gateways []string
gatewayTLSName string
gatewayCAFile string
emulatorURL string
apiKeys []string
runID string
@@ -48,17 +53,21 @@ type options struct {
reportPath string
realImageURLs []string
timeout time.Duration
mixedDuration time.Duration
imageRate float64
videoRate float64
}
type report struct {
RunID string `json:"runId"`
Profile string `json:"profile"`
StartedAt time.Time `json:"startedAt"`
FinishedAt time.Time `json:"finishedAt"`
Passed bool `json:"passed"`
Phases []phaseReport `json:"phases"`
Failure string `json:"failure,omitempty"`
SecretSafe bool `json:"secretSafe"`
SchemaVersion string `json:"schemaVersion"`
RunID string `json:"runId"`
Profile string `json:"profile"`
StartedAt time.Time `json:"startedAt"`
FinishedAt time.Time `json:"finishedAt"`
Passed bool `json:"passed"`
Phases []phaseReport `json:"phases"`
Failure string `json:"failure,omitempty"`
SecretSafe bool `json:"secretSafe"`
}
type phaseReport struct {
@@ -76,6 +85,9 @@ type phaseReport struct {
UniqueTaskIDs int `json:"uniqueTaskIds,omitempty"`
UniqueImageCombos int `json:"uniqueImageCombinations,omitempty"`
ForcedConversionTasks int `json:"forcedConversionTasks,omitempty"`
Throttled int `json:"throttled,omitempty"`
Unexpected5xx int `json:"unexpected5xx,omitempty"`
OfferedRatePerSecond float64 `json:"offeredRatePerSecond,omitempty"`
}
type phaseResult struct {
@@ -84,16 +96,32 @@ type phaseResult struct {
err error
}
type httpStatusError struct {
Status int
Body string
}
func (e *httpStatusError) Error() string {
return fmt.Sprintf("HTTP %d: %s", e.Status, e.Body)
}
func main() {
opts, err := parseOptions()
if err != nil {
fmt.Fprintln(os.Stderr, "acceptance load configuration error:", err)
os.Exit(2)
}
ctx, cancel := context.WithTimeout(context.Background(), opts.timeout)
signalContext, stopSignals := signal.NotifyContext(
context.Background(),
os.Interrupt,
syscall.SIGTERM,
)
defer stopSignals()
ctx, cancel := context.WithTimeout(signalContext, opts.timeout)
defer cancel()
result := report{
RunID: opts.runID, Profile: opts.profile, StartedAt: time.Now().UTC(), SecretSafe: true,
SchemaVersion: "acceptance-load-report/v1",
RunID: opts.runID, Profile: opts.profile, StartedAt: time.Now().UTC(), SecretSafe: true,
}
phaseResults, runErr := run(ctx, opts)
for _, phase := range phaseResults {
@@ -127,13 +155,20 @@ func parseOptions() (options, error) {
var profile string
var reportPath string
var timeout time.Duration
var mixedDuration time.Duration
var imageRate float64
var videoRate float64
flag.StringVar(&profile, "profile", env("AI_GATEWAY_ACCEPTANCE_PROFILE", "simulated-all"), "simulated-all, Gemini profile, video profile, or real-canary")
flag.StringVar(&reportPath, "report", env("AI_GATEWAY_ACCEPTANCE_REPORT", ""), "secret-safe JSON report path")
flag.DurationVar(&timeout, "timeout", 45*time.Minute, "overall timeout")
flag.DurationVar(&mixedDuration, "duration", envDuration("AI_GATEWAY_ACCEPTANCE_MIXED_DURATION", 10*time.Minute), "mixed workload duration")
flag.Float64Var(&imageRate, "image-rate", envFloat("AI_GATEWAY_ACCEPTANCE_IMAGE_RATE", 0), "mixed image requests per second")
flag.Float64Var(&videoRate, "video-rate", envFloat("AI_GATEWAY_ACCEPTANCE_VIDEO_RATE", 0), "mixed video requests per second")
flag.Parse()
opts := options{
gateways: splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAYS")),
gatewayTLSName: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME")),
gatewayCAFile: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE")),
emulatorURL: strings.TrimRight(strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_EMULATOR_URL")), "/"),
apiKeys: splitCSV(env("AI_GATEWAY_ACCEPTANCE_API_KEYS", os.Getenv("AI_GATEWAY_ACCEPTANCE_API_KEY"))),
runID: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_RUN_ID")),
@@ -144,6 +179,9 @@ func parseOptions() (options, error) {
reportPath: strings.TrimSpace(reportPath),
realImageURLs: splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS")),
timeout: timeout,
mixedDuration: mixedDuration,
imageRate: imageRate,
videoRate: videoRate,
}
if len(opts.gateways) != 2 {
return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAYS must contain exactly two API base URLs")
@@ -168,10 +206,16 @@ func parseOptions() (options, error) {
return options{}, errors.New("timeout must be positive")
}
switch opts.profile {
case "simulated-all", "gemini-baseline", "gemini-large", "gemini-peak", "video-throughput", "video-recovery":
case "simulated-smoke", "simulated-all", "gemini-baseline", "gemini-large", "gemini-peak", "video-throughput", "video-recovery",
"mixed-soak", "mixed-overload":
if opts.emulatorURL == "" {
return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_EMULATOR_URL is required for simulated profiles")
}
if (opts.profile == "mixed-soak" || opts.profile == "mixed-overload") &&
(opts.mixedDuration <= 0 || opts.imageRate < 0 || opts.videoRate < 0 ||
opts.imageRate+opts.videoRate <= 0) {
return options{}, errors.New("mixed profiles require a positive duration and offered rate")
}
case "real-canary":
if len(opts.realImageURLs) < 3 {
return options{}, errors.New("real-canary requires at least three public AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS")
@@ -184,10 +228,15 @@ func parseOptions() (options, error) {
func run(ctx context.Context, opts options) ([]phaseResult, error) {
var tlsConfig *tls.Config
if opts.gatewayTLSName != "" {
if opts.gatewayTLSName != "" || opts.gatewayCAFile != "" {
rootCAs, err := acceptanceRootCAs(opts.gatewayCAFile)
if err != nil {
return nil, err
}
tlsConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
ServerName: opts.gatewayTLSName,
RootCAs: rootCAs,
}
}
client := &http.Client{
@@ -207,14 +256,20 @@ func run(ctx context.Context, opts options) ([]phaseResult, error) {
result = runGemini(ctx, client, opts, name, profile.Requests, profile.InputBytes, profile.OutputBytes, false)
} else {
switch name {
case "smoke-gemini":
result = runGemini(ctx, client, opts, name, 4, 256<<10, 256<<10, false)
case "smoke-video":
result = runVideo(ctx, client, opts, name, 10, false, opts.emulatorFixtureURLs(), false, 0)
case "video-throughput":
result = runVideo(ctx, client, opts, name, 1200, false, opts.emulatorFixtureURLs(), false)
result = runVideo(ctx, client, opts, name, 1200, false, opts.emulatorFixtureURLs(), false, 0)
case "video-recovery":
result = runVideo(ctx, client, opts, name, 96, true, opts.emulatorFixtureURLs(), false)
result = runVideo(ctx, client, opts, name, 96, true, opts.emulatorFixtureURLs(), false, 0)
case "mixed-soak", "mixed-overload":
result = runMixed(ctx, client, opts, name == "mixed-overload")
case "real-gemini-canary":
result = runGemini(ctx, client, opts, name, 1, 256<<10, 0, true)
case "real-video-canary":
result = runVideo(ctx, client, opts, name, 1, false, opts.realImageURLs, true)
result = runVideo(ctx, client, opts, name, 1, false, opts.realImageURLs, true, 0)
default:
return fmt.Errorf("unknown phase %q", name)
}
@@ -223,7 +278,9 @@ func run(ctx context.Context, opts options) ([]phaseResult, error) {
return result.err
}
phases := []string{opts.profile}
if opts.profile == "simulated-all" {
if opts.profile == "simulated-smoke" {
phases = []string{"smoke-gemini", "smoke-video"}
} else if opts.profile == "simulated-all" {
phases = []string{"gemini-baseline", "gemini-large", "gemini-peak", "video-throughput", "video-recovery"}
} else if opts.profile == "real-canary" {
phases = []string{"real-gemini-canary", "real-video-canary"}
@@ -376,6 +433,7 @@ func runVideo(
longRun bool,
imageURLs []string,
realUpstream bool,
requestOffset int,
) phaseResult {
startedAt := time.Now()
combinationCount := 128
@@ -398,31 +456,32 @@ func runVideo(
wg.Add(1)
go func() {
defer wg.Done()
logicalIndex := index + requestOffset
requestStarted := time.Now()
startedTasks[index] = requestStarted
imageCount := acceptanceworkload.VideoImageCount(index)
combo := append([]string(nil), combinations[index%len(combinations)]...)
imageCount := acceptanceworkload.VideoImageCount(logicalIndex)
combo := append([]string(nil), combinations[logicalIndex%len(combinations)]...)
if len(combo) > imageCount {
combo = combo[:imageCount]
}
if !realUpstream && index%4 == 0 {
combo[0] = imageURLs[12+(index/4)%4]
if !realUpstream && logicalIndex%4 == 0 {
combo[0] = imageURLs[12+(logicalIndex/4)%4]
}
content := make([]any, 0, len(combo)+1)
prompt := "多参考图生成连续运镜视频，保持人物、服装和场景一致"
if longRun {
prompt += " acceptance-long-recovery"
}
if !realUpstream && index%4 == 0 {
if !realUpstream && logicalIndex%4 == 0 {
prompt += " acceptance-force-conversion"
}
content = append(content, map[string]any{"type": "text", "text": prompt})
for imageIndex, imageURL := range combo {
role := "reference_image"
if index%5 == 0 && imageIndex == 0 {
if logicalIndex%5 == 0 && imageIndex == 0 {
role = "first_frame"
}
if index%5 == 0 && len(combo) > 3 && imageIndex == len(combo)-1 {
if logicalIndex%5 == 0 && len(combo) > 3 && imageIndex == len(combo)-1 {
role = "last_frame"
}
content = append(content, map[string]any{
@@ -431,13 +490,13 @@ func runVideo(
})
}
body, _ := json.Marshal(map[string]any{
"model": opts.videoModel, "modelType": "omni_video", "content": content, "seed": index + 1,
"model": opts.videoModel, "modelType": "omni_video", "content": content, "seed": logicalIndex + 1,
"duration": 5, "ratio": "16:9", "resolution": "720p",
})
endpoint := opts.gateways[index%len(opts.gateways)] + "/api/v1/videos/generations"
endpoint := opts.gateways[logicalIndex%len(opts.gateways)] + "/api/v1/videos/generations"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
if err == nil {
opts.setHeaders(req, index, realUpstream)
opts.setHeaders(req, logicalIndex, realUpstream)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Async", "true")
var response *http.Response
@@ -460,7 +519,7 @@ func runVideo(
}
mu.Lock()
latencies[index] = time.Since(requestStarted)
if !realUpstream && index%4 == 0 {
if !realUpstream && logicalIndex%4 == 0 {
forcedConversions++
}
if err != nil {
@@ -492,6 +551,7 @@ func runVideo(
continue
}
index, taskID := index, taskID
logicalIndex := index + requestOffset
wg.Add(1)
go func() {
defer wg.Done()
@@ -501,7 +561,7 @@ func runVideo(
return
}
defer func() { <-pollSlots }()
err := pollVideoTask(ctx, client, opts, taskID, index, realUpstream)
err := pollVideoTask(ctx, client, opts, taskID, logicalIndex, realUpstream)
mu.Lock()
latencies[index] = time.Since(startedTasks[index])
if err != nil {
@@ -520,6 +580,9 @@ func runVideo(
for _, combo := range combinations {
comboSet[strings.Join(combo, "\x00")] = struct{}{}
}
if firstErr == nil && !realUpstream && len(comboSet) != 128 {
firstErr = fmt.Errorf("video image combinations=%d, want 128", len(comboSet))
}
elapsed := time.Since(startedAt)
result := phaseReport{
Name: name, Requests: requestCount, Completed: completed, Failed: failures,
@@ -533,6 +596,145 @@ func runVideo(
return phaseResult{report: result, latencies: latencies, err: firstErr}
}
func runMixed(
ctx context.Context,
client *http.Client,
opts options,
expectThrottling bool,
) phaseResult {
runCtx, stopRun := context.WithCancel(ctx)
defer stopRun()
startedAt := time.Now()
totalRate := opts.imageRate + opts.videoRate
interval := time.Duration(float64(time.Second) / totalRate)
if interval < time.Millisecond {
interval = time.Millisecond
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
stop := time.NewTimer(opts.mixedDuration)
defer stop.Stop()
slots := make(chan struct{}, 4096)
var wg sync.WaitGroup
var mu sync.Mutex
latencies := make([]time.Duration, 0, int(totalRate*opts.mixedDuration.Seconds()))
requests := 0
completed := 0
failed := 0
throttled := 0
unexpected5xx := 0
var firstErr error
imageAccumulator := float64(0)
generating := true
for generating {
select {
case <-runCtx.Done():
generating = false
if ctx.Err() != nil {
mu.Lock()
if firstErr == nil {
firstErr = ctx.Err()
}
mu.Unlock()
}
case <-stop.C:
generating = false
case <-ticker.C:
select {
case slots <- struct{}{}:
case <-runCtx.Done():
generating = false
continue
}
requestIndex := requests
requests++
imageAccumulator += opts.imageRate
isImage := imageAccumulator >= totalRate
if isImage {
imageAccumulator -= totalRate
}
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-slots }()
var result phaseResult
if isImage {
result = runGemini(runCtx, client, opts, "mixed-image", 1, 256<<10, 256<<10, false)
} else {
result = runVideo(
runCtx,
client,
opts,
"mixed-video",
1,
false,
opts.emulatorFixtureURLs(),
false,
requestIndex,
)
}
latency := time.Since(startedAt)
if len(result.latencies) > 0 {
latency = result.latencies[0]
}
mu.Lock()
defer mu.Unlock()
latencies = append(latencies, latency)
if result.err == nil {
completed++
return
}
var statusErr *httpStatusError
if errors.As(result.err, &statusErr) && statusErr.Status == http.StatusTooManyRequests {
throttled++
if !expectThrottling && firstErr == nil {
firstErr = result.err
failed++
stopRun()
}
return
}
if errors.As(result.err, &statusErr) && statusErr.Status >= 500 {
unexpected5xx++
}
failed++
if firstErr == nil {
firstErr = result.err
stopRun()
}
}()
}
}
wg.Wait()
if firstErr == nil && expectThrottling && throttled == 0 {
firstErr = errors.New("mixed overload did not receive any 429 response")
}
if firstErr == nil && completed+throttled != requests {
firstErr = fmt.Errorf(
"mixed workload accounted for %d of %d requests",
completed+throttled,
requests,
)
}
if firstErr == nil && unexpected5xx > 0 {
firstErr = fmt.Errorf("mixed workload received %d unexpected 5xx responses", unexpected5xx)
}
return phaseResult{
report: phaseReport{
Name: map[bool]string{false: "mixed-soak", true: "mixed-overload"}[expectThrottling],
Requests: requests,
Completed: completed,
Failed: failed,
DurationMS: time.Since(startedAt).Milliseconds(),
Throttled: throttled,
Unexpected5xx: unexpected5xx,
OfferedRatePerSecond: totalRate,
},
latencies: latencies,
err: firstErr,
}
}
func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskID string, index int, realUpstream bool) error {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
@@ -558,7 +760,7 @@ func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskI
if mediaURL == "" {
return fmt.Errorf("video task %s succeeded without a media URL", taskID)
}
return validateVideoAsset(ctx, http.DefaultClient, mediaURL)
return validateVideoAsset(ctx, client, mediaURL)
case "failed", "cancelled", "canceled":
return fmt.Errorf("video task %s finished with status %s", taskID, status)
}
@@ -782,7 +984,7 @@ func scanUntil(reader *bufio.Reader, target []byte, limit int64) error {
func responseStatusError(response *http.Response) error {
defer response.Body.Close()
payload, _ := io.ReadAll(io.LimitReader(response.Body, 512))
return fmt.Errorf("HTTP %d: %s", response.StatusCode, strings.TrimSpace(string(payload)))
return &httpStatusError{Status: response.StatusCode, Body: strings.TrimSpace(string(payload))}
}
func percentileMilliseconds(values []time.Duration, quantile float64) float64 {
@@ -837,6 +1039,31 @@ func redactError(message string, opts options) string {
return reportURLPattern.ReplaceAllString(message, "[REDACTED_URL]")
}
func acceptanceRootCAs(path string) (*x509.CertPool, error) {
if strings.TrimSpace(path) == "" {
return nil, nil
}
info, err := os.Lstat(path)
if err != nil {
return nil, fmt.Errorf("read Gateway CA file: %w", err)
}
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("Gateway CA file must be a regular non-symlink file")
}
payload, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read Gateway CA file: %w", err)
}
roots, err := x509.SystemCertPool()
if err != nil || roots == nil {
roots = x509.NewCertPool()
}
if !roots.AppendCertsFromPEM(payload) {
return nil, errors.New("Gateway CA file does not contain a PEM certificate")
}
return roots, nil
}
func splitCSV(value string) []string {
items := strings.Split(value, ",")
out := make([]string, 0, len(items))
@@ -854,3 +1081,27 @@ func env(name string, fallback string) string {
}
return fallback
}
func envDuration(name string, fallback time.Duration) time.Duration {
value := strings.TrimSpace(os.Getenv(name))
if value == "" {
return fallback
}
parsed, err := time.ParseDuration(value)
if err != nil {
return -1
}
return parsed
}
func envFloat(name string, fallback float64) float64 {
value := strings.TrimSpace(os.Getenv(name))
if value == "" {
return fallback
}
parsed, err := strconv.ParseFloat(value, 64)
if err != nil {
return -1
}
return parsed
}
apps/api/cmd/acceptance-load/main_test.go
+17
View File
@@ -9,6 +9,8 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync/atomic"
"testing"
)
@@ -190,3 +192,18 @@ func TestAcceptanceGatewayTLSNameSetsHostHeader(t *testing.T) {
t.Fatalf("Host=%q", request.Host)
}
}
func TestAcceptanceRootCAsRejectsSymlink(t *testing.T) {
root := t.TempDir()
target := filepath.Join(root, "ca.pem")
if err := os.WriteFile(target, []byte("not a certificate"), 0o600); err != nil {
t.Fatal(err)
}
link := filepath.Join(root, "ca-link.pem")
if err := os.Symlink(target, link); err != nil {
t.Fatal(err)
}
if _, err := acceptanceRootCAs(link); err == nil {
t.Fatal("symlink CA file was accepted")
}
}