easyai-ai-gateway/apps/api/cmd/acceptance-load/main.go

package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/sha256"
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"hash/crc32"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceworkload"
)

const (
	runHeader      = "X-EasyAI-Acceptance-Run"
	tokenHeader    = "X-EasyAI-Acceptance-Token"
	upstreamHeader = "X-EasyAI-Acceptance-Upstream"
)

const (
	geminiRequestPrefix     = `{"contents":[{"role":"user","parts":[{"text":"将多张参考图融合编辑为蓝色赛博朋克风格，保留主体结构"}`
	geminiRequestImageStart = `,{"inlineData":{"mimeType":"image/png","data":"`
	geminiRequestImageEnd   = `"}}`
	geminiRequestSuffix     = `]}],"generationConfig":{"responseModalities":["IMAGE"]}}`
)

var reportURLPattern = regexp.MustCompile(`(?i)(?:https?|postgres(?:ql)?):\/\/[^\s"'<>]+`)

type options struct {
	gateways       []string
	gatewayTLSName string
	gatewayCAFile  string
	emulatorURL    string
	apiKeys        []string
	runID          string
	runToken       string
	geminiModel    string
	videoModel     string
	profile        string
	reportPath     string
	realImageURLs  []string
	timeout        time.Duration
	mixedDuration  time.Duration
	imageRate      float64
	videoRate      float64
	shardIndex     int
	shardCount     int
	executionID    string
	requestCount   int
	externalClient *http.Client
}

type report struct {
	SchemaVersion    string        `json:"schemaVersion"`
	RunID            string        `json:"runId"`
	Profile          string        `json:"profile"`
	StartedAt        time.Time     `json:"startedAt"`
	FinishedAt       time.Time     `json:"finishedAt"`
	Passed           bool          `json:"passed"`
	Phases           []phaseReport `json:"phases"`
	Failure          string        `json:"failure,omitempty"`
	FailureOperation string        `json:"failureOperation,omitempty"`
	SecretSafe       bool          `json:"secretSafe"`
}

type phaseReport struct {
	Name                  string  `json:"name"`
	Requests              int     `json:"requests"`
	Completed             int     `json:"completed"`
	Failed                int     `json:"failed"`
	DurationMS            int64   `json:"durationMs"`
	SubmissionDurationMS  int64   `json:"submissionDurationMs,omitempty"`
	P50MS                 float64 `json:"p50Ms"`
	P95MS                 float64 `json:"p95Ms"`
	P99MS                 float64 `json:"p99Ms"`
	DecodedOutputBytes    int64   `json:"decodedOutputBytes,omitempty"`
	OutputSHA256          string  `json:"outputSha256,omitempty"`
	InputImagesPerRequest int     `json:"inputImagesPerRequest,omitempty"`
	UniqueTaskIDs         int     `json:"uniqueTaskIds,omitempty"`
	UniqueImageCombos     int     `json:"uniqueImageCombinations,omitempty"`
	ForcedConversionTasks int     `json:"forcedConversionTasks,omitempty"`
	Throttled             int     `json:"throttled,omitempty"`
	Unexpected5xx         int     `json:"unexpected5xx,omitempty"`
	OfferedRatePerSecond  float64 `json:"offeredRatePerSecond,omitempty"`
}

type phaseResult struct {
	report    phaseReport
	latencies []time.Duration
	err       error
}

type httpStatusError struct {
	Status int
	Body   string
}

type operationError struct {
	Operation string
	Err       error
}

func (e *operationError) Error() string { return e.Operation + ": " + e.Err.Error() }
func (e *operationError) Unwrap() error { return e.Err }

func withOperation(operation string, err error) error {
	if err == nil {
		return nil
	}
	var existing *operationError
	if errors.As(err, &existing) {
		return err
	}
	return &operationError{Operation: operation, Err: err}
}

func (e *httpStatusError) Error() string {
	return fmt.Sprintf("HTTP %d: %s", e.Status, e.Body)
}

func main() {
	opts, err := parseOptions()
	if err != nil {
		fmt.Fprintln(os.Stderr, "acceptance load configuration error:", err)
		os.Exit(2)
	}
	signalContext, stopSignals := signal.NotifyContext(
		context.Background(),
		os.Interrupt,
		syscall.SIGTERM,
	)
	defer stopSignals()
	ctx, cancel := context.WithTimeout(signalContext, opts.timeout)
	defer cancel()
	result := report{
		SchemaVersion: "acceptance-load-report/v2",
		RunID:         opts.runID, Profile: opts.profile, StartedAt: time.Now().UTC(), SecretSafe: true,
	}
	phaseResults, runErr := run(ctx, opts)
	for _, phase := range phaseResults {
		phase.report.P50MS = percentileMilliseconds(phase.latencies, 0.50)
		phase.report.P95MS = percentileMilliseconds(phase.latencies, 0.95)
		phase.report.P99MS = percentileMilliseconds(phase.latencies, 0.99)
		result.Phases = append(result.Phases, phase.report)
		if runErr == nil && phase.err != nil {
			runErr = phase.err
		}
	}
	result.FinishedAt = time.Now().UTC()
	result.Passed = runErr == nil
	if runErr != nil {
		result.Failure = redactError(runErr.Error(), opts)
		var operationErr *operationError
		if errors.As(runErr, &operationErr) {
			result.FailureOperation = operationErr.Operation
		}
	}
	payload, _ := json.MarshalIndent(result, "", "  ")
	fmt.Println(string(payload))
	if opts.reportPath != "" {
		if err := writeReportExclusive(opts.reportPath, payload); err != nil {
			fmt.Fprintln(os.Stderr, "write acceptance report failed:", err)
			os.Exit(1)
		}
	}
	if runErr != nil {
		os.Exit(1)
	}
}

func writeReportExclusive(path string, payload []byte) error {
	reportFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
	if err != nil {
		return err
	}
	if _, err := reportFile.Write(append(payload, '\n')); err != nil {
		_ = reportFile.Close()
		return err
	}
	return reportFile.Close()
}

func parseOptions() (options, error) {
	var profile string
	var reportPath string
	var timeout time.Duration
	var mixedDuration time.Duration
	var imageRate float64
	var videoRate float64
	var shardIndex int
	var shardCount int
	var executionID string
	var requestCount int
	flag.StringVar(&profile, "profile", env("AI_GATEWAY_ACCEPTANCE_PROFILE", "simulated-all"), "simulated-all, Gemini profile, video profile, or real-canary")
	flag.StringVar(&reportPath, "report", env("AI_GATEWAY_ACCEPTANCE_REPORT", ""), "secret-safe JSON report path")
	flag.DurationVar(&timeout, "timeout", 45*time.Minute, "overall timeout")
	flag.DurationVar(&mixedDuration, "duration", envDuration("AI_GATEWAY_ACCEPTANCE_MIXED_DURATION", 10*time.Minute), "mixed workload duration")
	flag.Float64Var(&imageRate, "image-rate", envFloat("AI_GATEWAY_ACCEPTANCE_IMAGE_RATE", 0), "mixed image requests per second")
	flag.Float64Var(&videoRate, "video-rate", envFloat("AI_GATEWAY_ACCEPTANCE_VIDEO_RATE", 0), "mixed video requests per second")
	flag.IntVar(&shardIndex, "shard-index", 0, "zero-based distributed load shard index")
	flag.IntVar(&shardCount, "shard-count", 1, "distributed load shard count")
	flag.StringVar(&executionID, "execution-id", "", "unique workload execution identifier")
	flag.IntVar(&requestCount, "requests", 0, "override the fixed profile request count")
	flag.Parse()
	opts := options{
		gateways:       splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAYS")),
		gatewayTLSName: strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME")),
		gatewayCAFile:  strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE")),
		emulatorURL:    strings.TrimRight(strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_EMULATOR_URL")), "/"),
		apiKeys:        splitCSV(env("AI_GATEWAY_ACCEPTANCE_API_KEYS", os.Getenv("AI_GATEWAY_ACCEPTANCE_API_KEY"))),
		runID:          strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_RUN_ID")),
		runToken:       strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_RUN_TOKEN")),
		geminiModel:    strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL")),
		videoModel:     strings.TrimSpace(os.Getenv("AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL")),
		profile:        strings.ToLower(strings.TrimSpace(profile)),
		reportPath:     strings.TrimSpace(reportPath),
		realImageURLs:  splitCSV(os.Getenv("AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS")),
		timeout:        timeout,
		mixedDuration:  mixedDuration,
		imageRate:      imageRate,
		videoRate:      videoRate,
		shardIndex:     shardIndex,
		shardCount:     shardCount,
		executionID:    strings.ToLower(strings.TrimSpace(executionID)),
		requestCount:   requestCount,
	}
	if opts.executionID == "" {
		opts.executionID = opts.profile
	}
	if matched, _ := regexp.MatchString(`^[a-z0-9][a-z0-9-]{0,127}$`, opts.executionID); !matched {
		return options{}, errors.New("execution-id must contain only lowercase letters, numbers, and hyphens")
	}
	if opts.shardCount < 1 || opts.shardCount > 32 || opts.shardIndex < 0 || opts.shardIndex >= opts.shardCount {
		return options{}, fmt.Errorf("invalid load shard %d/%d", opts.shardIndex, opts.shardCount)
	}
	if len(opts.gateways) < 1 || len(opts.gateways) > 2 {
		return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAYS must contain one or two API base URLs")
	}
	for index := range opts.gateways {
		opts.gateways[index] = strings.TrimRight(opts.gateways[index], "/")
		if _, err := url.ParseRequestURI(opts.gateways[index]); err != nil {
			return options{}, fmt.Errorf("gateway %d URL is invalid", index+1)
		}
	}
	if opts.gatewayTLSName != "" &&
		(strings.ContainsAny(opts.gatewayTLSName, "/:@") || !strings.Contains(opts.gatewayTLSName, ".")) {
		return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME must be a DNS hostname")
	}
	if len(opts.apiKeys) == 0 || opts.runID == "" || opts.runToken == "" {
		return options{}, errors.New("acceptance API key, Run ID, and Run Token environment variables are required")
	}
	if opts.geminiModel == "" || opts.videoModel == "" {
		return options{}, errors.New("Gemini and video model environment variables are required")
	}
	if opts.timeout <= 0 {
		return options{}, errors.New("timeout must be positive")
	}
	if opts.requestCount < 0 || opts.requestCount > 10000 {
		return options{}, errors.New("requests must be between 0 and 10000")
	}
	switch opts.profile {
	case "simulated-smoke", "simulated-all", "gemini-baseline", "gemini-multi-image", "gemini-large", "gemini-peak", "video-throughput", "video-recovery",
		"mixed-soak", "mixed-overload":
		if opts.emulatorURL == "" {
			return options{}, errors.New("AI_GATEWAY_ACCEPTANCE_EMULATOR_URL is required for simulated profiles")
		}
		if (opts.profile == "mixed-soak" || opts.profile == "mixed-overload") &&
			(opts.mixedDuration <= 0 || opts.imageRate < 0 || opts.videoRate < 0 ||
				opts.imageRate+opts.videoRate <= 0) {
			return options{}, errors.New("mixed profiles require a positive duration and offered rate")
		}
	case "real-canary":
		if len(opts.realImageURLs) < 3 {
			return options{}, errors.New("real-canary requires at least three public AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS")
		}
	default:
		return options{}, fmt.Errorf("unsupported profile %q", opts.profile)
	}
	return opts, nil
}

func (o options) shardRequestCount(total int) int {
	if total <= o.shardIndex {
		return 0
	}
	return (total-1-o.shardIndex)/o.shardCount + 1
}

func (o options) logicalRequestIndex(localIndex int) int {
	return o.shardIndex + localIndex*o.shardCount
}

func run(ctx context.Context, opts options) ([]phaseResult, error) {
	var tlsConfig *tls.Config
	if opts.gatewayTLSName != "" || opts.gatewayCAFile != "" {
		rootCAs, err := acceptanceRootCAs(opts.gatewayCAFile)
		if err != nil {
			return nil, err
		}
		tlsConfig = &tls.Config{
			MinVersion: tls.VersionTLS12,
			ServerName: opts.gatewayTLSName,
			RootCAs:    rootCAs,
		}
	}
	client := &http.Client{
		Timeout: opts.timeout,
		Transport: &http.Transport{
			MaxIdleConns:        2048,
			MaxIdleConnsPerHost: 1024,
			MaxConnsPerHost:     1024,
			ForceAttemptHTTP2:   true,
			TLSClientConfig:     tlsConfig,
		},
	}
	opts.externalClient = &http.Client{
		Timeout: opts.timeout,
		Transport: &http.Transport{
			MaxIdleConns:        512,
			MaxIdleConnsPerHost: 256,
			ForceAttemptHTTP2:   true,
		},
	}
	results := make([]phaseResult, 0, 5)
	runPhase := func(name string) error {
		var result phaseResult
		if profile, ok := acceptanceworkload.GeminiProfileByName(name); ok {
			requests := profile.Requests
			if opts.requestCount > 0 {
				requests = opts.requestCount
			}
			result = runGemini(
				ctx, client, opts, name, opts.shardRequestCount(requests),
				profile.InputImages, profile.InputBytes, profile.OutputBytes, false, opts.shardIndex, opts.shardCount,
			)
		} else {
			switch name {
			case "smoke-gemini":
				result = runGemini(ctx, client, opts, name, opts.shardRequestCount(4), 1, 256<<10, 256<<10, false, opts.shardIndex, opts.shardCount)
			case "smoke-video":
				result = runVideo(ctx, client, opts, name, opts.shardRequestCount(10), false, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
			case "video-throughput":
				requests := 1200
				if opts.requestCount > 0 {
					requests = opts.requestCount
				}
				result = runVideo(ctx, client, opts, name, opts.shardRequestCount(requests), false, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
			case "video-recovery":
				result = runVideo(ctx, client, opts, name, opts.shardRequestCount(96), true, opts.emulatorFixtureURLs(), false, opts.shardIndex, opts.shardCount)
			case "mixed-soak", "mixed-overload":
				result = runMixed(ctx, client, opts, name == "mixed-overload")
			case "real-gemini-canary":
				result = runGemini(ctx, client, opts, name, 1, 1, 256<<10, 0, true, 0, 1)
			case "real-video-canary":
				result = runVideo(ctx, client, opts, name, 1, false, opts.realImageURLs, true, 0, 1)
			default:
				return fmt.Errorf("unknown phase %q", name)
			}
		}
		results = append(results, result)
		return result.err
	}
	phases := []string{opts.profile}
	if opts.profile == "simulated-smoke" {
		phases = []string{"smoke-gemini", "smoke-video"}
	} else if opts.profile == "simulated-all" {
		phases = []string{"gemini-baseline", "gemini-large", "gemini-peak", "video-throughput", "video-recovery"}
	} else if opts.profile == "real-canary" {
		phases = []string{"real-gemini-canary", "real-video-canary"}
	}
	for _, phase := range phases {
		if err := runPhase(phase); err != nil {
			return results, err
		}
	}
	return results, nil
}

func runGemini(
	ctx context.Context,
	client *http.Client,
	opts options,
	name string,
	requestCount int,
	inputImages int,
	inputBytes int,
	expectedOutputBytes int,
	realUpstream bool,
	requestOffset int,
	requestStride int,
) phaseResult {
	if requestStride < 1 {
		requestStride = 1
	}
	startedAt := time.Now()
	expectedHash := ""
	if expectedOutputBytes > 0 {
		sum := sha256.Sum256(paddedPNG(expectedOutputBytes))
		expectedHash = hex.EncodeToString(sum[:])
	}
	latencies := make([]time.Duration, requestCount)
	var decodedBytes int64
	var completed int
	var failures int
	var firstErr error
	var mu sync.Mutex
	slots := make(chan struct{}, requestCount)
	var wg sync.WaitGroup
	for index := 0; index < requestCount; index++ {
		index := index
		wg.Add(1)
		go func() {
			defer wg.Done()
			logicalIndex := requestOffset + index*requestStride
			select {
			case slots <- struct{}{}:
			case <-ctx.Done():
				return
			}
			defer func() { <-slots }()
			requestStarted := time.Now()
			endpoint := opts.gateways[logicalIndex%len(opts.gateways)] + "/v1beta/models/" + url.PathEscape(opts.geminiModel) + ":generateContent"
			var req *http.Request
			var err error
			if name == acceptanceworkload.GeminiMultiImage.Name {
				req, err = newGeminiFileDataRequest(ctx, endpoint, geminiFixtureURLs(opts, inputImages, logicalIndex))
			} else {
				inputs := geminiInputs(inputImages, inputBytes, opts.runID+"/"+opts.executionID, logicalIndex)
				req, err = newGeminiRequest(ctx, endpoint, inputs)
			}
			if err == nil {
				opts.setHeaders(req, logicalIndex, realUpstream)
				req.Header.Set("Content-Type", "application/json")
				var response *http.Response
				response, err = client.Do(req)
				if err == nil {
					if response.StatusCode != http.StatusOK {
						err = withOperation("gemini_submit", responseStatusError(response))
					} else {
						var size int64
						var outputHash string
						size, outputHash, err = streamGeminiImageHash(response.Body)
						err = withOperation("gemini_response", err)
						_ = response.Body.Close()
						if err == nil && expectedOutputBytes > 0 && (size != int64(expectedOutputBytes) || outputHash != expectedHash) {
							err = withOperation("gemini_response", fmt.Errorf("Gemini output mismatch: bytes=%d hash=%s", size, outputHash))
						}
						if err == nil {
							mu.Lock()
							decodedBytes += size
							if expectedHash == "" {
								expectedHash = outputHash
							}
							mu.Unlock()
						}
					}
				}
			}
			err = withOperation("gemini_submit", err)
			latency := time.Since(requestStarted)
			mu.Lock()
			latencies[index] = latency
			if err != nil {
				failures++
				if firstErr == nil {
					firstErr = err
				}
			} else {
				completed++
			}
			mu.Unlock()
		}()
	}
	wg.Wait()
	elapsed := time.Since(startedAt)
	result := phaseReport{
		Name: name, Requests: requestCount, Completed: completed, Failed: failures,
		DurationMS: elapsed.Milliseconds(), DecodedOutputBytes: decodedBytes, OutputSHA256: expectedHash,
		InputImagesPerRequest: inputImages,
	}
	if firstErr == nil && name == "gemini-baseline" && elapsed > 8*time.Minute {
		firstErr = fmt.Errorf("Gemini baseline exceeded 8 minutes: %s", elapsed)
	}
	if firstErr == nil && completed != requestCount {
		firstErr = fmt.Errorf("Gemini completed %d of %d requests", completed, requestCount)
	}
	return phaseResult{report: result, latencies: latencies, err: firstErr}
}

func streamGeminiRequestBody(inputs [][]byte) io.ReadCloser {
	reader, writer := io.Pipe()
	go func() {
		closeWithError := func(err error) {
			_ = writer.CloseWithError(err)
		}
		if _, err := io.WriteString(writer, geminiRequestPrefix); err != nil {
			closeWithError(err)
			return
		}
		for _, input := range inputs {
			if _, err := io.WriteString(writer, geminiRequestImageStart); err != nil {
				closeWithError(err)
				return
			}
			encoder := base64.NewEncoder(base64.StdEncoding, writer)
			if _, err := encoder.Write(input); err != nil {
				closeWithError(err)
				return
			}
			if err := encoder.Close(); err != nil {
				closeWithError(err)
				return
			}
			if _, err := io.WriteString(writer, geminiRequestImageEnd); err != nil {
				closeWithError(err)
				return
			}
		}
		if _, err := io.WriteString(writer, geminiRequestSuffix); err != nil {
			closeWithError(err)
			return
		}
		_ = writer.Close()
	}()
	return reader
}

func newGeminiRequest(ctx context.Context, endpoint string, inputs [][]byte) (*http.Request, error) {
	getBody := func() (io.ReadCloser, error) {
		return streamGeminiRequestBody(inputs), nil
	}
	body, _ := getBody()
	request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, body)
	if err != nil {
		_ = body.Close()
		return nil, err
	}
	request.GetBody = getBody
	contentLength := len(geminiRequestPrefix) + len(geminiRequestSuffix)
	for _, input := range inputs {
		contentLength += len(geminiRequestImageStart) + base64.StdEncoding.EncodedLen(len(input)) + len(geminiRequestImageEnd)
	}
	request.ContentLength = int64(contentLength)
	return request, nil
}

func newGeminiFileDataRequest(ctx context.Context, endpoint string, imageURLs []string) (*http.Request, error) {
	parts := make([]any, 0, len(imageURLs)+1)
	parts = append(parts, map[string]any{"text": "将多张参考图融合编辑为蓝色赛博朋克风格，保留主体结构"})
	for _, imageURL := range imageURLs {
		parts = append(parts, map[string]any{"fileData": map[string]any{
			"mimeType": geminiFixtureMIMEType(imageURL),
			"fileUri":  imageURL,
		}})
	}
	body, err := json.Marshal(map[string]any{
		"contents":         []any{map[string]any{"role": "user", "parts": parts}},
		"generationConfig": map[string]any{"responseModalities": []string{"IMAGE"}},
	})
	if err != nil {
		return nil, err
	}
	return http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
}

func geminiFixtureMIMEType(imageURL string) string {
	lower := strings.ToLower(imageURL)
	switch {
	case strings.Contains(lower, ".webp"):
		return "image/webp"
	case strings.Contains(lower, ".jpg"), strings.Contains(lower, ".jpeg"):
		return "image/jpeg"
	default:
		return "image/png"
	}
}

func geminiFixtureURLs(opts options, count int, logicalIndex int) []string {
	fixtures := opts.emulatorFixtureURLs()
	if count > len(fixtures) {
		count = len(fixtures)
	}
	urls := make([]string, 0, count)
	for imageIndex := 0; imageIndex < count; imageIndex++ {
		urls = append(urls, fixtures[(logicalIndex*count+imageIndex)%len(fixtures)])
	}
	return urls
}

func runVideo(
	ctx context.Context,
	client *http.Client,
	opts options,
	name string,
	requestCount int,
	longRun bool,
	imageURLs []string,
	realUpstream bool,
	requestOffset int,
	requestStride int,
) phaseResult {
	if requestStride < 1 {
		requestStride = 1
	}
	startedAt := time.Now()
	combinationCount := 128
	if realUpstream {
		combinationCount = 1
	}
	combinations := videoCombinations(imageURLs, combinationCount)
	var taskIDs = make([]string, requestCount)
	latencies := make([]time.Duration, requestCount)
	startedTasks := make([]time.Time, requestCount)
	var completed int
	var failures int
	var forcedConversions int
	var firstErr error
	var mu sync.Mutex
	var wg sync.WaitGroup
	submitStartedAt := time.Now()
	for index := 0; index < requestCount; index++ {
		index := index
		wg.Add(1)
		go func() {
			defer wg.Done()
			logicalIndex := requestOffset + index*requestStride
			requestStarted := time.Now()
			startedTasks[index] = requestStarted
			imageCount := acceptanceworkload.VideoImageCount(logicalIndex)
			combo := append([]string(nil), combinations[logicalIndex%len(combinations)]...)
			if len(combo) > imageCount {
				combo = combo[:imageCount]
			}
			if !realUpstream && logicalIndex%4 == 0 {
				combo[0] = imageURLs[12+(logicalIndex/4)%4]
			}
			content := make([]any, 0, len(combo)+1)
			prompt := "多参考图生成连续运镜视频，保持人物、服装和场景一致"
			if longRun {
				prompt += " acceptance-long-recovery"
			}
			if !realUpstream && logicalIndex%4 == 0 {
				prompt += " acceptance-force-conversion"
			}
			content = append(content, map[string]any{"type": "text", "text": prompt})
			for imageIndex, imageURL := range combo {
				role := "reference_image"
				if logicalIndex%5 == 0 && imageIndex == 0 {
					role = "first_frame"
				}
				if logicalIndex%5 == 0 && len(combo) > 3 && imageIndex == len(combo)-1 {
					role = "last_frame"
				}
				content = append(content, map[string]any{
					"type": "image_url", "role": role,
					"image_url": map[string]any{"url": imageURL},
				})
			}
			body, _ := json.Marshal(map[string]any{
				"model": opts.videoModel, "modelType": "omni_video", "content": content, "seed": logicalIndex + 1,
				"duration": 5, "ratio": "16:9", "resolution": "720p",
			})
			endpoint := opts.gateways[logicalIndex%len(opts.gateways)] + "/api/v1/videos/generations"
			req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
			if err == nil {
				opts.setHeaders(req, logicalIndex, realUpstream)
				req.Header.Set("Content-Type", "application/json")
				req.Header.Set("X-Async", "true")
				var response *http.Response
				response, err = client.Do(req)
				if err == nil {
					if response.StatusCode != http.StatusAccepted {
						err = withOperation("video_submit", responseStatusError(response))
					} else {
						var accepted struct {
							TaskID string `json:"taskId"`
						}
						err = json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&accepted)
						_ = response.Body.Close()
						taskIDs[index] = strings.TrimSpace(accepted.TaskID)
						if err == nil && taskIDs[index] == "" {
							err = errors.New("video acceptance response did not include taskId")
						}
					}
				}
			}
			err = withOperation("video_submit", err)
			mu.Lock()
			latencies[index] = time.Since(requestStarted)
			if !realUpstream && logicalIndex%4 == 0 {
				forcedConversions++
			}
			if err != nil {
				failures++
				if firstErr == nil {
					firstErr = err
				}
			}
			mu.Unlock()
		}()
	}
	wg.Wait()
	submissionDuration := time.Since(submitStartedAt)
	if firstErr == nil && name == "video-throughput" && requestCount == 1200 && submissionDuration > 10*time.Second {
		firstErr = fmt.Errorf("1200 video submissions exceeded 10 seconds: %s", submissionDuration)
	}
	uniqueTaskIDs := map[string]struct{}{}
	for _, taskID := range taskIDs {
		if taskID != "" {
			uniqueTaskIDs[taskID] = struct{}{}
		}
	}
	if firstErr == nil && len(uniqueTaskIDs) != requestCount {
		firstErr = fmt.Errorf("video task IDs are not unique: unique=%d requests=%d", len(uniqueTaskIDs), requestCount)
	}
	pollSlots := make(chan struct{}, 256)
	for index, taskID := range taskIDs {
		if taskID == "" {
			continue
		}
		index, taskID := index, taskID
		logicalIndex := requestOffset + index*requestStride
		wg.Add(1)
		go func() {
			defer wg.Done()
			select {
			case pollSlots <- struct{}{}:
			case <-ctx.Done():
				return
			}
			defer func() { <-pollSlots }()
			err := pollVideoTask(ctx, client, opts, taskID, logicalIndex, realUpstream)
			mu.Lock()
			latencies[index] = time.Since(startedTasks[index])
			if err != nil {
				failures++
				if firstErr == nil {
					firstErr = err
				}
			} else {
				completed++
			}
			mu.Unlock()
		}()
	}
	wg.Wait()
	uniqueImageCombinations := requestedVideoCombinationCount(
		combinations,
		requestCount,
		requestOffset,
		requestStride,
	)
	expectedCombinations := min(requestCount, 128)
	if firstErr == nil && !realUpstream && uniqueImageCombinations < expectedCombinations {
		firstErr = fmt.Errorf(
			"video image combinations=%d, want at least %d",
			uniqueImageCombinations,
			expectedCombinations,
		)
	}
	elapsed := time.Since(startedAt)
	result := phaseReport{
		Name: name, Requests: requestCount, Completed: completed, Failed: failures,
		DurationMS: elapsed.Milliseconds(), SubmissionDurationMS: submissionDuration.Milliseconds(),
		UniqueTaskIDs: len(uniqueTaskIDs), UniqueImageCombos: uniqueImageCombinations,
		ForcedConversionTasks: forcedConversions,
	}
	if firstErr == nil && completed != requestCount {
		firstErr = fmt.Errorf("video completed %d of %d tasks", completed, requestCount)
	}
	return phaseResult{report: result, latencies: latencies, err: firstErr}
}

func runMixed(
	ctx context.Context,
	client *http.Client,
	opts options,
	expectThrottling bool,
) phaseResult {
	runCtx, stopRun := context.WithCancel(ctx)
	defer stopRun()
	startedAt := time.Now()
	totalRate := opts.imageRate + opts.videoRate
	interval := time.Duration(float64(time.Second) / totalRate)
	if interval < time.Millisecond {
		interval = time.Millisecond
	}
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	stop := time.NewTimer(opts.mixedDuration)
	defer stop.Stop()
	slots := make(chan struct{}, 4096)
	var wg sync.WaitGroup
	var mu sync.Mutex
	latencies := make([]time.Duration, 0, int(totalRate*opts.mixedDuration.Seconds()))
	requests := 0
	completed := 0
	failed := 0
	throttled := 0
	unexpected5xx := 0
	var firstErr error
	imageAccumulator := float64(0)
	generating := true
	for generating {
		select {
		case <-runCtx.Done():
			generating = false
			if ctx.Err() != nil {
				mu.Lock()
				if firstErr == nil {
					firstErr = ctx.Err()
				}
				mu.Unlock()
			}
		case <-stop.C:
			generating = false
		case <-ticker.C:
			select {
			case slots <- struct{}{}:
			case <-runCtx.Done():
				generating = false
				continue
			}
			requestIndex := opts.logicalRequestIndex(requests)
			requests++
			imageAccumulator += opts.imageRate
			isImage := imageAccumulator >= totalRate
			if isImage {
				imageAccumulator -= totalRate
			}
			wg.Add(1)
			go func() {
				defer wg.Done()
				defer func() { <-slots }()
				var result phaseResult
				if isImage {
					result = runGemini(runCtx, client, opts, "mixed-image", 1, 1, 256<<10, 256<<10, false, requestIndex, 1)
				} else {
					result = runVideo(
						runCtx,
						client,
						opts,
						"mixed-video",
						1,
						false,
						opts.emulatorFixtureURLs(),
						false,
						requestIndex,
						1,
					)
				}
				latency := time.Since(startedAt)
				if len(result.latencies) > 0 {
					latency = result.latencies[0]
				}
				mu.Lock()
				defer mu.Unlock()
				latencies = append(latencies, latency)
				if result.err == nil {
					completed++
					return
				}
				var statusErr *httpStatusError
				if errors.As(result.err, &statusErr) && statusErr.Status == http.StatusTooManyRequests {
					throttled++
					if !expectThrottling && firstErr == nil {
						firstErr = result.err
						failed++
						stopRun()
					}
					return
				}
				if errors.As(result.err, &statusErr) && statusErr.Status >= 500 {
					unexpected5xx++
				}
				failed++
				if firstErr == nil {
					firstErr = result.err
					stopRun()
				}
			}()
		}
	}
	wg.Wait()
	if firstErr == nil && expectThrottling && throttled == 0 {
		firstErr = errors.New("mixed overload did not receive any 429 response")
	}
	if firstErr == nil && completed+throttled != requests {
		firstErr = fmt.Errorf(
			"mixed workload accounted for %d of %d requests",
			completed+throttled,
			requests,
		)
	}
	if firstErr == nil && unexpected5xx > 0 {
		firstErr = fmt.Errorf("mixed workload received %d unexpected 5xx responses", unexpected5xx)
	}
	return phaseResult{
		report: phaseReport{
			Name:                 map[bool]string{false: "mixed-soak", true: "mixed-overload"}[expectThrottling],
			Requests:             requests,
			Completed:            completed,
			Failed:               failed,
			DurationMS:           time.Since(startedAt).Milliseconds(),
			Throttled:            throttled,
			Unexpected5xx:        unexpected5xx,
			OfferedRatePerSecond: totalRate,
		},
		latencies: latencies,
		err:       firstErr,
	}
}

func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskID string, index int, realUpstream bool) error {
	ticker := time.NewTicker(time.Second)
	defer ticker.Stop()
	for {
		endpoint := opts.gateways[index%len(opts.gateways)] + "/api/v1/ai/result/" + url.PathEscape(taskID)
		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
		if err != nil {
			return withOperation("video_poll", err)
		}
		opts.setHeaders(req, index, realUpstream)
		response, err := client.Do(req)
		if err == nil {
			if response.StatusCode != http.StatusOK {
				err = responseStatusError(response)
			} else {
				var payload map[string]any
				err = json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&payload)
				_ = response.Body.Close()
				status := strings.ToLower(strings.TrimSpace(fmt.Sprint(payload["status"])))
				switch status {
				case "succeeded", "success":
					mediaURL := findMediaURL(payload)
					if mediaURL == "" {
						return withOperation("video_poll", fmt.Errorf("video task %s succeeded without a media URL", taskID))
					}
					return withOperation("video_download", validateVideoAsset(ctx, client, opts, mediaURL, index))
				case "failed", "cancelled", "canceled":
					return withOperation("video_poll", fmt.Errorf("video task %s finished with status %s", taskID, status))
				}
			}
		}
		if err != nil {
			return withOperation("video_poll", err)
		}
		select {
		case <-ctx.Done():
			return withOperation("video_poll", ctx.Err())
		case <-ticker.C:
		}
	}
}

func findMediaURL(value any) string {
	switch typed := value.(type) {
	case map[string]any:
		preferredKeys := []string{"video_url", "url", "data", "content", "output", "result", "upload"}
		visited := make(map[string]struct{}, len(typed))
		for _, key := range preferredKeys {
			item, ok := typed[key]
			if !ok {
				continue
			}
			visited[key] = struct{}{}
			normalized := strings.ToLower(strings.TrimSpace(key))
			if (normalized == "url" || normalized == "video_url") && acceptanceMediaURL(strings.TrimSpace(fmt.Sprint(item))) {
				return strings.TrimSpace(fmt.Sprint(item))
			}
			if mediaURL := findMediaURL(item); mediaURL != "" {
				return mediaURL
			}
		}
		keys := make([]string, 0, len(typed)-len(visited))
		for key := range typed {
			if _, ok := visited[key]; !ok {
				keys = append(keys, key)
			}
		}
		sort.Strings(keys)
		for _, key := range keys {
			if mediaURL := findMediaURL(typed[key]); mediaURL != "" {
				return mediaURL
			}
		}
	case []any:
		for _, item := range typed {
			if mediaURL := findMediaURL(item); mediaURL != "" {
				return mediaURL
			}
		}
	}
	return ""
}

func acceptanceMediaURL(value string) bool {
	return strings.HasPrefix(value, "/") || strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://")
}

func validateVideoAsset(ctx context.Context, client *http.Client, opts options, mediaURL string, index int) error {
	requestURL, gatewayRequest, err := acceptanceMediaRequestURL(opts, mediaURL, index)
	if err != nil {
		return fmt.Errorf("resolve final video: %w", err)
	}
	request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
	if err != nil {
		return err
	}
	if gatewayRequest {
		opts.setHeaders(request, index, false)
	}
	mediaClient := client
	if !gatewayRequest && opts.gatewayTLSName != "" {
		mediaClient = opts.externalClient
		if mediaClient == nil {
			mediaClient = &http.Client{Timeout: client.Timeout}
		}
	}
	response, err := mediaClient.Do(request)
	if err != nil {
		return fmt.Errorf("download final video: %w", err)
	}
	defer response.Body.Close()
	if response.StatusCode < 200 || response.StatusCode >= 300 {
		return fmt.Errorf("download final video: HTTP %d", response.StatusCode)
	}
	payload, err := io.ReadAll(io.LimitReader(response.Body, 64<<10))
	if err != nil {
		return fmt.Errorf("read final video: %w", err)
	}
	contentType := strings.ToLower(strings.TrimSpace(response.Header.Get("Content-Type")))
	hasMP4Header := len(payload) >= 12 && string(payload[4:8]) == "ftyp"
	if len(payload) < 12 || (!strings.HasPrefix(contentType, "video/") && !hasMP4Header) {
		return fmt.Errorf("final video asset is invalid: bytes=%d content_type=%q", len(payload), contentType)
	}
	return nil
}

func acceptanceMediaRequestURL(opts options, mediaURL string, index int) (string, bool, error) {
	mediaURL = strings.TrimSpace(mediaURL)
	if mediaURL == "" {
		return "", false, errors.New("empty media URL")
	}
	if strings.HasPrefix(mediaURL, "/") {
		if len(opts.gateways) == 0 {
			return "", false, errors.New("no gateway URL configured")
		}
		base, err := url.Parse(opts.gateways[index%len(opts.gateways)])
		if err != nil {
			return "", false, err
		}
		reference, err := url.Parse(mediaURL)
		if err != nil {
			return "", false, err
		}
		return base.ResolveReference(reference).String(), true, nil
	}
	parsed, err := url.Parse(mediaURL)
	if err != nil {
		return "", false, err
	}
	if opts.gatewayTLSName == "" || !strings.EqualFold(parsed.Hostname(), opts.gatewayTLSName) {
		return parsed.String(), false, nil
	}
	if len(opts.gateways) == 0 {
		return "", false, errors.New("no gateway URL configured")
	}
	base, err := url.Parse(opts.gateways[index%len(opts.gateways)])
	if err != nil {
		return "", false, err
	}
	base.Path = parsed.Path
	base.RawPath = parsed.RawPath
	base.RawQuery = parsed.RawQuery
	base.Fragment = parsed.Fragment
	return base.String(), true, nil
}

func (o options) setHeaders(request *http.Request, requestIndex int, realUpstream bool) {
	if o.gatewayTLSName != "" {
		request.Host = o.gatewayTLSName
	}
	request.Header.Set("Authorization", "Bearer "+o.apiKeys[requestIndex%len(o.apiKeys)])
	request.Header.Set(runHeader, o.runID)
	request.Header.Set(tokenHeader, o.runToken)
	if request.Method == http.MethodPost {
		request.Header.Set("Idempotency-Key", fmt.Sprintf("acceptance-%s-%s-%d", o.runID, o.executionID, requestIndex))
	}
	if realUpstream {
		request.Header.Set(upstreamHeader, "real")
	}
}

func (o options) emulatorFixtureURLs() []string {
	out := make([]string, 0, 16)
	for index := 0; index < 4; index++ {
		out = append(out, fmt.Sprintf("%s/fixtures/image-%02d.png", o.emulatorURL, index))
	}
	for index := 4; index < 8; index++ {
		out = append(out, fmt.Sprintf("%s/fixtures/image-%02d.jpg", o.emulatorURL, index))
	}
	for index := 8; index < 12; index++ {
		out = append(out, fmt.Sprintf("%s/fixtures/image-%02d.webp", o.emulatorURL, index))
	}
	for index := 12; index < 16; index++ {
		out = append(out, fmt.Sprintf("%s/fixtures/image-%02d-oversized.jpg", o.emulatorURL, index))
	}
	return out
}

func videoCombinations(images []string, count int) [][]string {
	if len(images) < 3 {
		return nil
	}
	if len(images) < 9 {
		return [][]string{append([]string(nil), images[:3]...)}
	}
	out := make([][]string, 0, count)
	seen := map[string]struct{}{}
	for seed := 0; len(out) < count; seed++ {
		imageCount := acceptanceworkload.VideoImageCount(seed)
		combo := make([]string, 0, imageCount)
		used := map[int]struct{}{}
		state := uint64(seed + 1)
		for len(combo) < imageCount {
			state = state*6364136223846793005 + 1442695040888963407
			index := int((state >> 32) % uint64(len(images)))
			if _, exists := used[index]; exists {
				continue
			}
			used[index] = struct{}{}
			combo = append(combo, images[index])
		}
		signature := strings.Join(combo, "\x00")
		if _, exists := seen[signature]; exists {
			continue
		}
		seen[signature] = struct{}{}
		out = append(out, combo)
	}
	return out
}

func requestedVideoCombinationCount(
	combinations [][]string,
	requestCount int,
	requestOffset int,
	requestStride int,
) int {
	if len(combinations) == 0 || requestCount <= 0 {
		return 0
	}
	if requestStride < 1 {
		requestStride = 1
	}
	seen := map[string]struct{}{}
	for index := 0; index < requestCount; index++ {
		logicalIndex := requestOffset + index*requestStride
		imageCount := acceptanceworkload.VideoImageCount(logicalIndex)
		combo := combinations[logicalIndex%len(combinations)]
		if len(combo) > imageCount {
			combo = combo[:imageCount]
		}
		seen[strings.Join(combo, "\x00")] = struct{}{}
	}
	return len(seen)
}

func streamGeminiImageHash(reader io.Reader) (int64, string, error) {
	buffered := bufio.NewReaderSize(reader, 64<<10)
	if err := scanUntil(buffered, []byte(`"inlineData"`), 256<<20); err != nil {
		return 0, "", err
	}
	if err := scanUntil(buffered, []byte(`"data"`), 1<<20); err != nil {
		return 0, "", err
	}
	for {
		item, err := buffered.ReadByte()
		if err != nil {
			return 0, "", err
		}
		if item == ':' {
			break
		}
	}
	for {
		item, err := buffered.ReadByte()
		if err != nil {
			return 0, "", err
		}
		if item == '"' {
			break
		}
		if item != ' ' && item != '\t' && item != '\r' && item != '\n' {
			return 0, "", errors.New("Gemini inlineData.data is not a JSON string")
		}
	}
	jsonString := &base64JSONStringReader{reader: buffered}
	digest := sha256.New()
	decodedBytes, err := io.Copy(digest, base64.NewDecoder(base64.StdEncoding, jsonString))
	if err != nil {
		return 0, "", fmt.Errorf("decode Gemini Base64 output: %w", err)
	}
	if decodedBytes == 0 {
		return 0, "", errors.New("Gemini Base64 output is empty")
	}
	return decodedBytes, hex.EncodeToString(digest.Sum(nil)), nil
}

type base64JSONStringReader struct {
	reader *bufio.Reader
	done   bool
}

func (r *base64JSONStringReader) Read(payload []byte) (int, error) {
	if r.done {
		return 0, io.EOF
	}
	written := 0
	for written < len(payload) {
		item, err := r.reader.ReadByte()
		if err != nil {
			return written, err
		}
		if item == '"' {
			r.done = true
			if written == 0 {
				return 0, io.EOF
			}
			return written, nil
		}
		if item == '\\' {
			return written, errors.New("escaped Base64 JSON strings are not supported")
		}
		payload[written] = item
		written++
	}
	return written, nil
}

func scanUntil(reader *bufio.Reader, target []byte, limit int64) error {
	matched := 0
	for read := int64(0); read < limit; read++ {
		item, err := reader.ReadByte()
		if err != nil {
			return err
		}
		if item == target[matched] {
			matched++
			if matched == len(target) {
				return nil
			}
			continue
		}
		if item == target[0] {
			matched = 1
		} else {
			matched = 0
		}
	}
	return errors.New("Gemini inlineData output was not found")
}

func responseStatusError(response *http.Response) error {
	defer response.Body.Close()
	payload, _ := io.ReadAll(io.LimitReader(response.Body, 512))
	return &httpStatusError{Status: response.StatusCode, Body: strings.TrimSpace(string(payload))}
}

func percentileMilliseconds(values []time.Duration, quantile float64) float64 {
	filtered := make([]time.Duration, 0, len(values))
	for _, value := range values {
		if value > 0 {
			filtered = append(filtered, value)
		}
	}
	if len(filtered) == 0 {
		return 0
	}
	sort.Slice(filtered, func(i, j int) bool { return filtered[i] < filtered[j] })
	index := int(float64(len(filtered)-1) * quantile)
	return float64(filtered[index].Microseconds()) / 1000
}

func paddedPNG(size int) []byte {
	return paddedPNGVariant(size, 0)
}

func geminiInputs(count int, totalBytes int, executionID string, logicalIndex int) [][]byte {
	if count < 1 {
		count = 1
	}
	inputs := make([][]byte, 0, count)
	baseSize := totalBytes / count
	remainder := totalBytes % count
	for imageIndex := 0; imageIndex < count; imageIndex++ {
		size := baseSize
		if imageIndex < remainder {
			size++
		}
		variant := geminiInputVariant(executionID, logicalIndex*count+imageIndex)
		inputs = append(inputs, paddedPNGVariant(size, variant))
	}
	return inputs
}

func geminiInputVariant(runID string, logicalIndex int) int {
	digest := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", strings.TrimSpace(runID), logicalIndex)))
	return int(binary.BigEndian.Uint64(digest[:8]) & uint64(^uint(0)>>1))
}

func paddedPNGVariant(size int, variant int) []byte {
	base, _ := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
	if size <= len(base) {
		return base
	}
	iendOffset := len(base) - 12
	paddingLength := size - len(base) - 12
	if paddingLength < 0 {
		paddingLength = 0
	}
	chunk := make([]byte, 12+paddingLength)
	binary.BigEndian.PutUint32(chunk[:4], uint32(paddingLength))
	copy(chunk[4:8], []byte("teST"))
	if variant != 0 && paddingLength >= 8 {
		binary.BigEndian.PutUint64(chunk[8:16], uint64(variant))
	}
	binary.BigEndian.PutUint32(chunk[8+paddingLength:], crc32.ChecksumIEEE(chunk[4:8+paddingLength]))
	out := make([]byte, 0, size)
	out = append(out, base[:iendOffset]...)
	out = append(out, chunk...)
	out = append(out, base[iendOffset:]...)
	return out
}

func redactError(message string, opts options) string {
	for _, secret := range append(append([]string(nil), opts.apiKeys...), opts.runToken) {
		if secret != "" {
			message = strings.ReplaceAll(message, secret, "[REDACTED]")
		}
	}
	return reportURLPattern.ReplaceAllString(message, "[REDACTED_URL]")
}

func acceptanceRootCAs(path string) (*x509.CertPool, error) {
	if strings.TrimSpace(path) == "" {
		return nil, nil
	}
	info, err := os.Lstat(path)
	if err != nil {
		return nil, fmt.Errorf("read Gateway CA file: %w", err)
	}
	if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
		return nil, errors.New("Gateway CA file must be a regular non-symlink file")
	}
	payload, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read Gateway CA file: %w", err)
	}
	roots, err := x509.SystemCertPool()
	if err != nil || roots == nil {
		roots = x509.NewCertPool()
	}
	if !roots.AppendCertsFromPEM(payload) {
		return nil, errors.New("Gateway CA file does not contain a PEM certificate")
	}
	return roots, nil
}

func splitCSV(value string) []string {
	items := strings.Split(value, ",")
	out := make([]string, 0, len(items))
	for _, item := range items {
		if item = strings.TrimSpace(item); item != "" {
			out = append(out, item)
		}
	}
	return out
}

func env(name string, fallback string) string {
	if value := strings.TrimSpace(os.Getenv(name)); value != "" {
		return value
	}
	return fallback
}

func envDuration(name string, fallback time.Duration) time.Duration {
	value := strings.TrimSpace(os.Getenv(name))
	if value == "" {
		return fallback
	}
	parsed, err := time.ParseDuration(value)
	if err != nil {
		return -1
	}
	return parsed
}

func envFloat(name string, fallback float64) float64 {
	value := strings.TrimSpace(os.Getenv(name))
	if value == "" {
		return fallback
	}
	parsed, err := strconv.ParseFloat(value, 64)
	if err != nil {
		return -1
	}
	return parsed
}