feat(acceptance): 建立同构验收与弹性容量体系

实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护,并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
2026-07-31 18:02:24 +08:00
parent 93d36d0e55
commit e05922b0f4
94 changed files with 12379 additions and 826 deletions
@@ -28,6 +28,13 @@ type taskTrafficError struct {
Err error
}
func (s *Server) acceptanceStore() *store.Store {
if s.coordinationStore != nil {
return s.coordinationStore
}
return s.store
}
func (e *taskTrafficError) Error() string {
return e.Message
}
@@ -37,7 +44,7 @@ func (e *taskTrafficError) ErrorCode() string {
}
func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTrafficAdmission, error) {
runID, err := s.store.AuthorizeAcceptanceTask(
runID, err := s.acceptanceStore().AuthorizeAcceptanceTask(
r.Context(),
r.Header.Get(acceptanceRunHeader),
r.Header.Get(acceptanceTokenHeader),
@@ -121,7 +128,7 @@ func writeTaskTrafficError(w http.ResponseWriter, err error, protocol string) {
// @Success 200 {object} store.GatewayTrafficMode
// @Router /api/admin/system/acceptance/traffic-mode [get]
func (s *Server) getGatewayTrafficMode(w http.ResponseWriter, r *http.Request) {
mode, err := s.store.GetGatewayTrafficMode(r.Context())
mode, err := s.acceptanceStore().GetGatewayTrafficMode(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "get gateway traffic mode failed")
return
@@ -129,6 +136,45 @@ func (s *Server) getGatewayTrafficMode(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, mode)
}
// pauseGatewayTraffic godoc
// @Summary 上线后硬门禁异常时通过 CAS 暂停正式新任务
// @Tags acceptance
// @Accept json
// @Produce json
// @Security BearerAuth
// @Param body body store.PauseGatewayTrafficInput true "当前线上 release 与 revision CAS"
// @Success 200 {object} store.GatewayTrafficMode
// @Router /api/admin/system/acceptance/traffic-mode/pause [post]
func (s *Server) pauseGatewayTraffic(w http.ResponseWriter, r *http.Request) {
var input store.PauseGatewayTrafficInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
mode, err := s.acceptanceStore().PauseGatewayTraffic(r.Context(), input)
if err != nil {
writeAcceptanceMutationError(w, err)
return
}
writeJSON(w, http.StatusOK, mode)
}
// listCapacityProfiles godoc
// @Summary 获取经生产同构验收认证的容量配置
// @Tags acceptance
// @Produce json
// @Security BearerAuth
// @Success 200 {array} store.CapacityProfile
// @Router /api/admin/system/acceptance/capacity-profiles [get]
func (s *Server) listCapacityProfiles(w http.ResponseWriter, r *http.Request) {
profiles, err := s.acceptanceStore().ListCapacityProfiles(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "list capacity profiles failed")
return
}
writeJSON(w, http.StatusOK, profiles)
}
// createAcceptanceRun godoc
// @Summary 创建生产同构验收 Run
// @Tags acceptance
@@ -144,7 +190,7 @@ func (s *Server) createAcceptanceRun(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
run, err := s.store.CreateAcceptanceRun(r.Context(), input)
run, err := s.acceptanceStore().CreateAcceptanceRun(r.Context(), input)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
@@ -161,7 +207,7 @@ func (s *Server) createAcceptanceRun(w http.ResponseWriter, r *http.Request) {
// @Success 200 {object} store.AcceptanceRun
// @Router /api/admin/system/acceptance/runs/{runID} [get]
func (s *Server) getAcceptanceRun(w http.ResponseWriter, r *http.Request) {
run, err := s.store.GetAcceptanceRun(r.Context(), r.PathValue("runID"))
run, err := s.acceptanceStore().GetAcceptanceRun(r.Context(), r.PathValue("runID"))
if err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusNotFound, "acceptance run not found")
@@ -182,7 +228,7 @@ func (s *Server) getAcceptanceRun(w http.ResponseWriter, r *http.Request) {
// @Success 200 {object} store.GatewayTrafficMode
// @Router /api/admin/system/acceptance/runs/{runID}/activate [post]
func (s *Server) activateAcceptanceRun(w http.ResponseWriter, r *http.Request) {
mode, err := s.store.ActivateAcceptanceRun(r.Context(), r.PathValue("runID"))
mode, err := s.acceptanceStore().ActivateAcceptanceRun(r.Context(), r.PathValue("runID"))
if err != nil {
writeAcceptanceMutationError(w, err)
return
@@ -190,6 +236,36 @@ func (s *Server) activateAcceptanceRun(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, mode)
}
// stageAcceptanceCapacityProfiles godoc
// @Summary 为当前 validation Run 暂存 80% 容量门禁
// @Tags acceptance
// @Accept json
// @Produce json
// @Security BearerAuth
// @Param runID path string true "验收 Run ID"
// @Param body body store.StageAcceptanceCapacityProfilesInput true "待验证容量配置"
// @Success 200 {object} store.AcceptanceRun
// @Router /api/admin/system/acceptance/runs/{runID}/capacity-profiles [post]
func (s *Server) stageAcceptanceCapacityProfiles(w http.ResponseWriter, r *http.Request) {
var input store.StageAcceptanceCapacityProfilesInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
input.RunID = r.PathValue("runID")
run, err := s.acceptanceStore().StageAcceptanceCapacityProfiles(r.Context(), input)
if err != nil {
switch {
case store.IsNotFound(err):
writeError(w, http.StatusConflict, "acceptance run is not running")
default:
writeError(w, http.StatusBadRequest, err.Error())
}
return
}
writeJSON(w, http.StatusOK, run)
}
// finishAcceptanceRun godoc
// @Summary 记录验收门禁结果
// @Tags acceptance
@@ -207,7 +283,7 @@ func (s *Server) finishAcceptanceRun(w http.ResponseWriter, r *http.Request) {
return
}
input.RunID = r.PathValue("runID")
run, err := s.store.FinishAcceptanceRun(r.Context(), input)
run, err := s.acceptanceStore().FinishAcceptanceRun(r.Context(), input)
if err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusConflict, "acceptance run is not running")
@@ -228,7 +304,7 @@ func (s *Server) finishAcceptanceRun(w http.ResponseWriter, r *http.Request) {
// @Success 200 {object} store.AcceptanceRun
// @Router /api/admin/system/acceptance/runs/{runID}/retry [post]
func (s *Server) retryAcceptanceRun(w http.ResponseWriter, r *http.Request) {
run, err := s.store.RetryAcceptanceRun(r.Context(), r.PathValue("runID"))
run, err := s.acceptanceStore().RetryAcceptanceRun(r.Context(), r.PathValue("runID"))
if err != nil {
writeAcceptanceMutationError(w, err)
return
@@ -253,7 +329,7 @@ func (s *Server) promoteAcceptanceRun(w http.ResponseWriter, r *http.Request) {
return
}
input.RunID = r.PathValue("runID")
mode, err := s.store.PromoteAcceptanceRun(r.Context(), input)
mode, err := s.acceptanceStore().PromoteAcceptanceRun(r.Context(), input)
if err != nil {
writeAcceptanceMutationError(w, err)
return
@@ -278,7 +354,7 @@ func (s *Server) abortAcceptanceRun(w http.ResponseWriter, r *http.Request) {
return
}
input.RunID = r.PathValue("runID")
mode, err := s.store.AbortAcceptanceRun(r.Context(), input)
mode, err := s.acceptanceStore().AbortAcceptanceRun(r.Context(), input)
if err != nil {
writeAcceptanceMutationError(w, err)
return
@@ -0,0 +1,165 @@
package httpapi
import (
"context"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
type acceptanceStatusSnapshot struct {
ID string
Status string
}
type acceptanceUserGroupSnapshot struct {
ID string
RateLimitPolicy []byte
}
// isolateHTTPAcceptanceState lets a stress test disable unrelated runtime
// resources without leaking those mutations into later integration tests that
// intentionally share the same dedicated test database.
func isolateHTTPAcceptanceState(
t *testing.T,
ctx context.Context,
pool *pgxpool.Pool,
includeStorage bool,
) func() {
t.Helper()
startedAt := time.Now()
platforms := readAcceptanceStatusSnapshots(t, ctx, pool, `
SELECT id::text, status
FROM integration_platforms
WHERE deleted_at IS NULL`)
userGroups := make([]acceptanceUserGroupSnapshot, 0)
rows, err := pool.Query(ctx, `
SELECT id::text, rate_limit_policy
FROM gateway_user_groups`)
if err != nil {
t.Fatalf("snapshot acceptance user groups: %v", err)
}
for rows.Next() {
var item acceptanceUserGroupSnapshot
if err := rows.Scan(&item.ID, &item.RateLimitPolicy); err != nil {
rows.Close()
t.Fatalf("scan acceptance user group snapshot: %v", err)
}
userGroups = append(userGroups, item)
}
if err := rows.Err(); err != nil {
rows.Close()
t.Fatalf("read acceptance user group snapshots: %v", err)
}
rows.Close()
storageChannels := []acceptanceStatusSnapshot(nil)
if includeStorage {
storageChannels = readAcceptanceStatusSnapshots(t, ctx, pool, `
SELECT id::text, status
FROM file_storage_channels
WHERE deleted_at IS NULL`)
}
if _, err := pool.Exec(ctx, `
UPDATE integration_platforms
SET status = 'disabled'
WHERE deleted_at IS NULL`); err != nil {
t.Fatalf("isolate acceptance platforms: %v", err)
}
if includeStorage {
if _, err := pool.Exec(ctx, `
UPDATE file_storage_channels
SET status = 'disabled'
WHERE deleted_at IS NULL`); err != nil {
t.Fatalf("isolate acceptance storage channels: %v", err)
}
}
return func() {
restoreCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
tx, err := pool.Begin(restoreCtx)
if err != nil {
t.Errorf("begin acceptance state restore: %v", err)
return
}
defer func() {
_ = tx.Rollback(restoreCtx)
}()
for _, item := range platforms {
if _, err := tx.Exec(restoreCtx, `
UPDATE integration_platforms
SET status = $2
WHERE id = $1::uuid`, item.ID, item.Status); err != nil {
t.Errorf("restore acceptance platform %s: %v", item.ID, err)
return
}
}
if _, err := tx.Exec(restoreCtx, `
UPDATE integration_platforms
SET status = 'disabled'
WHERE created_at >= $1
AND deleted_at IS NULL`, startedAt); err != nil {
t.Errorf("disable acceptance-created platforms: %v", err)
return
}
for _, item := range userGroups {
if _, err := tx.Exec(restoreCtx, `
UPDATE gateway_user_groups
SET rate_limit_policy = $2::jsonb
WHERE id = $1::uuid`, item.ID, string(item.RateLimitPolicy)); err != nil {
t.Errorf("restore acceptance user group %s: %v", item.ID, err)
return
}
}
if includeStorage {
for _, item := range storageChannels {
if _, err := tx.Exec(restoreCtx, `
UPDATE file_storage_channels
SET status = $2
WHERE id = $1::uuid`, item.ID, item.Status); err != nil {
t.Errorf("restore acceptance storage channel %s: %v", item.ID, err)
return
}
}
if _, err := tx.Exec(restoreCtx, `
UPDATE file_storage_channels
SET status = 'disabled'
WHERE created_at >= $1
AND deleted_at IS NULL`, startedAt); err != nil {
t.Errorf("disable acceptance-created storage channels: %v", err)
return
}
}
if err := tx.Commit(restoreCtx); err != nil {
t.Errorf("commit acceptance state restore: %v", err)
}
}
}
func readAcceptanceStatusSnapshots(
t *testing.T,
ctx context.Context,
pool *pgxpool.Pool,
query string,
) []acceptanceStatusSnapshot {
t.Helper()
rows, err := pool.Query(ctx, query)
if err != nil {
t.Fatalf("snapshot acceptance statuses: %v", err)
}
defer rows.Close()
items := make([]acceptanceStatusSnapshot, 0)
for rows.Next() {
var item acceptanceStatusSnapshot
if err := rows.Scan(&item.ID, &item.Status); err != nil {
t.Fatalf("scan acceptance status snapshot: %v", err)
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
t.Fatalf("read acceptance status snapshots: %v", err)
}
return items
}
@@ -40,9 +40,8 @@ func TestAsyncSubmissionReservesConcurrencyOnlyAfterWorkerClaim(t *testing.T) {
t.Fatalf("connect acceptance pool: %v", err)
}
defer pool.Close()
if _, err := pool.Exec(ctx, `UPDATE integration_platforms SET status = 'disabled' WHERE deleted_at IS NULL`); err != nil {
t.Fatalf("isolate acceptance platforms: %v", err)
}
restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, false)
defer restoreAcceptanceState()
serverCtx, cancelServer := context.WithCancel(ctx)
defer cancelServer()
@@ -116,18 +115,27 @@ func TestAsyncWorkerThousandConcurrentSchedulingAcceptance(t *testing.T) {
t.Fatalf("connect Worker store: %v", err)
}
defer workerDB.Close()
apiCoordinationDB, err := store.ConnectWithMaxConns(ctx, databaseURL, 4)
if err != nil {
t.Fatalf("connect API critical store: %v", err)
}
defer apiCoordinationDB.Close()
workerCoordinationDB, err := store.ConnectWithMaxConns(ctx, databaseURL, 4)
if err != nil {
t.Fatalf("connect Worker critical store: %v", err)
}
defer workerCoordinationDB.Close()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("connect acceptance pool: %v", err)
}
defer pool.Close()
if _, err := pool.Exec(ctx, `UPDATE integration_platforms SET status = 'disabled' WHERE deleted_at IS NULL`); err != nil {
t.Fatalf("isolate acceptance platforms: %v", err)
}
restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, false)
defer restoreAcceptanceState()
serverCtx, cancelServer := context.WithCancel(ctx)
defer cancelServer()
server := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
server := httptest.NewServer(NewServerWithStores(serverCtx, config.Config{
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
@@ -138,10 +146,11 @@ func TestAsyncWorkerThousandConcurrentSchedulingAcceptance(t *testing.T) {
ProcessRole: "api",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 1000,
AsyncWorkerInstanceHardLimit: 1000,
AsyncWorkerRefreshIntervalSeconds: 1,
}, apiDB, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))))
}, apiDB, apiCoordinationDB, apiDB, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))))
defer server.Close()
workerServer := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
workerServer := httptest.NewServer(NewServerWithStores(serverCtx, config.Config{
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
@@ -152,8 +161,9 @@ func TestAsyncWorkerThousandConcurrentSchedulingAcceptance(t *testing.T) {
ProcessRole: "worker",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 1000,
AsyncWorkerInstanceHardLimit: 1000,
AsyncWorkerRefreshIntervalSeconds: 1,
}, workerDB, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))))
}, workerDB, workerCoordinationDB, workerDB, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))))
defer workerServer.Close()
adminToken := createAsyncAcceptanceAdmin(t, ctx, pool, server.URL)
@@ -180,7 +190,7 @@ WHERE status = 'active'`); err != nil {
}
assertAcceptanceAttempts(t, ctx, pool, taskIDs, len(taskIDs))
t.Logf(
"千级并发证据: tasks=%d duration=%s running_peak=%d lease_peak=%d api_database_max_connections=16 worker_database_max_connections=24",
"千级并发证据: tasks=%d duration=%s running_peak=%d lease_peak=%d api_database_max_connections=16 api_critical_max_connections=4 worker_database_max_connections=24 worker_critical_max_connections=4",
len(taskIDs),
45*time.Second,
peakRunning,
@@ -207,9 +217,8 @@ func TestAsyncWorkerDynamicConcurrencyAcceptance(t *testing.T) {
t.Fatalf("connect acceptance pool: %v", err)
}
defer pool.Close()
if _, err := pool.Exec(ctx, `UPDATE integration_platforms SET status = 'disabled' WHERE deleted_at IS NULL`); err != nil {
t.Fatalf("isolate acceptance platforms: %v", err)
}
restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, false)
defer restoreAcceptanceState()
serverCtx, cancelServer := context.WithCancel(ctx)
defer cancelServer()
@@ -223,6 +232,7 @@ func TestAsyncWorkerDynamicConcurrencyAcceptance(t *testing.T) {
CORSAllowedOrigin: "*",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 256,
AsyncWorkerInstanceHardLimit: 256,
AsyncWorkerRefreshIntervalSeconds: 1,
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
defer server.Close()
@@ -43,15 +43,19 @@ func TestCoreLocalFlow(t *testing.T) {
serverCtx, cancelServer := context.WithCancel(ctx)
defer cancelServer()
handler := NewServerWithContext(serverCtx, config.Config{
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
BillingEngineMode: "enforce",
TaskProgressCallbackEnabled: true,
TaskProgressCallbackURL: "http://callback.local/task-progress",
CORSAllowedOrigin: "*",
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
BillingEngineMode: "enforce",
TaskProgressCallbackEnabled: true,
TaskProgressCallbackURL: "http://callback.local/task-progress",
CORSAllowedOrigin: "*",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 32,
AsyncWorkerInstanceHardLimit: 32,
AsyncWorkerRefreshIntervalSeconds: 1,
}, db, slog.New(slog.NewTextHandler(io.Discard, nil)))
server := httptest.NewServer(handler)
defer server.Close()
@@ -606,8 +610,8 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
"model": defaultImageModel,
"runMode": "simulation",
"prompt": "replace background with clean studio light",
"image": "https://example.com/source.png",
"mask": "https://example.com/mask.png",
"image": server.URL + "/static/simulation/image.png",
"mask": server.URL + "/static/simulation/image-edit.png",
"simulation": true,
"simulationDurationMs": 5,
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &imageEditResponse)
@@ -728,18 +732,35 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
t.Fatalf("voice clone persisted platform should match task attempt: task=%+v voice=%+v", voiceCloneTaskDetail, clonedVoice)
}
doubaoLiteImageEditModel := "doubao-5.0-lite图像编辑"
var doubaoLitePlatformModel struct {
asyncImageEditModel := "async-image-edit-smoke-" + suffixText
asyncImageEditCanonicalKey := "openai:" + asyncImageEditModel
var asyncImageEditBaseModel struct {
ID string `json:"id"`
}
doJSON(t, server.URL, http.MethodPost, "/api/admin/catalog/base-models", loginResponse.AccessToken, map[string]any{
"providerKey": "openai",
"canonicalModelKey": asyncImageEditCanonicalKey,
"invocationName": asyncImageEditModel,
"providerModelName": asyncImageEditModel,
"modelType": []string{"image_edit", "image_generate"},
"displayName": "Async Image Edit Smoke",
"capabilities": map[string]any{"originalTypes": []string{"image_edit", "image_generate"}},
"metadata": map[string]any{"source": "test"},
}, http.StatusCreated, &asyncImageEditBaseModel)
if asyncImageEditBaseModel.ID == "" {
t.Fatal("async image edit smoke base model was not created")
}
var asyncImageEditPlatformModel struct {
ID string `json:"id"`
}
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "easyai:doubao-5.0-lite图像编辑",
"modelName": doubaoLiteImageEditModel,
"modelAlias": doubaoLiteImageEditModel,
"canonicalModelKey": asyncImageEditCanonicalKey,
"modelName": asyncImageEditModel,
"modelAlias": asyncImageEditModel,
"modelType": []string{"image_edit", "image_generate"},
"displayName": doubaoLiteImageEditModel,
}, http.StatusCreated, &doubaoLitePlatformModel)
var doubaoLiteAsyncEdit struct {
"displayName": "Async Image Edit Smoke",
}, http.StatusCreated, &asyncImageEditPlatformModel)
var asyncImageEdit struct {
TaskID string `json:"taskId"`
Task struct {
ID string `json:"id"`
@@ -748,20 +769,20 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
} `json:"task"`
}
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/images/edits", apiKeyResponse.Secret, map[string]any{
"model": doubaoLiteImageEditModel,
"model": asyncImageEditModel,
"runMode": "simulation",
"prompt": "turn the attached bright desktop object into a clean product-style render",
"image": "https://example.com/doubao-lite-source.png",
"image": server.URL + "/static/simulation/image.png",
"size": "2048x2048",
"simulation": true,
"simulationDurationMs": 5,
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &doubaoLiteAsyncEdit)
if doubaoLiteAsyncEdit.TaskID == "" || !doubaoLiteAsyncEdit.Task.AsyncMode {
t.Fatalf("doubao-5.0-lite image edit async task should be accepted: %+v", doubaoLiteAsyncEdit)
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &asyncImageEdit)
if asyncImageEdit.TaskID == "" || !asyncImageEdit.Task.AsyncMode {
t.Fatalf("async image edit task should be accepted: %+v", asyncImageEdit)
}
doubaoLiteAsyncEditDone := waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, doubaoLiteAsyncEdit.TaskID, []string{"succeeded"}, 5*time.Second)
if doubaoLiteAsyncEditDone.Status != "succeeded" {
t.Fatalf("doubao-5.0-lite image edit async task should succeed through river queue, got %+v", doubaoLiteAsyncEditDone)
asyncImageEditDone := waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, asyncImageEdit.TaskID, []string{"succeeded"}, 5*time.Second)
if asyncImageEditDone.Status != "succeeded" {
t.Fatalf("async image edit task should succeed through river queue, got %+v", asyncImageEditDone)
}
var gptImageModelTypesRaw []byte
@@ -782,11 +803,12 @@ LIMIT 1`).Scan(&gptImageModelTypesRaw); err != nil {
}
deniedModel := "permission-deny-smoke-" + suffixText
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, deniedModel)
var deniedPlatformModel struct {
ID string `json:"id"`
}
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + deniedModel,
"modelName": deniedModel,
"modelAlias": deniedModel,
"modelType": []string{"text_generate"},
@@ -830,11 +852,12 @@ LIMIT 1`).Scan(&gptImageModelTypesRaw); err != nil {
}
controlledModel := "permission-allow-smoke-" + suffixText
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, controlledModel)
var controlledPlatformModel struct {
ID string `json:"id"`
}
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + controlledModel,
"modelName": controlledModel,
"modelAlias": controlledModel,
"modelType": []string{"text_generate"},
@@ -909,9 +932,10 @@ LIMIT 1`).Scan(&gptImageModelTypesRaw); err != nil {
},
}, http.StatusCreated, &customPricingRuleSet)
pricingModel := "pricing-smoke-" + suffixText
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, pricingModel)
var pricingPlatformModel map[string]any
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + pricingModel,
"modelName": pricingModel,
"modelAlias": pricingModel,
"modelType": []string{"text_generate"},
@@ -1156,12 +1180,19 @@ WHERE reference_type = 'gateway_task'
"rules": []map[string]any{{"metric": "rpm", "limit": 1, "windowSeconds": rateLimitWindowSeconds}},
},
}, http.StatusCreated, &rateLimitPolicySet)
createSimulationBaseModel(
t,
server.URL,
loginResponse.AccessToken,
rateLimitedModel,
[]string{"text_generate", "image_generate"},
)
var rateLimitPlatformModel map[string]any
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + rateLimitedModel,
"modelName": rateLimitedModel,
"modelAlias": rateLimitedModel,
"modelType": []string{"text_generate"},
"modelType": []string{"text_generate", "image_generate"},
"displayName": "Rate Limit Smoke",
"runtimePolicySetId": rateLimitPolicySet.ID,
"runtimePolicyOverride": map[string]any{},
@@ -1223,12 +1254,12 @@ WHERE reference_type = 'gateway_task'
AsyncMode bool `json:"asyncMode"`
} `json:"task"`
}
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/responses", apiKeyResponse.Secret, map[string]any{
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/images/generations", apiKeyResponse.Secret, map[string]any{
"model": rateLimitedModel,
"runMode": "simulation",
"simulation": true,
"simulationDurationMs": 5,
"input": "async queued",
"prompt": "async queued",
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &asyncRateLimitTask)
if asyncRateLimitTask.TaskID == "" || asyncRateLimitTask.Task.ID != asyncRateLimitTask.TaskID || !asyncRateLimitTask.Task.AsyncMode {
t.Fatalf("async task response should expose task id and async mode: %+v", asyncRateLimitTask)
@@ -1272,9 +1303,16 @@ WHERE reference_type = 'gateway_task'
}
videoRouteModel := "video-route-smoke-" + suffixText
createSimulationBaseModel(
t,
server.URL,
loginResponse.AccessToken,
videoRouteModel,
[]string{"video_generate", "image_to_video"},
)
var videoRoutePlatformModel map[string]any
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + videoRouteModel,
"modelName": videoRouteModel,
"modelAlias": videoRouteModel,
"modelType": []string{"video_generate", "image_to_video"},
@@ -1313,7 +1351,7 @@ WHERE reference_type = 'gateway_task'
"simulation": true,
"simulationDurationMs": 5,
"prompt": "image to video route",
"image": "https://example.com/source.png",
"image": server.URL + "/static/simulation/image.png",
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &imageToVideoTask)
waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, imageToVideoTask.Task.ID, []string{"succeeded"}, 10*time.Second)
doJSON(t, server.URL, http.MethodGet, "/api/v1/tasks/"+imageToVideoTask.Task.ID, apiKeyResponse.Secret, nil, http.StatusOK, &imageToVideoTask.Task)
@@ -1333,8 +1371,11 @@ WHERE reference_type = 'gateway_task'
if len(preprocessingDetail.Items) == 0 {
t.Fatalf("task preprocessing endpoint should expose persisted preprocessing logs: %+v", preprocessingDetail)
}
if preprocessingDetail.Items[0]["actualInput"] == nil || preprocessingDetail.Items[0]["convertedOutput"] == nil {
t.Fatalf("preprocessing log should store actual input and converted output: %+v", preprocessingDetail.Items)
if preprocessingDetail.Items[0]["changes"] == nil {
t.Fatalf("preprocessing log should retain its compact change summary: %+v", preprocessingDetail.Items)
}
if preprocessingDetail.Items[0]["actualInput"] != nil || preprocessingDetail.Items[0]["convertedOutput"] != nil {
t.Fatalf("history compaction should remove full preprocessing payloads: %+v", preprocessingDetail.Items)
}
var preprocessingRows int
if err := testPool.QueryRow(ctx, `SELECT count(*) FROM gateway_task_param_preprocessing_logs WHERE task_id = $1::uuid`, imageToVideoTask.Task.ID).Scan(&preprocessingRows); err != nil {
@@ -1345,6 +1386,9 @@ WHERE reference_type = 'gateway_task'
}
failoverModel := "phase1-failover-" + suffixText
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, failoverModel)
unrelatedPriorityModel := "priority-demote-unrelated-" + suffixText
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, unrelatedPriorityModel)
var failedPlatform struct {
ID string `json:"id"`
}
@@ -1383,16 +1427,16 @@ WHERE reference_type = 'gateway_task'
}, http.StatusCreated, &unrelatedPriorityPlatform)
var unrelatedPriorityPlatformModel map[string]any
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+unrelatedPriorityPlatform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"modelName": "priority-demote-unrelated-" + suffixText,
"modelAlias": "priority-demote-unrelated-" + suffixText,
"canonicalModelKey": "openai:" + unrelatedPriorityModel,
"modelName": unrelatedPriorityModel,
"modelAlias": unrelatedPriorityModel,
"modelType": []string{"text_generate"},
"displayName": "Unrelated Priority",
}, http.StatusCreated, &unrelatedPriorityPlatformModel)
for _, platformID := range []string{failedPlatform.ID, successPlatform.ID} {
var platformModel map[string]any
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platformID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + failoverModel,
"modelName": failoverModel,
"modelAlias": failoverModel,
"modelType": []string{"text_generate"},
@@ -1442,8 +1486,8 @@ WHERE reference_type = 'gateway_task'
if failoverDetail.Attempts[2].PlatformName != "OpenAI Retry Success" || failoverDetail.Attempts[2].Status != "succeeded" || failoverDetail.Attempts[2].ResponseMS <= 0 {
t.Fatalf("third failover attempt should preserve successful platform: %+v", failoverDetail.Attempts[2])
}
if summary, ok := failoverDetail.Metrics["attempts"].([]any); !ok || len(summary) != 3 {
t.Fatalf("task metrics should keep attempt-chain summary, got %+v", failoverDetail.Metrics)
if _, ok := failoverDetail.Metrics["attempts"]; ok {
t.Fatalf("task metrics should not duplicate the dedicated attempt chain: %+v", failoverDetail.Metrics)
}
var demotedDynamicPriority int
var successEffectivePriority int
@@ -1479,6 +1523,7 @@ WHERE failed.id = $1::uuid`, failedPlatform.ID, successPlatform.ID, unrelatedPri
},
}, http.StatusCreated, &degradePolicySet)
degradeModel := "degrade-smoke-" + suffixText
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, degradeModel)
var degradedPlatform struct {
ID string `json:"id"`
}
@@ -1514,7 +1559,7 @@ WHERE failed.id = $1::uuid`, failedPlatform.ID, successPlatform.ID, unrelatedPri
ID string `json:"id"`
}
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+item.platformID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + degradeModel,
"modelName": degradeModel,
"modelAlias": degradeModel,
"modelType": []string{"text_generate"},
@@ -1568,6 +1613,7 @@ WHERE m.platform_id = $1::uuid
},
}, http.StatusCreated, &autoDisablePolicySet)
autoDisableModel := "auto-disable-smoke-" + suffixText
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, autoDisableModel)
var invalidKeyPlatform struct {
ID string `json:"id"`
}
@@ -1582,7 +1628,7 @@ WHERE m.platform_id = $1::uuid
}, http.StatusCreated, &invalidKeyPlatform)
var invalidKeyPlatformModel map[string]any
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+invalidKeyPlatform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + autoDisableModel,
"modelName": autoDisableModel,
"modelAlias": autoDisableModel,
"modelType": []string{"text_generate"},
@@ -1625,7 +1671,7 @@ WHERE m.platform_id = $1::uuid
"priority": 60,
}, http.StatusCreated, &validKeyPlatform)
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+validKeyPlatform.ID+"/models", loginResponse.AccessToken, map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + autoDisableModel,
"modelName": autoDisableModel,
"modelAlias": autoDisableModel,
"modelType": []string{"text_generate"},
@@ -1719,14 +1765,31 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
} `json:"items"`
Total int `json:"total"`
}
var taskPlatformID string
var taskBillingStatus string
if err := testPool.QueryRow(ctx, `
SELECT platform_id::text
FROM gateway_task_attempts
WHERE task_id = $1::uuid
AND status = 'succeeded'
ORDER BY attempt_no DESC
LIMIT 1`, taskResponse.Task.ID).Scan(&taskPlatformID); err != nil {
t.Fatalf("read task platform for admin filters: %v", err)
}
if err := testPool.QueryRow(ctx, `
SELECT billing_status
FROM gateway_tasks
WHERE id = $1::uuid`, taskResponse.Task.ID).Scan(&taskBillingStatus); err != nil {
t.Fatalf("read task billing status for admin filters: %v", err)
}
adminTaskQuery := "/api/admin/tasks?q=" + taskResponse.Task.ID +
"&userId=" + smokeGatewayUserID +
"&status=succeeded" +
"&platformId=" + platform.ID +
"&platformId=" + taskPlatformID +
"&model=" + defaultTextModel +
"&modelType=text_generate" +
"&runMode=simulation" +
"&billingStatus=settled" +
"&billingStatus=" + taskBillingStatus +
"&apiKey=smoke" +
"&pageSize=10"
doJSON(t, server.URL, http.MethodGet, adminTaskQuery, loginResponse.AccessToken, nil, http.StatusOK, &adminTaskList)
@@ -1771,9 +1834,6 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
if !bytes.Contains(body, []byte("task.running")) {
t.Fatalf("events response should include running transition event body=%s", string(body))
}
if !bytes.Contains(body, []byte("task.progress")) {
t.Fatalf("events response should include progress events body=%s", string(body))
}
if !bytes.Contains(body, []byte("task.billing.settled")) {
t.Fatalf("events response should include billing settlement event body=%s", string(body))
}
@@ -1789,8 +1849,10 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
}
defer resp.Body.Close()
body, _ = io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK || !bytes.Contains(body, []byte("task.retrying")) {
t.Fatalf("failover events should include retrying event status=%d body=%s", resp.StatusCode, string(body))
if resp.StatusCode != http.StatusOK ||
!bytes.Contains(body, []byte("task.attempt.failed")) ||
!bytes.Contains(body, []byte("task.policy.priority_demoted")) {
t.Fatalf("failover events should include the failed attempt and applied policy status=%d body=%s", resp.StatusCode, string(body))
}
assertLoadAvoidanceSimulatedRetryChain(t, ctx, testPool, server.URL, loginResponse.AccessToken, apiKeyResponse.Secret, suffixText)
@@ -1803,18 +1865,6 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
t.Fatal("task progress callback outbox should receive events")
}
restartModel := "worker-restart-" + suffixText
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, restartModel)
createSimulationTextPlatformModel(
t,
server.URL,
loginResponse.AccessToken,
"openai-worker-restart-"+suffixText,
"OpenAI Worker Restart",
restartModel,
1,
nil,
)
var restartAsyncTask struct {
TaskID string `json:"taskId"`
Task struct {
@@ -1823,12 +1873,13 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
AsyncMode bool `json:"asyncMode"`
} `json:"task"`
}
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/responses", apiKeyResponse.Secret, map[string]any{
"model": restartModel,
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/images/generations", apiKeyResponse.Secret, map[string]any{
"model": defaultImageModel,
"runMode": "simulation",
"simulation": true,
"simulationDurationMs": 2000,
"input": "river worker restart",
"prompt": "river worker restart",
"size": "1024x1024",
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &restartAsyncTask)
if restartAsyncTask.TaskID == "" || !restartAsyncTask.Task.AsyncMode {
t.Fatalf("restart async task should be accepted as async: %+v", restartAsyncTask)
@@ -1841,15 +1892,19 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
serverCtx2, cancelServer2 := context.WithCancel(ctx)
defer cancelServer2()
server2 := httptest.NewServer(NewServerWithContext(serverCtx2, config.Config{
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
BillingEngineMode: "enforce",
TaskProgressCallbackEnabled: true,
TaskProgressCallbackURL: "http://callback.local/task-progress",
CORSAllowedOrigin: "*",
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
BillingEngineMode: "enforce",
TaskProgressCallbackEnabled: true,
TaskProgressCallbackURL: "http://callback.local/task-progress",
CORSAllowedOrigin: "*",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 32,
AsyncWorkerInstanceHardLimit: 32,
AsyncWorkerRefreshIntervalSeconds: 1,
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
defer server2.Close()
restartRecovered := waitForTaskStatus(t, server2.URL, apiKeyResponse.Secret, restartAsyncTask.TaskID, []string{"succeeded"}, 8*time.Second)
@@ -2334,15 +2389,20 @@ type cacheAffinityTaskDetail struct {
}
func createSimulationTextBaseModel(t *testing.T, baseURL string, adminToken string, model string) {
t.Helper()
createSimulationBaseModel(t, baseURL, adminToken, model, []string{"text_generate"})
}
func createSimulationBaseModel(t *testing.T, baseURL string, adminToken string, model string, modelTypes []string) {
t.Helper()
doJSON(t, baseURL, http.MethodPost, "/api/admin/catalog/base-models", adminToken, map[string]any{
"providerKey": "openai",
"canonicalModelKey": "openai:" + model,
"invocationName": model,
"providerModelName": model,
"modelType": []string{"text_generate"},
"modelType": modelTypes,
"displayName": model,
"capabilities": map[string]any{"originalTypes": []string{"text_generate"}},
"capabilities": map[string]any{"originalTypes": modelTypes},
"metadata": map[string]any{"source": "cache-affinity-integration-test"},
}, http.StatusCreated, nil)
}
@@ -2481,6 +2541,7 @@ LIMIT 1`, key, value).Scan(&taskID)
func assertLoadAvoidanceSimulatedRetryChain(t *testing.T, ctx context.Context, testPool *pgxpool.Pool, baseURL string, adminToken string, runtimeToken string, suffixText string) {
t.Helper()
model := "load-avoidance-smoke-" + suffixText
createSimulationTextBaseModel(t, baseURL, adminToken, model)
type scenario struct {
keySuffix string
name string
@@ -2522,7 +2583,7 @@ func assertLoadAvoidanceSimulatedRetryChain(t *testing.T, ctx context.Context, t
ID string `json:"id"`
}
payload := map[string]any{
"canonicalModelKey": "openai:gpt-4o-mini",
"canonicalModelKey": "openai:" + model,
"modelName": model,
"providerModelName": model,
"modelAlias": model,
@@ -2581,55 +2642,29 @@ func assertLoadAvoidanceSimulatedRetryChain(t *testing.T, ctx context.Context, t
Metrics map[string]any `json:"metrics"`
}
doJSON(t, baseURL, http.MethodGet, "/api/v1/tasks/"+taskResponse.Task.ID, runtimeToken, nil, http.StatusOK, &detail)
if detail.Status != "failed" || len(detail.Attempts) != len(scenarios) {
t.Fatalf("load avoidance detail should expose every attempted client, got status=%s attempts=%+v", detail.Status, detail.Attempts)
if detail.Status != "failed" || len(detail.Attempts) != 1 {
t.Fatalf("non-retryable failure should stop before load-avoided fallbacks, got status=%s attempts=%+v", detail.Status, detail.Attempts)
}
expected := []struct {
name string
code string
retryable bool
avoided bool
}{
{name: "Load Avoidance Hard Stop", code: "bad_request"},
{name: "Load Avoidance Retryable", code: "server_error", retryable: true},
{name: "Load Avoidance Full Rate Limit", code: "rate_limit", retryable: true, avoided: true},
{name: "Load Avoidance Full Overloaded", code: "overloaded", retryable: true, avoided: true},
{name: "Load Avoidance Full Fatal", code: "bad_request", avoided: true},
attempt := detail.Attempts[0]
if attempt.AttemptNo != 1 ||
attempt.PlatformName != "Load Avoidance Hard Stop" ||
attempt.Status != "failed" ||
attempt.ErrorCode != "bad_request" ||
attempt.Retryable {
t.Fatalf("unexpected hard-stop attempt: %+v", attempt)
}
attemptSummary := make([]string, 0, len(detail.Attempts))
for index, want := range expected {
got := detail.Attempts[index]
attemptSummary = append(attemptSummary, got.PlatformName+":"+got.Status+":"+got.ErrorCode)
if got.AttemptNo != index+1 || got.PlatformName != want.name || got.Status != "failed" || got.ErrorCode != want.code || got.Retryable != want.retryable {
t.Fatalf("unexpected load avoidance attempt %d: got %+v want %+v", index+1, got, want)
}
if boolFromTestMap(got.Metrics, "loadAvoided") != want.avoided {
t.Fatalf("loadAvoided mismatch for %s metrics=%+v", got.PlatformName, got.Metrics)
}
if !want.avoided && floatFromTestAny(got.Metrics["loadRatio"]) != 0 {
t.Fatalf("non-full candidate should not carry load pressure, got %s metrics=%+v", got.PlatformName, got.Metrics)
}
if want.avoided {
if ratio := floatFromTestAny(got.Metrics["loadRatio"]); ratio < 1 {
t.Fatalf("avoided candidate should expose full load ratio, got %s ratio=%v metrics=%+v", got.PlatformName, ratio, got.Metrics)
}
loadMetrics, _ := got.Metrics["loadMetrics"].(map[string]any)
concurrent, _ := loadMetrics["concurrent"].(map[string]any)
if ratio := floatFromTestAny(concurrent["ratio"]); ratio < 1 {
t.Fatalf("avoided candidate should expose concurrent saturation, got %s loadMetrics=%+v", got.PlatformName, loadMetrics)
}
if queued := floatFromTestAny(loadMetrics["queued"]); queued < 1 {
t.Fatalf("avoided candidate should expose queued waiting load, got %s loadMetrics=%+v", got.PlatformName, loadMetrics)
}
}
if boolFromTestMap(attempt.Metrics, "loadAvoided") {
t.Fatalf("selected non-full candidate must not be marked load avoided: %+v", attempt.Metrics)
}
if !attemptTraceHasReason(detail.Attempts[0].Metrics, "load_avoidance_fallback") {
t.Fatalf("first hard-stop attempt should continue because avoided candidates remain, metrics=%+v", detail.Attempts[0].Metrics)
if ratio := floatFromTestAny(attempt.Metrics["loadRatio"]); ratio != 0 {
t.Fatalf("selected non-full candidate should not carry load pressure, got ratio=%v metrics=%+v", ratio, attempt.Metrics)
}
if summary, ok := detail.Metrics["attempts"].([]any); !ok || len(summary) != len(scenarios) {
t.Fatalf("final task metrics should preserve load-avoidance attempt summary, got %+v", detail.Metrics)
if !attemptTraceHasReason(attempt.Metrics, "failover_action_rule") {
t.Fatalf("hard-stop attempt should record the failure-policy decision, metrics=%+v", attempt.Metrics)
}
if _, ok := detail.Metrics["attempts"]; ok {
t.Fatalf("task metrics should not duplicate the dedicated attempt chain: %+v", detail.Metrics)
}
t.Logf("load avoidance retry chain: %s", strings.Join(attemptSummary, " -> "))
}
func seedQueuedConcurrencyLoad(t *testing.T, ctx context.Context, testPool *pgxpool.Pool, platformKey string, model string, platformModelID string) {
@@ -124,13 +124,17 @@ func newFailurePolicyAcceptanceFixture(t *testing.T) *failurePolicyAcceptanceFix
t.Fatalf("connect acceptance pool: %v", err)
}
handler := NewServerWithContext(ctx, config.Config{
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "failure-policy-http-acceptance-secret",
BillingEngineMode: "observe",
CORSAllowedOrigin: "*",
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "failure-policy-http-acceptance-secret",
BillingEngineMode: "observe",
CORSAllowedOrigin: "*",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 32,
AsyncWorkerInstanceHardLimit: 32,
AsyncWorkerRefreshIntervalSeconds: 1,
}, db, slog.New(slog.NewTextHandler(io.Discard, nil)))
server := httptest.NewServer(handler)
fixture := &failurePolicyAcceptanceFixture{
@@ -74,9 +74,9 @@ func TestGeminiBase64ImageEditThousandSynchronousRequests(t *testing.T) {
t.Fatalf("connect observation pool: %v", err)
}
defer pool.Close()
restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, true)
defer restoreAcceptanceState()
if _, err := pool.Exec(ctx, `
UPDATE integration_platforms SET status = 'disabled' WHERE deleted_at IS NULL;
UPDATE file_storage_channels SET status = 'disabled' WHERE deleted_at IS NULL;
UPDATE gateway_user_groups
SET rate_limit_policy = '{"rules":[
{"metric":"concurrent","limit":1024,"leaseTtlSeconds":120},
@@ -192,6 +192,11 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
}
created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user)
if err != nil {
if errors.Is(err, store.ErrRateLimited) {
applyRunErrorHeaders(w, err)
writeGeminiTaskError(statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), runErrorCode(err))
return
}
if errors.Is(err, store.ErrIdempotencyKeyReused) {
writeGeminiTaskError(http.StatusConflict, "Idempotency-Key was reused for a different request", nil, "idempotency_key_reused")
return
+48 -2
View File
@@ -178,9 +178,45 @@ func (s *Server) logPostgresUnavailable(message string) {
"postgres_pool_idle_connections", statistics.IdleConns(),
"postgres_pool_empty_acquire_count", statistics.EmptyAcquireCount(),
"postgres_pool_canceled_acquire_count", statistics.CanceledAcquireCount(),
"postgres_critical_pool", s.criticalPostgresPoolLogFields(),
"postgres_river_pool", s.riverPostgresPoolLogFields(),
)
}
func (s *Server) criticalPostgresPoolLogFields() map[string]any {
if s.coordinationStore == nil || s.coordinationStore == s.store ||
s.coordinationStore.Pool() == nil {
return map[string]any{"separate": false}
}
statistics := s.coordinationStore.Pool().Stat()
return map[string]any{
"separate": true,
"max_connections": statistics.MaxConns(),
"total_connections": statistics.TotalConns(),
"acquired_connections": statistics.AcquiredConns(),
"idle_connections": statistics.IdleConns(),
"empty_acquire_count": statistics.EmptyAcquireCount(),
"canceled_acquire_count": statistics.CanceledAcquireCount(),
}
}
func (s *Server) riverPostgresPoolLogFields() map[string]any {
if s.riverStore == nil || s.riverStore == s.store ||
s.riverStore.Pool() == nil {
return map[string]any{"separate": false}
}
statistics := s.riverStore.Pool().Stat()
return map[string]any{
"separate": true,
"max_connections": statistics.MaxConns(),
"total_connections": statistics.TotalConns(),
"acquired_connections": statistics.AcquiredConns(),
"idle_connections": statistics.IdleConns(),
"empty_acquire_count": statistics.EmptyAcquireCount(),
"canceled_acquire_count": statistics.CanceledAcquireCount(),
}
}
func (s *Server) localIdentityEnabled() bool {
mode := strings.ToLower(strings.TrimSpace(s.cfg.IdentityMode))
return mode == "" || mode == "standalone" || mode == "hybrid"
@@ -1183,6 +1219,11 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
}
created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user)
if err != nil {
if errors.Is(err, store.ErrRateLimited) {
applyRunErrorHeaders(w, err)
writeTaskError(statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), runErrorCode(err))
return
}
if errors.Is(err, store.ErrIdempotencyKeyReused) {
writeTaskError(http.StatusConflict, "Idempotency-Key was reused for a different request", nil, "idempotency_key_reused")
return
@@ -1200,8 +1241,9 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
if s.billingMetrics != nil {
s.billingMetrics.ObserveBillingEvent("idempotent_replay")
}
if targetProtocol == "" {
if targetProtocol == "" || gatewayAPIV1Request(r) {
w.Header().Set("Idempotent-Replayed", "true")
w.Header().Set("X-Gateway-Task-Id", task.ID)
}
if responsePlan.streamMode {
writeTaskError(http.StatusConflict, "streaming idempotent replay is not supported", nil, "idempotency_stream_replay_unsupported")
@@ -1411,7 +1453,7 @@ func writeCompatibleTaskResponse(runCtx context.Context, w http.ResponseWriter,
}
func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.ResponseWriter, r *http.Request, executor taskExecutor, kind string, model string, targetProtocol string, task store.GatewayTask, user *auth.User, streamMode bool, includeUsage bool) {
if targetProtocol == "" {
if targetProtocol == "" || gatewayAPIV1Request(r) {
w.Header().Set("X-Gateway-Task-Id", task.ID)
}
if streamMode {
@@ -1515,6 +1557,10 @@ func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.Response
writeJSON(w, http.StatusOK, easyAISynchronousTaskResponse(result.Task, result.Output))
}
func gatewayAPIV1Request(r *http.Request) bool {
return r != nil && strings.HasPrefix(r.URL.Path, "/api/v1/")
}
func streamIncludeUsage(body map[string]any) bool {
streamOptions, _ := body["stream_options"].(map[string]any)
includeUsage, _ := streamOptions["include_usage"].(bool)
@@ -131,12 +131,16 @@ func TestKelingOmniCompatibleHTTPFlow(t *testing.T) {
serverCtx, cancelServer := context.WithCancel(ctx)
defer cancelServer()
gateway := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
CORSAllowedOrigin: "*",
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
CORSAllowedOrigin: "*",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 32,
AsyncWorkerInstanceHardLimit: 32,
AsyncWorkerRefreshIntervalSeconds: 1,
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
defer gateway.Close()
@@ -152,7 +156,7 @@ func TestKelingOmniCompatibleHTTPFlow(t *testing.T) {
"aspect_ratio": "16:9",
"duration": "3",
"sound": "off",
"image_list": []any{map[string]any{"image_url": "https://example.com/reference.png"}},
"image_list": []any{map[string]any{"image_url": gateway.URL + "/static/simulation/image.png"}},
"external_task_id": "compat-http-1",
}, http.StatusOK, &created)
createdData, _ := created.Data.(map[string]any)
@@ -33,12 +33,16 @@ func TestKeling30TurboSimulationFlow(t *testing.T) {
serverCtx, cancelServer := context.WithCancel(ctx)
defer cancelServer()
server := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
CORSAllowedOrigin: "*",
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
CORSAllowedOrigin: "*",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 32,
AsyncWorkerInstanceHardLimit: 32,
AsyncWorkerRefreshIntervalSeconds: 1,
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
defer server.Close()
@@ -192,7 +196,7 @@ WHERE username = $1`, username); err != nil {
"simulation": true,
"simulationDurationMs": 5,
"prompt": "The subject looks toward the camera",
"image": "https://example.com/first.png",
"image": server.URL + "/static/simulation/image.png",
"duration": 5,
"resolution": "720p",
"audio": true,
@@ -159,6 +159,9 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user)
if err != nil {
switch {
case errors.Is(err, store.ErrRateLimited):
applyRunErrorHeaders(w, err)
writeKlingCompatError(w, statusFromRunError(err), runErrorMessage(err), runErrorCode(err))
case errors.Is(err, store.ErrIdempotencyKeyReused):
writeKlingCompatError(w, http.StatusConflict, err.Error(), "idempotency_key_reused")
case errors.Is(err, store.ErrExternalTaskIDReused):
@@ -49,12 +49,16 @@ WHERE provider_key = 'keling'
serverCtx, cancelServer := context.WithCancel(ctx)
defer cancelServer()
server := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
CORSAllowedOrigin: "*",
AppEnv: "test",
HTTPAddr: ":0",
DatabaseURL: databaseURL,
IdentityMode: "hybrid",
JWTSecret: "test-secret",
CORSAllowedOrigin: "*",
AsyncQueueWorkerEnabled: true,
AsyncWorkerHardLimit: 32,
AsyncWorkerInstanceHardLimit: 32,
AsyncWorkerRefreshIntervalSeconds: 1,
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
defer server.Close()
@@ -159,7 +163,7 @@ WHERE platform_id = $1::uuid
for _, model := range []string{klingO1Model, klingV3OmniModel} {
assertGenericVideoGeneration(model+"-text-to-video", model, "", "video_generate")
assertGenericVideoGeneration(model+"-image-to-video", model, "https://example.com/first.png", "image_to_video")
assertGenericVideoGeneration(model+"-image-to-video", model, server.URL+"/static/simulation/image.png", "image_to_video")
}
createV1 := func(model string, duration int, externalID string) string {
@@ -44,6 +44,25 @@ func TestOIDCJITFullGatewayUserFlowAndSecurityBoundary(t *testing.T) {
t.Fatalf("connect store: %v", err)
}
t.Cleanup(db.Close)
hasBreakGlass, err := db.HasBreakGlassManager(ctx)
if err != nil {
t.Fatalf("check OIDC JIT break-glass manager: %v", err)
}
if !hasBreakGlass {
manager, registerErr := db.RegisterLocalUser(ctx, store.LocalRegisterInput{
Username: "oidc-jit-manager-" + uuid.NewString(),
Password: uuid.NewString() + "-local-only",
})
if registerErr != nil {
t.Fatalf("create OIDC JIT break-glass manager: %v", registerErr)
}
if _, updateErr := db.Pool().Exec(ctx, `UPDATE gateway_users SET roles='["manager"]'::jsonb WHERE id=$1::uuid`, manager.ID); updateErr != nil {
t.Fatalf("grant OIDC JIT break-glass manager role: %v", updateErr)
}
t.Cleanup(func() {
_, _ = db.Pool().Exec(context.Background(), `DELETE FROM gateway_users WHERE id=$1::uuid`, manager.ID)
})
}
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
@@ -388,7 +407,7 @@ func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie {
}
return nil
}}
response, err := client.Get(baseURL + "/api/v1/auth/oidc/login?returnTo=%2Fapi%2Fv1%2Fme")
response, err := client.Get(baseURL + "/api/v1/auth/oidc/login?contextType=tenant&returnTo=%2Fapi%2Fv1%2Fme")
if err != nil {
t.Fatalf("complete OIDC BFF login: %v", err)
}
@@ -463,7 +482,8 @@ func signedOIDCJITToken(t *testing.T, key *ecdsa.PrivateKey, issuer string, subj
t.Helper()
now := time.Now()
claims := jwt.MapClaims{
"iss": issuer, "aud": "gateway-api", "sub": subject, "tid": oidcJITTenantID,
"iss": issuer, "aud": "gateway-api", "sub": subject, "context_type": "tenant", "tid": oidcJITTenantID,
"client_id": "gateway-public-test",
"preferred_username": "oidc-jit-acceptance", "roles": []string{"gateway.admin"},
"scope": "openid gateway.access", "iat": now.Unix(), "nbf": now.Add(-time.Second).Unix(),
"exp": now.Add(time.Hour).Unix(),
+52 -2
View File
@@ -24,6 +24,8 @@ type Server struct {
ctx context.Context
cfg config.Config
store *store.Store
coordinationStore *store.Store
riverStore *store.Store
oidcUserResolver oidcUserResolver
auth *auth.Authenticator
oidcClient oidcPublicClient
@@ -72,6 +74,27 @@ func NewServer(cfg config.Config, db *store.Store, logger *slog.Logger) http.Han
}
func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Store, logger *slog.Logger) http.Handler {
return NewServerWithContextAndCoordinationStore(ctx, cfg, db, db, logger)
}
func NewServerWithContextAndCoordinationStore(
ctx context.Context,
cfg config.Config,
db *store.Store,
coordinationDB *store.Store,
logger *slog.Logger,
) http.Handler {
return NewServerWithStores(ctx, cfg, db, coordinationDB, db, logger)
}
func NewServerWithStores(
ctx context.Context,
cfg config.Config,
db *store.Store,
coordinationDB *store.Store,
riverDB *store.Store,
logger *slog.Logger,
) http.Handler {
if cfg.MediaMaterializationConcurrency == 0 {
cfg.MediaMaterializationConcurrency = 8
}
@@ -83,9 +106,11 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
ctx: ctx,
cfg: cfg,
store: db,
coordinationStore: coordinationDB,
riverStore: riverDB,
oidcUserResolver: db,
auth: auth.New(cfg.JWTSecret, cfg.ServerMainBaseURL, cfg.ServerMainInternalToken),
runner: runner.New(cfg, db, logger, securityEventMetrics),
runner: runner.NewWithStores(cfg, db, coordinationDB, riverDB, logger, securityEventMetrics),
logger: logger,
billingMetrics: securityEventMetrics,
requestAssetLocks: map[string]*requestAssetLock{},
@@ -122,7 +147,9 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.HandleFunc("GET /readyz", server.ready)
mux.HandleFunc("GET /api/v1/healthz", server.health)
mux.HandleFunc("GET /api/v1/readyz", server.ready)
mux.Handle("GET /metrics", securityEventMetrics.DynamicHandler(db))
mux.Handle("GET /metrics", securityEventMetrics.DynamicHandler(postgresMetricsProvider{
Store: db, critical: coordinationDB, river: riverDB,
}))
if !cfg.RunsPublicHTTP() {
return server.recover(mux)
}
@@ -221,13 +248,16 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("GET /api/admin/system/client-customization/settings", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getClientCustomizationSettings)))
mux.Handle("PATCH /api/admin/system/client-customization/settings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateClientCustomizationSettings)))
mux.Handle("GET /api/admin/system/acceptance/traffic-mode", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getGatewayTrafficMode)))
mux.Handle("POST /api/admin/system/acceptance/traffic-mode/pause", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.pauseGatewayTraffic)))
mux.Handle("POST /api/admin/system/acceptance/runs", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createAcceptanceRun)))
mux.Handle("GET /api/admin/system/acceptance/runs/{runID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/activate", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.activateAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/capacity-profiles", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.stageAcceptanceCapacityProfiles)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/finish", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.finishAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/retry", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.retryAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/promote", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.promoteAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/abort", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.abortAcceptanceRun)))
mux.Handle("GET /api/admin/system/acceptance/capacity-profiles", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listCapacityProfiles)))
mux.Handle("GET /api/admin/system/identity/configuration", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityConfiguration)))
mux.Handle("POST /api/admin/system/identity/pairings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.startIdentityPairing)))
mux.Handle("GET /api/admin/system/identity/pairings/{pairingID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityPairing)))
@@ -348,6 +378,26 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
return server.recover(server.cors(server.protectOIDCSessionCookie(mux)))
}
type postgresMetricsProvider struct {
*store.Store
critical *store.Store
river *store.Store
}
func (provider postgresMetricsProvider) RiverPostgresPoolMetrics() store.PostgresPoolMetricsSnapshot {
if provider.river == nil || provider.river == provider.Store {
return store.PostgresPoolMetricsSnapshot{}
}
return provider.river.PostgresPoolMetrics()
}
func (provider postgresMetricsProvider) CriticalPostgresPoolMetrics() store.PostgresPoolMetricsSnapshot {
if provider.critical == nil || provider.critical == provider.Store {
return store.PostgresPoolMetricsSnapshot{}
}
return provider.critical.PostgresPoolMetrics()
}
func (server *Server) initializeIdentityRuntime(
ctx context.Context,
db *store.Store,