feat(acceptance): 建立同构验收与弹性容量体系
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护,并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
@@ -28,6 +28,13 @@ type taskTrafficError struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (s *Server) acceptanceStore() *store.Store {
|
||||
if s.coordinationStore != nil {
|
||||
return s.coordinationStore
|
||||
}
|
||||
return s.store
|
||||
}
|
||||
|
||||
func (e *taskTrafficError) Error() string {
|
||||
return e.Message
|
||||
}
|
||||
@@ -37,7 +44,7 @@ func (e *taskTrafficError) ErrorCode() string {
|
||||
}
|
||||
|
||||
func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTrafficAdmission, error) {
|
||||
runID, err := s.store.AuthorizeAcceptanceTask(
|
||||
runID, err := s.acceptanceStore().AuthorizeAcceptanceTask(
|
||||
r.Context(),
|
||||
r.Header.Get(acceptanceRunHeader),
|
||||
r.Header.Get(acceptanceTokenHeader),
|
||||
@@ -121,7 +128,7 @@ func writeTaskTrafficError(w http.ResponseWriter, err error, protocol string) {
|
||||
// @Success 200 {object} store.GatewayTrafficMode
|
||||
// @Router /api/admin/system/acceptance/traffic-mode [get]
|
||||
func (s *Server) getGatewayTrafficMode(w http.ResponseWriter, r *http.Request) {
|
||||
mode, err := s.store.GetGatewayTrafficMode(r.Context())
|
||||
mode, err := s.acceptanceStore().GetGatewayTrafficMode(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "get gateway traffic mode failed")
|
||||
return
|
||||
@@ -129,6 +136,45 @@ func (s *Server) getGatewayTrafficMode(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, mode)
|
||||
}
|
||||
|
||||
// pauseGatewayTraffic godoc
|
||||
// @Summary 上线后硬门禁异常时通过 CAS 暂停正式新任务
|
||||
// @Tags acceptance
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param body body store.PauseGatewayTrafficInput true "当前线上 release 与 revision CAS"
|
||||
// @Success 200 {object} store.GatewayTrafficMode
|
||||
// @Router /api/admin/system/acceptance/traffic-mode/pause [post]
|
||||
func (s *Server) pauseGatewayTraffic(w http.ResponseWriter, r *http.Request) {
|
||||
var input store.PauseGatewayTrafficInput
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||
return
|
||||
}
|
||||
mode, err := s.acceptanceStore().PauseGatewayTraffic(r.Context(), input)
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, mode)
|
||||
}
|
||||
|
||||
// listCapacityProfiles godoc
|
||||
// @Summary 获取经生产同构验收认证的容量配置
|
||||
// @Tags acceptance
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Success 200 {array} store.CapacityProfile
|
||||
// @Router /api/admin/system/acceptance/capacity-profiles [get]
|
||||
func (s *Server) listCapacityProfiles(w http.ResponseWriter, r *http.Request) {
|
||||
profiles, err := s.acceptanceStore().ListCapacityProfiles(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "list capacity profiles failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, profiles)
|
||||
}
|
||||
|
||||
// createAcceptanceRun godoc
|
||||
// @Summary 创建生产同构验收 Run
|
||||
// @Tags acceptance
|
||||
@@ -144,7 +190,7 @@ func (s *Server) createAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||
return
|
||||
}
|
||||
run, err := s.store.CreateAcceptanceRun(r.Context(), input)
|
||||
run, err := s.acceptanceStore().CreateAcceptanceRun(r.Context(), input)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
@@ -161,7 +207,7 @@ func (s *Server) createAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
// @Success 200 {object} store.AcceptanceRun
|
||||
// @Router /api/admin/system/acceptance/runs/{runID} [get]
|
||||
func (s *Server) getAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
run, err := s.store.GetAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
run, err := s.acceptanceStore().GetAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
if err != nil {
|
||||
if store.IsNotFound(err) {
|
||||
writeError(w, http.StatusNotFound, "acceptance run not found")
|
||||
@@ -182,7 +228,7 @@ func (s *Server) getAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
// @Success 200 {object} store.GatewayTrafficMode
|
||||
// @Router /api/admin/system/acceptance/runs/{runID}/activate [post]
|
||||
func (s *Server) activateAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
mode, err := s.store.ActivateAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
mode, err := s.acceptanceStore().ActivateAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
@@ -190,6 +236,36 @@ func (s *Server) activateAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, mode)
|
||||
}
|
||||
|
||||
// stageAcceptanceCapacityProfiles godoc
|
||||
// @Summary 为当前 validation Run 暂存 80% 容量门禁
|
||||
// @Tags acceptance
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param runID path string true "验收 Run ID"
|
||||
// @Param body body store.StageAcceptanceCapacityProfilesInput true "待验证容量配置"
|
||||
// @Success 200 {object} store.AcceptanceRun
|
||||
// @Router /api/admin/system/acceptance/runs/{runID}/capacity-profiles [post]
|
||||
func (s *Server) stageAcceptanceCapacityProfiles(w http.ResponseWriter, r *http.Request) {
|
||||
var input store.StageAcceptanceCapacityProfilesInput
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||
return
|
||||
}
|
||||
input.RunID = r.PathValue("runID")
|
||||
run, err := s.acceptanceStore().StageAcceptanceCapacityProfiles(r.Context(), input)
|
||||
if err != nil {
|
||||
switch {
|
||||
case store.IsNotFound(err):
|
||||
writeError(w, http.StatusConflict, "acceptance run is not running")
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, run)
|
||||
}
|
||||
|
||||
// finishAcceptanceRun godoc
|
||||
// @Summary 记录验收门禁结果
|
||||
// @Tags acceptance
|
||||
@@ -207,7 +283,7 @@ func (s *Server) finishAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
input.RunID = r.PathValue("runID")
|
||||
run, err := s.store.FinishAcceptanceRun(r.Context(), input)
|
||||
run, err := s.acceptanceStore().FinishAcceptanceRun(r.Context(), input)
|
||||
if err != nil {
|
||||
if store.IsNotFound(err) {
|
||||
writeError(w, http.StatusConflict, "acceptance run is not running")
|
||||
@@ -228,7 +304,7 @@ func (s *Server) finishAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
// @Success 200 {object} store.AcceptanceRun
|
||||
// @Router /api/admin/system/acceptance/runs/{runID}/retry [post]
|
||||
func (s *Server) retryAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
run, err := s.store.RetryAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
run, err := s.acceptanceStore().RetryAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
@@ -253,7 +329,7 @@ func (s *Server) promoteAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
input.RunID = r.PathValue("runID")
|
||||
mode, err := s.store.PromoteAcceptanceRun(r.Context(), input)
|
||||
mode, err := s.acceptanceStore().PromoteAcceptanceRun(r.Context(), input)
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
@@ -278,7 +354,7 @@ func (s *Server) abortAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
input.RunID = r.PathValue("runID")
|
||||
mode, err := s.store.AbortAcceptanceRun(r.Context(), input)
|
||||
mode, err := s.acceptanceStore().AbortAcceptanceRun(r.Context(), input)
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type acceptanceStatusSnapshot struct {
|
||||
ID string
|
||||
Status string
|
||||
}
|
||||
|
||||
type acceptanceUserGroupSnapshot struct {
|
||||
ID string
|
||||
RateLimitPolicy []byte
|
||||
}
|
||||
|
||||
// isolateHTTPAcceptanceState lets a stress test disable unrelated runtime
|
||||
// resources without leaking those mutations into later integration tests that
|
||||
// intentionally share the same dedicated test database.
|
||||
func isolateHTTPAcceptanceState(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
pool *pgxpool.Pool,
|
||||
includeStorage bool,
|
||||
) func() {
|
||||
t.Helper()
|
||||
startedAt := time.Now()
|
||||
platforms := readAcceptanceStatusSnapshots(t, ctx, pool, `
|
||||
SELECT id::text, status
|
||||
FROM integration_platforms
|
||||
WHERE deleted_at IS NULL`)
|
||||
userGroups := make([]acceptanceUserGroupSnapshot, 0)
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT id::text, rate_limit_policy
|
||||
FROM gateway_user_groups`)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot acceptance user groups: %v", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var item acceptanceUserGroupSnapshot
|
||||
if err := rows.Scan(&item.ID, &item.RateLimitPolicy); err != nil {
|
||||
rows.Close()
|
||||
t.Fatalf("scan acceptance user group snapshot: %v", err)
|
||||
}
|
||||
userGroups = append(userGroups, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
t.Fatalf("read acceptance user group snapshots: %v", err)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
storageChannels := []acceptanceStatusSnapshot(nil)
|
||||
if includeStorage {
|
||||
storageChannels = readAcceptanceStatusSnapshots(t, ctx, pool, `
|
||||
SELECT id::text, status
|
||||
FROM file_storage_channels
|
||||
WHERE deleted_at IS NULL`)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE integration_platforms
|
||||
SET status = 'disabled'
|
||||
WHERE deleted_at IS NULL`); err != nil {
|
||||
t.Fatalf("isolate acceptance platforms: %v", err)
|
||||
}
|
||||
if includeStorage {
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE file_storage_channels
|
||||
SET status = 'disabled'
|
||||
WHERE deleted_at IS NULL`); err != nil {
|
||||
t.Fatalf("isolate acceptance storage channels: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return func() {
|
||||
restoreCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
tx, err := pool.Begin(restoreCtx)
|
||||
if err != nil {
|
||||
t.Errorf("begin acceptance state restore: %v", err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback(restoreCtx)
|
||||
}()
|
||||
for _, item := range platforms {
|
||||
if _, err := tx.Exec(restoreCtx, `
|
||||
UPDATE integration_platforms
|
||||
SET status = $2
|
||||
WHERE id = $1::uuid`, item.ID, item.Status); err != nil {
|
||||
t.Errorf("restore acceptance platform %s: %v", item.ID, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(restoreCtx, `
|
||||
UPDATE integration_platforms
|
||||
SET status = 'disabled'
|
||||
WHERE created_at >= $1
|
||||
AND deleted_at IS NULL`, startedAt); err != nil {
|
||||
t.Errorf("disable acceptance-created platforms: %v", err)
|
||||
return
|
||||
}
|
||||
for _, item := range userGroups {
|
||||
if _, err := tx.Exec(restoreCtx, `
|
||||
UPDATE gateway_user_groups
|
||||
SET rate_limit_policy = $2::jsonb
|
||||
WHERE id = $1::uuid`, item.ID, string(item.RateLimitPolicy)); err != nil {
|
||||
t.Errorf("restore acceptance user group %s: %v", item.ID, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if includeStorage {
|
||||
for _, item := range storageChannels {
|
||||
if _, err := tx.Exec(restoreCtx, `
|
||||
UPDATE file_storage_channels
|
||||
SET status = $2
|
||||
WHERE id = $1::uuid`, item.ID, item.Status); err != nil {
|
||||
t.Errorf("restore acceptance storage channel %s: %v", item.ID, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(restoreCtx, `
|
||||
UPDATE file_storage_channels
|
||||
SET status = 'disabled'
|
||||
WHERE created_at >= $1
|
||||
AND deleted_at IS NULL`, startedAt); err != nil {
|
||||
t.Errorf("disable acceptance-created storage channels: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(restoreCtx); err != nil {
|
||||
t.Errorf("commit acceptance state restore: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readAcceptanceStatusSnapshots(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
pool *pgxpool.Pool,
|
||||
query string,
|
||||
) []acceptanceStatusSnapshot {
|
||||
t.Helper()
|
||||
rows, err := pool.Query(ctx, query)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot acceptance statuses: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]acceptanceStatusSnapshot, 0)
|
||||
for rows.Next() {
|
||||
var item acceptanceStatusSnapshot
|
||||
if err := rows.Scan(&item.ID, &item.Status); err != nil {
|
||||
t.Fatalf("scan acceptance status snapshot: %v", err)
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("read acceptance status snapshots: %v", err)
|
||||
}
|
||||
return items
|
||||
}
|
||||
@@ -40,9 +40,8 @@ func TestAsyncSubmissionReservesConcurrencyOnlyAfterWorkerClaim(t *testing.T) {
|
||||
t.Fatalf("connect acceptance pool: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if _, err := pool.Exec(ctx, `UPDATE integration_platforms SET status = 'disabled' WHERE deleted_at IS NULL`); err != nil {
|
||||
t.Fatalf("isolate acceptance platforms: %v", err)
|
||||
}
|
||||
restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, false)
|
||||
defer restoreAcceptanceState()
|
||||
|
||||
serverCtx, cancelServer := context.WithCancel(ctx)
|
||||
defer cancelServer()
|
||||
@@ -116,18 +115,27 @@ func TestAsyncWorkerThousandConcurrentSchedulingAcceptance(t *testing.T) {
|
||||
t.Fatalf("connect Worker store: %v", err)
|
||||
}
|
||||
defer workerDB.Close()
|
||||
apiCoordinationDB, err := store.ConnectWithMaxConns(ctx, databaseURL, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("connect API critical store: %v", err)
|
||||
}
|
||||
defer apiCoordinationDB.Close()
|
||||
workerCoordinationDB, err := store.ConnectWithMaxConns(ctx, databaseURL, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("connect Worker critical store: %v", err)
|
||||
}
|
||||
defer workerCoordinationDB.Close()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect acceptance pool: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if _, err := pool.Exec(ctx, `UPDATE integration_platforms SET status = 'disabled' WHERE deleted_at IS NULL`); err != nil {
|
||||
t.Fatalf("isolate acceptance platforms: %v", err)
|
||||
}
|
||||
restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, false)
|
||||
defer restoreAcceptanceState()
|
||||
|
||||
serverCtx, cancelServer := context.WithCancel(ctx)
|
||||
defer cancelServer()
|
||||
server := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
|
||||
server := httptest.NewServer(NewServerWithStores(serverCtx, config.Config{
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
@@ -138,10 +146,11 @@ func TestAsyncWorkerThousandConcurrentSchedulingAcceptance(t *testing.T) {
|
||||
ProcessRole: "api",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 1000,
|
||||
AsyncWorkerInstanceHardLimit: 1000,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, apiDB, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))))
|
||||
}, apiDB, apiCoordinationDB, apiDB, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))))
|
||||
defer server.Close()
|
||||
workerServer := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
|
||||
workerServer := httptest.NewServer(NewServerWithStores(serverCtx, config.Config{
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
@@ -152,8 +161,9 @@ func TestAsyncWorkerThousandConcurrentSchedulingAcceptance(t *testing.T) {
|
||||
ProcessRole: "worker",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 1000,
|
||||
AsyncWorkerInstanceHardLimit: 1000,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, workerDB, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))))
|
||||
}, workerDB, workerCoordinationDB, workerDB, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))))
|
||||
defer workerServer.Close()
|
||||
|
||||
adminToken := createAsyncAcceptanceAdmin(t, ctx, pool, server.URL)
|
||||
@@ -180,7 +190,7 @@ WHERE status = 'active'`); err != nil {
|
||||
}
|
||||
assertAcceptanceAttempts(t, ctx, pool, taskIDs, len(taskIDs))
|
||||
t.Logf(
|
||||
"千级并发证据: tasks=%d duration=%s running_peak=%d lease_peak=%d api_database_max_connections=16 worker_database_max_connections=24",
|
||||
"千级并发证据: tasks=%d duration=%s running_peak=%d lease_peak=%d api_database_max_connections=16 api_critical_max_connections=4 worker_database_max_connections=24 worker_critical_max_connections=4",
|
||||
len(taskIDs),
|
||||
45*time.Second,
|
||||
peakRunning,
|
||||
@@ -207,9 +217,8 @@ func TestAsyncWorkerDynamicConcurrencyAcceptance(t *testing.T) {
|
||||
t.Fatalf("connect acceptance pool: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if _, err := pool.Exec(ctx, `UPDATE integration_platforms SET status = 'disabled' WHERE deleted_at IS NULL`); err != nil {
|
||||
t.Fatalf("isolate acceptance platforms: %v", err)
|
||||
}
|
||||
restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, false)
|
||||
defer restoreAcceptanceState()
|
||||
|
||||
serverCtx, cancelServer := context.WithCancel(ctx)
|
||||
defer cancelServer()
|
||||
@@ -223,6 +232,7 @@ func TestAsyncWorkerDynamicConcurrencyAcceptance(t *testing.T) {
|
||||
CORSAllowedOrigin: "*",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 256,
|
||||
AsyncWorkerInstanceHardLimit: 256,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
defer server.Close()
|
||||
|
||||
@@ -43,15 +43,19 @@ func TestCoreLocalFlow(t *testing.T) {
|
||||
serverCtx, cancelServer := context.WithCancel(ctx)
|
||||
defer cancelServer()
|
||||
handler := NewServerWithContext(serverCtx, config.Config{
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
BillingEngineMode: "enforce",
|
||||
TaskProgressCallbackEnabled: true,
|
||||
TaskProgressCallbackURL: "http://callback.local/task-progress",
|
||||
CORSAllowedOrigin: "*",
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
BillingEngineMode: "enforce",
|
||||
TaskProgressCallbackEnabled: true,
|
||||
TaskProgressCallbackURL: "http://callback.local/task-progress",
|
||||
CORSAllowedOrigin: "*",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 32,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, db, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
@@ -606,8 +610,8 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
|
||||
"model": defaultImageModel,
|
||||
"runMode": "simulation",
|
||||
"prompt": "replace background with clean studio light",
|
||||
"image": "https://example.com/source.png",
|
||||
"mask": "https://example.com/mask.png",
|
||||
"image": server.URL + "/static/simulation/image.png",
|
||||
"mask": server.URL + "/static/simulation/image-edit.png",
|
||||
"simulation": true,
|
||||
"simulationDurationMs": 5,
|
||||
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &imageEditResponse)
|
||||
@@ -728,18 +732,35 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
|
||||
t.Fatalf("voice clone persisted platform should match task attempt: task=%+v voice=%+v", voiceCloneTaskDetail, clonedVoice)
|
||||
}
|
||||
|
||||
doubaoLiteImageEditModel := "doubao-5.0-lite图像编辑"
|
||||
var doubaoLitePlatformModel struct {
|
||||
asyncImageEditModel := "async-image-edit-smoke-" + suffixText
|
||||
asyncImageEditCanonicalKey := "openai:" + asyncImageEditModel
|
||||
var asyncImageEditBaseModel struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/catalog/base-models", loginResponse.AccessToken, map[string]any{
|
||||
"providerKey": "openai",
|
||||
"canonicalModelKey": asyncImageEditCanonicalKey,
|
||||
"invocationName": asyncImageEditModel,
|
||||
"providerModelName": asyncImageEditModel,
|
||||
"modelType": []string{"image_edit", "image_generate"},
|
||||
"displayName": "Async Image Edit Smoke",
|
||||
"capabilities": map[string]any{"originalTypes": []string{"image_edit", "image_generate"}},
|
||||
"metadata": map[string]any{"source": "test"},
|
||||
}, http.StatusCreated, &asyncImageEditBaseModel)
|
||||
if asyncImageEditBaseModel.ID == "" {
|
||||
t.Fatal("async image edit smoke base model was not created")
|
||||
}
|
||||
var asyncImageEditPlatformModel struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "easyai:doubao-5.0-lite图像编辑",
|
||||
"modelName": doubaoLiteImageEditModel,
|
||||
"modelAlias": doubaoLiteImageEditModel,
|
||||
"canonicalModelKey": asyncImageEditCanonicalKey,
|
||||
"modelName": asyncImageEditModel,
|
||||
"modelAlias": asyncImageEditModel,
|
||||
"modelType": []string{"image_edit", "image_generate"},
|
||||
"displayName": doubaoLiteImageEditModel,
|
||||
}, http.StatusCreated, &doubaoLitePlatformModel)
|
||||
var doubaoLiteAsyncEdit struct {
|
||||
"displayName": "Async Image Edit Smoke",
|
||||
}, http.StatusCreated, &asyncImageEditPlatformModel)
|
||||
var asyncImageEdit struct {
|
||||
TaskID string `json:"taskId"`
|
||||
Task struct {
|
||||
ID string `json:"id"`
|
||||
@@ -748,20 +769,20 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
|
||||
} `json:"task"`
|
||||
}
|
||||
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/images/edits", apiKeyResponse.Secret, map[string]any{
|
||||
"model": doubaoLiteImageEditModel,
|
||||
"model": asyncImageEditModel,
|
||||
"runMode": "simulation",
|
||||
"prompt": "turn the attached bright desktop object into a clean product-style render",
|
||||
"image": "https://example.com/doubao-lite-source.png",
|
||||
"image": server.URL + "/static/simulation/image.png",
|
||||
"size": "2048x2048",
|
||||
"simulation": true,
|
||||
"simulationDurationMs": 5,
|
||||
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &doubaoLiteAsyncEdit)
|
||||
if doubaoLiteAsyncEdit.TaskID == "" || !doubaoLiteAsyncEdit.Task.AsyncMode {
|
||||
t.Fatalf("doubao-5.0-lite image edit async task should be accepted: %+v", doubaoLiteAsyncEdit)
|
||||
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &asyncImageEdit)
|
||||
if asyncImageEdit.TaskID == "" || !asyncImageEdit.Task.AsyncMode {
|
||||
t.Fatalf("async image edit task should be accepted: %+v", asyncImageEdit)
|
||||
}
|
||||
doubaoLiteAsyncEditDone := waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, doubaoLiteAsyncEdit.TaskID, []string{"succeeded"}, 5*time.Second)
|
||||
if doubaoLiteAsyncEditDone.Status != "succeeded" {
|
||||
t.Fatalf("doubao-5.0-lite image edit async task should succeed through river queue, got %+v", doubaoLiteAsyncEditDone)
|
||||
asyncImageEditDone := waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, asyncImageEdit.TaskID, []string{"succeeded"}, 5*time.Second)
|
||||
if asyncImageEditDone.Status != "succeeded" {
|
||||
t.Fatalf("async image edit task should succeed through river queue, got %+v", asyncImageEditDone)
|
||||
}
|
||||
|
||||
var gptImageModelTypesRaw []byte
|
||||
@@ -782,11 +803,12 @@ LIMIT 1`).Scan(&gptImageModelTypesRaw); err != nil {
|
||||
}
|
||||
|
||||
deniedModel := "permission-deny-smoke-" + suffixText
|
||||
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, deniedModel)
|
||||
var deniedPlatformModel struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + deniedModel,
|
||||
"modelName": deniedModel,
|
||||
"modelAlias": deniedModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
@@ -830,11 +852,12 @@ LIMIT 1`).Scan(&gptImageModelTypesRaw); err != nil {
|
||||
}
|
||||
|
||||
controlledModel := "permission-allow-smoke-" + suffixText
|
||||
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, controlledModel)
|
||||
var controlledPlatformModel struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + controlledModel,
|
||||
"modelName": controlledModel,
|
||||
"modelAlias": controlledModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
@@ -909,9 +932,10 @@ LIMIT 1`).Scan(&gptImageModelTypesRaw); err != nil {
|
||||
},
|
||||
}, http.StatusCreated, &customPricingRuleSet)
|
||||
pricingModel := "pricing-smoke-" + suffixText
|
||||
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, pricingModel)
|
||||
var pricingPlatformModel map[string]any
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + pricingModel,
|
||||
"modelName": pricingModel,
|
||||
"modelAlias": pricingModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
@@ -1156,12 +1180,19 @@ WHERE reference_type = 'gateway_task'
|
||||
"rules": []map[string]any{{"metric": "rpm", "limit": 1, "windowSeconds": rateLimitWindowSeconds}},
|
||||
},
|
||||
}, http.StatusCreated, &rateLimitPolicySet)
|
||||
createSimulationBaseModel(
|
||||
t,
|
||||
server.URL,
|
||||
loginResponse.AccessToken,
|
||||
rateLimitedModel,
|
||||
[]string{"text_generate", "image_generate"},
|
||||
)
|
||||
var rateLimitPlatformModel map[string]any
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + rateLimitedModel,
|
||||
"modelName": rateLimitedModel,
|
||||
"modelAlias": rateLimitedModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
"modelType": []string{"text_generate", "image_generate"},
|
||||
"displayName": "Rate Limit Smoke",
|
||||
"runtimePolicySetId": rateLimitPolicySet.ID,
|
||||
"runtimePolicyOverride": map[string]any{},
|
||||
@@ -1223,12 +1254,12 @@ WHERE reference_type = 'gateway_task'
|
||||
AsyncMode bool `json:"asyncMode"`
|
||||
} `json:"task"`
|
||||
}
|
||||
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/responses", apiKeyResponse.Secret, map[string]any{
|
||||
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/images/generations", apiKeyResponse.Secret, map[string]any{
|
||||
"model": rateLimitedModel,
|
||||
"runMode": "simulation",
|
||||
"simulation": true,
|
||||
"simulationDurationMs": 5,
|
||||
"input": "async queued",
|
||||
"prompt": "async queued",
|
||||
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &asyncRateLimitTask)
|
||||
if asyncRateLimitTask.TaskID == "" || asyncRateLimitTask.Task.ID != asyncRateLimitTask.TaskID || !asyncRateLimitTask.Task.AsyncMode {
|
||||
t.Fatalf("async task response should expose task id and async mode: %+v", asyncRateLimitTask)
|
||||
@@ -1272,9 +1303,16 @@ WHERE reference_type = 'gateway_task'
|
||||
}
|
||||
|
||||
videoRouteModel := "video-route-smoke-" + suffixText
|
||||
createSimulationBaseModel(
|
||||
t,
|
||||
server.URL,
|
||||
loginResponse.AccessToken,
|
||||
videoRouteModel,
|
||||
[]string{"video_generate", "image_to_video"},
|
||||
)
|
||||
var videoRoutePlatformModel map[string]any
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + videoRouteModel,
|
||||
"modelName": videoRouteModel,
|
||||
"modelAlias": videoRouteModel,
|
||||
"modelType": []string{"video_generate", "image_to_video"},
|
||||
@@ -1313,7 +1351,7 @@ WHERE reference_type = 'gateway_task'
|
||||
"simulation": true,
|
||||
"simulationDurationMs": 5,
|
||||
"prompt": "image to video route",
|
||||
"image": "https://example.com/source.png",
|
||||
"image": server.URL + "/static/simulation/image.png",
|
||||
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &imageToVideoTask)
|
||||
waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, imageToVideoTask.Task.ID, []string{"succeeded"}, 10*time.Second)
|
||||
doJSON(t, server.URL, http.MethodGet, "/api/v1/tasks/"+imageToVideoTask.Task.ID, apiKeyResponse.Secret, nil, http.StatusOK, &imageToVideoTask.Task)
|
||||
@@ -1333,8 +1371,11 @@ WHERE reference_type = 'gateway_task'
|
||||
if len(preprocessingDetail.Items) == 0 {
|
||||
t.Fatalf("task preprocessing endpoint should expose persisted preprocessing logs: %+v", preprocessingDetail)
|
||||
}
|
||||
if preprocessingDetail.Items[0]["actualInput"] == nil || preprocessingDetail.Items[0]["convertedOutput"] == nil {
|
||||
t.Fatalf("preprocessing log should store actual input and converted output: %+v", preprocessingDetail.Items)
|
||||
if preprocessingDetail.Items[0]["changes"] == nil {
|
||||
t.Fatalf("preprocessing log should retain its compact change summary: %+v", preprocessingDetail.Items)
|
||||
}
|
||||
if preprocessingDetail.Items[0]["actualInput"] != nil || preprocessingDetail.Items[0]["convertedOutput"] != nil {
|
||||
t.Fatalf("history compaction should remove full preprocessing payloads: %+v", preprocessingDetail.Items)
|
||||
}
|
||||
var preprocessingRows int
|
||||
if err := testPool.QueryRow(ctx, `SELECT count(*) FROM gateway_task_param_preprocessing_logs WHERE task_id = $1::uuid`, imageToVideoTask.Task.ID).Scan(&preprocessingRows); err != nil {
|
||||
@@ -1345,6 +1386,9 @@ WHERE reference_type = 'gateway_task'
|
||||
}
|
||||
|
||||
failoverModel := "phase1-failover-" + suffixText
|
||||
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, failoverModel)
|
||||
unrelatedPriorityModel := "priority-demote-unrelated-" + suffixText
|
||||
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, unrelatedPriorityModel)
|
||||
var failedPlatform struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
@@ -1383,16 +1427,16 @@ WHERE reference_type = 'gateway_task'
|
||||
}, http.StatusCreated, &unrelatedPriorityPlatform)
|
||||
var unrelatedPriorityPlatformModel map[string]any
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+unrelatedPriorityPlatform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"modelName": "priority-demote-unrelated-" + suffixText,
|
||||
"modelAlias": "priority-demote-unrelated-" + suffixText,
|
||||
"canonicalModelKey": "openai:" + unrelatedPriorityModel,
|
||||
"modelName": unrelatedPriorityModel,
|
||||
"modelAlias": unrelatedPriorityModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
"displayName": "Unrelated Priority",
|
||||
}, http.StatusCreated, &unrelatedPriorityPlatformModel)
|
||||
for _, platformID := range []string{failedPlatform.ID, successPlatform.ID} {
|
||||
var platformModel map[string]any
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+platformID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + failoverModel,
|
||||
"modelName": failoverModel,
|
||||
"modelAlias": failoverModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
@@ -1442,8 +1486,8 @@ WHERE reference_type = 'gateway_task'
|
||||
if failoverDetail.Attempts[2].PlatformName != "OpenAI Retry Success" || failoverDetail.Attempts[2].Status != "succeeded" || failoverDetail.Attempts[2].ResponseMS <= 0 {
|
||||
t.Fatalf("third failover attempt should preserve successful platform: %+v", failoverDetail.Attempts[2])
|
||||
}
|
||||
if summary, ok := failoverDetail.Metrics["attempts"].([]any); !ok || len(summary) != 3 {
|
||||
t.Fatalf("task metrics should keep attempt-chain summary, got %+v", failoverDetail.Metrics)
|
||||
if _, ok := failoverDetail.Metrics["attempts"]; ok {
|
||||
t.Fatalf("task metrics should not duplicate the dedicated attempt chain: %+v", failoverDetail.Metrics)
|
||||
}
|
||||
var demotedDynamicPriority int
|
||||
var successEffectivePriority int
|
||||
@@ -1479,6 +1523,7 @@ WHERE failed.id = $1::uuid`, failedPlatform.ID, successPlatform.ID, unrelatedPri
|
||||
},
|
||||
}, http.StatusCreated, °radePolicySet)
|
||||
degradeModel := "degrade-smoke-" + suffixText
|
||||
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, degradeModel)
|
||||
var degradedPlatform struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
@@ -1514,7 +1559,7 @@ WHERE failed.id = $1::uuid`, failedPlatform.ID, successPlatform.ID, unrelatedPri
|
||||
ID string `json:"id"`
|
||||
}
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+item.platformID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + degradeModel,
|
||||
"modelName": degradeModel,
|
||||
"modelAlias": degradeModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
@@ -1568,6 +1613,7 @@ WHERE m.platform_id = $1::uuid
|
||||
},
|
||||
}, http.StatusCreated, &autoDisablePolicySet)
|
||||
autoDisableModel := "auto-disable-smoke-" + suffixText
|
||||
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, autoDisableModel)
|
||||
var invalidKeyPlatform struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
@@ -1582,7 +1628,7 @@ WHERE m.platform_id = $1::uuid
|
||||
}, http.StatusCreated, &invalidKeyPlatform)
|
||||
var invalidKeyPlatformModel map[string]any
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+invalidKeyPlatform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + autoDisableModel,
|
||||
"modelName": autoDisableModel,
|
||||
"modelAlias": autoDisableModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
@@ -1625,7 +1671,7 @@ WHERE m.platform_id = $1::uuid
|
||||
"priority": 60,
|
||||
}, http.StatusCreated, &validKeyPlatform)
|
||||
doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+validKeyPlatform.ID+"/models", loginResponse.AccessToken, map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + autoDisableModel,
|
||||
"modelName": autoDisableModel,
|
||||
"modelAlias": autoDisableModel,
|
||||
"modelType": []string{"text_generate"},
|
||||
@@ -1719,14 +1765,31 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
|
||||
} `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
var taskPlatformID string
|
||||
var taskBillingStatus string
|
||||
if err := testPool.QueryRow(ctx, `
|
||||
SELECT platform_id::text
|
||||
FROM gateway_task_attempts
|
||||
WHERE task_id = $1::uuid
|
||||
AND status = 'succeeded'
|
||||
ORDER BY attempt_no DESC
|
||||
LIMIT 1`, taskResponse.Task.ID).Scan(&taskPlatformID); err != nil {
|
||||
t.Fatalf("read task platform for admin filters: %v", err)
|
||||
}
|
||||
if err := testPool.QueryRow(ctx, `
|
||||
SELECT billing_status
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid`, taskResponse.Task.ID).Scan(&taskBillingStatus); err != nil {
|
||||
t.Fatalf("read task billing status for admin filters: %v", err)
|
||||
}
|
||||
adminTaskQuery := "/api/admin/tasks?q=" + taskResponse.Task.ID +
|
||||
"&userId=" + smokeGatewayUserID +
|
||||
"&status=succeeded" +
|
||||
"&platformId=" + platform.ID +
|
||||
"&platformId=" + taskPlatformID +
|
||||
"&model=" + defaultTextModel +
|
||||
"&modelType=text_generate" +
|
||||
"&runMode=simulation" +
|
||||
"&billingStatus=settled" +
|
||||
"&billingStatus=" + taskBillingStatus +
|
||||
"&apiKey=smoke" +
|
||||
"&pageSize=10"
|
||||
doJSON(t, server.URL, http.MethodGet, adminTaskQuery, loginResponse.AccessToken, nil, http.StatusOK, &adminTaskList)
|
||||
@@ -1771,9 +1834,6 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
|
||||
if !bytes.Contains(body, []byte("task.running")) {
|
||||
t.Fatalf("events response should include running transition event body=%s", string(body))
|
||||
}
|
||||
if !bytes.Contains(body, []byte("task.progress")) {
|
||||
t.Fatalf("events response should include progress events body=%s", string(body))
|
||||
}
|
||||
if !bytes.Contains(body, []byte("task.billing.settled")) {
|
||||
t.Fatalf("events response should include billing settlement event body=%s", string(body))
|
||||
}
|
||||
@@ -1789,8 +1849,10 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ = io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK || !bytes.Contains(body, []byte("task.retrying")) {
|
||||
t.Fatalf("failover events should include retrying event status=%d body=%s", resp.StatusCode, string(body))
|
||||
if resp.StatusCode != http.StatusOK ||
|
||||
!bytes.Contains(body, []byte("task.attempt.failed")) ||
|
||||
!bytes.Contains(body, []byte("task.policy.priority_demoted")) {
|
||||
t.Fatalf("failover events should include the failed attempt and applied policy status=%d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
assertLoadAvoidanceSimulatedRetryChain(t, ctx, testPool, server.URL, loginResponse.AccessToken, apiKeyResponse.Secret, suffixText)
|
||||
@@ -1803,18 +1865,6 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
|
||||
t.Fatal("task progress callback outbox should receive events")
|
||||
}
|
||||
|
||||
restartModel := "worker-restart-" + suffixText
|
||||
createSimulationTextBaseModel(t, server.URL, loginResponse.AccessToken, restartModel)
|
||||
createSimulationTextPlatformModel(
|
||||
t,
|
||||
server.URL,
|
||||
loginResponse.AccessToken,
|
||||
"openai-worker-restart-"+suffixText,
|
||||
"OpenAI Worker Restart",
|
||||
restartModel,
|
||||
1,
|
||||
nil,
|
||||
)
|
||||
var restartAsyncTask struct {
|
||||
TaskID string `json:"taskId"`
|
||||
Task struct {
|
||||
@@ -1823,12 +1873,13 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
|
||||
AsyncMode bool `json:"asyncMode"`
|
||||
} `json:"task"`
|
||||
}
|
||||
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/responses", apiKeyResponse.Secret, map[string]any{
|
||||
"model": restartModel,
|
||||
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/images/generations", apiKeyResponse.Secret, map[string]any{
|
||||
"model": defaultImageModel,
|
||||
"runMode": "simulation",
|
||||
"simulation": true,
|
||||
"simulationDurationMs": 2000,
|
||||
"input": "river worker restart",
|
||||
"prompt": "river worker restart",
|
||||
"size": "1024x1024",
|
||||
}, map[string]string{"X-Async": "true"}, http.StatusAccepted, &restartAsyncTask)
|
||||
if restartAsyncTask.TaskID == "" || !restartAsyncTask.Task.AsyncMode {
|
||||
t.Fatalf("restart async task should be accepted as async: %+v", restartAsyncTask)
|
||||
@@ -1841,15 +1892,19 @@ WHERE id = $1::uuid`, taskResponse.Task.ID); err != nil {
|
||||
serverCtx2, cancelServer2 := context.WithCancel(ctx)
|
||||
defer cancelServer2()
|
||||
server2 := httptest.NewServer(NewServerWithContext(serverCtx2, config.Config{
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
BillingEngineMode: "enforce",
|
||||
TaskProgressCallbackEnabled: true,
|
||||
TaskProgressCallbackURL: "http://callback.local/task-progress",
|
||||
CORSAllowedOrigin: "*",
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
BillingEngineMode: "enforce",
|
||||
TaskProgressCallbackEnabled: true,
|
||||
TaskProgressCallbackURL: "http://callback.local/task-progress",
|
||||
CORSAllowedOrigin: "*",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 32,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
defer server2.Close()
|
||||
restartRecovered := waitForTaskStatus(t, server2.URL, apiKeyResponse.Secret, restartAsyncTask.TaskID, []string{"succeeded"}, 8*time.Second)
|
||||
@@ -2334,15 +2389,20 @@ type cacheAffinityTaskDetail struct {
|
||||
}
|
||||
|
||||
func createSimulationTextBaseModel(t *testing.T, baseURL string, adminToken string, model string) {
|
||||
t.Helper()
|
||||
createSimulationBaseModel(t, baseURL, adminToken, model, []string{"text_generate"})
|
||||
}
|
||||
|
||||
func createSimulationBaseModel(t *testing.T, baseURL string, adminToken string, model string, modelTypes []string) {
|
||||
t.Helper()
|
||||
doJSON(t, baseURL, http.MethodPost, "/api/admin/catalog/base-models", adminToken, map[string]any{
|
||||
"providerKey": "openai",
|
||||
"canonicalModelKey": "openai:" + model,
|
||||
"invocationName": model,
|
||||
"providerModelName": model,
|
||||
"modelType": []string{"text_generate"},
|
||||
"modelType": modelTypes,
|
||||
"displayName": model,
|
||||
"capabilities": map[string]any{"originalTypes": []string{"text_generate"}},
|
||||
"capabilities": map[string]any{"originalTypes": modelTypes},
|
||||
"metadata": map[string]any{"source": "cache-affinity-integration-test"},
|
||||
}, http.StatusCreated, nil)
|
||||
}
|
||||
@@ -2481,6 +2541,7 @@ LIMIT 1`, key, value).Scan(&taskID)
|
||||
func assertLoadAvoidanceSimulatedRetryChain(t *testing.T, ctx context.Context, testPool *pgxpool.Pool, baseURL string, adminToken string, runtimeToken string, suffixText string) {
|
||||
t.Helper()
|
||||
model := "load-avoidance-smoke-" + suffixText
|
||||
createSimulationTextBaseModel(t, baseURL, adminToken, model)
|
||||
type scenario struct {
|
||||
keySuffix string
|
||||
name string
|
||||
@@ -2522,7 +2583,7 @@ func assertLoadAvoidanceSimulatedRetryChain(t *testing.T, ctx context.Context, t
|
||||
ID string `json:"id"`
|
||||
}
|
||||
payload := map[string]any{
|
||||
"canonicalModelKey": "openai:gpt-4o-mini",
|
||||
"canonicalModelKey": "openai:" + model,
|
||||
"modelName": model,
|
||||
"providerModelName": model,
|
||||
"modelAlias": model,
|
||||
@@ -2581,55 +2642,29 @@ func assertLoadAvoidanceSimulatedRetryChain(t *testing.T, ctx context.Context, t
|
||||
Metrics map[string]any `json:"metrics"`
|
||||
}
|
||||
doJSON(t, baseURL, http.MethodGet, "/api/v1/tasks/"+taskResponse.Task.ID, runtimeToken, nil, http.StatusOK, &detail)
|
||||
if detail.Status != "failed" || len(detail.Attempts) != len(scenarios) {
|
||||
t.Fatalf("load avoidance detail should expose every attempted client, got status=%s attempts=%+v", detail.Status, detail.Attempts)
|
||||
if detail.Status != "failed" || len(detail.Attempts) != 1 {
|
||||
t.Fatalf("non-retryable failure should stop before load-avoided fallbacks, got status=%s attempts=%+v", detail.Status, detail.Attempts)
|
||||
}
|
||||
expected := []struct {
|
||||
name string
|
||||
code string
|
||||
retryable bool
|
||||
avoided bool
|
||||
}{
|
||||
{name: "Load Avoidance Hard Stop", code: "bad_request"},
|
||||
{name: "Load Avoidance Retryable", code: "server_error", retryable: true},
|
||||
{name: "Load Avoidance Full Rate Limit", code: "rate_limit", retryable: true, avoided: true},
|
||||
{name: "Load Avoidance Full Overloaded", code: "overloaded", retryable: true, avoided: true},
|
||||
{name: "Load Avoidance Full Fatal", code: "bad_request", avoided: true},
|
||||
attempt := detail.Attempts[0]
|
||||
if attempt.AttemptNo != 1 ||
|
||||
attempt.PlatformName != "Load Avoidance Hard Stop" ||
|
||||
attempt.Status != "failed" ||
|
||||
attempt.ErrorCode != "bad_request" ||
|
||||
attempt.Retryable {
|
||||
t.Fatalf("unexpected hard-stop attempt: %+v", attempt)
|
||||
}
|
||||
attemptSummary := make([]string, 0, len(detail.Attempts))
|
||||
for index, want := range expected {
|
||||
got := detail.Attempts[index]
|
||||
attemptSummary = append(attemptSummary, got.PlatformName+":"+got.Status+":"+got.ErrorCode)
|
||||
if got.AttemptNo != index+1 || got.PlatformName != want.name || got.Status != "failed" || got.ErrorCode != want.code || got.Retryable != want.retryable {
|
||||
t.Fatalf("unexpected load avoidance attempt %d: got %+v want %+v", index+1, got, want)
|
||||
}
|
||||
if boolFromTestMap(got.Metrics, "loadAvoided") != want.avoided {
|
||||
t.Fatalf("loadAvoided mismatch for %s metrics=%+v", got.PlatformName, got.Metrics)
|
||||
}
|
||||
if !want.avoided && floatFromTestAny(got.Metrics["loadRatio"]) != 0 {
|
||||
t.Fatalf("non-full candidate should not carry load pressure, got %s metrics=%+v", got.PlatformName, got.Metrics)
|
||||
}
|
||||
if want.avoided {
|
||||
if ratio := floatFromTestAny(got.Metrics["loadRatio"]); ratio < 1 {
|
||||
t.Fatalf("avoided candidate should expose full load ratio, got %s ratio=%v metrics=%+v", got.PlatformName, ratio, got.Metrics)
|
||||
}
|
||||
loadMetrics, _ := got.Metrics["loadMetrics"].(map[string]any)
|
||||
concurrent, _ := loadMetrics["concurrent"].(map[string]any)
|
||||
if ratio := floatFromTestAny(concurrent["ratio"]); ratio < 1 {
|
||||
t.Fatalf("avoided candidate should expose concurrent saturation, got %s loadMetrics=%+v", got.PlatformName, loadMetrics)
|
||||
}
|
||||
if queued := floatFromTestAny(loadMetrics["queued"]); queued < 1 {
|
||||
t.Fatalf("avoided candidate should expose queued waiting load, got %s loadMetrics=%+v", got.PlatformName, loadMetrics)
|
||||
}
|
||||
}
|
||||
if boolFromTestMap(attempt.Metrics, "loadAvoided") {
|
||||
t.Fatalf("selected non-full candidate must not be marked load avoided: %+v", attempt.Metrics)
|
||||
}
|
||||
if !attemptTraceHasReason(detail.Attempts[0].Metrics, "load_avoidance_fallback") {
|
||||
t.Fatalf("first hard-stop attempt should continue because avoided candidates remain, metrics=%+v", detail.Attempts[0].Metrics)
|
||||
if ratio := floatFromTestAny(attempt.Metrics["loadRatio"]); ratio != 0 {
|
||||
t.Fatalf("selected non-full candidate should not carry load pressure, got ratio=%v metrics=%+v", ratio, attempt.Metrics)
|
||||
}
|
||||
if summary, ok := detail.Metrics["attempts"].([]any); !ok || len(summary) != len(scenarios) {
|
||||
t.Fatalf("final task metrics should preserve load-avoidance attempt summary, got %+v", detail.Metrics)
|
||||
if !attemptTraceHasReason(attempt.Metrics, "failover_action_rule") {
|
||||
t.Fatalf("hard-stop attempt should record the failure-policy decision, metrics=%+v", attempt.Metrics)
|
||||
}
|
||||
if _, ok := detail.Metrics["attempts"]; ok {
|
||||
t.Fatalf("task metrics should not duplicate the dedicated attempt chain: %+v", detail.Metrics)
|
||||
}
|
||||
t.Logf("load avoidance retry chain: %s", strings.Join(attemptSummary, " -> "))
|
||||
}
|
||||
|
||||
func seedQueuedConcurrencyLoad(t *testing.T, ctx context.Context, testPool *pgxpool.Pool, platformKey string, model string, platformModelID string) {
|
||||
|
||||
@@ -124,13 +124,17 @@ func newFailurePolicyAcceptanceFixture(t *testing.T) *failurePolicyAcceptanceFix
|
||||
t.Fatalf("connect acceptance pool: %v", err)
|
||||
}
|
||||
handler := NewServerWithContext(ctx, config.Config{
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "failure-policy-http-acceptance-secret",
|
||||
BillingEngineMode: "observe",
|
||||
CORSAllowedOrigin: "*",
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "failure-policy-http-acceptance-secret",
|
||||
BillingEngineMode: "observe",
|
||||
CORSAllowedOrigin: "*",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 32,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, db, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
server := httptest.NewServer(handler)
|
||||
fixture := &failurePolicyAcceptanceFixture{
|
||||
|
||||
@@ -74,9 +74,9 @@ func TestGeminiBase64ImageEditThousandSynchronousRequests(t *testing.T) {
|
||||
t.Fatalf("connect observation pool: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, true)
|
||||
defer restoreAcceptanceState()
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE integration_platforms SET status = 'disabled' WHERE deleted_at IS NULL;
|
||||
UPDATE file_storage_channels SET status = 'disabled' WHERE deleted_at IS NULL;
|
||||
UPDATE gateway_user_groups
|
||||
SET rate_limit_policy = '{"rules":[
|
||||
{"metric":"concurrent","limit":1024,"leaseTtlSeconds":120},
|
||||
|
||||
@@ -192,6 +192,11 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrRateLimited) {
|
||||
applyRunErrorHeaders(w, err)
|
||||
writeGeminiTaskError(statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), runErrorCode(err))
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrIdempotencyKeyReused) {
|
||||
writeGeminiTaskError(http.StatusConflict, "Idempotency-Key was reused for a different request", nil, "idempotency_key_reused")
|
||||
return
|
||||
|
||||
@@ -178,9 +178,45 @@ func (s *Server) logPostgresUnavailable(message string) {
|
||||
"postgres_pool_idle_connections", statistics.IdleConns(),
|
||||
"postgres_pool_empty_acquire_count", statistics.EmptyAcquireCount(),
|
||||
"postgres_pool_canceled_acquire_count", statistics.CanceledAcquireCount(),
|
||||
"postgres_critical_pool", s.criticalPostgresPoolLogFields(),
|
||||
"postgres_river_pool", s.riverPostgresPoolLogFields(),
|
||||
)
|
||||
}
|
||||
|
||||
func (s *Server) criticalPostgresPoolLogFields() map[string]any {
|
||||
if s.coordinationStore == nil || s.coordinationStore == s.store ||
|
||||
s.coordinationStore.Pool() == nil {
|
||||
return map[string]any{"separate": false}
|
||||
}
|
||||
statistics := s.coordinationStore.Pool().Stat()
|
||||
return map[string]any{
|
||||
"separate": true,
|
||||
"max_connections": statistics.MaxConns(),
|
||||
"total_connections": statistics.TotalConns(),
|
||||
"acquired_connections": statistics.AcquiredConns(),
|
||||
"idle_connections": statistics.IdleConns(),
|
||||
"empty_acquire_count": statistics.EmptyAcquireCount(),
|
||||
"canceled_acquire_count": statistics.CanceledAcquireCount(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) riverPostgresPoolLogFields() map[string]any {
|
||||
if s.riverStore == nil || s.riverStore == s.store ||
|
||||
s.riverStore.Pool() == nil {
|
||||
return map[string]any{"separate": false}
|
||||
}
|
||||
statistics := s.riverStore.Pool().Stat()
|
||||
return map[string]any{
|
||||
"separate": true,
|
||||
"max_connections": statistics.MaxConns(),
|
||||
"total_connections": statistics.TotalConns(),
|
||||
"acquired_connections": statistics.AcquiredConns(),
|
||||
"idle_connections": statistics.IdleConns(),
|
||||
"empty_acquire_count": statistics.EmptyAcquireCount(),
|
||||
"canceled_acquire_count": statistics.CanceledAcquireCount(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) localIdentityEnabled() bool {
|
||||
mode := strings.ToLower(strings.TrimSpace(s.cfg.IdentityMode))
|
||||
return mode == "" || mode == "standalone" || mode == "hybrid"
|
||||
@@ -1183,6 +1219,11 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
|
||||
}
|
||||
created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrRateLimited) {
|
||||
applyRunErrorHeaders(w, err)
|
||||
writeTaskError(statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), runErrorCode(err))
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrIdempotencyKeyReused) {
|
||||
writeTaskError(http.StatusConflict, "Idempotency-Key was reused for a different request", nil, "idempotency_key_reused")
|
||||
return
|
||||
@@ -1200,8 +1241,9 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
|
||||
if s.billingMetrics != nil {
|
||||
s.billingMetrics.ObserveBillingEvent("idempotent_replay")
|
||||
}
|
||||
if targetProtocol == "" {
|
||||
if targetProtocol == "" || gatewayAPIV1Request(r) {
|
||||
w.Header().Set("Idempotent-Replayed", "true")
|
||||
w.Header().Set("X-Gateway-Task-Id", task.ID)
|
||||
}
|
||||
if responsePlan.streamMode {
|
||||
writeTaskError(http.StatusConflict, "streaming idempotent replay is not supported", nil, "idempotency_stream_replay_unsupported")
|
||||
@@ -1411,7 +1453,7 @@ func writeCompatibleTaskResponse(runCtx context.Context, w http.ResponseWriter,
|
||||
}
|
||||
|
||||
func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.ResponseWriter, r *http.Request, executor taskExecutor, kind string, model string, targetProtocol string, task store.GatewayTask, user *auth.User, streamMode bool, includeUsage bool) {
|
||||
if targetProtocol == "" {
|
||||
if targetProtocol == "" || gatewayAPIV1Request(r) {
|
||||
w.Header().Set("X-Gateway-Task-Id", task.ID)
|
||||
}
|
||||
if streamMode {
|
||||
@@ -1515,6 +1557,10 @@ func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.Response
|
||||
writeJSON(w, http.StatusOK, easyAISynchronousTaskResponse(result.Task, result.Output))
|
||||
}
|
||||
|
||||
func gatewayAPIV1Request(r *http.Request) bool {
|
||||
return r != nil && strings.HasPrefix(r.URL.Path, "/api/v1/")
|
||||
}
|
||||
|
||||
func streamIncludeUsage(body map[string]any) bool {
|
||||
streamOptions, _ := body["stream_options"].(map[string]any)
|
||||
includeUsage, _ := streamOptions["include_usage"].(bool)
|
||||
|
||||
@@ -131,12 +131,16 @@ func TestKelingOmniCompatibleHTTPFlow(t *testing.T) {
|
||||
serverCtx, cancelServer := context.WithCancel(ctx)
|
||||
defer cancelServer()
|
||||
gateway := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
CORSAllowedOrigin: "*",
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
CORSAllowedOrigin: "*",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 32,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
defer gateway.Close()
|
||||
|
||||
@@ -152,7 +156,7 @@ func TestKelingOmniCompatibleHTTPFlow(t *testing.T) {
|
||||
"aspect_ratio": "16:9",
|
||||
"duration": "3",
|
||||
"sound": "off",
|
||||
"image_list": []any{map[string]any{"image_url": "https://example.com/reference.png"}},
|
||||
"image_list": []any{map[string]any{"image_url": gateway.URL + "/static/simulation/image.png"}},
|
||||
"external_task_id": "compat-http-1",
|
||||
}, http.StatusOK, &created)
|
||||
createdData, _ := created.Data.(map[string]any)
|
||||
|
||||
@@ -33,12 +33,16 @@ func TestKeling30TurboSimulationFlow(t *testing.T) {
|
||||
serverCtx, cancelServer := context.WithCancel(ctx)
|
||||
defer cancelServer()
|
||||
server := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
CORSAllowedOrigin: "*",
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
CORSAllowedOrigin: "*",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 32,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
defer server.Close()
|
||||
|
||||
@@ -192,7 +196,7 @@ WHERE username = $1`, username); err != nil {
|
||||
"simulation": true,
|
||||
"simulationDurationMs": 5,
|
||||
"prompt": "The subject looks toward the camera",
|
||||
"image": "https://example.com/first.png",
|
||||
"image": server.URL + "/static/simulation/image.png",
|
||||
"duration": 5,
|
||||
"resolution": "720p",
|
||||
"audio": true,
|
||||
|
||||
@@ -159,6 +159,9 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
|
||||
created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, store.ErrRateLimited):
|
||||
applyRunErrorHeaders(w, err)
|
||||
writeKlingCompatError(w, statusFromRunError(err), runErrorMessage(err), runErrorCode(err))
|
||||
case errors.Is(err, store.ErrIdempotencyKeyReused):
|
||||
writeKlingCompatError(w, http.StatusConflict, err.Error(), "idempotency_key_reused")
|
||||
case errors.Is(err, store.ErrExternalTaskIDReused):
|
||||
|
||||
@@ -49,12 +49,16 @@ WHERE provider_key = 'keling'
|
||||
serverCtx, cancelServer := context.WithCancel(ctx)
|
||||
defer cancelServer()
|
||||
server := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
CORSAllowedOrigin: "*",
|
||||
AppEnv: "test",
|
||||
HTTPAddr: ":0",
|
||||
DatabaseURL: databaseURL,
|
||||
IdentityMode: "hybrid",
|
||||
JWTSecret: "test-secret",
|
||||
CORSAllowedOrigin: "*",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 32,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
defer server.Close()
|
||||
|
||||
@@ -159,7 +163,7 @@ WHERE platform_id = $1::uuid
|
||||
|
||||
for _, model := range []string{klingO1Model, klingV3OmniModel} {
|
||||
assertGenericVideoGeneration(model+"-text-to-video", model, "", "video_generate")
|
||||
assertGenericVideoGeneration(model+"-image-to-video", model, "https://example.com/first.png", "image_to_video")
|
||||
assertGenericVideoGeneration(model+"-image-to-video", model, server.URL+"/static/simulation/image.png", "image_to_video")
|
||||
}
|
||||
|
||||
createV1 := func(model string, duration int, externalID string) string {
|
||||
|
||||
@@ -44,6 +44,25 @@ func TestOIDCJITFullGatewayUserFlowAndSecurityBoundary(t *testing.T) {
|
||||
t.Fatalf("connect store: %v", err)
|
||||
}
|
||||
t.Cleanup(db.Close)
|
||||
hasBreakGlass, err := db.HasBreakGlassManager(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("check OIDC JIT break-glass manager: %v", err)
|
||||
}
|
||||
if !hasBreakGlass {
|
||||
manager, registerErr := db.RegisterLocalUser(ctx, store.LocalRegisterInput{
|
||||
Username: "oidc-jit-manager-" + uuid.NewString(),
|
||||
Password: uuid.NewString() + "-local-only",
|
||||
})
|
||||
if registerErr != nil {
|
||||
t.Fatalf("create OIDC JIT break-glass manager: %v", registerErr)
|
||||
}
|
||||
if _, updateErr := db.Pool().Exec(ctx, `UPDATE gateway_users SET roles='["manager"]'::jsonb WHERE id=$1::uuid`, manager.ID); updateErr != nil {
|
||||
t.Fatalf("grant OIDC JIT break-glass manager role: %v", updateErr)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.Pool().Exec(context.Background(), `DELETE FROM gateway_users WHERE id=$1::uuid`, manager.ID)
|
||||
})
|
||||
}
|
||||
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
@@ -388,7 +407,7 @@ func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie {
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
response, err := client.Get(baseURL + "/api/v1/auth/oidc/login?returnTo=%2Fapi%2Fv1%2Fme")
|
||||
response, err := client.Get(baseURL + "/api/v1/auth/oidc/login?contextType=tenant&returnTo=%2Fapi%2Fv1%2Fme")
|
||||
if err != nil {
|
||||
t.Fatalf("complete OIDC BFF login: %v", err)
|
||||
}
|
||||
@@ -463,7 +482,8 @@ func signedOIDCJITToken(t *testing.T, key *ecdsa.PrivateKey, issuer string, subj
|
||||
t.Helper()
|
||||
now := time.Now()
|
||||
claims := jwt.MapClaims{
|
||||
"iss": issuer, "aud": "gateway-api", "sub": subject, "tid": oidcJITTenantID,
|
||||
"iss": issuer, "aud": "gateway-api", "sub": subject, "context_type": "tenant", "tid": oidcJITTenantID,
|
||||
"client_id": "gateway-public-test",
|
||||
"preferred_username": "oidc-jit-acceptance", "roles": []string{"gateway.admin"},
|
||||
"scope": "openid gateway.access", "iat": now.Unix(), "nbf": now.Add(-time.Second).Unix(),
|
||||
"exp": now.Add(time.Hour).Unix(),
|
||||
|
||||
@@ -24,6 +24,8 @@ type Server struct {
|
||||
ctx context.Context
|
||||
cfg config.Config
|
||||
store *store.Store
|
||||
coordinationStore *store.Store
|
||||
riverStore *store.Store
|
||||
oidcUserResolver oidcUserResolver
|
||||
auth *auth.Authenticator
|
||||
oidcClient oidcPublicClient
|
||||
@@ -72,6 +74,27 @@ func NewServer(cfg config.Config, db *store.Store, logger *slog.Logger) http.Han
|
||||
}
|
||||
|
||||
func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Store, logger *slog.Logger) http.Handler {
|
||||
return NewServerWithContextAndCoordinationStore(ctx, cfg, db, db, logger)
|
||||
}
|
||||
|
||||
func NewServerWithContextAndCoordinationStore(
|
||||
ctx context.Context,
|
||||
cfg config.Config,
|
||||
db *store.Store,
|
||||
coordinationDB *store.Store,
|
||||
logger *slog.Logger,
|
||||
) http.Handler {
|
||||
return NewServerWithStores(ctx, cfg, db, coordinationDB, db, logger)
|
||||
}
|
||||
|
||||
func NewServerWithStores(
|
||||
ctx context.Context,
|
||||
cfg config.Config,
|
||||
db *store.Store,
|
||||
coordinationDB *store.Store,
|
||||
riverDB *store.Store,
|
||||
logger *slog.Logger,
|
||||
) http.Handler {
|
||||
if cfg.MediaMaterializationConcurrency == 0 {
|
||||
cfg.MediaMaterializationConcurrency = 8
|
||||
}
|
||||
@@ -83,9 +106,11 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
|
||||
ctx: ctx,
|
||||
cfg: cfg,
|
||||
store: db,
|
||||
coordinationStore: coordinationDB,
|
||||
riverStore: riverDB,
|
||||
oidcUserResolver: db,
|
||||
auth: auth.New(cfg.JWTSecret, cfg.ServerMainBaseURL, cfg.ServerMainInternalToken),
|
||||
runner: runner.New(cfg, db, logger, securityEventMetrics),
|
||||
runner: runner.NewWithStores(cfg, db, coordinationDB, riverDB, logger, securityEventMetrics),
|
||||
logger: logger,
|
||||
billingMetrics: securityEventMetrics,
|
||||
requestAssetLocks: map[string]*requestAssetLock{},
|
||||
@@ -122,7 +147,9 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
|
||||
mux.HandleFunc("GET /readyz", server.ready)
|
||||
mux.HandleFunc("GET /api/v1/healthz", server.health)
|
||||
mux.HandleFunc("GET /api/v1/readyz", server.ready)
|
||||
mux.Handle("GET /metrics", securityEventMetrics.DynamicHandler(db))
|
||||
mux.Handle("GET /metrics", securityEventMetrics.DynamicHandler(postgresMetricsProvider{
|
||||
Store: db, critical: coordinationDB, river: riverDB,
|
||||
}))
|
||||
if !cfg.RunsPublicHTTP() {
|
||||
return server.recover(mux)
|
||||
}
|
||||
@@ -221,13 +248,16 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
|
||||
mux.Handle("GET /api/admin/system/client-customization/settings", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getClientCustomizationSettings)))
|
||||
mux.Handle("PATCH /api/admin/system/client-customization/settings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateClientCustomizationSettings)))
|
||||
mux.Handle("GET /api/admin/system/acceptance/traffic-mode", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getGatewayTrafficMode)))
|
||||
mux.Handle("POST /api/admin/system/acceptance/traffic-mode/pause", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.pauseGatewayTraffic)))
|
||||
mux.Handle("POST /api/admin/system/acceptance/runs", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createAcceptanceRun)))
|
||||
mux.Handle("GET /api/admin/system/acceptance/runs/{runID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getAcceptanceRun)))
|
||||
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/activate", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.activateAcceptanceRun)))
|
||||
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/capacity-profiles", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.stageAcceptanceCapacityProfiles)))
|
||||
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/finish", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.finishAcceptanceRun)))
|
||||
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/retry", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.retryAcceptanceRun)))
|
||||
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/promote", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.promoteAcceptanceRun)))
|
||||
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/abort", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.abortAcceptanceRun)))
|
||||
mux.Handle("GET /api/admin/system/acceptance/capacity-profiles", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listCapacityProfiles)))
|
||||
mux.Handle("GET /api/admin/system/identity/configuration", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityConfiguration)))
|
||||
mux.Handle("POST /api/admin/system/identity/pairings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.startIdentityPairing)))
|
||||
mux.Handle("GET /api/admin/system/identity/pairings/{pairingID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityPairing)))
|
||||
@@ -348,6 +378,26 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
|
||||
return server.recover(server.cors(server.protectOIDCSessionCookie(mux)))
|
||||
}
|
||||
|
||||
type postgresMetricsProvider struct {
|
||||
*store.Store
|
||||
critical *store.Store
|
||||
river *store.Store
|
||||
}
|
||||
|
||||
func (provider postgresMetricsProvider) RiverPostgresPoolMetrics() store.PostgresPoolMetricsSnapshot {
|
||||
if provider.river == nil || provider.river == provider.Store {
|
||||
return store.PostgresPoolMetricsSnapshot{}
|
||||
}
|
||||
return provider.river.PostgresPoolMetrics()
|
||||
}
|
||||
|
||||
func (provider postgresMetricsProvider) CriticalPostgresPoolMetrics() store.PostgresPoolMetricsSnapshot {
|
||||
if provider.critical == nil || provider.critical == provider.Store {
|
||||
return store.PostgresPoolMetricsSnapshot{}
|
||||
}
|
||||
return provider.critical.PostgresPoolMetrics()
|
||||
}
|
||||
|
||||
func (server *Server) initializeIdentityRuntime(
|
||||
ctx context.Context,
|
||||
db *store.Store,
|
||||
|
||||
Reference in New Issue
Block a user