BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

迁移音频生成与语音合成到 gateway 并补充 simulation 测试

Browse Source
This commit is contained in:
wangbo
2026-06-07 10:26:57 +08:00
parent 78ab867a9f
commit dc14866210
22 changed files with 2475 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/httpapi/chat_completions_mode_test.go
+44
View File
@@ -72,6 +72,50 @@ func TestPlanTaskResponseKeepsAsyncTaskModeForOtherAPIV1Tasks(t *testing.T) {
}
}
func TestPlanTaskResponseKeepsCompatibleSyncForAudioOpenAPIUnlessAsync(t *testing.T) {
for _, item := range []struct {
kind string
path string
}{
{kind: "song.generations", path: "/api/v1/song/generations"},
{kind: "music.generations", path: "/api/v1/music/generations"},
{kind: "speech.generations", path: "/api/v1/speech/generations"},
} {
t.Run(item.kind, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, item.path, nil)
plan := planTaskResponse(item.kind, true, map[string]any{"stream": true}, req)
if plan.asyncMode {
t.Fatalf("%s should default to synchronous compatible response", item.path)
}
if !plan.compatibleMode {
t.Fatalf("%s should return compatible response payloads", item.path)
}
if plan.streamMode {
t.Fatal("audio OpenAPI endpoints should stay JSON-only even when stream=true is present")
}
asyncReq := httptest.NewRequest(http.MethodPost, item.path, nil)
asyncReq.Header.Set("X-Async", "true")
asyncPlan := planTaskResponse(item.kind, true, map[string]any{}, asyncReq)
if !asyncPlan.asyncMode || !asyncPlan.compatibleMode {
t.Fatalf("%s should support X-Async while keeping compatible mode, got %+v", item.path, asyncPlan)
}
})
}
}
func TestAPIKeyScopeAllowedRecognizesAudioAndMusicAliases(t *testing.T) {
if !apiKeyScopeAllowed(&auth.User{APIKeyID: "key", APIKeyScopes: []string{"audio_generate"}}, "song.generations") {
t.Fatal("audio_generate scope should allow song generations")
}
if !apiKeyScopeAllowed(&auth.User{APIKeyID: "key", APIKeyScopes: []string{"text_to_speech"}}, "speech.generations") {
t.Fatal("text_to_speech scope should allow speech generations")
}
if apiKeyScopeAllowed(&auth.User{APIKeyID: "key", APIKeyScopes: []string{"image"}}, "speech.generations") {
t.Fatal("image scope should not allow speech generations")
}
}
func TestWriteCompatibleTaskResponseReturnsJSONWhenStreamIsFalse(t *testing.T) {
executor := &fakeTaskExecutor{output: map[string]any{"id": "chatcmpl-test", "object": "chat.completion"}}
req := httptest.NewRequest(http.MethodPost, "/api/v1/chat/completions", nil)
apps/api/internal/httpapi/core_flow_integration_test.go
+77 -7
View File
@@ -106,7 +106,7 @@ func TestCoreLocalFlow(t *testing.T) {
}
doJSON(t, server.URL, http.MethodPost, "/api/v1/api-keys", loginResponse.AccessToken, map[string]any{
"name": "smoke key",
"scopes": []string{"chat", "image", "video"},
"scopes": []string{"chat", "image", "video", "music", "audio"},
}, http.StatusCreated, &apiKeyResponse)
if !strings.HasPrefix(apiKeyResponse.Secret, "sk-gw-") || apiKeyResponse.APIKey.Status != "active" {
t.Fatalf("unexpected api key response: %+v", apiKeyResponse)
@@ -444,6 +444,71 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
t.Fatalf("unexpected image edit task: %+v", imageEditResponse.Task)
}
songMarker := "song-simulation-" + suffixText
var songResult map[string]any
doJSON(t, server.URL, http.MethodPost, "/api/v1/song/generations", apiKeyResponse.Secret, map[string]any{
"model": "chirp-v5-0",
"runMode": "simulation",
"prompt": "city lights and soft drums",
"tags": "pop, synth",
"negativeTags": "noise",
"simulation": true,
"simulationDurationMs": 5,
"integrationTestMarker": songMarker,
}, http.StatusOK, &songResult)
songData, _ := songResult["data"].([]any)
if songResult["status"] != "success" || len(songData) == 0 {
t.Fatalf("unexpected song generation compatible result: %+v", songResult)
}
songItem, _ := songData[0].(map[string]any)
if songItem["type"] != "audio" || songItem["audio_url"] != "/static/simulation/audio.wav" {
t.Fatalf("song simulation should return audio asset data: %+v", songItem)
}
var songTaskDetail struct {
Status string `json:"status"`
ModelType string `json:"modelType"`
Result map[string]any `json:"result"`
FinalChargeAmount float64 `json:"finalChargeAmount"`
}
songTaskID := waitForTaskIDByRequestField(t, ctx, testPool, "integrationTestMarker", songMarker, 2*time.Second)
doJSON(t, server.URL, http.MethodGet, "/api/v1/tasks/"+songTaskID, apiKeyResponse.Secret, nil, http.StatusOK, &songTaskDetail)
if songTaskDetail.Status != "succeeded" || songTaskDetail.ModelType != "audio_generate" || songTaskDetail.FinalChargeAmount <= 0 {
t.Fatalf("song simulation task should succeed with audio_generate billing: %+v", songTaskDetail)
}
speechMarker := "speech-simulation-" + suffixText
var speechResult map[string]any
doJSON(t, server.URL, http.MethodPost, "/api/v1/speech/generations", apiKeyResponse.Secret, map[string]any{
"model": "speech-2.6-turbo",
"runMode": "simulation",
"text": "hello gateway speech",
"voice_id": "female-shaonv",
"speed": 1,
"vol": 1,
"pitch": 0,
"simulation": true,
"simulationDurationMs": 5,
"integrationTestMarker": speechMarker,
}, http.StatusOK, &speechResult)
speechData, _ := speechResult["data"].([]any)
if speechResult["status"] != "success" || len(speechData) == 0 {
t.Fatalf("unexpected speech generation compatible result: %+v", speechResult)
}
speechItem, _ := speechData[0].(map[string]any)
if speechItem["type"] != "audio" || speechItem["audio_url"] != "/static/simulation/audio.wav" || speechItem["revised_text"] != "hello gateway speech" {
t.Fatalf("speech simulation should return audio asset data: %+v", speechItem)
}
var speechTaskDetail struct {
Status string `json:"status"`
ModelType string `json:"modelType"`
FinalChargeAmount float64 `json:"finalChargeAmount"`
}
speechTaskID := waitForTaskIDByRequestField(t, ctx, testPool, "integrationTestMarker", speechMarker, 2*time.Second)
doJSON(t, server.URL, http.MethodGet, "/api/v1/tasks/"+speechTaskID, apiKeyResponse.Secret, nil, http.StatusOK, &speechTaskDetail)
if speechTaskDetail.Status != "succeeded" || speechTaskDetail.ModelType != "text_to_speech" || speechTaskDetail.FinalChargeAmount <= 0 {
t.Fatalf("speech simulation task should succeed with text_to_speech billing: %+v", speechTaskDetail)
}
doubaoLiteImageEditModel := "doubao-5.0-lite图像编辑"
var doubaoLitePlatformModel struct {
ID string `json:"id"`
@@ -838,21 +903,26 @@ WHERE reference_type = 'gateway_task'
}
var modelRateLimits struct {
Items []struct {
ModelName string `json:"modelName"`
ModelAlias string `json:"modelAlias"`
ModelName string `json:"modelName"`
ModelAlias string `json:"modelAlias"`
Concurrent struct {
CurrentValue float64 `json:"currentValue"`
} `json:"concurrent"`
QueuedTasks float64 `json:"queuedTasks"`
} `json:"items"`
}
doJSON(t, server.URL, http.MethodGet, "/api/admin/runtime/model-rate-limits", loginResponse.AccessToken, nil, http.StatusOK, &modelRateLimits)
var queuedTasks float64
var runningTasks float64
for _, item := range modelRateLimits.Items {
if item.ModelName == rateLimitedModel || item.ModelAlias == rateLimitedModel {
queuedTasks = item.QueuedTasks
runningTasks = item.Concurrent.CurrentValue
break
}
}
if queuedTasks < 1 {
t.Fatalf("realtime load should count async rate-limited task as queued, got %v in %+v", queuedTasks, modelRateLimits.Items)
if queuedTasks+runningTasks < 1 && asyncRateLimitDetail.Status != "queued" {
t.Fatalf("realtime load should count async rate-limited task as queued or running, got queued=%v running=%v in %+v", queuedTasks, runningTasks, modelRateLimits.Items)
}
asyncRateLimitCompleted := waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, asyncRateLimitTask.TaskID, []string{"succeeded"}, time.Duration(rateLimitWindowSeconds+3)*time.Second)
if asyncRateLimitCompleted.Status != "succeeded" {
@@ -1227,7 +1297,7 @@ WHERE m.platform_id = $1::uuid
ErrorMessage string `json:"errorMessage"`
} `json:"items"`
}
doJSON(t, server.URL, http.MethodGet, "/api/v1/tasks?limit=20", loginResponse.AccessToken, nil, http.StatusOK, &taskList)
doJSON(t, server.URL, http.MethodGet, "/api/v1/tasks?limit=50", loginResponse.AccessToken, nil, http.StatusOK, &taskList)
if !taskListContains(taskList.Items, taskResponse.Task.ID) || !taskListContains(taskList.Items, pricingTask.Task.ID) {
t.Fatalf("task list should include persisted task records, got %+v", taskList.Items)
}
@@ -1242,7 +1312,7 @@ WHERE m.platform_id = $1::uuid
ErrorMessage string `json:"errorMessage"`
} `json:"items"`
}
doJSON(t, server.URL, http.MethodGet, "/api/workspace/tasks?limit=20", loginResponse.AccessToken, nil, http.StatusOK, &workspaceTaskList)
doJSON(t, server.URL, http.MethodGet, "/api/workspace/tasks?limit=50", loginResponse.AccessToken, nil, http.StatusOK, &workspaceTaskList)
if !taskListContains(workspaceTaskList.Items, taskResponse.Task.ID) || !taskListContains(workspaceTaskList.Items, pricingTask.Task.ID) {
t.Fatalf("workspace task list should include persisted task records, got %+v", workspaceTaskList.Items)
}
apps/api/internal/httpapi/handlers.go
+19
View File
@@ -881,6 +881,9 @@ func (s *Server) listModelRateLimitStatuses(w http.ResponseWriter, r *http.Reque
// @Router /api/v1/images/generations [post]
// @Router /api/v1/images/edits [post]
// @Router /api/v1/videos/generations [post]
// @Router /api/v1/song/generations [post]
// @Router /api/v1/music/generations [post]
// @Router /api/v1/speech/generations [post]
// @Router /chat/completions [post]
// @Router /v1/chat/completions [post]
// @Router /responses [post]
@@ -893,6 +896,12 @@ func (s *Server) listModelRateLimitStatuses(w http.ResponseWriter, r *http.Reque
// @Router /v1/images/generations [post]
// @Router /images/edits [post]
// @Router /v1/images/edits [post]
// @Router /song/generations [post]
// @Router /v1/song/generations [post]
// @Router /music/generations [post]
// @Router /v1/music/generations [post]
// @Router /speech/generations [post]
// @Router /v1/speech/generations [post]
func (s *Server) createTask(kind string, compatible bool) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, ok := auth.UserFromContext(r.Context())
@@ -1153,6 +1162,12 @@ func apiKeyScopeAllowed(user *auth.User, kind string) bool {
if required == "rerank" && scope == "text_rerank" {
return true
}
if required == "music" && (scope == "audio_generate" || scope == "music_generate" || scope == "song") {
return true
}
if required == "audio" && (scope == "text_to_speech" || scope == "speech" || scope == "tts") {
return true
}
}
return false
}
@@ -1169,6 +1184,10 @@ func scopeForTaskKind(kind string) string {
return "image"
case "videos.generations":
return "video"
case "song.generations", "music.generations":
return "music"
case "speech.generations":
return "audio"
default:
return kind
}
apps/api/internal/httpapi/openapi_models.go
+28 -11
View File
@@ -172,18 +172,35 @@ type PricingEstimateResponse struct {
}
type TaskRequest struct {
Model string `json:"model" example:"gpt-4o-mini"`
Messages []ChatMessage `json:"messages,omitempty"`
Input string `json:"input,omitempty" example:"Tell me a short story"`
Prompt string `json:"prompt,omitempty" example:"A watercolor robot reading a book"`
Stream bool `json:"stream,omitempty" example:"false"`
RunMode string `json:"runMode,omitempty" example:"simulation"`
MaxTokens int `json:"max_tokens,omitempty" example:"512"`
Model string `json:"model" example:"gpt-4o-mini"`
Messages []ChatMessage `json:"messages,omitempty"`
Input string `json:"input,omitempty" example:"Tell me a short story"`
Prompt string `json:"prompt,omitempty" example:"A watercolor robot reading a book"`
Text string `json:"text,omitempty" example:"Hello from EasyAI audio synthesis."`
TextFileID string `json:"text_file_id,omitempty" example:""`
VoiceID string `json:"voice_id,omitempty" example:"female-shaonv"`
Stream bool `json:"stream,omitempty" example:"false"`
RunMode string `json:"runMode,omitempty" example:"simulation"`
MaxTokens int `json:"max_tokens,omitempty" example:"512"`
// ReasoningEffort 推理深度，OpenAI-compatible 请求字段；开放字符串，取值随 provider 和模型能力而定，常见值为 none、minimal、low、medium、high、xhigh，也可配置 max 等供应商自定义值。
ReasoningEffort string `json:"reasoning_effort,omitempty" example:"medium"`
Size string `json:"size,omitempty" example:"1024x1024"`
Duration int `json:"duration,omitempty" example:"5"`
Resolution string `json:"resolution,omitempty" example:"720p"`
ReasoningEffort string `json:"reasoning_effort,omitempty" example:"medium"`
Size string `json:"size,omitempty" example:"1024x1024"`
Duration int `json:"duration,omitempty" example:"5"`
Resolution string `json:"resolution,omitempty" example:"720p"`
MakeInstrumental bool `json:"makeInstrumental,omitempty" example:"false"`
CustomMode bool `json:"customMode,omitempty" example:"false"`
Style string `json:"style,omitempty" example:"city pop, bright synth"`
Title string `json:"title,omitempty" example:"Useful Tools"`
Tags string `json:"tags,omitempty" example:"city pop, synth"`
NegativeTags string `json:"negativeTags,omitempty" example:"noise"`
VocalGender string `json:"vocalGender,omitempty" example:"f"`
StyleWeight float64 `json:"styleWeight,omitempty" example:"0.65"`
WeirdnessConstraint float64 `json:"weirdnessConstraint,omitempty" example:"0.35"`
AudioWeight float64 `json:"audioWeight,omitempty" example:"0.65"`
Speed float64 `json:"speed,omitempty" example:"1"`
Vol float64 `json:"vol,omitempty" example:"1"`
Pitch float64 `json:"pitch,omitempty" example:"0"`
Emotion string `json:"emotion,omitempty" example:"happy"`
}
type ChatCompletionRequest struct {
apps/api/internal/httpapi/server.go
+9
View File
@@ -135,6 +135,9 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("POST /api/v1/images/generations", server.auth.Require(auth.PermissionBasic, server.createTask("images.generations", false)))
mux.Handle("POST /api/v1/images/edits", server.auth.Require(auth.PermissionBasic, server.createTask("images.edits", false)))
mux.Handle("POST /api/v1/videos/generations", server.auth.Require(auth.PermissionBasic, server.createTask("videos.generations", false)))
mux.Handle("POST /api/v1/song/generations", server.auth.Require(auth.PermissionBasic, server.createTask("song.generations", true)))
mux.Handle("POST /api/v1/music/generations", server.auth.Require(auth.PermissionBasic, server.createTask("music.generations", true)))
mux.Handle("POST /api/v1/speech/generations", server.auth.Require(auth.PermissionBasic, server.createTask("speech.generations", true)))
mux.Handle("POST /api/v1/files/upload", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.uploadFile)))
mux.Handle("GET /api/v1/tasks", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.listTasks)))
mux.Handle("GET /api/v1/tasks/{taskID}", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.getTask)))
@@ -152,6 +155,12 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("POST /v1/images/generations", server.auth.Require(auth.PermissionBasic, server.createTask("images.generations", true)))
mux.Handle("POST /images/edits", server.auth.Require(auth.PermissionBasic, server.createTask("images.edits", true)))
mux.Handle("POST /v1/images/edits", server.auth.Require(auth.PermissionBasic, server.createTask("images.edits", true)))
mux.Handle("POST /song/generations", server.auth.Require(auth.PermissionBasic, server.createTask("song.generations", true)))
mux.Handle("POST /v1/song/generations", server.auth.Require(auth.PermissionBasic, server.createTask("song.generations", true)))
mux.Handle("POST /music/generations", server.auth.Require(auth.PermissionBasic, server.createTask("music.generations", true)))
mux.Handle("POST /v1/music/generations", server.auth.Require(auth.PermissionBasic, server.createTask("music.generations", true)))
mux.Handle("POST /speech/generations", server.auth.Require(auth.PermissionBasic, server.createTask("speech.generations", true)))
mux.Handle("POST /v1/speech/generations", server.auth.Require(auth.PermissionBasic, server.createTask("speech.generations", true)))
mux.Handle("POST /v1/files/upload", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.uploadFile)))
return server.recover(server.cors(mux))
apps/api/internal/httpapi/simulation_assets.go
+9 -2
View File
@@ -18,13 +18,18 @@ const simulationVideoMP4Base64 = "AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAA
var simulationVideoMP4 = mustDecodeSimulationAsset(simulationVideoMP4Base64)
const simulationAudioWAVBase64 = "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
var simulationAudioWAV = mustDecodeSimulationAsset(simulationAudioWAVBase64)
// serveSimulationAsset godoc
// @Summary 获取模拟资源
// @Description 返回本地模拟模式使用的图片、视频封面或短视频资源。
// @Description 返回本地模拟模式使用的图片、视频封面、短视频或音频资源。
// @Tags simulation
// @Produce image/svg+xml
// @Produce video/mp4
// @Param asset path string true "资源文件名，可选 image.svg、image.png、image-edit.svg、image-edit.png、video-poster.svg、video.mp4"
// @Produce audio/wav
// @Param asset path string true "资源文件名，可选 image.svg、image.png、image-edit.svg、image-edit.png、video-poster.svg、video.mp4、audio.wav"
// @Success 200 {file} binary
// @Failure 404 {string} string "Not Found"
// @Router /static/simulation/{asset} [get]
@@ -39,6 +44,8 @@ func serveSimulationAsset(w http.ResponseWriter, r *http.Request) {
serveSimulationContent(w, r, "video-poster.svg", "image/svg+xml; charset=utf-8", []byte(simulationVideoPosterSVG))
case "video.mp4":
serveSimulationContent(w, r, "video.mp4", "video/mp4", simulationVideoMP4)
case "audio.wav":
serveSimulationContent(w, r, "audio.wav", "audio/wav", simulationAudioWAV)
default:
http.NotFound(w, r)
}