迁移音频生成与语音合成到 gateway 并补充 simulation 测试

This commit is contained in:
2026-06-07 10:26:57 +08:00
parent 78ab867a9f
commit dc14866210
22 changed files with 2475 additions and 55 deletions
+102
View File
@@ -2,6 +2,7 @@ package clients
import (
"context"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -65,6 +66,35 @@ func TestSimulationClientReturnsVideoDemoAssets(t *testing.T) {
}
}
func TestSimulationClientReturnsAudioDemoAssets(t *testing.T) {
response, err := (SimulationClient{}).Run(context.Background(), Request{
Kind: "speech.generations",
ModelType: "text_to_speech",
Model: "speech-2.6-turbo",
Body: map[string]any{
"text": "hello from simulation",
"voice_id": "female-shaonv",
"count": 2,
"simulationDurationMs": 5,
},
Candidate: store.RuntimeModelCandidate{Provider: "simulation"},
})
if err != nil {
t.Fatalf("run simulation audio client: %v", err)
}
data, _ := response.Result["data"].([]any)
if len(data) != 2 || response.Result["status"] != "success" {
t.Fatalf("unexpected simulated audio response: %+v", response.Result)
}
item, _ := data[0].(map[string]any)
if item["type"] != "audio" || item["url"] != "/static/simulation/audio.wav" || item["audio_url"] != "/static/simulation/audio.wav" {
t.Fatalf("unexpected simulated audio item: %+v", item)
}
if item["revised_text"] != "hello from simulation" || item["assetSource"] != "simulation" {
t.Fatalf("unexpected simulated audio metadata: %+v", item)
}
}
func TestSimulationDurationDefaultsByMediaType(t *testing.T) {
imageDuration := simulationDuration(Request{Kind: "images.generations"})
if imageDuration < 10*time.Second || imageDuration > 30*time.Second {
@@ -74,12 +104,84 @@ func TestSimulationDurationDefaultsByMediaType(t *testing.T) {
if videoDuration < 2*time.Minute || videoDuration > 3*time.Minute {
t.Fatalf("video simulation duration should default to 2-3m, got %s", videoDuration)
}
audioDuration := simulationDuration(Request{Kind: "speech.generations"})
if audioDuration < 2*time.Second || audioDuration > 6*time.Second {
t.Fatalf("audio simulation duration should default to 2-6s, got %s", audioDuration)
}
textDuration := simulationDuration(Request{Kind: "chat.completions"})
if textDuration < 800*time.Millisecond || textDuration > 2400*time.Millisecond {
t.Fatalf("text simulation duration should keep short defaults, got %s", textDuration)
}
}
func TestMinimaxClientSpeechUsesT2AV2AndNormalizesAudio(t *testing.T) {
var captured map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/t2a_v2" {
t.Fatalf("unexpected request: %s %s", r.Method, r.URL.String())
}
if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
t.Fatalf("unexpected auth header: %q", got)
}
if err := json.NewDecoder(r.Body).Decode(&captured); err != nil {
t.Fatalf("decode request: %v", err)
}
w.Header().Set("x-request-id", "req-minimax-speech")
_ = json.NewEncoder(w).Encode(map[string]any{
"data": map[string]any{"audio": "68656c6c6f"},
"base_resp": map[string]any{"status_code": 0},
})
}))
defer server.Close()
response, err := (MinimaxClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
Kind: "speech.generations",
Model: "MiniMax Speech 2.6 Turbo",
Body: map[string]any{
"text": "hello",
"voice_id": "female-shaonv",
"speed": 1.2,
"vol": 0.8,
"pitch": -1,
"emotion": "happy",
},
Candidate: store.RuntimeModelCandidate{
Provider: "minimax",
BaseURL: server.URL,
ProviderModelName: "speech-2.6-turbo",
Credentials: map[string]any{"apiKey": "test-key"},
},
})
if err != nil {
t.Fatalf("run minimax speech client: %v", err)
}
if captured["model"] != "speech-2.6-turbo" || captured["text"] != "hello" {
t.Fatalf("unexpected minimax speech payload: %+v", captured)
}
if _, ok := captured["voice_id"]; ok {
t.Fatalf("voice_id should be moved into voice_setting: %+v", captured)
}
voiceSetting, ok := captured["voice_setting"].(map[string]any)
if !ok {
t.Fatalf("missing voice_setting: %+v", captured)
}
if voiceSetting["voice_id"] != "female-shaonv" || voiceSetting["speed"] != 1.2 || voiceSetting["vol"] != 0.8 || voiceSetting["pitch"] != float64(-1) || voiceSetting["emotion"] != "happy" {
t.Fatalf("unexpected voice_setting: %+v", voiceSetting)
}
data, _ := response.Result["data"].([]any)
if len(data) != 1 {
t.Fatalf("unexpected minimax speech response: %+v", response.Result)
}
item, _ := data[0].(map[string]any)
expectedContent := "data:audio/mpeg;base64," + base64.StdEncoding.EncodeToString([]byte("hello"))
if item["type"] != "audio" || item["content"] != expectedContent || item["mime_type"] != "audio/mpeg" {
t.Fatalf("unexpected normalized audio item: %+v", item)
}
if response.RequestID != "req-minimax-speech" {
t.Fatalf("unexpected request id: %q", response.RequestID)
}
}
func TestSimulationDurationCanBeControlledByParams(t *testing.T) {
fixedDuration := simulationDuration(Request{Body: map[string]any{"simulationDurationSeconds": 7}})
if fixedDuration != 7*time.Second {
+119
View File
@@ -2,8 +2,11 @@ package clients
import (
"context"
"encoding/base64"
"encoding/hex"
"net/http"
"strings"
"time"
)
type JimengClient struct{ HTTPClient *http.Client }
@@ -15,6 +18,7 @@ type MidjourneyClient struct{ HTTPClient *http.Client }
type ViduClient struct{ HTTPClient *http.Client }
type AliyunBailianClient struct{ HTTPClient *http.Client }
type NewAPIClient struct{ HTTPClient *http.Client }
type SunoClient struct{ HTTPClient *http.Client }
func (c JimengClient) Run(ctx context.Context, request Request) (Response, error) {
return providerTaskClient{HTTPClient: c.HTTPClient, Spec: jimengSpec()}.Run(ctx, request)
@@ -33,6 +37,9 @@ func (c HunyuanVideoClient) Run(ctx context.Context, request Request) (Response,
}
func (c MinimaxClient) Run(ctx context.Context, request Request) (Response, error) {
if request.Kind == "speech.generations" {
return c.runSpeech(ctx, request)
}
return providerTaskClient{HTTPClient: c.HTTPClient, Spec: minimaxSpec()}.Run(ctx, request)
}
@@ -52,6 +59,10 @@ func (c NewAPIClient) Run(ctx context.Context, request Request) (Response, error
return providerTaskClient{HTTPClient: c.HTTPClient, Spec: newAPISpec()}.Run(ctx, request)
}
func (c SunoClient) Run(ctx context.Context, request Request) (Response, error) {
return providerTaskClient{HTTPClient: c.HTTPClient, Spec: sunoSpec()}.Run(ctx, request)
}
func jimengSpec() providerTaskSpec {
return providerTaskSpec{
Name: "jimeng",
@@ -149,6 +160,114 @@ func minimaxSpec() providerTaskSpec {
}
}
func (c MinimaxClient) runSpeech(ctx context.Context, request Request) (Response, error) {
startedAt := time.Now()
payload := minimaxSpeechPayload(request)
result, requestID, err := providerPostJSON(ctx, httpClient(request.HTTPClient, c.HTTPClient), providerURL(request.Candidate.BaseURL, "/t2a_v2"), payload, request.Candidate.Credentials, "bearer")
finishedAt := time.Now()
if err != nil {
return Response{}, annotateResponseError(err, requestID, startedAt, finishedAt)
}
audioHex := strings.TrimSpace(stringFromPathValue(valueAtPath(result, "data.audio")))
if audioHex == "" {
message := firstNonEmptyString(valueAtPath(result, "base_resp.status_msg"), valueAtPath(result, "message"), "minimax speech audio is missing")
return Response{}, &ClientError{Code: "invalid_response", Message: message, RequestID: firstNonEmptyString(requestID, requestIDFromResult(result)), ResponseStartedAt: startedAt, ResponseFinishedAt: finishedAt, ResponseDurationMS: responseDurationMS(startedAt, finishedAt), Retryable: false}
}
audioBytes, err := hex.DecodeString(audioHex)
if err != nil {
return Response{}, &ClientError{Code: "invalid_response", Message: "minimax speech audio hex is invalid: " + err.Error(), RequestID: firstNonEmptyString(requestID, requestIDFromResult(result)), ResponseStartedAt: startedAt, ResponseFinishedAt: finishedAt, ResponseDurationMS: responseDurationMS(startedAt, finishedAt), Retryable: false}
}
normalized := cloneMapAny(result)
normalized["status"] = "success"
normalized["created"] = time.Now().UnixMilli()
normalized["model"] = request.Model
normalized["raw_data"] = cloneMapAny(result)
normalized["data"] = []any{map[string]any{
"type": "audio",
"content": "data:audio/mpeg;base64," + base64.StdEncoding.EncodeToString(audioBytes),
"mime_type": "audio/mpeg",
"uploaded": false,
}}
return Response{
Result: normalized,
RequestID: firstNonEmptyString(requestID, requestIDFromResult(result)),
Progress: providerProgress(request),
ResponseStartedAt: startedAt,
ResponseFinishedAt: finishedAt,
ResponseDurationMS: responseDurationMS(startedAt, finishedAt),
}, nil
}
func minimaxSpeechPayload(request Request) map[string]any {
body := cloneBody(request.Body)
body["model"] = upstreamModelName(request.Candidate)
voiceID := firstNonEmptyString(body["voice_id"], body["voiceId"])
speed := firstPresent(body["speed"], float64(1))
vol := firstPresent(body["vol"], body["volume"], float64(1))
pitch := firstPresent(body["pitch"], float64(0))
voiceSetting := map[string]any{
"voice_id": voiceID,
"speed": speed,
"vol": vol,
"pitch": pitch,
}
if emotion := firstNonEmptyString(body["emotion"]); emotion != "" {
voiceSetting["emotion"] = emotion
}
delete(body, "voice_id")
delete(body, "voiceId")
delete(body, "speed")
delete(body, "vol")
delete(body, "volume")
delete(body, "pitch")
delete(body, "emotion")
body["voice_setting"] = voiceSetting
return body
}
func sunoSpec() providerTaskSpec {
return providerTaskSpec{
Name: "suno",
SubmitPath: func(Request, map[string]any) string { return "/generator/suno" },
PollPath: func(_ Request, upstreamTaskID string, _ map[string]any) string {
return "/v2/sunoinfo?id=" + upstreamTaskID
},
Auth: "bearer",
TaskIDPaths: []string{"data"},
StatusPaths: []string{"data.status"},
SuccessStatuses: []string{"succeeded", "complete", "completed"},
FailureStatuses: []string{"failed"},
DefaultSubmitBody: func(request Request, body map[string]any) map[string]any {
body["task"] = "create"
body["model"] = sunoMappedModel(upstreamModelName(request.Candidate))
if body["customMode"] == nil {
body["customMode"] = false
}
if body["makeInstrumental"] == nil {
body["makeInstrumental"] = false
}
return body
},
}
}
func sunoMappedModel(model string) string {
switch strings.TrimSpace(model) {
case "chirp-v3-0", "chirp-v3-5":
return "v40"
case "chirp-v4-0":
return "v40"
case "chirp-v4-5":
return "v45"
case "chirp-v4-5+":
return "v45+"
case "chirp-v5-0":
return "v50"
default:
return model
}
}
func midjourneySpec() providerTaskSpec {
return providerTaskSpec{
Name: "midjourney",
+22 -3
View File
@@ -29,7 +29,7 @@ type providerTaskClient struct {
}
func (c providerTaskClient) Run(ctx context.Context, request Request) (Response, error) {
if request.Kind != "images.generations" && request.Kind != "images.edits" && request.Kind != "videos.generations" {
if !providerTaskKindSupported(request.Kind) {
return Response{}, &ClientError{Code: "unsupported_kind", Message: "unsupported " + c.Spec.Name + " request kind", Retryable: false}
}
startedAt := time.Now()
@@ -119,6 +119,15 @@ func (c providerTaskClient) Run(ctx context.Context, request Request) (Response,
}
}
func providerTaskKindSupported(kind string) bool {
switch kind {
case "images.generations", "images.edits", "videos.generations", "song.generations", "music.generations", "speech.generations":
return true
default:
return false
}
}
func (c providerTaskClient) submit(ctx context.Context, request Request, payload map[string]any) (map[string]any, string, error) {
path := c.Spec.SubmitPath(request, payload)
return providerPostJSON(ctx, httpClient(request.HTTPClient, c.HTTPClient), providerURL(request.Candidate.BaseURL, path), payload, request.Candidate.Credentials, c.Spec.Auth)
@@ -287,7 +296,7 @@ func containsStatus(values []string, status string) bool {
}
func hasProviderTaskResult(result map[string]any) bool {
return result["data"] != nil || valueAtPath(result, "output.image_urls") != nil || valueAtPath(result, "output.video_url") != nil || valueAtPath(result, "Response.ResultVideoUrl") != nil || valueAtPath(result, "Response.ResultImages") != nil || result["urls"] != nil
return result["data"] != nil || valueAtPath(result, "data.result") != nil || valueAtPath(result, "data.audio") != nil || valueAtPath(result, "output.image_urls") != nil || valueAtPath(result, "output.video_url") != nil || valueAtPath(result, "Response.ResultVideoUrl") != nil || valueAtPath(result, "Response.ResultImages") != nil || result["audio_url"] != nil || result["urls"] != nil
}
func normalizeProviderTaskResult(request Request, spec providerTaskSpec, result map[string]any, upstreamTaskID string) map[string]any {
@@ -316,9 +325,19 @@ func providerTaskData(request Request, result map[string]any) []any {
if request.Kind == "videos.generations" || strings.Contains(request.ModelType, "video") {
fileType = "video"
}
if request.Kind == "song.generations" || request.Kind == "music.generations" || request.Kind == "speech.generations" || strings.Contains(request.ModelType, "audio") || strings.Contains(request.ModelType, "speech") {
fileType = "audio"
}
urlValues := []any{}
for _, path := range []string{
"urls",
"audio_url",
"audioUrl",
"data.audio_url",
"data.audioUrl",
"data.result",
"data.result.audio_url",
"data.result.audioUrl",
"image_urls",
"data.image_urls",
"data.images",
@@ -368,7 +387,7 @@ func appendURLValues(out *[]any, value any) {
*out = append(*out, item)
}
case map[string]any:
for _, key := range []string{"url", "image_url", "imageUrl", "video_url", "videoUrl", "content", "output"} {
for _, key := range []string{"url", "audio_url", "audioUrl", "image_url", "imageUrl", "video_url", "videoUrl", "content", "output"} {
if item := strings.TrimSpace(fmt.Sprint(typed[key])); item != "" && item != "<nil>" {
*out = append(*out, item)
return
@@ -2,6 +2,7 @@ package clients
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
@@ -227,6 +228,80 @@ func TestProviderTaskClientsSubmitAndPoll(t *testing.T) {
}
}
func TestSunoClientSubmitsAndPollsAudioGeneration(t *testing.T) {
var submitted map[string]any
var submittedRemoteTaskID string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
t.Fatalf("unexpected auth header: %q", got)
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("x-request-id", "req-suno")
switch {
case r.Method == http.MethodPost && r.URL.Path == "/generator/suno":
if err := json.NewDecoder(r.Body).Decode(&submitted); err != nil {
t.Fatalf("decode suno submit request: %v", err)
}
_, _ = w.Write([]byte(`{"code":200,"data":"suno-task"}`))
case r.Method == http.MethodGet && r.URL.Path == "/v2/sunoinfo" && r.URL.Query().Get("id") == "suno-task":
_, _ = w.Write([]byte(`{"code":200,"data":{"status":"succeeded","result":[{"audio_url":"https://cdn.example/song.mp3"}]}}`))
default:
t.Fatalf("unexpected request: %s %s", r.Method, r.URL.String())
}
}))
defer server.Close()
response, err := (SunoClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
Kind: "song.generations",
ModelType: "audio_generate",
Model: "Suno V5",
Body: map[string]any{
"prompt": "city lights",
"tags": "pop",
"negativeTags": "noise",
},
Candidate: store.RuntimeModelCandidate{
Provider: "suno",
SpecType: "suno",
BaseURL: server.URL,
Credentials: map[string]any{"apiKey": "test-key"},
PlatformConfig: map[string]any{"pollIntervalMs": 1, "pollTimeoutMs": 1000},
ProviderModelName: "chirp-v5-0",
ModelType: "audio_generate",
},
OnRemoteTaskSubmitted: func(remoteTaskID string, payload map[string]any) error {
submittedRemoteTaskID = remoteTaskID
if payload["payload"] == nil || payload["submit"] == nil {
t.Fatalf("missing remote payload: %#v", payload)
}
return nil
},
})
if err != nil {
t.Fatalf("run suno client: %v", err)
}
if submittedRemoteTaskID != "suno-task" {
t.Fatalf("unexpected remote task id: %q", submittedRemoteTaskID)
}
if submitted["task"] != "create" || submitted["model"] != "v50" || submitted["prompt"] != "city lights" {
t.Fatalf("unexpected suno submit payload: %+v", submitted)
}
if submitted["customMode"] != false || submitted["makeInstrumental"] != false {
t.Fatalf("suno defaults should match main-server style payload: %+v", submitted)
}
data, _ := response.Result["data"].([]any)
if len(data) != 1 {
t.Fatalf("unexpected suno response: %+v", response.Result)
}
first, _ := data[0].(map[string]any)
if first["type"] != "audio" || first["url"] != "https://cdn.example/song.mp3" {
t.Fatalf("unexpected suno normalized audio item: %+v", first)
}
if response.RequestID != "req-suno" {
t.Fatalf("unexpected request id: %q", response.RequestID)
}
}
func TestProviderTaskClientFailureAndRetryableErrors(t *testing.T) {
t.Run("poll failure", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+66
View File
@@ -15,6 +15,8 @@ const (
defaultSimulationTextMaxDuration = 2400 * time.Millisecond
defaultSimulationImageMinDuration = 10 * time.Second
defaultSimulationImageMaxDuration = 30 * time.Second
defaultSimulationAudioMinDuration = 2 * time.Second
defaultSimulationAudioMaxDuration = 6 * time.Second
defaultSimulationVideoMinDuration = 2 * time.Minute
defaultSimulationVideoMaxDuration = 3 * time.Minute
maxSimulationDuration = 10 * time.Minute
@@ -156,6 +158,24 @@ func simulatedResult(request Request) map[string]any {
"model": request.Model,
"data": simulatedVideoData(request),
}
case "song.generations", "music.generations":
return map[string]any{
"id": "song-simulated",
"created": nowUnix(),
"model": request.Model,
"status": "success",
"data": simulatedAudioData(request, "simulation music"),
"message": "simulation music generated",
}
case "speech.generations":
return map[string]any{
"id": "speech-simulated",
"created": nowUnix(),
"model": request.Model,
"status": "success",
"data": simulatedAudioData(request, "simulation speech"),
"message": "simulation speech generated",
}
default:
modelType := strings.ToLower(request.ModelType)
kind := strings.ToLower(request.Kind)
@@ -167,6 +187,15 @@ func simulatedResult(request Request) map[string]any {
"data": simulatedVideoData(request),
}
}
if strings.Contains(modelType, "audio") || strings.Contains(modelType, "speech") || strings.Contains(kind, "audio") || strings.Contains(kind, "song") || strings.Contains(kind, "music") || strings.Contains(kind, "speech") {
return map[string]any{
"id": "audio-simulated",
"created": nowUnix(),
"model": request.Model,
"status": "success",
"data": simulatedAudioData(request, "simulation audio"),
}
}
return map[string]any{
"id": "img-simulated",
"created": nowUnix(),
@@ -307,6 +336,24 @@ func simulatedVideoData(request Request) []any {
return items
}
func simulatedAudioData(request Request, fallbackPrompt string) []any {
count := simulatedOutputCount(request.Body)
items := make([]any, 0, count)
for index := 0; index < count; index += 1 {
items = append(items, map[string]any{
"type": "audio",
"url": "/static/simulation/audio.wav",
"audio_url": "/static/simulation/audio.wav",
"duration": simulatedAudioDurationSeconds(request),
"assetSource": "simulation",
"index": index,
"prompt": firstNonEmptyPrompt(request.Body, fallbackPrompt),
"revised_text": firstNonEmptyString(stringValue(request.Body, "text"), firstNonEmptyPrompt(request.Body, fallbackPrompt)),
})
}
return items
}
func simulatedUsage(request Request) Usage {
if request.ModelType == "chat" || request.ModelType == "text_generate" || request.Kind == "responses" {
return Usage{InputTokens: 12, OutputTokens: 8, TotalTokens: 20}
@@ -368,6 +415,9 @@ func defaultSimulationDurationRange(request Request) (time.Duration, time.Durati
if simulationImageRequest(request) {
return defaultSimulationImageMinDuration, defaultSimulationImageMaxDuration
}
if simulationAudioRequest(request) {
return defaultSimulationAudioMinDuration, defaultSimulationAudioMaxDuration
}
return defaultSimulationTextMinDuration, defaultSimulationTextMaxDuration
}
@@ -383,6 +433,12 @@ func simulationImageRequest(request Request) bool {
return strings.Contains(kind, "image") || strings.Contains(modelType, "image")
}
func simulationAudioRequest(request Request) bool {
kind := strings.ToLower(request.Kind)
modelType := strings.ToLower(request.ModelType)
return strings.Contains(kind, "audio") || strings.Contains(kind, "song") || strings.Contains(kind, "music") || strings.Contains(kind, "speech") || strings.Contains(modelType, "audio") || strings.Contains(modelType, "speech")
}
func simulationDurationSeconds(request Request, keys ...string) int {
for _, source := range []map[string]any{request.Body, request.Candidate.PlatformConfig, request.Candidate.Credentials} {
for _, key := range keys {
@@ -440,6 +496,16 @@ func simulatedVideoDurationSeconds(request Request) int {
return 5
}
func simulatedAudioDurationSeconds(request Request) int {
if duration := intValue(request.Body, "duration", 0); duration > 0 {
return duration
}
if seconds := len([]rune(stringValue(request.Body, "text"))) / 8; seconds > 0 {
return seconds
}
return 3
}
func firstNonEmptyPrompt(body map[string]any, fallback string) string {
for _, key := range []string{"prompt", "input"} {
if value := strings.TrimSpace(stringValue(body, key)); value != "" {