diff --git a/apps/api/internal/clients/clients_test.go b/apps/api/internal/clients/clients_test.go index 6112dee..1b201f3 100644 --- a/apps/api/internal/clients/clients_test.go +++ b/apps/api/internal/clients/clients_test.go @@ -344,6 +344,79 @@ func TestOpenAIClientChatContract(t *testing.T) { } } +func TestOpenAIClientAliyunChatSendsStandardInputAudioAndNormalizesLegacyAudioURL(t *testing.T) { + var captured map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&captured); err != nil { + t.Fatalf("decode request: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "chatcmpl-audio", + "object": "chat.completion", + "model": captured["model"], + "choices": []any{map[string]any{ + "message": map[string]any{"role": "assistant", "content": "ok"}, + }}, + }) + })) + defer server.Close() + + _, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{ + Kind: "chat.completions", + Model: "qwen-omni", + Body: map[string]any{ + "model": "qwen-omni", + "messages": []any{ + map[string]any{ + "role": "user", + "content": []any{ + map[string]any{ + "type": "input_audio", + "input_audio": map[string]any{ + "data": "https://cdn.example.com/audio/standard.wav", + "format": "wav", + }, + }, + map[string]any{ + "type": "audio_url", + "audio_url": map[string]any{"url": "https://cdn.example.com/audio/legacy.m4a"}, + }, + }, + }, + }, + }, + Candidate: store.RuntimeModelCandidate{ + Provider: "aliyun-bailian-openai", + BaseURL: server.URL, + ProviderModelName: "qwen-omni", + Credentials: map[string]any{"apiKey": "test-key"}, + }, + }) + if err != nil { + t.Fatalf("run aliyun openai-compatible client: %v", err) + } + + messages, _ := captured["messages"].([]any) + message, _ := messages[0].(map[string]any) + content, _ := message["content"].([]any) + standard, _ := content[0].(map[string]any) + if standard["type"] != "input_audio" { + t.Fatalf("standard input_audio was not preserved: %+v", standard) + } + standardAudio, _ := standard["input_audio"].(map[string]any) + if standardAudio["data"] != "https://cdn.example.com/audio/standard.wav" || standardAudio["format"] != "wav" { + t.Fatalf("unexpected standard input_audio: %+v", standardAudio) + } + legacy, _ := content[1].(map[string]any) + if legacy["type"] != "input_audio" { + t.Fatalf("legacy audio_url was not normalized: %+v", legacy) + } + legacyAudio, _ := legacy["input_audio"].(map[string]any) + if legacyAudio["data"] != "https://cdn.example.com/audio/legacy.m4a" || legacyAudio["format"] != "m4a" { + t.Fatalf("unexpected normalized legacy audio: %+v", legacyAudio) + } +} + func TestOpenAIClientImageEditUsesMultipartWithMultipleImages(t *testing.T) { var contentType string var receivedFields map[string][]string diff --git a/apps/api/internal/clients/openai.go b/apps/api/internal/clients/openai.go index 1be5776..2f96131 100644 --- a/apps/api/internal/clients/openai.go +++ b/apps/api/internal/clients/openai.go @@ -48,6 +48,11 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error } if endpointKind == "chat.completions" { body = NormalizeChatCompletionRequestBody(body) + normalizedBody, normalizeErr := normalizeOpenAIChatAudioParts(body, request.Candidate) + if normalizeErr != nil { + return Response{}, normalizeErr + } + body = normalizedBody applyOpenAIChatReasoningParams(body, request.Candidate) body = FilterOpenAIChatRequestBody(body) } else if request.Kind == "responses" { diff --git a/apps/api/internal/clients/openai_chat_audio.go b/apps/api/internal/clients/openai_chat_audio.go new file mode 100644 index 0000000..c0e9d8b --- /dev/null +++ b/apps/api/internal/clients/openai_chat_audio.go @@ -0,0 +1,169 @@ +package clients + +import ( + "net/http" + "net/url" + "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +var openAIChatAudioFormatByMIME = map[string]string{ + "audio/aac": "aac", + "audio/flac": "flac", + "audio/m4a": "m4a", + "audio/mp4": "m4a", + "audio/mpeg": "mp3", + "audio/ogg": "ogg", + "audio/opus": "opus", + "audio/wav": "wav", + "audio/webm": "webm", + "audio/x-m4a": "m4a", + "audio/x-wav": "wav", +} + +var openAIChatAudioFormatByExtension = map[string]string{ + "aac": "aac", + "flac": "flac", + "m4a": "m4a", + "mp3": "mp3", + "mp4": "m4a", + "oga": "ogg", + "ogg": "ogg", + "opus": "opus", + "wav": "wav", + "webm": "webm", +} + +func normalizeOpenAIChatAudioParts(body map[string]any, candidate store.RuntimeModelCandidate) (map[string]any, error) { + if !isAliyunBailianOpenAI(candidate) { + return body, nil + } + messages, ok := body["messages"].([]any) + if !ok { + return body, nil + } + + normalizedMessages := make([]any, 0, len(messages)) + for _, rawMessage := range messages { + message, ok := rawMessage.(map[string]any) + if !ok { + normalizedMessages = append(normalizedMessages, rawMessage) + continue + } + content, ok := message["content"].([]any) + if !ok { + normalizedMessages = append(normalizedMessages, rawMessage) + continue + } + + normalizedContent := make([]any, 0, len(content)) + for _, rawPart := range content { + part, ok := rawPart.(map[string]any) + if !ok { + normalizedContent = append(normalizedContent, rawPart) + continue + } + normalizedPart, err := normalizeAliyunOpenAIChatAudioPart(part) + if err != nil { + return nil, err + } + normalizedContent = append(normalizedContent, normalizedPart) + } + copiedMessage := cloneMapAny(message) + copiedMessage["content"] = normalizedContent + normalizedMessages = append(normalizedMessages, copiedMessage) + } + + out := cloneMapAny(body) + out["messages"] = normalizedMessages + return out, nil +} + +func normalizeAliyunOpenAIChatAudioPart(part map[string]any) (map[string]any, error) { + switch strings.TrimSpace(stringFromAny(part["type"])) { + case "audio_url": + audioURL, _ := part["audio_url"].(map[string]any) + data := strings.TrimSpace(stringFromAny(audioURL["url"])) + format := resolveOpenAIChatAudioFormat(data, audioURL["format"], audioURL["mime_type"], audioURL["mimeType"]) + if data == "" || format == "" { + return nil, invalidOpenAIChatAudioError() + } + return map[string]any{ + "type": "input_audio", + "input_audio": map[string]any{ + "data": data, + "format": format, + }, + }, nil + case "input_audio": + inputAudio, _ := part["input_audio"].(map[string]any) + data := firstNonEmptyString(inputAudio["data"], inputAudio["url"]) + format := resolveOpenAIChatAudioFormat(data, inputAudio["format"], inputAudio["mime_type"], inputAudio["mimeType"]) + if data == "" || format == "" { + return nil, invalidOpenAIChatAudioError() + } + copiedInput := cloneMapAny(inputAudio) + copiedInput["data"] = data + copiedInput["format"] = format + delete(copiedInput, "url") + copiedPart := cloneMapAny(part) + copiedPart["input_audio"] = copiedInput + return copiedPart, nil + default: + return part, nil + } +} + +func resolveOpenAIChatAudioFormat(source string, values ...any) string { + if explicit := strings.ToLower(strings.TrimSpace(firstNonEmptyString(values...))); explicit != "" { + if normalized := openAIChatAudioFormatByExtension[explicit]; normalized != "" { + return normalized + } + if normalized := openAIChatAudioFormatByMIME[strings.TrimSpace(strings.Split(explicit, ";")[0])]; normalized != "" { + return normalized + } + return explicit + } + if dataMIME := openAIChatAudioDataMIME(source); dataMIME != "" { + if normalized := openAIChatAudioFormatByMIME[dataMIME]; normalized != "" { + return normalized + } + } + if extension := openAIChatAudioSourceExtension(source); extension != "" { + return openAIChatAudioFormatByExtension[extension] + } + return "" +} + +func openAIChatAudioDataMIME(source string) string { + if !strings.HasPrefix(strings.ToLower(source), "data:") { + return "" + } + header := strings.TrimPrefix(source[:strings.Index(source+",", ",")], "data:") + return strings.ToLower(strings.TrimSpace(strings.Split(header, ";")[0])) +} + +func openAIChatAudioSourceExtension(source string) string { + if source == "" || strings.HasPrefix(strings.ToLower(source), "data:") { + return "" + } + if parsed, err := url.Parse(source); err == nil && parsed.Path != "" { + source = parsed.Path + } + source = strings.TrimSuffix(strings.Split(strings.Split(source, "?")[0], "#")[0], "/") + if index := strings.LastIndex(source, "."); index >= 0 && index+1 < len(source) { + return strings.ToLower(source[index+1:]) + } + return "" +} + +func invalidOpenAIChatAudioError() error { + return &ClientError{ + Code: "invalid_parameter", + Message: "input_audio requires data and a resolvable format", + Param: "messages.content.input_audio", + StatusCode: http.StatusBadRequest, + Retryable: false, + } +} diff --git a/apps/api/internal/runner/request_assets_test.go b/apps/api/internal/runner/request_assets_test.go index 0f3f542..1d05812 100644 --- a/apps/api/internal/runner/request_assets_test.go +++ b/apps/api/internal/runner/request_assets_test.go @@ -45,6 +45,56 @@ func TestHydrateProviderRequestAssetsConvertsStrictBase64Field(t *testing.T) { } } +func TestHydrateProviderRequestAssetsConvertsInputAudioDataToRawBase64(t *testing.T) { + storageDir := t.TempDir() + fileName := "gateway-request-asset-audio.mp3" + payload := []byte("audio bytes") + if err := os.WriteFile(filepath.Join(storageDir, fileName), payload, 0o644); err != nil { + t.Fatalf("write request asset: %v", err) + } + service := &Service{cfg: config.Config{LocalUploadedStorageDir: storageDir}} + body := map[string]any{ + "messages": []any{ + map[string]any{ + "role": "user", + "content": []any{ + map[string]any{ + "type": "input_audio", + "input_audio": map[string]any{ + "data": map[string]any{ + "assetRef": map[string]any{ + "sha256": "sha-input-audio", + "contentType": "audio/mpeg", + "url": "/static/uploaded/" + fileName, + "storageProvider": "local_static", + }, + "url": "/static/uploaded/" + fileName, + }, + "format": "mp3", + }, + }, + }, + }, + }, + } + + hydrated, err := service.hydrateProviderRequestAssets(context.Background(), body, store.RuntimeModelCandidate{}) + if err != nil { + t.Fatalf("hydrate request assets: %v", err) + } + messages := hydrated["messages"].([]any) + message := messages[0].(map[string]any) + content := message["content"].([]any) + audioPart := content[0].(map[string]any) + inputAudio := audioPart["input_audio"].(map[string]any) + if got, want := stringFromAny(inputAudio["data"]), base64.StdEncoding.EncodeToString(payload); got != want { + t.Fatalf("unexpected hydrated input audio: got %q want %q", got, want) + } + if inputAudio["format"] != "mp3" { + t.Fatalf("input audio format was not preserved: %+v", inputAudio) + } +} + func TestHydrateProviderRequestAssetsConvertsBase64ArrayField(t *testing.T) { storageDir := t.TempDir() fileName := "gateway-request-asset-array.png"