fix(audio): 兼容百炼 OpenAI 音频输入

为阿里云百炼 OpenAI-compatible Chat 统一保留标准 input_audio，并将旧版 audio_url 转换为 input_audio，同时根据 MIME 或扩展名补齐音频格式。补充请求资产水合测试，确认 input_audio.data 会转换为裸 Base64 且保留 format。验证：在 apps/api 执行 env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1 全部通过。
2026-07-27 23:50:17 +08:00
parent 95776c84f6
commit 046c16fc69
4 changed files with 297 additions and 0 deletions
@@ -344,6 +344,79 @@ func TestOpenAIClientChatContract(t *testing.T) {
 	}
 }

+func TestOpenAIClientAliyunChatSendsStandardInputAudioAndNormalizesLegacyAudioURL(t *testing.T) {
+	var captured map[string]any
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if err := json.NewDecoder(r.Body).Decode(&captured); err != nil {
+			t.Fatalf("decode request: %v", err)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"id":     "chatcmpl-audio",
+			"object": "chat.completion",
+			"model":  captured["model"],
+			"choices": []any{map[string]any{
+				"message": map[string]any{"role": "assistant", "content": "ok"},
+			}},
+		})
+	}))
+	defer server.Close()
+
+	_, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
+		Kind:  "chat.completions",
+		Model: "qwen-omni",
+		Body: map[string]any{
+			"model": "qwen-omni",
+			"messages": []any{
+				map[string]any{
+					"role": "user",
+					"content": []any{
+						map[string]any{
+							"type": "input_audio",
+							"input_audio": map[string]any{
+								"data":   "https://cdn.example.com/audio/standard.wav",
+								"format": "wav",
+							},
+						},
+						map[string]any{
+							"type":      "audio_url",
+							"audio_url": map[string]any{"url": "https://cdn.example.com/audio/legacy.m4a"},
+						},
+					},
+				},
+			},
+		},
+		Candidate: store.RuntimeModelCandidate{
+			Provider:          "aliyun-bailian-openai",
+			BaseURL:           server.URL,
+			ProviderModelName: "qwen-omni",
+			Credentials:       map[string]any{"apiKey": "test-key"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("run aliyun openai-compatible client: %v", err)
+	}
+
+	messages, _ := captured["messages"].([]any)
+	message, _ := messages[0].(map[string]any)
+	content, _ := message["content"].([]any)
+	standard, _ := content[0].(map[string]any)
+	if standard["type"] != "input_audio" {
+		t.Fatalf("standard input_audio was not preserved: %+v", standard)
+	}
+	standardAudio, _ := standard["input_audio"].(map[string]any)
+	if standardAudio["data"] != "https://cdn.example.com/audio/standard.wav" || standardAudio["format"] != "wav" {
+		t.Fatalf("unexpected standard input_audio: %+v", standardAudio)
+	}
+	legacy, _ := content[1].(map[string]any)
+	if legacy["type"] != "input_audio" {
+		t.Fatalf("legacy audio_url was not normalized: %+v", legacy)
+	}
+	legacyAudio, _ := legacy["input_audio"].(map[string]any)
+	if legacyAudio["data"] != "https://cdn.example.com/audio/legacy.m4a" || legacyAudio["format"] != "m4a" {
+		t.Fatalf("unexpected normalized legacy audio: %+v", legacyAudio)
+	}
+}
+
 func TestOpenAIClientImageEditUsesMultipartWithMultipleImages(t *testing.T) {
 	var contentType string
 	var receivedFields map[string][]string
@@ -48,6 +48,11 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error
 	}
 	if endpointKind == "chat.completions" {
 		body = NormalizeChatCompletionRequestBody(body)
+		normalizedBody, normalizeErr := normalizeOpenAIChatAudioParts(body, request.Candidate)
+		if normalizeErr != nil {
+			return Response{}, normalizeErr
+		}
+		body = normalizedBody
 		applyOpenAIChatReasoningParams(body, request.Candidate)
 		body = FilterOpenAIChatRequestBody(body)
 	} else if request.Kind == "responses" {
@@ -0,0 +1,169 @@
+package clients
+
+import (
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+)
+
+var openAIChatAudioFormatByMIME = map[string]string{
+	"audio/aac":   "aac",
+	"audio/flac":  "flac",
+	"audio/m4a":   "m4a",
+	"audio/mp4":   "m4a",
+	"audio/mpeg":  "mp3",
+	"audio/ogg":   "ogg",
+	"audio/opus":  "opus",
+	"audio/wav":   "wav",
+	"audio/webm":  "webm",
+	"audio/x-m4a": "m4a",
+	"audio/x-wav": "wav",
+}
+
+var openAIChatAudioFormatByExtension = map[string]string{
+	"aac":  "aac",
+	"flac": "flac",
+	"m4a":  "m4a",
+	"mp3":  "mp3",
+	"mp4":  "m4a",
+	"oga":  "ogg",
+	"ogg":  "ogg",
+	"opus": "opus",
+	"wav":  "wav",
+	"webm": "webm",
+}
+
+func normalizeOpenAIChatAudioParts(body map[string]any, candidate store.RuntimeModelCandidate) (map[string]any, error) {
+	if !isAliyunBailianOpenAI(candidate) {
+		return body, nil
+	}
+	messages, ok := body["messages"].([]any)
+	if !ok {
+		return body, nil
+	}
+
+	normalizedMessages := make([]any, 0, len(messages))
+	for _, rawMessage := range messages {
+		message, ok := rawMessage.(map[string]any)
+		if !ok {
+			normalizedMessages = append(normalizedMessages, rawMessage)
+			continue
+		}
+		content, ok := message["content"].([]any)
+		if !ok {
+			normalizedMessages = append(normalizedMessages, rawMessage)
+			continue
+		}
+
+		normalizedContent := make([]any, 0, len(content))
+		for _, rawPart := range content {
+			part, ok := rawPart.(map[string]any)
+			if !ok {
+				normalizedContent = append(normalizedContent, rawPart)
+				continue
+			}
+			normalizedPart, err := normalizeAliyunOpenAIChatAudioPart(part)
+			if err != nil {
+				return nil, err
+			}
+			normalizedContent = append(normalizedContent, normalizedPart)
+		}
+		copiedMessage := cloneMapAny(message)
+		copiedMessage["content"] = normalizedContent
+		normalizedMessages = append(normalizedMessages, copiedMessage)
+	}
+
+	out := cloneMapAny(body)
+	out["messages"] = normalizedMessages
+	return out, nil
+}
+
+func normalizeAliyunOpenAIChatAudioPart(part map[string]any) (map[string]any, error) {
+	switch strings.TrimSpace(stringFromAny(part["type"])) {
+	case "audio_url":
+		audioURL, _ := part["audio_url"].(map[string]any)
+		data := strings.TrimSpace(stringFromAny(audioURL["url"]))
+		format := resolveOpenAIChatAudioFormat(data, audioURL["format"], audioURL["mime_type"], audioURL["mimeType"])
+		if data == "" || format == "" {
+			return nil, invalidOpenAIChatAudioError()
+		}
+		return map[string]any{
+			"type": "input_audio",
+			"input_audio": map[string]any{
+				"data":   data,
+				"format": format,
+			},
+		}, nil
+	case "input_audio":
+		inputAudio, _ := part["input_audio"].(map[string]any)
+		data := firstNonEmptyString(inputAudio["data"], inputAudio["url"])
+		format := resolveOpenAIChatAudioFormat(data, inputAudio["format"], inputAudio["mime_type"], inputAudio["mimeType"])
+		if data == "" || format == "" {
+			return nil, invalidOpenAIChatAudioError()
+		}
+		copiedInput := cloneMapAny(inputAudio)
+		copiedInput["data"] = data
+		copiedInput["format"] = format
+		delete(copiedInput, "url")
+		copiedPart := cloneMapAny(part)
+		copiedPart["input_audio"] = copiedInput
+		return copiedPart, nil
+	default:
+		return part, nil
+	}
+}
+
+func resolveOpenAIChatAudioFormat(source string, values ...any) string {
+	if explicit := strings.ToLower(strings.TrimSpace(firstNonEmptyString(values...))); explicit != "" {
+		if normalized := openAIChatAudioFormatByExtension[explicit]; normalized != "" {
+			return normalized
+		}
+		if normalized := openAIChatAudioFormatByMIME[strings.TrimSpace(strings.Split(explicit, ";")[0])]; normalized != "" {
+			return normalized
+		}
+		return explicit
+	}
+	if dataMIME := openAIChatAudioDataMIME(source); dataMIME != "" {
+		if normalized := openAIChatAudioFormatByMIME[dataMIME]; normalized != "" {
+			return normalized
+		}
+	}
+	if extension := openAIChatAudioSourceExtension(source); extension != "" {
+		return openAIChatAudioFormatByExtension[extension]
+	}
+	return ""
+}
+
+func openAIChatAudioDataMIME(source string) string {
+	if !strings.HasPrefix(strings.ToLower(source), "data:") {
+		return ""
+	}
+	header := strings.TrimPrefix(source[:strings.Index(source+",", ",")], "data:")
+	return strings.ToLower(strings.TrimSpace(strings.Split(header, ";")[0]))
+}
+
+func openAIChatAudioSourceExtension(source string) string {
+	if source == "" || strings.HasPrefix(strings.ToLower(source), "data:") {
+		return ""
+	}
+	if parsed, err := url.Parse(source); err == nil && parsed.Path != "" {
+		source = parsed.Path
+	}
+	source = strings.TrimSuffix(strings.Split(strings.Split(source, "?")[0], "#")[0], "/")
+	if index := strings.LastIndex(source, "."); index >= 0 && index+1 < len(source) {
+		return strings.ToLower(source[index+1:])
+	}
+	return ""
+}
+
+func invalidOpenAIChatAudioError() error {
+	return &ClientError{
+		Code:       "invalid_parameter",
+		Message:    "input_audio requires data and a resolvable format",
+		Param:      "messages.content.input_audio",
+		StatusCode: http.StatusBadRequest,
+		Retryable:  false,
+	}
+}
@@ -45,6 +45,56 @@ func TestHydrateProviderRequestAssetsConvertsStrictBase64Field(t *testing.T) {
 	}
 }

+func TestHydrateProviderRequestAssetsConvertsInputAudioDataToRawBase64(t *testing.T) {
+	storageDir := t.TempDir()
+	fileName := "gateway-request-asset-audio.mp3"
+	payload := []byte("audio bytes")
+	if err := os.WriteFile(filepath.Join(storageDir, fileName), payload, 0o644); err != nil {
+		t.Fatalf("write request asset: %v", err)
+	}
+	service := &Service{cfg: config.Config{LocalUploadedStorageDir: storageDir}}
+	body := map[string]any{
+		"messages": []any{
+			map[string]any{
+				"role": "user",
+				"content": []any{
+					map[string]any{
+						"type": "input_audio",
+						"input_audio": map[string]any{
+							"data": map[string]any{
+								"assetRef": map[string]any{
+									"sha256":          "sha-input-audio",
+									"contentType":     "audio/mpeg",
+									"url":             "/static/uploaded/" + fileName,
+									"storageProvider": "local_static",
+								},
+								"url": "/static/uploaded/" + fileName,
+							},
+							"format": "mp3",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	hydrated, err := service.hydrateProviderRequestAssets(context.Background(), body, store.RuntimeModelCandidate{})
+	if err != nil {
+		t.Fatalf("hydrate request assets: %v", err)
+	}
+	messages := hydrated["messages"].([]any)
+	message := messages[0].(map[string]any)
+	content := message["content"].([]any)
+	audioPart := content[0].(map[string]any)
+	inputAudio := audioPart["input_audio"].(map[string]any)
+	if got, want := stringFromAny(inputAudio["data"]), base64.StdEncoding.EncodeToString(payload); got != want {
+		t.Fatalf("unexpected hydrated input audio: got %q want %q", got, want)
+	}
+	if inputAudio["format"] != "mp3" {
+		t.Fatalf("input audio format was not preserved: %+v", inputAudio)
+	}
+}
+
 func TestHydrateProviderRequestAssetsConvertsBase64ArrayField(t *testing.T) {
 	storageDir := t.TempDir()
 	fileName := "gateway-request-asset-array.png"