fix: preprocess message content by model capability

2026-05-13 23:39:11 +08:00
parent b8a716169f
commit f1535a94c2
3 changed files with 362 additions and 0 deletions
@@ -123,6 +123,163 @@ func TestParamProcessorOmniCapabilityLogUsesActualCapabilityKey(t *testing.T) {
 	t.Fatalf("expected log to reference capabilities.omni.input_audio, got %+v", result.Log.Changes)
 }

+func TestParamProcessorChatConvertsUnsupportedMediaMessageContentToText(t *testing.T) {
+	body := map[string]any{
+		"model": "text-only",
+		"messages": []any{
+			map[string]any{
+				"role": "user",
+				"content": []any{
+					map[string]any{"type": "text", "text": "describe these"},
+					map[string]any{"type": "image_url", "image_url": map[string]any{"url": "https://example.com/image.png"}},
+					map[string]any{"type": "video_url", "video_url": map[string]any{"url": "https://example.com/video.mp4"}},
+					map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": "https://example.com/audio.mp3"}},
+					map[string]any{"type": "input_audio", "input_audio": map[string]any{"data": "https://example.com/input.wav"}},
+				},
+			},
+		},
+	}
+	candidate := store.RuntimeModelCandidate{
+		ModelType: "text_generate",
+		Capabilities: map[string]any{
+			"text_generate": map[string]any{},
+			"originalTypes": []any{"text_generate"},
+		},
+	}
+
+	result := preprocessRequestWithLog("chat.completions", body, candidate)
+	messages, _ := result.Body["messages"].([]any)
+	if len(messages) != 1 {
+		t.Fatalf("expected one message, got %+v", result.Body["messages"])
+	}
+	message, _ := messages[0].(map[string]any)
+	content, _ := message["content"].([]any)
+	if len(content) != 5 {
+		t.Fatalf("expected five content parts, got %+v", message["content"])
+	}
+	expectedText := []string{
+		"describe these",
+		"Image link: https://example.com/image.png",
+		"video URL: https://example.com/video.mp4",
+		"audio URL: https://example.com/audio.mp3",
+		"audio URL: https://example.com/input.wav",
+	}
+	for index, expected := range expectedText {
+		part, _ := content[index].(map[string]any)
+		if stringFromAny(part["text"]) != expected {
+			t.Fatalf("content[%d] text = %q, want %q; all=%+v", index, stringFromAny(part["text"]), expected, content)
+		}
+	}
+	if len(result.Log.Changes) != 4 {
+		t.Fatalf("expected four media conversion changes, got %+v", result.Log.Changes)
+	}
+	expectedCapabilityPaths := map[string]bool{
+		"capabilities.image_analysis":      false,
+		"capabilities.video_understanding": false,
+		"capabilities.audio_understanding": false,
+	}
+	for _, change := range result.Log.Changes {
+		if _, ok := expectedCapabilityPaths[change.CapabilityPath]; ok {
+			expectedCapabilityPaths[change.CapabilityPath] = true
+		}
+	}
+	for path, found := range expectedCapabilityPaths {
+		if !found {
+			t.Fatalf("expected conversion log for %s, got %+v", path, result.Log.Changes)
+		}
+	}
+}
+
+func TestParamProcessorChatKeepsOmniMessageContent(t *testing.T) {
+	body := map[string]any{
+		"model": "omni",
+		"messages": []any{
+			map[string]any{
+				"role": "user",
+				"content": []any{
+					map[string]any{"type": "image_url", "image_url": map[string]any{"url": "https://example.com/image.png"}},
+					map[string]any{"type": "video_url", "video_url": map[string]any{"url": "https://example.com/video.mp4"}},
+					map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": "https://example.com/audio.mp3"}},
+				},
+			},
+		},
+	}
+	candidate := store.RuntimeModelCandidate{
+		ModelType: "text_generate",
+		Capabilities: map[string]any{
+			"text_generate": map[string]any{},
+			"omni":          map[string]any{},
+			"originalTypes": []any{"text_generate", "omni"},
+		},
+	}
+
+	result := preprocessRequestWithLog("chat.completions", body, candidate)
+	if result.Log.Changed {
+		t.Fatalf("omni model should keep message media content unchanged, got %+v", result.Log.Changes)
+	}
+	messages, _ := result.Body["messages"].([]any)
+	message, _ := messages[0].(map[string]any)
+	content, _ := message["content"].([]any)
+	for _, item := range content {
+		part, _ := item.(map[string]any)
+		if stringFromAny(part["type"]) == "text" {
+			t.Fatalf("media content should not be converted for omni model: %+v", content)
+		}
+	}
+}
+
+func TestParamProcessorChatConvertsOnlyUnsupportedModalities(t *testing.T) {
+	body := map[string]any{
+		"model": "vision-only",
+		"messages": []any{
+			map[string]any{
+				"role": "user",
+				"content": []any{
+					map[string]any{"type": "image_url", "image_url": map[string]any{"url": "https://example.com/image.png"}},
+					map[string]any{"type": "video_url", "video_url": map[string]any{"url": "https://example.com/video.mp4"}},
+				},
+			},
+		},
+	}
+	candidate := store.RuntimeModelCandidate{
+		ModelType: "text_generate",
+		Capabilities: map[string]any{
+			"text_generate":  map[string]any{},
+			"image_analysis": map[string]any{},
+			"originalTypes":  []any{"text_generate", "image_analysis"},
+		},
+	}
+
+	result := preprocessRequestWithLog("chat.completions", body, candidate)
+	messages, _ := result.Body["messages"].([]any)
+	message, _ := messages[0].(map[string]any)
+	content, _ := message["content"].([]any)
+	first, _ := content[0].(map[string]any)
+	second, _ := content[1].(map[string]any)
+	if stringFromAny(first["type"]) != "image_url" {
+		t.Fatalf("image content should be kept when image_analysis is supported: %+v", content)
+	}
+	if stringFromAny(second["text"]) != "video URL: https://example.com/video.mp4" {
+		t.Fatalf("video content should be converted, got %+v", second)
+	}
+	if len(result.Log.Changes) != 1 || result.Log.Changes[0].CapabilityPath != "capabilities.video_understanding" {
+		t.Fatalf("expected only video conversion to be logged, got %+v", result.Log.Changes)
+	}
+}
+
+func TestSkipTaskParameterPreprocessingLogForTextModelTypes(t *testing.T) {
+	for _, modelType := range []string{"text_generate", "chat", "responses", "text"} {
+		if !skipTaskParameterPreprocessingLog(modelType) {
+			t.Fatalf("%s should skip task parameter preprocessing log", modelType)
+		}
+	}
+	for _, modelType := range []string{"image_generate", "image_edit", "video_generate", "omni_video"} {
+		if skipTaskParameterPreprocessingLog(modelType) {
+			t.Fatalf("%s should keep task parameter preprocessing log", modelType)
+		}
+	}
+}
+
 func TestParamProcessorVideoCapabilitiesNormalizeAndFilter(t *testing.T) {
 	body := map[string]any{
 		"model":        "Seedance",