feat(api): 添加多媒体内容支持并优化钱包计费系统

- 在 API 接口定义中为 video_url 和 audio_url 类型添加 mime_type 字段 - 实现 Google Gemini 客户端对视频和音频内容的支持，包括媒体类型检测和数据传输 - 添加 Gemini 客户端测试用例验证多媒体内容转换功能 - 重构 Playground 页面的媒体上传逻辑以支持 MIME 类型传递 - 实现钱包计费预留机制，确保任务执行前余额充足 - 添加钱包冻结余额管理，防止并发操作导致的超扣问题 - 实现计费预留释放逻辑，处理任务失败或取消情况下的资金返还 - 优化数据库事务处理，确保计费操作的原子性和一致性 - 添加数据库集成测试验证迁移脚本执行流程 - 统一 Google Gemini 相关模型提供商标识符映射
2026-05-22 23:46:08 +08:00
parent af9b281d34
commit 8ad5b06c18
12 changed files with 1104 additions and 82 deletions
@@ -569,6 +569,70 @@ func TestGeminiClientChatContract(t *testing.T) {
 	}
 }

+func TestGeminiClientChatConvertsMediaContentParts(t *testing.T) {
+	var captured map[string]any
+	var gotPath string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		if err := json.NewDecoder(r.Body).Decode(&captured); err != nil {
+			t.Fatalf("decode request: %v", err)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"candidates": []any{map[string]any{
+				"content": map[string]any{"parts": []any{map[string]any{"text": "video ok"}}},
+			}},
+		})
+	}))
+	defer server.Close()
+
+	_, err := (GeminiClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
+		Kind:  "chat.completions",
+		Model: "gemini:gemini-2.5-flash",
+		Body: map[string]any{
+			"model": "gemini:gemini-2.5-flash",
+			"messages": []any{map[string]any{
+				"role": "user",
+				"content": []any{
+					map[string]any{"type": "text", "text": "analyze this video"},
+					map[string]any{"type": "video_url", "video_url": map[string]any{"url": "https://cdn.example.com/input.mov", "mime_type": "video/quicktime"}},
+					map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": "data:audio/wav;base64,UklGRg=="}},
+				},
+			}},
+		},
+		Candidate: store.RuntimeModelCandidate{
+			BaseURL:           server.URL + "/v1beta/openai",
+			ProviderModelName: "gemini-2.5-flash",
+			ModelType:         "chat",
+			Credentials:       map[string]any{"apiKey": "gemini-key"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("run gemini client: %v", err)
+	}
+	if gotPath != "/v1beta/models/gemini-2.5-flash:generateContent" {
+		t.Fatalf("Gemini OpenAI-compatible base URL should normalize to native endpoint, got %s", gotPath)
+	}
+	contents, _ := captured["contents"].([]any)
+	if len(contents) != 1 {
+		t.Fatalf("unexpected Gemini contents: %+v", captured)
+	}
+	turn, _ := contents[0].(map[string]any)
+	parts, _ := turn["parts"].([]any)
+	if len(parts) != 3 {
+		t.Fatalf("expected text, video, and audio parts, got %+v", turn)
+	}
+	video, _ := parts[1].(map[string]any)
+	videoFile, _ := video["fileData"].(map[string]any)
+	if videoFile["fileUri"] != "https://cdn.example.com/input.mov" || videoFile["mimeType"] != "video/quicktime" {
+		t.Fatalf("video_url should become Gemini fileData, got %+v", video)
+	}
+	audio, _ := parts[2].(map[string]any)
+	audioInline, _ := audio["inlineData"].(map[string]any)
+	if audioInline["mimeType"] != "audio/wav" || audioInline["data"] != "UklGRg==" {
+		t.Fatalf("audio data URL should become Gemini inlineData, got %+v", audio)
+	}
+}
+
 func TestGeminiClientChatRestoresToolContext(t *testing.T) {
 	var captured map[string]any
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -5,8 +5,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"mime"
 	"net/http"
 	"net/url"
+	"path"
 	"strings"
 	"time"
 )
@@ -58,6 +60,7 @@ func geminiURL(baseURL string, model string, apiKey string) string {
 	if base == "" {
 		base = "https://generativelanguage.googleapis.com"
 	}
+	base = strings.TrimSuffix(base, "/openai")
 	if strings.HasSuffix(base, "/v1beta") {
 		base = strings.TrimSuffix(base, "/v1beta")
 	}
@@ -121,7 +124,7 @@ func geminiContentsFromMessages(body map[string]any) []any {
 			})
 			continue
 		}
-		parts := geminiTextParts(message["content"])
+		parts := geminiContentParts(message["content"])
 		if role == "assistant" {
 			for _, rawToolCall := range toolCallsSlice(message["tool_calls"]) {
 				toolCall, _ := rawToolCall.(map[string]any)
@@ -157,7 +160,7 @@ func geminiRole(role string) string {
 	return "user"
 }

-func geminiTextParts(content any) []any {
+func geminiContentParts(content any) []any {
 	parts := make([]any, 0)
 	switch typed := content.(type) {
 	case string:
@@ -167,14 +170,146 @@ func geminiTextParts(content any) []any {
 	case []any:
 		for _, rawPart := range typed {
 			part, _ := rawPart.(map[string]any)
-			if text := stringFromAny(firstPresent(part["text"], part["content"])); strings.TrimSpace(text) != "" {
-				parts = append(parts, map[string]any{"text": text})
+			if len(part) == 0 {
+				continue
+			}
+			switch stringFromAny(part["type"]) {
+			case "text":
+				if text := strings.TrimSpace(stringFromAny(firstPresent(part["text"], part["content"]))); text != "" {
+					parts = append(parts, map[string]any{"text": text})
+				}
+			case "image_url":
+				if media := geminiMediaPart(part, "image_url", "image"); media != nil {
+					parts = append(parts, media)
+				}
+			case "video_url":
+				if media := geminiMediaPart(part, "video_url", "video"); media != nil {
+					parts = append(parts, media)
+				}
+			case "audio_url":
+				if media := geminiMediaPart(part, "audio_url", "audio"); media != nil {
+					parts = append(parts, media)
+				}
+			case "input_audio":
+				if media := geminiInputAudioPart(part); media != nil {
+					parts = append(parts, media)
+				}
+			default:
+				if text := strings.TrimSpace(stringFromAny(firstPresent(part["text"], part["content"]))); text != "" {
+					parts = append(parts, map[string]any{"text": text})
+				}
 			}
 		}
 	}
 	return parts
 }

+func geminiMediaPart(part map[string]any, key string, mediaType string) map[string]any {
+	nested := mapFromAny(part[key])
+	uri := firstNonEmptyString(nested["url"], part["url"], part[key])
+	if uri == "" {
+		return nil
+	}
+	mimeType := firstNonEmptyString(nested["mime_type"], nested["mimeType"], part["mime_type"], part["mimeType"])
+	return geminiMediaURLPart(uri, mimeType, mediaType)
+}
+
+func geminiInputAudioPart(part map[string]any) map[string]any {
+	audio := mapFromAny(part["input_audio"])
+	uri := firstNonEmptyString(audio["data"], audio["url"])
+	if uri == "" {
+		return nil
+	}
+	mimeType := firstNonEmptyString(audio["mime_type"], audio["mimeType"])
+	if mimeType == "" {
+		format := strings.ToLower(strings.TrimPrefix(stringFromAny(audio["format"]), "."))
+		if strings.Contains(format, "/") {
+			mimeType = format
+		} else if format == "mp3" {
+			mimeType = "audio/mpeg"
+		} else if format != "" {
+			mimeType = "audio/" + format
+		}
+	}
+	return geminiMediaURLPart(uri, mimeType, "audio")
+}
+
+func geminiMediaURLPart(uri string, explicitMimeType string, mediaType string) map[string]any {
+	if parsed := geminiDataURL(uri); parsed != nil {
+		return map[string]any{"inlineData": map[string]any{
+			"mimeType": geminiMediaMime(firstNonEmptyString(explicitMimeType, parsed.mimeType), mediaType),
+			"data":     parsed.data,
+		}}
+	}
+	return map[string]any{"fileData": map[string]any{
+		"fileUri":  uri,
+		"mimeType": geminiMediaMime(firstNonEmptyString(explicitMimeType, mimeFromURI(uri)), mediaType),
+	}}
+}
+
+type geminiParsedDataURL struct {
+	mimeType string
+	data     string
+}
+
+func geminiDataURL(value string) *geminiParsedDataURL {
+	if !strings.HasPrefix(value, "data:") {
+		return nil
+	}
+	prefix, data, ok := strings.Cut(value, ",")
+	if !ok || !strings.Contains(prefix, ";base64") {
+		return nil
+	}
+	mimeType := strings.TrimPrefix(strings.Split(prefix, ";")[0], "data:")
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+	return &geminiParsedDataURL{mimeType: mimeType, data: data}
+}
+
+func mimeFromURI(value string) string {
+	pathValue := value
+	if parsed, err := url.Parse(value); err == nil && parsed.Path != "" {
+		pathValue = parsed.Path
+	}
+	extension := strings.ToLower(path.Ext(pathValue))
+	if extension == "" {
+		return ""
+	}
+	return mime.TypeByExtension(extension)
+}
+
+func geminiMediaMime(mimeType string, mediaType string) string {
+	normalized := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))
+	switch mediaType {
+	case "image":
+		if strings.HasPrefix(normalized, "image/") && normalized != "image/svg+xml" {
+			return normalized
+		}
+		return "image/png"
+	case "video":
+		switch normalized {
+		case "video/x-msvideo":
+			return "video/avi"
+		case "video/quicktime", "video/mpeg", "video/mp4", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp":
+			return normalized
+		default:
+			return "video/mp4"
+		}
+	case "audio":
+		switch normalized {
+		case "audio/x-wav", "audio/wave":
+			return "audio/wav"
+		case "audio/mpeg", "audio/mp3", "audio/wav", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mp4", "audio/webm":
+			return normalized
+		default:
+			return "audio/mpeg"
+		}
+	default:
+		return "application/octet-stream"
+	}
+}
+
 func toolCallsSlice(value any) []any {
 	switch typed := value.(type) {
 	case []any: