feat(api): 添加多媒体内容支持并优化钱包计费系统

- 在 API 接口定义中为 video_url 和 audio_url 类型添加 mime_type 字段
- 实现 Google Gemini 客户端对视频和音频内容的支持,包括媒体类型检测和数据传输
- 添加 Gemini 客户端测试用例验证多媒体内容转换功能
- 重构 Playground 页面的媒体上传逻辑以支持 MIME 类型传递
- 实现钱包计费预留机制,确保任务执行前余额充足
- 添加钱包冻结余额管理,防止并发操作导致的超扣问题
- 实现计费预留释放逻辑,处理任务失败或取消情况下的资金返还
- 优化数据库事务处理,确保计费操作的原子性和一致性
- 添加数据库集成测试验证迁移脚本执行流程
- 统一 Google Gemini 相关模型提供商标识符映射
This commit is contained in:
2026-05-22 23:46:08 +08:00
parent af9b281d34
commit 8ad5b06c18
12 changed files with 1104 additions and 82 deletions
+64
View File
@@ -569,6 +569,70 @@ func TestGeminiClientChatContract(t *testing.T) {
}
}
func TestGeminiClientChatConvertsMediaContentParts(t *testing.T) {
var captured map[string]any
var gotPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := json.NewDecoder(r.Body).Decode(&captured); err != nil {
t.Fatalf("decode request: %v", err)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"candidates": []any{map[string]any{
"content": map[string]any{"parts": []any{map[string]any{"text": "video ok"}}},
}},
})
}))
defer server.Close()
_, err := (GeminiClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
Kind: "chat.completions",
Model: "gemini:gemini-2.5-flash",
Body: map[string]any{
"model": "gemini:gemini-2.5-flash",
"messages": []any{map[string]any{
"role": "user",
"content": []any{
map[string]any{"type": "text", "text": "analyze this video"},
map[string]any{"type": "video_url", "video_url": map[string]any{"url": "https://cdn.example.com/input.mov", "mime_type": "video/quicktime"}},
map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": "data:audio/wav;base64,UklGRg=="}},
},
}},
},
Candidate: store.RuntimeModelCandidate{
BaseURL: server.URL + "/v1beta/openai",
ProviderModelName: "gemini-2.5-flash",
ModelType: "chat",
Credentials: map[string]any{"apiKey": "gemini-key"},
},
})
if err != nil {
t.Fatalf("run gemini client: %v", err)
}
if gotPath != "/v1beta/models/gemini-2.5-flash:generateContent" {
t.Fatalf("Gemini OpenAI-compatible base URL should normalize to native endpoint, got %s", gotPath)
}
contents, _ := captured["contents"].([]any)
if len(contents) != 1 {
t.Fatalf("unexpected Gemini contents: %+v", captured)
}
turn, _ := contents[0].(map[string]any)
parts, _ := turn["parts"].([]any)
if len(parts) != 3 {
t.Fatalf("expected text, video, and audio parts, got %+v", turn)
}
video, _ := parts[1].(map[string]any)
videoFile, _ := video["fileData"].(map[string]any)
if videoFile["fileUri"] != "https://cdn.example.com/input.mov" || videoFile["mimeType"] != "video/quicktime" {
t.Fatalf("video_url should become Gemini fileData, got %+v", video)
}
audio, _ := parts[2].(map[string]any)
audioInline, _ := audio["inlineData"].(map[string]any)
if audioInline["mimeType"] != "audio/wav" || audioInline["data"] != "UklGRg==" {
t.Fatalf("audio data URL should become Gemini inlineData, got %+v", audio)
}
}
func TestGeminiClientChatRestoresToolContext(t *testing.T) {
var captured map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {