feat(api): 添加多媒体内容支持并优化钱包计费系统
- 在 API 接口定义中为 video_url 和 audio_url 类型添加 mime_type 字段 - 实现 Google Gemini 客户端对视频和音频内容的支持,包括媒体类型检测和数据传输 - 添加 Gemini 客户端测试用例验证多媒体内容转换功能 - 重构 Playground 页面的媒体上传逻辑以支持 MIME 类型传递 - 实现钱包计费预留机制,确保任务执行前余额充足 - 添加钱包冻结余额管理,防止并发操作导致的超扣问题 - 实现计费预留释放逻辑,处理任务失败或取消情况下的资金返还 - 优化数据库事务处理,确保计费操作的原子性和一致性 - 添加数据库集成测试验证迁移脚本执行流程 - 统一 Google Gemini 相关模型提供商标识符映射
This commit is contained in:
@@ -684,11 +684,13 @@ export interface VideoGenerationContent {
|
||||
};
|
||||
video_url?: {
|
||||
url: string;
|
||||
mime_type?: string;
|
||||
refer_type?: 'feature' | 'base';
|
||||
keep_original_sound?: 'yes' | 'no';
|
||||
};
|
||||
audio_url?: {
|
||||
url: string;
|
||||
mime_type?: string;
|
||||
};
|
||||
role?: VideoGenerationContentRole;
|
||||
shot_index?: number;
|
||||
|
||||
@@ -32,8 +32,8 @@ export interface PlaygroundUpload {
|
||||
export type OpenAIChatContentPart =
|
||||
| { type: 'text'; text: string }
|
||||
| { type: 'image_url'; image_url: { url: string } }
|
||||
| { type: 'video_url'; video_url: { url: string } }
|
||||
| { type: 'audio_url'; audio_url: { url: string } }
|
||||
| { type: 'video_url'; video_url: { mime_type?: string; url: string } }
|
||||
| { type: 'audio_url'; audio_url: { mime_type?: string; url: string } }
|
||||
| { type: 'file_url'; file_url: { filename: string; url: string } };
|
||||
|
||||
export const mediaUploadAccept = 'image/*,video/*,audio/*';
|
||||
@@ -518,11 +518,17 @@ export function openAIContentFromPromptAndUploads(prompt: string, uploads: Playg
|
||||
function openAIContentPartFromUpload(item: PlaygroundUpload): OpenAIChatContentPart | undefined {
|
||||
if (!item.url) return undefined;
|
||||
if (item.kind === 'image') return { type: 'image_url', image_url: { url: item.url } };
|
||||
if (item.kind === 'video') return { type: 'video_url', video_url: { url: item.url } };
|
||||
if (item.kind === 'audio') return { type: 'audio_url', audio_url: { url: item.url } };
|
||||
if (item.kind === 'video') return { type: 'video_url', video_url: mediaURLPayload(item) };
|
||||
if (item.kind === 'audio') return { type: 'audio_url', audio_url: mediaURLPayload(item) };
|
||||
return { type: 'file_url', file_url: { filename: item.name, url: item.url } };
|
||||
}
|
||||
|
||||
function mediaURLPayload(item: PlaygroundUpload) {
|
||||
const payload: { mime_type?: string; url: string } = { url: item.url };
|
||||
if (item.contentType) payload.mime_type = item.contentType;
|
||||
return payload;
|
||||
}
|
||||
|
||||
export function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>) {
|
||||
const images = uploads.filter((item) => item.kind === 'image').map((item) => item.url);
|
||||
const payload: Record<string, string | string[]> = {};
|
||||
@@ -570,10 +576,10 @@ function videoGenerationContentFromUpload(item: PlaygroundUpload): VideoGenerati
|
||||
return { type: 'image_url', role: 'reference_image', image_url: { url: item.url } };
|
||||
}
|
||||
if (item.kind === 'video') {
|
||||
return { type: 'video_url', role: 'reference_video', video_url: { url: item.url, refer_type: 'feature' } };
|
||||
return { type: 'video_url', role: 'reference_video', video_url: { ...mediaURLPayload(item), refer_type: 'feature' } };
|
||||
}
|
||||
if (item.kind === 'audio') {
|
||||
return { type: 'audio_url', role: 'reference_audio', audio_url: { url: item.url } };
|
||||
return { type: 'audio_url', role: 'reference_audio', audio_url: mediaURLPayload(item) };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user