BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: optimize playground media references

Browse Source
This commit is contained in:
wangbo
2026-05-13 22:06:13 +08:00
parent 5868dd7e68
commit 0cd4e6fed1
6 changed files with 1522 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/web/src/pages/PlaygroundPage.tsx
+524 -37
View File
@@ -19,10 +19,15 @@ import { code } from '@streamdown/code';
import { math } from '@streamdown/math';
import { mermaid } from '@streamdown/mermaid';
import type { GatewayApiKey, GatewayTask, PlatformModel } from '@easyai-ai-gateway/contracts';
import { Bot, ChevronDown, FileText, Image as ImageIcon, LoaderCircle, MessageSquarePlus, Music2, Paperclip, Send, Sparkles, Video, X } from 'lucide-react';
import { Bot, ChevronDown, FileText, Image as ImageIcon, LoaderCircle, MessageSquarePlus, Music2, Paperclip, Plus, Repeat2, Send, Sparkles, Video, X } from 'lucide-react';
import { Badge, Button, Select, Textarea } from '../components/ui';
import { GatewayApiError, createImageEditTask, createImageGenerationTask, createVideoGenerationTask, pollTaskUntilSettled, streamChatCompletionText, taskIsPending, uploadFileToStorage } from '../api';
import type { PlaygroundMode } from '../types';
import {
PlaygroundPromptMentionInput,
removeInvalidPlaygroundResourceTokens,
replacePlaygroundResourceTokens,
} from './playground-prompt-mention';
import {
defaultMediaGenerationSettings,
deriveMediaModelCapabilities,
@@ -59,6 +64,7 @@ interface ModelOption {
}
type PlaygroundUploadKind = 'audio' | 'file' | 'image' | 'video';
type PlaygroundUploadRole = 'first_frame' | 'last_frame';
interface PlaygroundUpload {
contentType: string;
@@ -66,6 +72,7 @@ interface PlaygroundUpload {
kind: PlaygroundUploadKind;
name: string;
raw: Record<string, unknown>;
role?: PlaygroundUploadRole;
size: number;
url: string;
}
@@ -95,6 +102,7 @@ const quickPrompts: Record<PlaygroundMode, string[]> = {
};
const mediaUploadAccept = 'image/*,video/*,audio/*';
const imageOnlyUploadAccept = 'image/*';
const chatUploadAccept = [
mediaUploadAccept,
'.csv',
@@ -167,7 +175,9 @@ export function PlaygroundPage(props: {
const pendingMediaModelRef = useRef('');
const resumedTaskIdsRef = useRef(new Set<string>());
const activeMode = useMemo(() => modeOptions.find((item) => item.value === props.mode) ?? modeOptions[0], [props.mode]);
const mediaUploadAcceptValue = mediaUploadAcceptForMode(props.mode, videoMode);
const effectiveImageHasReference = imageHasReference || (props.mode === 'image' && mediaUploads.some((item) => item.kind === 'image'));
const mediaUploadSignature = useMemo(() => mediaUploads.map((item) => `${item.id}:${item.kind}:${item.role ?? ''}`).join('|'), [mediaUploads]);
const modelOptions = useMemo(
() => buildModelOptions(filterModelsForMode(props.models, props.mode, effectiveImageHasReference, videoMode)),
[effectiveImageHasReference, props.mode, props.models, videoMode],
@@ -215,7 +225,31 @@ export function PlaygroundPage(props: {
writeStoredMediaRuns(mediaRuns);
}, [mediaRuns]);
async function uploadMediaFiles(files: File[]) {
useEffect(() => {
if (props.mode === 'chat') return;
setPrompt((current) => removeInvalidPlaygroundResourceTokens(current, mediaUploads));
}, [mediaUploadSignature, props.mode]);
useEffect(() => {
if (props.mode === 'image') {
setMediaUploads((current) => current.some((item) => item.kind !== 'image') ? current.filter((item) => item.kind === 'image') : current);
return;
}
if (props.mode === 'video' && videoMode === 'first_last_frame') {
setMediaUploads((current) => normalizeFirstLastFrameUploads(current));
}
}, [props.mode, videoMode]);
useEffect(() => {
if (props.mode !== 'video') return;
setVideoMode((current) => {
if (mediaUploads.length > 0 && current === 'text_to_video') return 'omni_reference';
if (mediaUploads.length === 0 && current === 'omni_reference') return 'text_to_video';
return current;
});
}, [mediaUploads.length, props.mode]);
async function uploadMediaFiles(files: File[], targetRole?: PlaygroundUploadRole) {
if (!files.length) return;
const credential = activeApiKeySecret || props.token;
if (!props.token) {
@@ -231,15 +265,19 @@ export function PlaygroundPage(props: {
try {
const { items, warnings } = await uploadPlaygroundFiles(credential, files, {
allowFiles: false,
allowedKinds: allowedMediaUploadKinds(props.mode, videoMode),
source: `ai-gateway-playground-${props.mode}`,
});
if (items.length) {
setMediaUploads((current) => [...current, ...items]);
setMediaUploads((current) => mergeMediaUploadsForMode(current, items, props.mode, videoMode, targetRole));
if (props.mode === 'image' && items.some((item) => item.kind === 'image')) {
setImageHasReference(true);
}
if (props.mode === 'video' && videoMode === 'text_to_video') {
setVideoMode('omni_reference');
}
}
setMediaUploadMessage(warnings[0] ?? (items.length ? `已上传 ${items.length} 个参考素材。` : ''));
setMediaUploadMessage(warnings[0] ?? '');
} catch (err) {
setMediaUploadMessage(err instanceof Error ? err.message : '文件上传失败');
} finally {
@@ -305,7 +343,8 @@ export function PlaygroundPage(props: {
const localId = newLocalId();
const runUploads = overrides ? [] : mediaUploads;
const modelLabel = modelOptions.find((item) => item.value === runModel)?.label ?? runModel;
const runModelOption = modelOptions.find((item) => item.value === runModel);
const modelLabel = runModelOption?.label ?? runModel;
const run: MediaGenerationRun = {
createdAt: new Date().toISOString(),
localId,
@@ -320,11 +359,13 @@ export function PlaygroundPage(props: {
setMediaRuns((current) => [...current, run]);
setMediaMessage('');
try {
const uploadPayload = mediaUploadRequestPayload(runUploads, runMode);
const requestPrompt = replacePlaygroundResourceTokens(trimmedPrompt, runUploads, runMode);
const uploadPayload = mediaUploadRequestPayload(runUploads, runMode, videoMode);
const requestPayload = {
model: runModel,
prompt: promptWithUploadSummary(trimmedPrompt, runUploads),
prompt: requestPrompt,
...mediaRequestPayload(runSettings, runMode),
...videoModeRequestPayload(runMode, videoMode, runUploads, runModelOption),
...uploadPayload,
};
const response = runMode === 'video'
@@ -415,7 +456,7 @@ export function PlaygroundPage(props: {
imageHasReference={effectiveImageHasReference}
mediaSettings={mediaSettings}
mediaCapabilities={mediaCapabilities}
uploadAccept={mediaUploadAccept}
uploadAccept={mediaUploadAcceptValue}
uploadMessage={mediaUploadMessage}
uploads={mediaUploads}
uploading={mediaUploading}
@@ -434,8 +475,9 @@ export function PlaygroundPage(props: {
}
return next;
})}
onSwapFrameUploads={() => setMediaUploads((current) => swapFirstLastFrameUploads(current))}
onSubmit={() => void submitMediaTask()}
onUploadFiles={(files) => void uploadMediaFiles(files)}
onUploadFiles={(files, targetRole) => void uploadMediaFiles(files, targetRole)}
onVideoModeChange={setVideoMode}
/>
);
@@ -1120,34 +1162,64 @@ function Composer(props: {
onModelChange: (value: string) => void;
onPromptChange: (value: string) => void;
onRemoveUpload?: (id: string) => void;
onSwapFrameUploads?: () => void;
onSubmit?: () => void;
onUploadFiles?: (files: File[]) => void;
onUploadFiles?: (files: File[], targetRole?: PlaygroundUploadRole) => void;
onVideoModeChange?: (value: VideoCreateMode) => void;
}) {
const quickItems = quickPrompts[props.mode];
const apiKeyNotice = props.apiKeys && props.apiKeySecretsById ? apiKeyNoticeText(props.apiKeys, props.apiKeySecretsById) : '';
const hasMediaReferencePicker = props.mode !== 'chat' && Boolean(props.onUploadFiles);
const mediaReferenceMessage = hasMediaReferencePicker
? props.uploadMessage || mediaUploadSummaryMessage(props.uploads ?? [], props.mode, props.videoMode ?? 'text_to_video')
: props.uploadMessage;
return (
<div className={props.compact ? 'playgroundComposer compact' : 'playgroundComposer'}>
<div className="composerBody">
<ComposerUploadButton
accept={props.uploadAccept ?? mediaUploadAccept}
active={Boolean(props.uploads?.length) || props.imageHasReference === true}
disabled={!props.onUploadFiles}
uploading={props.uploading}
onFiles={props.onUploadFiles}
/>
<div className="composerInputStack">
<Textarea
size={props.compact ? 'sm' : 'md'}
value={props.prompt}
placeholder={placeholderByMode[props.mode]}
onChange={(event) => props.onPromptChange(event.target.value)}
/>
<UploadAttachmentList
message={props.uploadMessage}
<div className={hasMediaReferencePicker ? 'composerBody composerBodyWithReferences' : 'composerBody'} data-frame-mode={props.mode === 'video' && props.videoMode === 'first_last_frame'}>
{hasMediaReferencePicker ? (
<MediaReferencePicker
accept={props.uploadAccept ?? mediaUploadAccept}
mode={props.mode as Exclude<PlaygroundMode, 'chat'>}
uploads={props.uploads ?? []}
uploading={props.uploading}
videoMode={props.videoMode ?? 'text_to_video'}
onFiles={props.onUploadFiles}
onRemove={props.onRemoveUpload}
onSwapFrames={props.onSwapFrameUploads}
/>
) : (
<ComposerUploadButton
accept={props.uploadAccept ?? mediaUploadAccept}
active={Boolean(props.uploads?.length) || props.imageHasReference === true}
disabled={!props.onUploadFiles}
uploading={props.uploading}
onFiles={props.onUploadFiles}
/>
)}
<div className="composerInputStack">
{hasMediaReferencePicker ? (
<PlaygroundPromptMentionInput
value={props.prompt}
placeholder={mediaPromptPlaceholder(props.mode)}
uploads={props.uploads ?? []}
onChange={props.onPromptChange}
/>
) : (
<Textarea
size={props.compact ? 'sm' : 'md'}
value={props.prompt}
placeholder={placeholderByMode[props.mode]}
onChange={(event) => props.onPromptChange(event.target.value)}
/>
)}
{!hasMediaReferencePicker && (
<UploadAttachmentList
message={props.uploadMessage}
uploads={props.uploads ?? []}
onRemove={props.onRemoveUpload}
/>
)}
{hasMediaReferencePicker && mediaReferenceMessage && <div className="composerUploadMessage">{mediaReferenceMessage}</div>}
</div>
</div>
<div className="composerFooter">
@@ -1235,6 +1307,260 @@ function ComposerUploadButton(props: {
);
}
function MediaReferencePicker(props: {
accept: string;
mode: Exclude<PlaygroundMode, 'chat'>;
uploads: PlaygroundUpload[];
uploading?: boolean;
videoMode: VideoCreateMode;
onFiles?: (files: File[], targetRole?: PlaygroundUploadRole) => void;
onRemove?: (id: string) => void;
onSwapFrames?: () => void;
}) {
if (props.mode === 'video' && props.videoMode === 'first_last_frame') {
return (
<FirstLastFramePicker
accept={props.accept}
uploads={props.uploads}
uploading={props.uploading}
onFiles={props.onFiles}
onRemove={props.onRemove}
onSwapFrames={props.onSwapFrames}
/>
);
}
return (
<StackedReferencePicker
accept={props.accept}
uploads={props.uploads}
uploading={props.uploading}
onFiles={props.onFiles}
onRemove={props.onRemove}
/>
);
}
function StackedReferencePicker(props: {
accept: string;
uploads: PlaygroundUpload[];
uploading?: boolean;
onFiles?: (files: File[]) => void;
onRemove?: (id: string) => void;
}) {
const inputRef = useRef<HTMLInputElement>(null);
const [hoveredId, setHoveredId] = useState('');
const [expanded, setExpanded] = useState(false);
const hoveredUpload = props.uploads.find((item) => item.id === hoveredId);
const disabled = props.uploading || !props.onFiles;
const uploadCardIndex = props.uploads.length;
return (
<div
className="mediaReferenceStack"
data-expanded={expanded}
onMouseLeave={() => {
setExpanded(false);
setHoveredId('');
}}
>
{hoveredUpload && <div className="mediaReferenceTooltip">{hoveredUpload.name}</div>}
<div
className="mediaReferenceStackCards"
data-empty={!props.uploads.length}
style={{ '--reference-count': Math.max(1, props.uploads.length + 1) } as React.CSSProperties}
>
{props.uploads.map((item, index) => (
<div
className="mediaReferenceCard"
data-hovered={hoveredId === item.id}
data-kind={item.kind}
key={item.id}
style={referenceCardStyle(index)}
title={item.name}
onMouseEnter={() => {
setExpanded(true);
setHoveredId(item.id);
}}
>
<ReferencePreview item={item} />
{item.kind === 'video' && <span className="mediaReferenceDuration">视频</span>}
{props.onRemove && hoveredId === item.id && (
<button type="button" className="mediaReferenceRemove" aria-label={`删除 ${item.name}`} onClick={() => props.onRemove?.(item.id)}>
<X size={13} />
</button>
)}
</div>
))}
<button
type="button"
className="mediaReferenceCard mediaReferenceUploadCard"
aria-label="追加参考资源"
data-has-uploads={props.uploads.length > 0}
data-uploading={Boolean(props.uploading)}
disabled={disabled}
style={referenceCardStyle(uploadCardIndex)}
title="追加参考资源"
onClick={() => inputRef.current?.click()}
onMouseEnter={() => setHoveredId('')}
>
{props.uploading ? <LoaderCircle className="composerUploadSpinner" size={18} /> : <Plus size={20} />}
<span>参考内容</span>
</button>
{props.uploads.length > 0 && (
<button
type="button"
className="mediaReferenceAdd"
aria-label="追加参考资源"
data-uploading={Boolean(props.uploading)}
disabled={disabled}
title="追加参考资源"
onClick={() => inputRef.current?.click()}
onMouseEnter={() => {
setExpanded(false);
setHoveredId('');
}}
>
{props.uploading ? <LoaderCircle className="composerUploadSpinner" size={15} /> : <Plus size={17} />}
</button>
)}
</div>
<input
ref={inputRef}
type="file"
multiple
hidden
accept={props.accept}
disabled={disabled}
onChange={(event) => {
const files = Array.from(event.currentTarget.files ?? []);
event.currentTarget.value = '';
props.onFiles?.(files);
}}
/>
</div>
);
}
const referenceTiltValues = [-7, 6, -3, 8, -5, 4, -8, 5];
const referenceXValues = [0, -5, 6, -3, 4, -6, 3, -4];
const referenceYValues = [0, 3, -1, 4, 1, 5, 2, -2];
function referenceCardStyle(index: number) {
const valueIndex = index % referenceTiltValues.length;
return {
'--reference-index': index,
'--reference-tilt': `${referenceTiltValues[valueIndex]}deg`,
'--reference-x': `${referenceXValues[valueIndex]}px`,
'--reference-y': `${referenceYValues[valueIndex]}px`,
} as React.CSSProperties;
}
function FirstLastFramePicker(props: {
accept: string;
uploads: PlaygroundUpload[];
uploading?: boolean;
onFiles?: (files: File[], targetRole?: PlaygroundUploadRole) => void;
onRemove?: (id: string) => void;
onSwapFrames?: () => void;
}) {
const first = frameUploadByRole(props.uploads, 'first_frame');
const last = frameUploadByRole(props.uploads, 'last_frame');
const canSwap = Boolean(first && last);
return (
<div className="firstLastFramePicker">
<FrameSlot
accept={props.accept}
item={first}
label="首帧"
role="first_frame"
uploading={props.uploading}
onFiles={props.onFiles}
onRemove={props.onRemove}
/>
<button type="button" className="frameSwapButton" aria-label="交换首尾帧" disabled={!canSwap} onClick={props.onSwapFrames}>
<Repeat2 size={19} />
</button>
<FrameSlot
accept={props.accept}
item={last}
label="尾帧"
role="last_frame"
uploading={props.uploading}
onFiles={props.onFiles}
onRemove={props.onRemove}
/>
</div>
);
}
function FrameSlot(props: {
accept: string;
item?: PlaygroundUpload;
label: string;
role: PlaygroundUploadRole;
uploading?: boolean;
onFiles?: (files: File[], targetRole?: PlaygroundUploadRole) => void;
onRemove?: (id: string) => void;
}) {
const inputRef = useRef<HTMLInputElement>(null);
const disabled = props.uploading || !props.onFiles;
return (
<div className="frameSlot">
<button
type="button"
className="frameSlotButton"
data-filled={Boolean(props.item)}
disabled={disabled}
title={props.item?.name ?? props.label}
onClick={() => inputRef.current?.click()}
>
{props.item ? <ReferencePreview item={props.item} /> : <Plus size={20} />}
<span>{props.label}</span>
</button>
{props.item && props.onRemove && (
<button type="button" className="frameSlotRemove" aria-label={`删除 ${props.label}`} onClick={() => props.onRemove?.(props.item!.id)}>
<X size={13} />
</button>
)}
<input
ref={inputRef}
type="file"
hidden
accept={props.accept}
disabled={disabled}
onChange={(event) => {
const files = Array.from(event.currentTarget.files ?? []);
event.currentTarget.value = '';
props.onFiles?.(files, props.role);
}}
/>
</div>
);
}
function ReferencePreview(props: { item: PlaygroundUpload }) {
if (props.item.kind === 'image') {
return <img src={props.item.url} alt="" draggable={false} />;
}
if (props.item.kind === 'video') {
return <video src={props.item.url} muted playsInline preload="metadata" />;
}
if (props.item.kind === 'audio') {
return (
<span className="mediaReferencePlaceholder">
<Music2 size={18} />
</span>
);
}
return (
<span className="mediaReferencePlaceholder">
<FileText size={18} />
</span>
);
}
function UploadAttachmentList(props: {
message?: string;
uploads: PlaygroundUpload[];
@@ -1301,16 +1627,17 @@ function ApiKeySelect(props: {
async function uploadPlaygroundFiles(
token: string,
files: File[],
options: { allowFiles: boolean; source: string },
options: { allowFiles: boolean; allowedKinds?: PlaygroundUploadKind[]; source: string },
): Promise<{ items: PlaygroundUpload[]; warnings: string[] }> {
const allowedKinds = options.allowedKinds ?? (options.allowFiles ? ['audio', 'file', 'image', 'video'] : ['audio', 'image', 'video']);
const accepted: Array<{ file: File; kind: PlaygroundUploadKind }> = [];
const warnings: string[] = [];
files.forEach((file) => {
const kind = acceptedUploadKind(file, options.allowFiles);
if (!kind) {
if (!kind || !allowedKinds.includes(kind)) {
warnings.push(options.allowFiles
? `已跳过 ${file.name}，聊天仅支持图片、视频、音频和常见文档。`
: `已跳过 ${file.name}，当前场景仅支持图片、视频和音频。`);
: `已跳过 ${file.name}，当前场景仅支持${allowedUploadKindLabel(allowedKinds)}。`);
return;
}
accepted.push({ file, kind });
@@ -1399,7 +1726,7 @@ function promptWithUploadSummary(prompt: string, uploads: PlaygroundUpload[]) {
return `${prompt}\n\n参考附件：\n${lines.join('\n')}`;
}
function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>) {
function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>, videoMode: VideoCreateMode) {
const images = uploads.filter((item) => item.kind === 'image').map((item) => item.url);
const videos = uploads.filter((item) => item.kind === 'video').map((item) => item.url);
const audios = uploads.filter((item) => item.kind === 'audio').map((item) => item.url);
@@ -1411,19 +1738,25 @@ function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exclude<Pl
}
return payload;
}
if (videoMode === 'first_last_frame') {
const first = frameUploadByRole(uploads, 'first_frame');
const last = frameUploadByRole(uploads, 'last_frame');
if (first) {
payload.first_frame = first.url;
}
if (last) {
payload.last_frame = last.url;
}
return payload;
}
if (images.length) {
payload.image = singleOrMany(images);
payload.image_url = images[0];
payload.images = images;
payload.reference_image = singleOrMany(images);
}
if (videos.length) {
payload.reference_video = singleOrMany(videos);
payload.video_url = videos[0];
}
if (audios.length) {
payload.reference_audio = singleOrMany(audios);
payload.audio_url = audios[0];
}
return payload;
}
@@ -1439,6 +1772,11 @@ function uploadKindLabel(kind: PlaygroundUploadKind) {
return '文件';
}
function allowedUploadKindLabel(kinds: PlaygroundUploadKind[]) {
const labels = kinds.map(uploadKindLabel);
return labels.length ? labels.join('、') : '当前文件类型';
}
function formatFileSize(size: number) {
if (!Number.isFinite(size) || size <= 0) return '';
if (size < 1024) return `${size} B`;
@@ -1446,6 +1784,155 @@ function formatFileSize(size: number) {
return `${(size / 1024 / 1024).toFixed(1)} MB`;
}
function mediaPromptPlaceholder(mode: PlaygroundMode) {
if (mode === 'image') return '输入画面描述，可用 @ 快速引用图片资源，例如：让 @图像 1 保持人物一致...';
if (mode === 'video') return '输入镜头、运动和风格，可用 @ 引用图片、视频或音频资源...';
return placeholderByMode.chat;
}
function mediaUploadAcceptForMode(mode: PlaygroundMode, videoMode: VideoCreateMode) {
if (mode === 'image') return imageOnlyUploadAccept;
if (mode === 'video' && videoMode === 'first_last_frame') return imageOnlyUploadAccept;
return mediaUploadAccept;
}
function allowedMediaUploadKinds(mode: PlaygroundMode, videoMode: VideoCreateMode): PlaygroundUploadKind[] {
if (mode === 'image') return ['image'];
if (mode === 'video' && videoMode === 'first_last_frame') return ['image'];
if (mode === 'video') return ['audio', 'image', 'video'];
return ['audio', 'file', 'image', 'video'];
}
function mediaUploadSummaryMessage(uploads: PlaygroundUpload[], mode: PlaygroundMode, videoMode: VideoCreateMode) {
if (!uploads.length) return '';
const images = uploads.filter((item) => item.kind === 'image').length;
const videos = uploads.filter((item) => item.kind === 'video').length;
const audios = uploads.filter((item) => item.kind === 'audio').length;
const files = uploads.filter((item) => item.kind === 'file').length;
if (mode === 'image') {
return `已上传 ${images} 张参考图。`;
}
if (mode === 'video' && videoMode === 'first_last_frame') {
const first = frameUploadByRole(uploads, 'first_frame');
const last = frameUploadByRole(uploads, 'last_frame');
if (first && last) return '已上传首帧、尾帧参考图。';
if (first) return '已上传首帧参考图。';
if (last) return '已上传尾帧参考图。';
return `已上传 ${images} 张首尾帧参考图。`;
}
const parts = [
images ? `${images} 张图片` : '',
videos ? `${videos} 个视频` : '',
audios ? `${audios} 段音频` : '',
files ? `${files} 个文件` : '',
].filter(Boolean);
return parts.length ? `已上传 ${parts.join('、')}。` : '';
}
function mergeMediaUploadsForMode(
current: PlaygroundUpload[],
items: PlaygroundUpload[],
mode: PlaygroundMode,
videoMode: VideoCreateMode,
targetRole?: PlaygroundUploadRole,
) {
if (mode === 'image') {
return [...current.filter((item) => item.kind === 'image'), ...items.filter((item) => item.kind === 'image')];
}
if (mode === 'video' && videoMode === 'first_last_frame') {
return mergeFirstLastFrameUploads(current, items, targetRole);
}
return [...current, ...items.filter((item) => item.kind === 'image' || item.kind === 'video' || item.kind === 'audio')];
}
function normalizeFirstLastFrameUploads(uploads: PlaygroundUpload[]) {
const images = uploads.filter((item) => item.kind === 'image');
if (!images.length) return uploads.length ? [] : uploads;
const first = frameUploadByRole(images, 'first_frame') ?? images[0];
const last = frameUploadByRole(images, 'last_frame') ?? images.find((item) => item.id !== first?.id);
const next: PlaygroundUpload[] = [];
if (first) next.push({ ...first, role: 'first_frame' });
if (last) next.push({ ...last, role: 'last_frame' });
return uploadListsEqual(uploads, next) ? uploads : next;
}
function mergeFirstLastFrameUploads(current: PlaygroundUpload[], items: PlaygroundUpload[], targetRole?: PlaygroundUploadRole) {
const incoming = items.filter((item) => item.kind === 'image');
let next = normalizeFirstLastFrameUploads(current);
if (!incoming.length) return next;
const assignUpload = (item: PlaygroundUpload, role: PlaygroundUploadRole) => {
next = next.filter((upload) => upload.role !== role);
next.push({ ...item, role });
};
if (targetRole) {
assignUpload(incoming[0]!, targetRole);
const oppositeRole: PlaygroundUploadRole = targetRole === 'first_frame' ? 'last_frame' : 'first_frame';
incoming.slice(1).forEach((item) => {
if (!frameUploadByRole(next, oppositeRole)) assignUpload(item, oppositeRole);
});
return sortFrameUploads(next);
}
incoming.forEach((item) => {
if (!frameUploadByRole(next, 'first_frame')) {
assignUpload(item, 'first_frame');
} else if (!frameUploadByRole(next, 'last_frame')) {
assignUpload(item, 'last_frame');
}
});
return sortFrameUploads(next);
}
function swapFirstLastFrameUploads(uploads: PlaygroundUpload[]) {
return sortFrameUploads(uploads.map((item) => {
if (item.role === 'first_frame') return { ...item, role: 'last_frame' as const };
if (item.role === 'last_frame') return { ...item, role: 'first_frame' as const };
return item;
}));
}
function sortFrameUploads(uploads: PlaygroundUpload[]) {
const first = frameUploadByRole(uploads, 'first_frame');
const last = frameUploadByRole(uploads, 'last_frame');
return [first, last].filter((item): item is PlaygroundUpload => Boolean(item));
}
function frameUploadByRole(uploads: PlaygroundUpload[], role: PlaygroundUploadRole) {
return uploads.find((item) => item.role === role);
}
function uploadListsEqual(left: PlaygroundUpload[], right: PlaygroundUpload[]) {
if (left.length !== right.length) return false;
return left.every((item, index) => {
const next = right[index];
return next && item.id === next.id && item.role === next.role && item.kind === next.kind;
});
}
function videoModeRequestPayload(
mode: Exclude<PlaygroundMode, 'chat'>,
videoMode: VideoCreateMode,
uploads: PlaygroundUpload[],
modelOption?: ModelOption,
) {
if (mode !== 'video') return {};
const modelTypes = new Set(modelOption?.models.flatMap((model) => model.modelType) ?? []);
if (videoMode === 'first_last_frame') {
const modelType = modelTypes.has('video_first_last_frame') ? 'video_first_last_frame' : 'image_to_video';
return { capabilityType: modelType, model_type: modelType };
}
if (videoMode === 'omni_reference' || uploads.length > 0) {
const modelType = modelTypes.has('omni_video')
? 'omni_video'
: modelTypes.has('video_reference')
? 'video_reference'
: modelTypes.has('image_to_video')
? 'image_to_video'
: 'video_generate';
return { capabilityType: modelType, model_type: modelType };
}
return {};
}
function filterModelsForMode(models: PlatformModel[], mode: PlaygroundMode, hasReference: boolean, videoMode: VideoCreateMode) {
if (mode === 'chat') {
return filterWithFallback(models, ['text_generate', 'chat', 'responses', 'text']);
@@ -1455,7 +1942,7 @@ function filterModelsForMode(models: PlatformModel[], mode: PlaygroundMode, hasR
return filterWithFallback(models, [...preferredTypes, 'image']);
}
const videoTypesByMode: Record<VideoCreateMode, string[]> = {
first_last_frame: ['image_to_video', 'video_first_last_frame', 'video_generate'],
first_last_frame: ['video_first_last_frame', 'image_to_video', 'video_generate'],
omni_reference: ['omni_video', 'video_reference', 'video_generate'],
text_to_video: ['text_to_video', 'video_generate'],
};