diff --git a/apps/web/src/pages/PlaygroundPage.tsx b/apps/web/src/pages/PlaygroundPage.tsx index 7d9432e..9c06163 100644 --- a/apps/web/src/pages/PlaygroundPage.tsx +++ b/apps/web/src/pages/PlaygroundPage.tsx @@ -2,10 +2,11 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import type { GatewayApiKey, GatewayTask, PlatformModel } from '@easyai-ai-gateway/contracts'; import { ChevronDown, MessageSquarePlus, Send, Sparkles } from 'lucide-react'; import { Badge, Button, Select, Textarea } from '../components/ui'; -import { createImageEditTask, createImageGenerationTask, createVideoGenerationTask, pollTaskUntilSettled, taskIsPending } from '../api'; +import { createImageEditTask, createImageGenerationTask, createVideoGenerationTask, pollTaskUntilSettled, resolveApiAssetUrl, taskIsPending } from '../api'; import type { PlaygroundMode } from '../types'; import { PlaygroundPromptMentionInput, + buildPlaygroundResourceToken, removeInvalidPlaygroundResourceTokens, replacePlaygroundResourceTokens, } from './playground-prompt-mention'; @@ -240,6 +241,8 @@ export function PlaygroundPage(props: { mode?: Exclude; prompt?: string; settings?: MediaGenerationSettings; + uploads?: PlaygroundUpload[]; + videoMode?: VideoCreateMode; }) { const runMode = overrides?.mode ?? props.mode; if (runMode === 'chat') return; @@ -266,7 +269,8 @@ export function PlaygroundPage(props: { } const localId = newLocalId(); - const runUploads = overrides ? [] : mediaUploads; + const runUploads = sanitizeMediaRunUploads(overrides?.uploads ?? mediaUploads); + const runVideoMode = overrides?.videoMode ?? videoMode; const runModelOption = modelOptions.find((item) => item.value === runModel); const modelLabel = runModelOption?.label ?? runModel; const run: MediaGenerationRun = { @@ -278,6 +282,8 @@ export function PlaygroundPage(props: { prompt: trimmedPrompt, settings: runSettings, status: 'submitting', + uploads: runUploads, + videoMode: runMode === 'video' ? runVideoMode : undefined, }; setMediaRuns((current) => [...current, run]); @@ -288,7 +294,7 @@ export function PlaygroundPage(props: { if (runMode === 'video') { response = await createVideoGenerationTask(credential, { model: runModel, - content: sharedVideoGenerationContentFromPromptAndUploads(requestPrompt, runUploads, videoMode), + content: sharedVideoGenerationContentFromPromptAndUploads(requestPrompt, runUploads, runVideoMode), ...mediaRequestPayload(runSettings, 'video'), }); } else { @@ -353,8 +359,15 @@ export function PlaygroundPage(props: { } function editMediaRun(run: MediaGenerationRun) { - setPrompt(run.prompt); + const runUploads = editableMediaRunUploads(run); + const editablePrompt = editableMediaRunPrompt(run, runUploads); + setPrompt(editablePrompt); setMediaSettings(run.settings); + setMediaUploads(runUploads); + setImageHasReference(run.mode === 'image' && runUploads.some((item) => item.kind === 'image')); + if (run.mode === 'video') { + setVideoMode(run.videoMode ?? inferVideoModeFromUploads(runUploads)); + } selectMediaRunModel(run); if (props.mode !== run.mode) { props.onModeChange(run.mode); @@ -363,15 +376,23 @@ export function PlaygroundPage(props: { } function rerunMediaRun(run: MediaGenerationRun) { - setPrompt(run.prompt); + const runUploads = editableMediaRunUploads(run); + const editablePrompt = editableMediaRunPrompt(run, runUploads); + const runVideoMode = run.videoMode ?? inferVideoModeFromUploads(runUploads); + setPrompt(editablePrompt); setMediaSettings(run.settings); + setMediaUploads(runUploads); + setImageHasReference(run.mode === 'image' && runUploads.some((item) => item.kind === 'image')); + if (run.mode === 'video') { + setVideoMode(runVideoMode); + } const runModel = selectMediaRunModel(run); if (props.mode !== run.mode) { props.onModeChange(run.mode); setMediaMessage('已切换到对应模式并带入模型和参数，请确认后再次生成。'); return; } - void submitMediaTask({ mode: run.mode, model: runModel, prompt: run.prompt, settings: run.settings }); + void submitMediaTask({ mode: run.mode, model: runModel, prompt: editablePrompt, settings: run.settings, uploads: runUploads, videoMode: runVideoMode }); } const mediaComposer = props.mode === 'chat' ? null : ( @@ -676,8 +697,8 @@ function Composer(props: { } function mediaPromptPlaceholder(mode: PlaygroundMode) { - if (mode === 'image') return '输入画面描述，可用 @ 快速引用图片资源，例如：让 @图像 1 保持人物一致...'; - if (mode === 'video') return '输入镜头、运动和风格，可用 @ 引用图片、视频或音频资源...'; + if (mode === 'image') return '输入画面描述，可用 @ 或 @资产 快速引用图片资源，例如：让 @图像 1 保持人物一致...'; + if (mode === 'video') return '输入镜头、运动和风格，可用 @ 或 @资产 引用图片、视频或音频资源...'; return placeholderByMode.chat; } @@ -771,6 +792,211 @@ function updateMediaRun(runs: MediaGenerationRun[], localId: string, patch: Part return runs.map((run) => run.localId === localId ? { ...run, ...patch } : run); } +const playgroundResourceTokenPattern = /<<]+)>>>/g; + +function editableMediaRunUploads(run: MediaGenerationRun): PlaygroundUpload[] { + const storedUploads = sanitizeMediaRunUploads(run.uploads ?? []); + if (storedUploads.length) return storedUploads; + return mediaRunUploadsFromTaskRequest(run.task?.request, run.mode); +} + +function editableMediaRunPrompt(run: MediaGenerationRun, uploads: PlaygroundUpload[]) { + return restorePromptResourceTokens(run.prompt, uploads, run.mode); +} + +function restorePromptResourceTokens(raw: string, uploads: PlaygroundUpload[], mode: Exclude) { + if (!uploads.length) return raw; + if (raw.includes('<< [item.id, item])); + const usedIds = new Set(); + let fallbackIndex = 0; + return raw.replace(playgroundResourceTokenPattern, (full, id: string) => { + let item = byId.get(id); + if (item) { + usedIds.add(item.id); + } else { + while (fallbackIndex < uploads.length && usedIds.has(uploads[fallbackIndex]!.id)) { + fallbackIndex += 1; + } + item = uploads[fallbackIndex]; + fallbackIndex += 1; + if (item) usedIds.add(item.id); + } + return item ? buildPlaygroundResourceToken(item.id) : full; + }); +} + +function restorePromptLabelsToResourceTokens(raw: string, uploads: PlaygroundUpload[], mode: Exclude) { + return uploads.reduce((text, item) => { + const index = uploadKindIndex(item, uploads); + return promptResourceRestorePatterns(item.kind, index, mode).reduce( + (current, pattern) => current.replace(pattern, buildPlaygroundResourceToken(item.id)), + text, + ); + }, raw); +} + +function promptResourceRestorePatterns(kind: PlaygroundUpload['kind'], index: number, mode: Exclude) { + const englishKind = mode === 'image' ? 'image' : uploadKindEnglish(kind); + const patterns = [ + new RegExp(`\\b${englishKind}\\s+${index}\\b`, 'g'), + ]; + if (kind === 'image') { + patterns.push(new RegExp(`@(?:图像|图片)\\s*${index}(?!\\d)`, 'g')); + } else { + patterns.push(new RegExp(`@${uploadKindTitle(kind)}\\s*${index}(?!\\d)`, 'g')); + } + patterns.push(new RegExp(`@(?:素材|资产)\\s*${index}(?!\\d)`, 'g')); + return patterns; +} + +function mediaRunUploadsFromTaskRequest(request: unknown, mode: Exclude): PlaygroundUpload[] { + const record = recordFromUnknown(request); + if (!record) return []; + const uploads: PlaygroundUpload[] = []; + if (Array.isArray(record.content)) { + record.content.forEach((item) => { + const part = recordFromUnknown(item); + if (!part) return; + appendUploadFromContentPart(uploads, part); + }); + } + if (mode === 'image') { + appendImageUploadsFromRequestValue(uploads, record.image); + appendImageUploadsFromRequestValue(uploads, record.images); + appendImageUploadsFromRequestValue(uploads, record.input_image); + appendImageUploadsFromRequestValue(uploads, record.input_images); + } + return dedupeMediaRunUploads(uploads); +} + +function appendUploadFromContentPart(uploads: PlaygroundUpload[], part: Record) { + const role = part.role === 'first_frame' || part.role === 'last_frame' ? part.role : undefined; + const type = stringFromUnknown(part.type); + if (type === 'image_url') { + appendMediaRunUploadUrl(uploads, 'image', firstString(nestedString(part.image_url, 'url'), part.url), role); + return; + } + if (type === 'video_url') { + appendMediaRunUploadUrl(uploads, 'video', firstString(nestedString(part.video_url, 'url'), part.url), role); + return; + } + if (type === 'audio_url') { + appendMediaRunUploadUrl(uploads, 'audio', firstString(nestedString(part.audio_url, 'url'), part.url), role); + } +} + +function appendImageUploadsFromRequestValue(uploads: PlaygroundUpload[], value: unknown) { + if (!value) return; + if (typeof value === 'string') { + appendMediaRunUploadUrl(uploads, 'image', value); + return; + } + if (Array.isArray(value)) { + value.forEach((item) => appendImageUploadsFromRequestValue(uploads, item)); + return; + } + const record = recordFromUnknown(value); + if (!record) return; + appendMediaRunUploadUrl(uploads, 'image', firstString(record.url, nestedString(record.image_url, 'url'), record.image_url, record.imageUrl, record.path)); +} + +function appendMediaRunUploadUrl(uploads: PlaygroundUpload[], kind: PlaygroundUpload['kind'], value: unknown, role?: PlaygroundUpload['role']) { + const rawUrl = stringFromUnknown(value); + if (!rawUrl) return; + const url = resolveApiAssetUrl(rawUrl); + uploads.push({ + contentType: '', + id: `${kind}-${uploads.length}-${url}`, + kind, + name: `${uploadKindTitle(kind)} ${uploads.filter((item) => item.kind === kind).length + 1}`, + raw: {}, + role, + size: 0, + url, + }); +} + +function dedupeMediaRunUploads(uploads: PlaygroundUpload[]) { + const seen = new Set(); + return uploads.filter((item) => { + const key = `${item.kind}:${item.url}:${item.role ?? ''}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function sanitizeMediaRunUploads(uploads: unknown[]): PlaygroundUpload[] { + return uploads + .map((item, index) => sanitizeMediaRunUpload(item, index)) + .filter((item): item is PlaygroundUpload => Boolean(item)); +} + +function sanitizeMediaRunUpload(value: unknown, index: number): PlaygroundUpload | undefined { + const record = recordFromUnknown(value); + if (!record) return undefined; + const url = stringFromUnknown(record.url); + const kind = uploadKindFromUnknown(record.kind); + if (!url || !kind) return undefined; + const size = numericFromUnknown(record.size); + return { + contentType: stringFromUnknown(record.contentType), + id: stringFromUnknown(record.id) || `${kind}-${index}-${url}`, + kind, + name: stringFromUnknown(record.name) || `${uploadKindTitle(kind)} ${index + 1}`, + raw: recordFromUnknown(record.raw) ?? {}, + role: record.role === 'first_frame' || record.role === 'last_frame' ? record.role : undefined, + size: size && size > 0 ? Math.round(size) : 0, + url, + }; +} + +function numericFromUnknown(value: unknown) { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (typeof value === 'string' && value.trim()) { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + return undefined; +} + +function uploadKindFromUnknown(value: unknown): PlaygroundUpload['kind'] | undefined { + if (value === 'audio' || value === 'file' || value === 'image' || value === 'video') return value; + return undefined; +} + +function uploadKindTitle(kind: PlaygroundUpload['kind']) { + if (kind === 'image') return '图像'; + if (kind === 'video') return '视频'; + if (kind === 'audio') return '音频'; + return '文件'; +} + +function uploadKindEnglish(kind: PlaygroundUpload['kind']) { + if (kind === 'image') return 'image'; + if (kind === 'video') return 'video'; + if (kind === 'audio') return 'audio'; + return 'file'; +} + +function uploadKindIndex(item: PlaygroundUpload, uploads: PlaygroundUpload[]) { + const sameKind = uploads.filter((upload) => upload.kind === item.kind); + return Math.max(1, sameKind.findIndex((upload) => upload.id === item.id) + 1); +} + +function inferVideoModeFromUploads(uploads: PlaygroundUpload[]): VideoCreateMode { + if (uploads.some((item) => item.role === 'first_frame' || item.role === 'last_frame')) return 'first_last_frame'; + if (uploads.length > 0) return 'omni_reference'; + return 'text_to_video'; +} + function readStoredMediaRuns(): MediaGenerationRun[] { if (typeof window === 'undefined') return []; try { @@ -796,7 +1022,9 @@ function writeStoredMediaRuns(runs: MediaGenerationRun[]) { window.localStorage.removeItem(MEDIA_RUNS_STORAGE_KEY); return; } - const storedRuns = sortMediaRunsByCreatedAt(runs).slice(-MEDIA_RUNS_STORAGE_LIMIT); + const storedRuns = sortMediaRunsByCreatedAt(runs) + .slice(-MEDIA_RUNS_STORAGE_LIMIT) + .map((run) => ({ ...run, uploads: sanitizeMediaRunUploads(run.uploads ?? []) })); window.localStorage.setItem(MEDIA_RUNS_STORAGE_KEY, JSON.stringify({ runs: storedRuns, version: 1, @@ -823,6 +1051,7 @@ function mediaRunFromStorage(value: unknown, index: number): MediaGenerationRun status = 'failed'; error = error || '任务提交已中断，请重新生成。'; } + const uploads = sanitizeMediaRunUploads(Array.isArray(record.uploads) ? record.uploads : []); return { createdAt, error, @@ -834,6 +1063,8 @@ function mediaRunFromStorage(value: unknown, index: number): MediaGenerationRun settings: mediaSettingsFromStorage(record.settings), status, task, + uploads, + videoMode: mode === 'video' ? videoModeFromStorage(record.videoMode, uploads) : undefined, }; } @@ -858,6 +1089,11 @@ function mediaSettingsFromStorage(value: unknown): MediaGenerationSettings { }; } +function videoModeFromStorage(value: unknown, uploads: PlaygroundUpload[]): VideoCreateMode { + if (value === 'text_to_video' || value === 'first_last_frame' || value === 'omni_reference') return value; + return inferVideoModeFromUploads(uploads); +} + function countPresetFromStorage(value: unknown, fallback: MediaGenerationSettings['countPreset']) { if (value === 'custom') return value; const numeric = Number(value); @@ -880,6 +1116,11 @@ function stringFromUnknown(value: unknown) { return typeof value === 'string' ? value.trim() : ''; } +function nestedString(value: unknown, key: string) { + const record = recordFromUnknown(value); + return record ? stringFromUnknown(record[key]) : ''; +} + function dateStringFromUnknown(value: unknown) { const raw = stringFromUnknown(value); if (!raw) return undefined; diff --git a/apps/web/src/pages/playground-media.tsx b/apps/web/src/pages/playground-media.tsx index c042b7e..2a25ec8 100644 --- a/apps/web/src/pages/playground-media.tsx +++ b/apps/web/src/pages/playground-media.tsx @@ -5,16 +5,19 @@ import Slider from 'antd/es/slider'; import { Download, Edit3, + FileText, Image as ImageIcon, Images, Link2, LoaderCircle, + Music2, Sparkles, Square, } from 'lucide-react'; import { resolveApiAssetUrl } from '../api'; import { Button, Input, Popover, PopoverContent, PopoverTrigger } from '../components/ui'; import type { PlaygroundMode } from '../types'; +import type { PlaygroundUpload, PlaygroundUploadKind, PlaygroundVideoCreateMode } from './playground-upload'; export type MediaOutputMode = 'single' | 'group'; export type MediaCountPreset = 1 | 2 | 3 | 4 | 'custom'; @@ -46,6 +49,8 @@ export interface MediaGenerationRun { settings: MediaGenerationSettings; status: GatewayTask['status'] | 'submitting'; task?: GatewayTask; + uploads?: PlaygroundUpload[]; + videoMode?: PlaygroundVideoCreateMode; } export function gatewayTaskErrorText(task: GatewayTask | undefined, fallback = '任务失败') { @@ -513,20 +518,40 @@ function MediaTaskCard(props: { const isPending = props.run.status === 'submitting' || props.run.status === 'queued' || props.run.status === 'running'; const backdropItem = expectedCount === 1 && items[0]?.type === 'image' ? items[0] : undefined; const errorText = mediaRunErrorText(props.run); + const references = mediaReferenceItems(props.run); + const promptParts = promptDisplayParts(props.run.prompt, references); + const taskMeta = mediaTaskMetaText(props.run); return (
-
-

- {props.run.prompt} - {props.run.mode === 'video' ? '视频' : '图片'} {props.run.modelLabel} {props.run.settings.aspectRatio} {props.run.settings.resolution} -

- +
+ {references.length > 0 && } +
+
+ + + + {props.run.modelLabel} + {props.run.mode === 'video' ? '视频生成' : references.length ? '图像编辑' : '图像生成'} + + {taskMeta} +
+
+ {promptParts} +
+
{status}
+ {errorText && ( +
+ 错误详情 + {errorText} +
+ )} +
{backdropItem && }
@@ -543,12 +568,6 @@ function MediaTaskCard(props: {
- {errorText && ( -
- 错误详情 - {errorText} -
- )}