feat(web): improve playground media resource prompts
This commit is contained in:
@@ -2,10 +2,11 @@ import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
import type { GatewayApiKey, GatewayTask, PlatformModel } from '@easyai-ai-gateway/contracts';
|
||||
import { ChevronDown, MessageSquarePlus, Send, Sparkles } from 'lucide-react';
|
||||
import { Badge, Button, Select, Textarea } from '../components/ui';
|
||||
import { createImageEditTask, createImageGenerationTask, createVideoGenerationTask, pollTaskUntilSettled, taskIsPending } from '../api';
|
||||
import { createImageEditTask, createImageGenerationTask, createVideoGenerationTask, pollTaskUntilSettled, resolveApiAssetUrl, taskIsPending } from '../api';
|
||||
import type { PlaygroundMode } from '../types';
|
||||
import {
|
||||
PlaygroundPromptMentionInput,
|
||||
buildPlaygroundResourceToken,
|
||||
removeInvalidPlaygroundResourceTokens,
|
||||
replacePlaygroundResourceTokens,
|
||||
} from './playground-prompt-mention';
|
||||
@@ -240,6 +241,8 @@ export function PlaygroundPage(props: {
|
||||
mode?: Exclude<PlaygroundMode, 'chat'>;
|
||||
prompt?: string;
|
||||
settings?: MediaGenerationSettings;
|
||||
uploads?: PlaygroundUpload[];
|
||||
videoMode?: VideoCreateMode;
|
||||
}) {
|
||||
const runMode = overrides?.mode ?? props.mode;
|
||||
if (runMode === 'chat') return;
|
||||
@@ -266,7 +269,8 @@ export function PlaygroundPage(props: {
|
||||
}
|
||||
|
||||
const localId = newLocalId();
|
||||
const runUploads = overrides ? [] : mediaUploads;
|
||||
const runUploads = sanitizeMediaRunUploads(overrides?.uploads ?? mediaUploads);
|
||||
const runVideoMode = overrides?.videoMode ?? videoMode;
|
||||
const runModelOption = modelOptions.find((item) => item.value === runModel);
|
||||
const modelLabel = runModelOption?.label ?? runModel;
|
||||
const run: MediaGenerationRun = {
|
||||
@@ -278,6 +282,8 @@ export function PlaygroundPage(props: {
|
||||
prompt: trimmedPrompt,
|
||||
settings: runSettings,
|
||||
status: 'submitting',
|
||||
uploads: runUploads,
|
||||
videoMode: runMode === 'video' ? runVideoMode : undefined,
|
||||
};
|
||||
|
||||
setMediaRuns((current) => [...current, run]);
|
||||
@@ -288,7 +294,7 @@ export function PlaygroundPage(props: {
|
||||
if (runMode === 'video') {
|
||||
response = await createVideoGenerationTask(credential, {
|
||||
model: runModel,
|
||||
content: sharedVideoGenerationContentFromPromptAndUploads(requestPrompt, runUploads, videoMode),
|
||||
content: sharedVideoGenerationContentFromPromptAndUploads(requestPrompt, runUploads, runVideoMode),
|
||||
...mediaRequestPayload(runSettings, 'video'),
|
||||
});
|
||||
} else {
|
||||
@@ -353,8 +359,15 @@ export function PlaygroundPage(props: {
|
||||
}
|
||||
|
||||
function editMediaRun(run: MediaGenerationRun) {
|
||||
setPrompt(run.prompt);
|
||||
const runUploads = editableMediaRunUploads(run);
|
||||
const editablePrompt = editableMediaRunPrompt(run, runUploads);
|
||||
setPrompt(editablePrompt);
|
||||
setMediaSettings(run.settings);
|
||||
setMediaUploads(runUploads);
|
||||
setImageHasReference(run.mode === 'image' && runUploads.some((item) => item.kind === 'image'));
|
||||
if (run.mode === 'video') {
|
||||
setVideoMode(run.videoMode ?? inferVideoModeFromUploads(runUploads));
|
||||
}
|
||||
selectMediaRunModel(run);
|
||||
if (props.mode !== run.mode) {
|
||||
props.onModeChange(run.mode);
|
||||
@@ -363,15 +376,23 @@ export function PlaygroundPage(props: {
|
||||
}
|
||||
|
||||
function rerunMediaRun(run: MediaGenerationRun) {
|
||||
setPrompt(run.prompt);
|
||||
const runUploads = editableMediaRunUploads(run);
|
||||
const editablePrompt = editableMediaRunPrompt(run, runUploads);
|
||||
const runVideoMode = run.videoMode ?? inferVideoModeFromUploads(runUploads);
|
||||
setPrompt(editablePrompt);
|
||||
setMediaSettings(run.settings);
|
||||
setMediaUploads(runUploads);
|
||||
setImageHasReference(run.mode === 'image' && runUploads.some((item) => item.kind === 'image'));
|
||||
if (run.mode === 'video') {
|
||||
setVideoMode(runVideoMode);
|
||||
}
|
||||
const runModel = selectMediaRunModel(run);
|
||||
if (props.mode !== run.mode) {
|
||||
props.onModeChange(run.mode);
|
||||
setMediaMessage('已切换到对应模式并带入模型和参数，请确认后再次生成。');
|
||||
return;
|
||||
}
|
||||
void submitMediaTask({ mode: run.mode, model: runModel, prompt: run.prompt, settings: run.settings });
|
||||
void submitMediaTask({ mode: run.mode, model: runModel, prompt: editablePrompt, settings: run.settings, uploads: runUploads, videoMode: runVideoMode });
|
||||
}
|
||||
|
||||
const mediaComposer = props.mode === 'chat' ? null : (
|
||||
@@ -676,8 +697,8 @@ function Composer(props: {
|
||||
}
|
||||
|
||||
function mediaPromptPlaceholder(mode: PlaygroundMode) {
|
||||
if (mode === 'image') return '输入画面描述，可用 @ 快速引用图片资源，例如：让 @图像 1 保持人物一致...';
|
||||
if (mode === 'video') return '输入镜头、运动和风格，可用 @ 引用图片、视频或音频资源...';
|
||||
if (mode === 'image') return '输入画面描述，可用 @ 或 @资产 快速引用图片资源，例如：让 @图像 1 保持人物一致...';
|
||||
if (mode === 'video') return '输入镜头、运动和风格，可用 @ 或 @资产 引用图片、视频或音频资源...';
|
||||
return placeholderByMode.chat;
|
||||
}
|
||||
|
||||
@@ -771,6 +792,211 @@ function updateMediaRun(runs: MediaGenerationRun[], localId: string, patch: Part
|
||||
return runs.map((run) => run.localId === localId ? { ...run, ...patch } : run);
|
||||
}
|
||||
|
||||
const playgroundResourceTokenPattern = /<<<playground-resource:([^>]+)>>>/g;
|
||||
|
||||
function editableMediaRunUploads(run: MediaGenerationRun): PlaygroundUpload[] {
|
||||
const storedUploads = sanitizeMediaRunUploads(run.uploads ?? []);
|
||||
if (storedUploads.length) return storedUploads;
|
||||
return mediaRunUploadsFromTaskRequest(run.task?.request, run.mode);
|
||||
}
|
||||
|
||||
function editableMediaRunPrompt(run: MediaGenerationRun, uploads: PlaygroundUpload[]) {
|
||||
return restorePromptResourceTokens(run.prompt, uploads, run.mode);
|
||||
}
|
||||
|
||||
function restorePromptResourceTokens(raw: string, uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>) {
|
||||
if (!uploads.length) return raw;
|
||||
if (raw.includes('<<<playground-resource:')) {
|
||||
return restoreSerializedPromptResourceTokens(raw, uploads);
|
||||
}
|
||||
return restorePromptLabelsToResourceTokens(raw, uploads, mode);
|
||||
}
|
||||
|
||||
function restoreSerializedPromptResourceTokens(raw: string, uploads: PlaygroundUpload[]) {
|
||||
const byId = new Map(uploads.map((item) => [item.id, item]));
|
||||
const usedIds = new Set<string>();
|
||||
let fallbackIndex = 0;
|
||||
return raw.replace(playgroundResourceTokenPattern, (full, id: string) => {
|
||||
let item = byId.get(id);
|
||||
if (item) {
|
||||
usedIds.add(item.id);
|
||||
} else {
|
||||
while (fallbackIndex < uploads.length && usedIds.has(uploads[fallbackIndex]!.id)) {
|
||||
fallbackIndex += 1;
|
||||
}
|
||||
item = uploads[fallbackIndex];
|
||||
fallbackIndex += 1;
|
||||
if (item) usedIds.add(item.id);
|
||||
}
|
||||
return item ? buildPlaygroundResourceToken(item.id) : full;
|
||||
});
|
||||
}
|
||||
|
||||
function restorePromptLabelsToResourceTokens(raw: string, uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>) {
|
||||
return uploads.reduce((text, item) => {
|
||||
const index = uploadKindIndex(item, uploads);
|
||||
return promptResourceRestorePatterns(item.kind, index, mode).reduce(
|
||||
(current, pattern) => current.replace(pattern, buildPlaygroundResourceToken(item.id)),
|
||||
text,
|
||||
);
|
||||
}, raw);
|
||||
}
|
||||
|
||||
function promptResourceRestorePatterns(kind: PlaygroundUpload['kind'], index: number, mode: Exclude<PlaygroundMode, 'chat'>) {
|
||||
const englishKind = mode === 'image' ? 'image' : uploadKindEnglish(kind);
|
||||
const patterns = [
|
||||
new RegExp(`\\b${englishKind}\\s+${index}\\b`, 'g'),
|
||||
];
|
||||
if (kind === 'image') {
|
||||
patterns.push(new RegExp(`@(?:图像|图片)\\s*${index}(?!\\d)`, 'g'));
|
||||
} else {
|
||||
patterns.push(new RegExp(`@${uploadKindTitle(kind)}\\s*${index}(?!\\d)`, 'g'));
|
||||
}
|
||||
patterns.push(new RegExp(`@(?:素材|资产)\\s*${index}(?!\\d)`, 'g'));
|
||||
return patterns;
|
||||
}
|
||||
|
||||
function mediaRunUploadsFromTaskRequest(request: unknown, mode: Exclude<PlaygroundMode, 'chat'>): PlaygroundUpload[] {
|
||||
const record = recordFromUnknown(request);
|
||||
if (!record) return [];
|
||||
const uploads: PlaygroundUpload[] = [];
|
||||
if (Array.isArray(record.content)) {
|
||||
record.content.forEach((item) => {
|
||||
const part = recordFromUnknown(item);
|
||||
if (!part) return;
|
||||
appendUploadFromContentPart(uploads, part);
|
||||
});
|
||||
}
|
||||
if (mode === 'image') {
|
||||
appendImageUploadsFromRequestValue(uploads, record.image);
|
||||
appendImageUploadsFromRequestValue(uploads, record.images);
|
||||
appendImageUploadsFromRequestValue(uploads, record.input_image);
|
||||
appendImageUploadsFromRequestValue(uploads, record.input_images);
|
||||
}
|
||||
return dedupeMediaRunUploads(uploads);
|
||||
}
|
||||
|
||||
function appendUploadFromContentPart(uploads: PlaygroundUpload[], part: Record<string, unknown>) {
|
||||
const role = part.role === 'first_frame' || part.role === 'last_frame' ? part.role : undefined;
|
||||
const type = stringFromUnknown(part.type);
|
||||
if (type === 'image_url') {
|
||||
appendMediaRunUploadUrl(uploads, 'image', firstString(nestedString(part.image_url, 'url'), part.url), role);
|
||||
return;
|
||||
}
|
||||
if (type === 'video_url') {
|
||||
appendMediaRunUploadUrl(uploads, 'video', firstString(nestedString(part.video_url, 'url'), part.url), role);
|
||||
return;
|
||||
}
|
||||
if (type === 'audio_url') {
|
||||
appendMediaRunUploadUrl(uploads, 'audio', firstString(nestedString(part.audio_url, 'url'), part.url), role);
|
||||
}
|
||||
}
|
||||
|
||||
function appendImageUploadsFromRequestValue(uploads: PlaygroundUpload[], value: unknown) {
|
||||
if (!value) return;
|
||||
if (typeof value === 'string') {
|
||||
appendMediaRunUploadUrl(uploads, 'image', value);
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((item) => appendImageUploadsFromRequestValue(uploads, item));
|
||||
return;
|
||||
}
|
||||
const record = recordFromUnknown(value);
|
||||
if (!record) return;
|
||||
appendMediaRunUploadUrl(uploads, 'image', firstString(record.url, nestedString(record.image_url, 'url'), record.image_url, record.imageUrl, record.path));
|
||||
}
|
||||
|
||||
function appendMediaRunUploadUrl(uploads: PlaygroundUpload[], kind: PlaygroundUpload['kind'], value: unknown, role?: PlaygroundUpload['role']) {
|
||||
const rawUrl = stringFromUnknown(value);
|
||||
if (!rawUrl) return;
|
||||
const url = resolveApiAssetUrl(rawUrl);
|
||||
uploads.push({
|
||||
contentType: '',
|
||||
id: `${kind}-${uploads.length}-${url}`,
|
||||
kind,
|
||||
name: `${uploadKindTitle(kind)} ${uploads.filter((item) => item.kind === kind).length + 1}`,
|
||||
raw: {},
|
||||
role,
|
||||
size: 0,
|
||||
url,
|
||||
});
|
||||
}
|
||||
|
||||
function dedupeMediaRunUploads(uploads: PlaygroundUpload[]) {
|
||||
const seen = new Set<string>();
|
||||
return uploads.filter((item) => {
|
||||
const key = `${item.kind}:${item.url}:${item.role ?? ''}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function sanitizeMediaRunUploads(uploads: unknown[]): PlaygroundUpload[] {
|
||||
return uploads
|
||||
.map((item, index) => sanitizeMediaRunUpload(item, index))
|
||||
.filter((item): item is PlaygroundUpload => Boolean(item));
|
||||
}
|
||||
|
||||
function sanitizeMediaRunUpload(value: unknown, index: number): PlaygroundUpload | undefined {
|
||||
const record = recordFromUnknown(value);
|
||||
if (!record) return undefined;
|
||||
const url = stringFromUnknown(record.url);
|
||||
const kind = uploadKindFromUnknown(record.kind);
|
||||
if (!url || !kind) return undefined;
|
||||
const size = numericFromUnknown(record.size);
|
||||
return {
|
||||
contentType: stringFromUnknown(record.contentType),
|
||||
id: stringFromUnknown(record.id) || `${kind}-${index}-${url}`,
|
||||
kind,
|
||||
name: stringFromUnknown(record.name) || `${uploadKindTitle(kind)} ${index + 1}`,
|
||||
raw: recordFromUnknown(record.raw) ?? {},
|
||||
role: record.role === 'first_frame' || record.role === 'last_frame' ? record.role : undefined,
|
||||
size: size && size > 0 ? Math.round(size) : 0,
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
function numericFromUnknown(value: unknown) {
|
||||
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed)) return parsed;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function uploadKindFromUnknown(value: unknown): PlaygroundUpload['kind'] | undefined {
|
||||
if (value === 'audio' || value === 'file' || value === 'image' || value === 'video') return value;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function uploadKindTitle(kind: PlaygroundUpload['kind']) {
|
||||
if (kind === 'image') return '图像';
|
||||
if (kind === 'video') return '视频';
|
||||
if (kind === 'audio') return '音频';
|
||||
return '文件';
|
||||
}
|
||||
|
||||
function uploadKindEnglish(kind: PlaygroundUpload['kind']) {
|
||||
if (kind === 'image') return 'image';
|
||||
if (kind === 'video') return 'video';
|
||||
if (kind === 'audio') return 'audio';
|
||||
return 'file';
|
||||
}
|
||||
|
||||
function uploadKindIndex(item: PlaygroundUpload, uploads: PlaygroundUpload[]) {
|
||||
const sameKind = uploads.filter((upload) => upload.kind === item.kind);
|
||||
return Math.max(1, sameKind.findIndex((upload) => upload.id === item.id) + 1);
|
||||
}
|
||||
|
||||
function inferVideoModeFromUploads(uploads: PlaygroundUpload[]): VideoCreateMode {
|
||||
if (uploads.some((item) => item.role === 'first_frame' || item.role === 'last_frame')) return 'first_last_frame';
|
||||
if (uploads.length > 0) return 'omni_reference';
|
||||
return 'text_to_video';
|
||||
}
|
||||
|
||||
function readStoredMediaRuns(): MediaGenerationRun[] {
|
||||
if (typeof window === 'undefined') return [];
|
||||
try {
|
||||
@@ -796,7 +1022,9 @@ function writeStoredMediaRuns(runs: MediaGenerationRun[]) {
|
||||
window.localStorage.removeItem(MEDIA_RUNS_STORAGE_KEY);
|
||||
return;
|
||||
}
|
||||
const storedRuns = sortMediaRunsByCreatedAt(runs).slice(-MEDIA_RUNS_STORAGE_LIMIT);
|
||||
const storedRuns = sortMediaRunsByCreatedAt(runs)
|
||||
.slice(-MEDIA_RUNS_STORAGE_LIMIT)
|
||||
.map((run) => ({ ...run, uploads: sanitizeMediaRunUploads(run.uploads ?? []) }));
|
||||
window.localStorage.setItem(MEDIA_RUNS_STORAGE_KEY, JSON.stringify({
|
||||
runs: storedRuns,
|
||||
version: 1,
|
||||
@@ -823,6 +1051,7 @@ function mediaRunFromStorage(value: unknown, index: number): MediaGenerationRun
|
||||
status = 'failed';
|
||||
error = error || '任务提交已中断，请重新生成。';
|
||||
}
|
||||
const uploads = sanitizeMediaRunUploads(Array.isArray(record.uploads) ? record.uploads : []);
|
||||
return {
|
||||
createdAt,
|
||||
error,
|
||||
@@ -834,6 +1063,8 @@ function mediaRunFromStorage(value: unknown, index: number): MediaGenerationRun
|
||||
settings: mediaSettingsFromStorage(record.settings),
|
||||
status,
|
||||
task,
|
||||
uploads,
|
||||
videoMode: mode === 'video' ? videoModeFromStorage(record.videoMode, uploads) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -858,6 +1089,11 @@ function mediaSettingsFromStorage(value: unknown): MediaGenerationSettings {
|
||||
};
|
||||
}
|
||||
|
||||
function videoModeFromStorage(value: unknown, uploads: PlaygroundUpload[]): VideoCreateMode {
|
||||
if (value === 'text_to_video' || value === 'first_last_frame' || value === 'omni_reference') return value;
|
||||
return inferVideoModeFromUploads(uploads);
|
||||
}
|
||||
|
||||
function countPresetFromStorage(value: unknown, fallback: MediaGenerationSettings['countPreset']) {
|
||||
if (value === 'custom') return value;
|
||||
const numeric = Number(value);
|
||||
@@ -880,6 +1116,11 @@ function stringFromUnknown(value: unknown) {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function nestedString(value: unknown, key: string) {
|
||||
const record = recordFromUnknown(value);
|
||||
return record ? stringFromUnknown(record[key]) : '';
|
||||
}
|
||||
|
||||
function dateStringFromUnknown(value: unknown) {
|
||||
const raw = stringFromUnknown(value);
|
||||
if (!raw) return undefined;
|
||||
|
||||
Reference in New Issue
Block a user