feat(web): improve media playground controls

This commit is contained in:
2026-05-11 00:40:02 +08:00
parent ada765d90e
commit 9f7c9f6581
4 changed files with 1952 additions and 27 deletions
+483 -26
View File
@@ -1,4 +1,4 @@
import { useEffect, useMemo, useState, type ReactNode } from 'react';
import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react';
import {
AssistantRuntimeProvider,
ComposerPrimitive,
@@ -6,25 +6,51 @@ import {
ThreadPrimitive,
useMessagePartText,
useLocalRuntime,
useThread,
type ChatModelAdapter,
type ThreadMessage,
type ThreadMessageLike,
} from '@assistant-ui/react';
import { StreamdownTextPrimitive } from '@assistant-ui/react-streamdown';
import { cjk } from '@streamdown/cjk';
import { code } from '@streamdown/code';
import { math } from '@streamdown/math';
import { mermaid } from '@streamdown/mermaid';
import type { GatewayApiKey, PlatformModel } from '@easyai-ai-gateway/contracts';
import type { GatewayApiKey, GatewayTask, PlatformModel } from '@easyai-ai-gateway/contracts';
import { Bot, ChevronDown, Image as ImageIcon, MessageSquarePlus, Paperclip, Send, Sparkles, Video } from 'lucide-react';
import { Badge, Button, Select, Textarea } from '../components/ui';
import { streamChatCompletionText } from '../api';
import { createImageGenerationTask, createVideoGenerationTask, getTask, streamChatCompletionText } from '../api';
import type { PlaygroundMode } from '../types';
import {
defaultMediaGenerationSettings,
deriveMediaModelCapabilities,
mediaRequestPayload,
MediaSettingsPopover,
MediaTaskBoard,
normalizeMediaSettingsForCapabilities,
type MediaGenerationRun,
type MediaGenerationSettings,
type MediaModelCapabilities,
} from './playground-media';
type VideoCreateMode = 'text_to_video' | 'first_last_frame' | 'omni_reference';
const MEDIA_RUNS_STORAGE_KEY = 'easyai:playground:media-runs:v1';
const MEDIA_RUNS_STORAGE_LIMIT = 50;
const CHAT_MESSAGES_STORAGE_KEY = 'easyai:playground:chat-messages:v1';
const CHAT_MESSAGES_STORAGE_LIMIT = 100;
interface StoredChatMessage {
content: string;
createdAt: string;
id: string;
role: 'assistant' | 'user';
}
interface ModelOption {
count: number;
label: string;
models: PlatformModel[];
provider: string;
value: string;
}
@@ -87,16 +113,195 @@ export function PlaygroundPage(props: {
const [imageHasReference, setImageHasReference] = useState(false);
const [videoMode, setVideoMode] = useState<VideoCreateMode>('text_to_video');
const [threadKey, setThreadKey] = useState(0);
const [mediaSettings, setMediaSettings] = useState(defaultMediaGenerationSettings);
const [mediaRuns, setMediaRuns] = useState<MediaGenerationRun[]>(readStoredMediaRuns);
const [mediaSubmitting, setMediaSubmitting] = useState(false);
const [mediaMessage, setMediaMessage] = useState('');
const isMountedRef = useRef(false);
const resumedTaskIdsRef = useRef(new Set<string>());
const activeMode = useMemo(() => modeOptions.find((item) => item.value === props.mode) ?? modeOptions[0], [props.mode]);
const modelOptions = useMemo(
() => buildModelOptions(filterModelsForMode(props.models, props.mode, imageHasReference, videoMode)),
[imageHasReference, props.mode, props.models, videoMode],
);
const activeApiKeyId = resolveSelectedApiKeyId(props.apiKeys, props.apiKeySecretsById, props.selectedApiKeyId);
const activeApiKeySecret = activeApiKeyId ? props.apiKeySecretsById[activeApiKeyId] ?? '' : '';
const activeModelOption = useMemo(() => modelOptions.find((item) => item.value === selectedModel), [modelOptions, selectedModel]);
const mediaCapabilities = useMemo(
() => props.mode === 'chat'
? undefined
: deriveMediaModelCapabilities(activeModelOption?.models, props.mode, videoMode),
[activeModelOption, props.mode, videoMode],
);
useEffect(() => {
setSelectedModel((current) => modelOptions.some((item) => item.value === current) ? current : modelOptions[0]?.value ?? '');
}, [modelOptions]);
useEffect(() => {
isMountedRef.current = true;
return () => {
isMountedRef.current = false;
};
}, []);
useEffect(() => {
if (props.mode === 'chat' || !mediaCapabilities) return;
const mediaMode = props.mode;
setMediaSettings((current) => normalizeMediaSettingsForCapabilities(current, mediaCapabilities, mediaMode));
}, [mediaCapabilities, props.mode]);
useEffect(() => {
writeStoredMediaRuns(mediaRuns);
}, [mediaRuns]);
useEffect(() => {
const credential = activeApiKeySecret || props.token;
if (!credential) return;
const resumableRuns = mediaRuns.filter((run) => (
run.task?.id
&& taskIsPending(run.status)
&& !resumedTaskIdsRef.current.has(run.task.id)
));
resumableRuns.forEach((run) => {
if (!run.task?.id) return;
resumedTaskIdsRef.current.add(run.task.id);
void pollTaskUntilSettled(credential, run.task)
.then((detail) => {
if (!isMountedRef.current) return;
setMediaRuns((current) => updateMediaRun(current, run.localId, {
error: detail.error,
status: detail.status,
task: detail,
}));
})
.catch((err) => {
if (!isMountedRef.current) return;
const errorMessage = err instanceof Error ? err.message : '任务状态同步失败';
setMediaRuns((current) => updateMediaRun(current, run.localId, { error: errorMessage, status: 'failed' }));
});
});
}, [activeApiKeySecret, mediaRuns, props.token]);
async function submitMediaTask(overrides?: {
mode?: Exclude<PlaygroundMode, 'chat'>;
prompt?: string;
settings?: MediaGenerationSettings;
}) {
const runMode = overrides?.mode ?? props.mode;
if (runMode === 'chat') return;
const sourceSettings = overrides?.settings ?? mediaSettings;
const runSettings = mediaCapabilities ? normalizeMediaSettingsForCapabilities(sourceSettings, mediaCapabilities, runMode) : sourceSettings;
const trimmedPrompt = (overrides?.prompt ?? prompt).trim();
const credential = activeApiKeySecret || props.token;
if (!props.token) {
props.onLogin();
return;
}
if (!credential) {
setMediaMessage('请选择可用于测试的 API Key;如果列表为空,请先创建一个 Key。');
return;
}
if (!selectedModel) {
setMediaMessage('当前没有可用模型,请确认用户组权限或平台模型配置。');
return;
}
if (!trimmedPrompt) {
setMediaMessage(runMode === 'video' ? '请输入视频提示词。' : '请输入图片提示词。');
return;
}
const localId = newLocalId();
const modelLabel = modelOptions.find((item) => item.value === selectedModel)?.label ?? selectedModel;
const run: MediaGenerationRun = {
createdAt: new Date().toISOString(),
localId,
mode: runMode,
modelLabel,
prompt: trimmedPrompt,
settings: runSettings,
status: 'submitting',
};
setMediaRuns((current) => [...current, run]);
setMediaMessage('');
setMediaSubmitting(true);
try {
const requestPayload = {
model: selectedModel,
prompt: trimmedPrompt,
...mediaRequestPayload(runSettings),
};
const response = runMode === 'video'
? await createVideoGenerationTask(credential, requestPayload)
: await createImageGenerationTask(credential, requestPayload);
setMediaRuns((current) => updateMediaRun(current, localId, { status: response.task.status, task: response.task }));
const detail = await pollTaskUntilSettled(credential, response.task);
setMediaRuns((current) => updateMediaRun(current, localId, {
error: detail.error,
status: detail.status,
task: detail,
}));
} catch (err) {
const errorMessage = err instanceof Error ? err.message : '生成任务提交失败';
setMediaMessage(errorMessage);
setMediaRuns((current) => updateMediaRun(current, localId, { error: errorMessage, status: 'failed' }));
} finally {
setMediaSubmitting(false);
}
}
function editMediaRun(run: MediaGenerationRun) {
setPrompt(run.prompt);
setMediaSettings(run.settings);
if (props.mode !== run.mode) {
props.onModeChange(run.mode);
}
setMediaMessage('已带入这条任务的提示词和参数,可调整后再次生成。');
}
function rerunMediaRun(run: MediaGenerationRun) {
setPrompt(run.prompt);
setMediaSettings(run.settings);
if (props.mode !== run.mode) {
props.onModeChange(run.mode);
setMediaMessage('已切换到对应模式并带入参数,请确认模型后再次生成。');
return;
}
void submitMediaTask({ mode: run.mode, prompt: run.prompt, settings: run.settings });
}
const mediaComposer = props.mode === 'chat' ? null : (
<Composer
apiKeySecretsById={props.apiKeySecretsById}
apiKeys={props.apiKeys}
isSubmitting={mediaSubmitting}
mode={props.mode}
modelOptions={modelOptions}
prompt={prompt}
selectedApiKeyId={activeApiKeyId}
selectedModel={selectedModel}
imageHasReference={imageHasReference}
mediaSettings={mediaSettings}
mediaCapabilities={mediaCapabilities}
videoMode={videoMode}
onApiKeyChange={props.onApiKeyChange}
onCreateApiKey={props.onCreateApiKey}
onImageReferenceChange={setImageHasReference}
onMediaSettingsChange={setMediaSettings}
onModeChange={props.onModeChange}
onModelChange={setSelectedModel}
onPromptChange={setPrompt}
onSubmit={() => void submitMediaTask()}
onVideoModeChange={setVideoMode}
/>
);
function startNewThread() {
clearStoredChatMessages();
setThreadKey((value) => value + 1);
}
return (
<div className="playgroundPage">
<aside className="playgroundSidebar">
@@ -104,7 +309,7 @@ export function PlaygroundPage(props: {
<strong></strong>
<Badge variant="secondary">Test</Badge>
</div>
<button type="button" className="playgroundSideItem active" onClick={() => setThreadKey((value) => value + 1)}>
<button type="button" className="playgroundSideItem active" onClick={startNewThread}>
<MessageSquarePlus size={15} />
</button>
@@ -115,7 +320,7 @@ export function PlaygroundPage(props: {
</aside>
<main className="playgroundStage">
<section className="playgroundHero" data-chat={props.mode === 'chat'}>
<section className="playgroundHero" data-chat={props.mode === 'chat'} data-media-board={props.mode !== 'chat' && mediaRuns.length > 0}>
{props.mode === 'chat' ? (
<AssistantChatPlayground
key={threadKey}
@@ -131,29 +336,20 @@ export function PlaygroundPage(props: {
onModelChange={setSelectedModel}
onLogin={props.onLogin}
/>
) : mediaRuns.length > 0 && mediaComposer ? (
<MediaTaskBoard
composer={mediaComposer}
message={mediaMessage}
runs={mediaRuns}
onEditRun={editMediaRun}
onRerun={rerunMediaRun}
/>
) : (
<>
<ModeSwitch activeMode={props.mode} onModeChange={props.onModeChange} />
<PlaygroundGreeting activeMode={activeMode} />
<Composer
apiKeySecretsById={props.apiKeySecretsById}
apiKeys={props.apiKeys}
mode={props.mode}
modelOptions={modelOptions}
prompt={prompt}
selectedApiKeyId={props.selectedApiKeyId}
selectedModel={selectedModel}
imageHasReference={imageHasReference}
videoMode={videoMode}
onApiKeyChange={props.onApiKeyChange}
onCreateApiKey={props.onCreateApiKey}
onImageReferenceChange={setImageHasReference}
onModeChange={props.onModeChange}
onModelChange={setSelectedModel}
onPromptChange={setPrompt}
onSubmit={() => undefined}
onVideoModeChange={setVideoMode}
/>
{mediaMessage && <p className="playgroundError">{mediaMessage}</p>}
{mediaComposer}
</>
)}
</section>
@@ -250,6 +446,7 @@ function AssistantChatPlayground(props: {
const activeApiKeySecret = activeApiKeyId ? props.apiKeySecretsById[activeApiKeyId] ?? '' : '';
const canRun = Boolean(props.token && props.selectedModel && activeApiKeySecret);
const apiKeyNotice = apiKeyNoticeText(props.apiKeys, props.apiKeySecretsById);
const initialMessages = useMemo(() => readStoredChatMessages(), []);
const adapter = useMemo<ChatModelAdapter>(() => ({
async *run({ abortSignal, messages }) {
if (!props.token) {
@@ -282,10 +479,11 @@ function AssistantChatPlayground(props: {
};
},
}), [activeApiKeySecret, props]);
const runtime = useLocalRuntime(adapter);
const runtime = useLocalRuntime(adapter, { initialMessages });
return (
<AssistantRuntimeProvider runtime={runtime}>
<AssistantChatPersistenceBridge />
<ThreadPrimitive.Root className="assistantThreadRoot">
<ThreadPrimitive.Empty>
<div className="assistantEmptyStage">
@@ -345,6 +543,21 @@ function AssistantChatPlayground(props: {
);
}
function AssistantChatPersistenceBridge() {
const messages = useThread((state) => state.messages);
const skipInitialEmptyWriteRef = useRef(true);
useEffect(() => {
if (skipInitialEmptyWriteRef.current) {
skipInitialEmptyWriteRef.current = false;
if (!messages.length && hasStoredChatMessages()) return;
}
writeStoredChatMessages(messages);
}, [messages]);
return null;
}
function ModeSwitch(props: {
activeMode: PlaygroundMode;
onModeChange: (mode: PlaygroundMode) => void;
@@ -535,6 +748,90 @@ function threadMessageText(message: ThreadMessage) {
.trim();
}
function readStoredChatMessages(): ThreadMessageLike[] {
if (typeof window === 'undefined') return [];
try {
const raw = window.localStorage.getItem(CHAT_MESSAGES_STORAGE_KEY);
if (!raw) return [];
const parsed = JSON.parse(raw) as unknown;
const record = recordFromUnknown(parsed);
const source = Array.isArray(parsed) ? parsed : record?.messages;
if (!Array.isArray(source)) return [];
return source
.map(chatMessageLikeFromStorage)
.filter((item): item is ThreadMessageLike => Boolean(item))
.slice(-CHAT_MESSAGES_STORAGE_LIMIT);
} catch {
return [];
}
}
function hasStoredChatMessages() {
if (typeof window === 'undefined') return false;
try {
return Boolean(window.localStorage.getItem(CHAT_MESSAGES_STORAGE_KEY));
} catch {
return false;
}
}
function writeStoredChatMessages(messages: readonly ThreadMessage[]) {
if (typeof window === 'undefined') return;
try {
const storedMessages = messages
.map(storedChatMessageFromThread)
.filter((item): item is StoredChatMessage => Boolean(item))
.slice(-CHAT_MESSAGES_STORAGE_LIMIT);
if (!storedMessages.length) {
window.localStorage.removeItem(CHAT_MESSAGES_STORAGE_KEY);
return;
}
window.localStorage.setItem(CHAT_MESSAGES_STORAGE_KEY, JSON.stringify({
messages: storedMessages,
version: 1,
}));
} catch {
// Best effort only: local chat history should not block sending messages.
}
}
function clearStoredChatMessages() {
if (typeof window === 'undefined') return;
try {
window.localStorage.removeItem(CHAT_MESSAGES_STORAGE_KEY);
} catch {
// Ignore storage errors.
}
}
function chatMessageLikeFromStorage(value: unknown): ThreadMessageLike | undefined {
const record = recordFromUnknown(value);
if (!record) return undefined;
const role = record.role === 'assistant' || record.role === 'user' ? record.role : undefined;
const content = stringFromUnknown(record.content);
if (!role || !content) return undefined;
const createdAt = dateStringFromUnknown(record.createdAt);
return {
content,
createdAt: createdAt ? new Date(createdAt) : undefined,
id: stringFromUnknown(record.id) || undefined,
role,
status: role === 'assistant' ? { type: 'complete', reason: 'stop' } : undefined,
};
}
function storedChatMessageFromThread(message: ThreadMessage): StoredChatMessage | undefined {
if (message.role !== 'assistant' && message.role !== 'user') return undefined;
const content = threadMessageText(message);
if (!content) return undefined;
return {
content,
createdAt: message.createdAt.toISOString(),
id: message.id,
role: message.role,
};
}
function resolveSelectedApiKeyId(apiKeys: GatewayApiKey[], secretsById: Record<string, string>, selectedApiKeyId: string) {
if (selectedApiKeyId && secretsById[selectedApiKeyId]) return selectedApiKeyId;
const firstUsable = apiKeys.find((item) => Boolean(secretsById[item.id]));
@@ -561,6 +858,9 @@ function Composer(props: {
apiKeys?: GatewayApiKey[];
compact?: boolean;
imageHasReference?: boolean;
isSubmitting?: boolean;
mediaCapabilities?: MediaModelCapabilities;
mediaSettings?: MediaGenerationSettings;
mode: PlaygroundMode;
modelOptions: ModelOption[];
prompt: string;
@@ -570,6 +870,7 @@ function Composer(props: {
onApiKeyChange?: (apiKeyId: string) => void;
onCreateApiKey?: () => void;
onImageReferenceChange?: (value: boolean) => void;
onMediaSettingsChange?: (settings: MediaGenerationSettings) => void;
onModeChange: (mode: PlaygroundMode) => void;
onModelChange: (value: string) => void;
onPromptChange: (value: string) => void;
@@ -611,6 +912,14 @@ function Composer(props: {
<option value={item.value} key={item.value}>{modelOptionLabel(item)}</option>
)) : <option value=""></option>}
</Select>
{props.mode !== 'chat' && props.mediaSettings && props.onMediaSettingsChange && (
<MediaSettingsPopover
capabilities={props.mediaCapabilities}
mode={props.mode}
settings={props.mediaSettings}
onChange={props.onMediaSettingsChange}
/>
)}
{props.apiKeys && props.apiKeySecretsById && props.onApiKeyChange && (
<ApiKeySelect
apiKeySecretsById={props.apiKeySecretsById}
@@ -627,7 +936,7 @@ function Composer(props: {
<div className="composerQuickPrompts">
{quickItems.map((item) => <button type="button" key={item} onClick={() => props.onPromptChange(item)}>{item}</button>)}
</div>
<Button type="button" size="icon" aria-label="发送测试" onClick={props.onSubmit}>
<Button type="button" size="icon" aria-label="发送测试" disabled={props.isSubmitting} onClick={props.onSubmit}>
<Send size={15} />
</Button>
</div>
@@ -691,6 +1000,7 @@ function buildModelOptions(models: PlatformModel[]): ModelOption[] {
const current = grouped.get(value);
if (current) {
current.count += 1;
current.models.push(model);
if (!current.provider.includes(model.provider || '')) {
current.provider = [current.provider, model.provider].filter(Boolean).join(' / ');
}
@@ -699,6 +1009,7 @@ function buildModelOptions(models: PlatformModel[]): ModelOption[] {
grouped.set(value, {
count: 1,
label: model.modelAlias || model.displayName || model.modelName,
models: [model],
provider: model.provider || model.platformName || '',
value,
});
@@ -711,3 +1022,149 @@ function modelOptionLabel(option: ModelOption) {
const provider = option.provider ? ` · ${option.provider}` : '';
return `${option.label}${provider}${count}`;
}
function updateMediaRun(runs: MediaGenerationRun[], localId: string, patch: Partial<MediaGenerationRun>) {
return runs.map((run) => run.localId === localId ? { ...run, ...patch } : run);
}
async function pollTaskUntilSettled(token: string, task: GatewayTask) {
let detail = task;
for (let attempt = 0; attempt < 20; attempt += 1) {
detail = await getTask(token, detail.id);
if (!taskIsPending(detail.status)) return detail;
await delay(1200);
}
return detail;
}
function taskIsPending(status: string) {
return status === 'queued' || status === 'running' || status === 'submitting';
}
function readStoredMediaRuns(): MediaGenerationRun[] {
if (typeof window === 'undefined') return [];
try {
const raw = window.localStorage.getItem(MEDIA_RUNS_STORAGE_KEY);
if (!raw) return [];
const parsed = JSON.parse(raw) as unknown;
const record = recordFromUnknown(parsed);
const source = Array.isArray(parsed) ? parsed : record?.runs;
if (!Array.isArray(source)) return [];
const runs = source
.map((item, index) => mediaRunFromStorage(item, index))
.filter((item): item is MediaGenerationRun => Boolean(item));
return sortMediaRunsByCreatedAt(runs).slice(-MEDIA_RUNS_STORAGE_LIMIT);
} catch {
return [];
}
}
function writeStoredMediaRuns(runs: MediaGenerationRun[]) {
if (typeof window === 'undefined') return;
try {
if (!runs.length) {
window.localStorage.removeItem(MEDIA_RUNS_STORAGE_KEY);
return;
}
const storedRuns = sortMediaRunsByCreatedAt(runs).slice(-MEDIA_RUNS_STORAGE_LIMIT);
window.localStorage.setItem(MEDIA_RUNS_STORAGE_KEY, JSON.stringify({
runs: storedRuns,
version: 1,
}));
} catch {
// Best effort only: private mode or full storage should not break generation.
}
}
function mediaRunFromStorage(value: unknown, index: number): MediaGenerationRun | undefined {
const record = recordFromUnknown(value);
if (!record) return undefined;
const mode = record.mode === 'image' || record.mode === 'video' ? record.mode : undefined;
const prompt = stringFromUnknown(record.prompt);
if (!mode || !prompt) return undefined;
const task = taskFromStorage(record.task);
const createdAt = dateStringFromUnknown(record.createdAt) ?? new Date().toISOString();
const localId = stringFromUnknown(record.localId) || task?.id || `stored-${index}-${createdAt}`;
const modelLabel = stringFromUnknown(record.modelLabel) || task?.model || '未知模型';
let status: MediaGenerationRun['status'] = stringFromUnknown(record.status) || task?.status || 'failed';
let error = stringFromUnknown(record.error);
if (status === 'submitting' && !task?.id) {
status = 'failed';
error = error || '任务提交已中断,请重新生成。';
}
return {
createdAt,
error,
localId,
mode,
modelLabel,
prompt,
settings: mediaSettingsFromStorage(record.settings),
status,
task,
};
}
function sortMediaRunsByCreatedAt(runs: MediaGenerationRun[]) {
return [...runs].sort((left, right) => Date.parse(left.createdAt) - Date.parse(right.createdAt));
}
function mediaSettingsFromStorage(value: unknown): MediaGenerationSettings {
const fallback = defaultMediaGenerationSettings();
const record = recordFromUnknown(value);
if (!record) return fallback;
return {
aspectRatio: stringFromUnknown(record.aspectRatio) || fallback.aspectRatio,
countPreset: countPresetFromStorage(record.countPreset, fallback.countPreset),
customCount: numberFromUnknown(record.customCount, fallback.customCount, 1, 20),
height: numberFromUnknown(record.height, fallback.height, 128, 8192),
outputMode: record.outputMode === 'group' ? 'group' : 'single',
resolution: stringFromUnknown(record.resolution) || fallback.resolution,
width: numberFromUnknown(record.width, fallback.width, 128, 8192),
};
}
function countPresetFromStorage(value: unknown, fallback: MediaGenerationSettings['countPreset']) {
if (value === 'custom') return value;
const numeric = Number(value);
if (numeric === 1 || numeric === 2 || numeric === 3 || numeric === 4) return numeric;
return fallback;
}
function taskFromStorage(value: unknown): GatewayTask | undefined {
const record = recordFromUnknown(value);
if (!record || !stringFromUnknown(record.id) || !stringFromUnknown(record.status)) return undefined;
return record as unknown as GatewayTask;
}
function recordFromUnknown(value: unknown): Record<string, unknown> | undefined {
if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
return value as Record<string, unknown>;
}
function stringFromUnknown(value: unknown) {
return typeof value === 'string' ? value.trim() : '';
}
function dateStringFromUnknown(value: unknown) {
const raw = stringFromUnknown(value);
if (!raw) return undefined;
const time = Date.parse(raw);
return Number.isFinite(time) ? raw : undefined;
}
function numberFromUnknown(value: unknown, fallback: number, min: number, max: number) {
const number = typeof value === 'number' ? value : Number(value);
if (!Number.isFinite(number)) return fallback;
return Math.min(max, Math.max(min, Math.round(number)));
}
function delay(ms: number) {
return new Promise((resolve) => window.setTimeout(resolve, ms));
}
function newLocalId() {
return typeof crypto !== 'undefined' && 'randomUUID' in crypto
? crypto.randomUUID()
: `${Date.now()}-${Math.random().toString(36).slice(2)}`;
}