fix: align video generation payloads

apps/web/src/pages/PlaygroundPage.tsx

@@ -33,6 +33,7 @@ import {
   swapFirstLastFrameUploads as sharedSwapFirstLastFrameUploads,
   uploadPlaygroundFiles as sharedUploadPlaygroundFiles,
   UploadAttachmentList as SharedUploadAttachmentList,
+  videoGenerationContentFromPromptAndUploads as sharedVideoGenerationContentFromPromptAndUploads,
   allowedMediaUploadKinds as sharedAllowedMediaUploadKinds,
   type PlaygroundUpload,
   type PlaygroundUploadRole,
@@ -283,19 +284,25 @@ export function PlaygroundPage(props: {
     setMediaMessage('');
     try {
       const requestPrompt = replacePlaygroundResourceTokens(trimmedPrompt, runUploads, runMode);
-      const uploadPayload = sharedMediaUploadRequestPayload(runUploads, runMode, videoMode);
-      const requestPayload = {
-        model: runModel,
-        prompt: requestPrompt,
-        ...mediaRequestPayload(runSettings, runMode),
-        ...videoModeRequestPayload(runMode, videoMode, runUploads, runModelOption),
-        ...uploadPayload,
-      };
-      const response = runMode === 'video'
-        ? await createVideoGenerationTask(credential, requestPayload)
-        : runUploads.some((item) => item.kind === 'image')
+      let response: { task: GatewayTask; next: Record<string, string> };
+      if (runMode === 'video') {
+        response = await createVideoGenerationTask(credential, {
+          model: runModel,
+          content: sharedVideoGenerationContentFromPromptAndUploads(requestPrompt, runUploads, videoMode),
+          ...mediaRequestPayload(runSettings, 'video'),
+        });
+      } else {
+        const uploadPayload = sharedMediaUploadRequestPayload(runUploads, 'image');
+        const requestPayload = {
+          model: runModel,
+          prompt: requestPrompt,
+          ...mediaRequestPayload(runSettings, 'image'),
+          ...uploadPayload,
+        };
+        response = runUploads.some((item) => item.kind === 'image')
           ? await createImageEditTask(credential, requestPayload)
           : await createImageGenerationTask(credential, requestPayload);
+      }
       setMediaRuns((current) => updateMediaRun(current, localId, { status: response.task.status, task: response.task }));
       if (!overrides) {
         setMediaUploads([]);
@@ -674,31 +681,6 @@ function mediaPromptPlaceholder(mode: PlaygroundMode) {
   return placeholderByMode.chat;
 }
 
-function videoModeRequestPayload(
-  mode: Exclude<PlaygroundMode, 'chat'>,
-  videoMode: VideoCreateMode,
-  uploads: PlaygroundUpload[],
-  modelOption?: ModelOption,
-) {
-  if (mode !== 'video') return {};
-  const modelTypes = new Set(modelOption?.models.flatMap((model) => model.modelType) ?? []);
-  if (videoMode === 'first_last_frame') {
-    const modelType = modelTypes.has('video_first_last_frame') ? 'video_first_last_frame' : 'image_to_video';
-    return { capabilityType: modelType, model_type: modelType };
-  }
-  if (videoMode === 'omni_reference' || uploads.length > 0) {
-    const modelType = modelTypes.has('omni_video')
-      ? 'omni_video'
-      : modelTypes.has('video_reference')
-        ? 'video_reference'
-        : modelTypes.has('image_to_video')
-          ? 'image_to_video'
-          : 'video_generate';
-    return { capabilityType: modelType, model_type: modelType };
-  }
-  return {};
-}
-
 function filterModelsForMode(models: PlatformModel[], mode: PlaygroundMode, hasReference: boolean, videoMode: VideoCreateMode) {
   if (mode === 'chat') {
     return filterWithFallback(models, ['text_generate', 'chat', 'responses', 'text']);

apps/web/src/pages/playground-media.tsx

@@ -159,8 +159,6 @@ export function mediaRequestPayload(settings: MediaGenerationSettings, mode: Exc
       aspect_ratio: settings.aspectRatio === 'auto' ? undefined : settings.aspectRatio,
       audio: settings.outputAudio,
       duration: settings.durationSeconds,
-      duration_seconds: settings.durationSeconds,
-      output_audio: settings.outputAudio,
       resolution: settings.resolution,
     };
   }

apps/web/src/pages/playground-upload.tsx

@@ -11,6 +11,7 @@ import {
   X,
 } from 'lucide-react';
 import { uploadFileToStorage } from '../api';
+import type { VideoGenerationContent } from '../api';
 import type { PlaygroundMode } from '../types';
 
 export type PlaygroundUploadKind = 'audio' | 'file' | 'image' | 'video';
@@ -522,10 +523,8 @@ function openAIContentPartFromUpload(item: PlaygroundUpload): OpenAIChatContentP
   return { type: 'file_url', file_url: { filename: item.name, url: item.url } };
 }
 
-export function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>, videoMode: PlaygroundVideoCreateMode) {
+export function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>) {
   const images = uploads.filter((item) => item.kind === 'image').map((item) => item.url);
-  const videos = uploads.filter((item) => item.kind === 'video').map((item) => item.url);
-  const audios = uploads.filter((item) => item.kind === 'audio').map((item) => item.url);
   const payload: Record<string, string | string[]> = {};
   if (mode === 'image') {
     if (images.length) {
@@ -534,27 +533,49 @@ export function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exc
     }
     return payload;
   }
+  return payload;
+}
+
+export function videoGenerationContentFromPromptAndUploads(
+  prompt: string,
+  uploads: PlaygroundUpload[],
+  videoMode: PlaygroundVideoCreateMode,
+): VideoGenerationContent[] {
+  const content: VideoGenerationContent[] = [];
+  const text = prompt.trim();
+  if (text) {
+    content.push({ type: 'text', text });
+  }
   if (videoMode === 'first_last_frame') {
     const first = frameUploadByRole(uploads, 'first_frame');
     const last = frameUploadByRole(uploads, 'last_frame');
-    if (first) {
-      payload.first_frame = first.url;
+    if (first?.url) {
+      content.push({ type: 'image_url', role: 'first_frame', image_url: { url: first.url } });
     }
-    if (last) {
-      payload.last_frame = last.url;
+    if (last?.url) {
+      content.push({ type: 'image_url', role: 'last_frame', image_url: { url: last.url } });
     }
-    return payload;
+    return content.length ? content : [{ type: 'text', text: '' }];
   }
-  if (images.length) {
-    payload.reference_image = singleOrMany(images);
+  uploads.forEach((item) => {
+    const part = videoGenerationContentFromUpload(item);
+    if (part) content.push(part);
+  });
+  return content.length ? content : [{ type: 'text', text: '' }];
+}
+
+function videoGenerationContentFromUpload(item: PlaygroundUpload): VideoGenerationContent | undefined {
+  if (!item.url) return undefined;
+  if (item.kind === 'image') {
+    return { type: 'image_url', role: 'reference_image', image_url: { url: item.url } };
   }
-  if (videos.length) {
-    payload.reference_video = singleOrMany(videos);
+  if (item.kind === 'video') {
+    return { type: 'video_url', role: 'reference_video', video_url: { url: item.url, refer_type: 'feature' } };
   }
-  if (audios.length) {
-    payload.reference_audio = singleOrMany(audios);
+  if (item.kind === 'audio') {
+    return { type: 'audio_url', role: 'reference_audio', audio_url: { url: item.url } };
   }
-  return payload;
+  return undefined;
 }
 
 function singleOrMany(values: string[]) {