fix: align video generation payloads

2026-05-14 00:14:54 +08:00
parent f254551522
commit 3225833f96
11 changed files with 702 additions and 188 deletions
@@ -662,46 +662,83 @@ export async function createImageEditTask(
  });
 }

+export type VideoGenerationContentRole =
+  | 'first_frame'
+  | 'last_frame'
+  | 'reference_image'
+  | 'reference_video'
+  | 'reference_audio'
+  | 'digital_human_frame'
+  | 'reference'
+  | 'element'
+  | 'video_feature'
+  | 'video_base'
+  | 'shot_prompt';
+
+export interface VideoGenerationContent {
+  type: 'text' | 'image_url' | 'audio_url' | 'video_url' | 'element';
+  text?: string;
+  image_url?: {
+    url: string;
+  };
+  video_url?: {
+    url: string;
+    refer_type?: 'feature' | 'base';
+    keep_original_sound?: 'yes' | 'no';
+  };
+  audio_url?: {
+    url: string;
+  };
+  role?: VideoGenerationContentRole;
+  shot_index?: number;
+  duration?: number;
+  name?: string;
+  element?: {
+    system_element_id?: string;
+    inline_element?: {
+      name: string;
+      description?: string;
+      frontal_image_url: string;
+      refer_images: Array<{ url: string; slot_key?: string }>;
+      tags?: string[];
+    };
+  };
+}
+
+export interface VideoGenerationParams {
+  content: VideoGenerationContent[];
+  model: string;
+  aspect_ratio?: string;
+  resolution?: string;
+  duration?: number;
+  audio_list?: Array<{
+    url?: string;
+    audio_url?: string;
+    name?: string;
+  }>;
+  audio?: boolean;
+  framespersecond?: number;
+  watermark?: boolean;
+  seed?: number;
+  camerafixed?: boolean;
+  camera_control?: string;
+  camera_control_strength?: number;
+  prompt_extend?: boolean;
+  size?: string;
+  task_id?: string;
+  conversation_id?: string;
+  histories?: string;
+  callback_url?: string;
+  prompt_optimizer?: boolean;
+  fast_pretreatment?: boolean;
+  mode?: 'std' | 'pro';
+  negative_prompt?: string;
+  cfg_scale?: number;
+}
+
 export async function createVideoGenerationTask(
  token: string,
-  input: {
-    audio?: boolean;
-    audioUrl?: string | string[];
-    audio_url?: string | string[];
-    capabilityType?: string;
-    content?: Array<Record<string, unknown>>;
-    firstFrame?: string;
-    first_frame?: string;
-    model: string;
-    model_type?: string;
-    prompt: string;
-    aspect_ratio?: string;
-    count?: number;
-    duration?: number;
-    duration_seconds?: number;
-    height?: number;
-    image?: string | string[];
-    imageUrl?: string | string[];
-    image_url?: string | string[];
-    imageUrls?: string[];
-    image_urls?: string[];
-    lastFrame?: string;
-    last_frame?: string;
-    n?: number;
-    output_audio?: boolean;
-    referenceAudio?: string | string[];
-    referenceVideo?: string | string[];
-    reference_audio?: string | string[];
-    reference_image?: string | string[];
-    reference_video?: string | string[];
-    resolution?: string;
-    runMode?: string;
-    simulation?: boolean;
-    size?: string;
-    videoUrl?: string | string[];
-    video_url?: string | string[];
-    width?: number;
-  },
+  input: VideoGenerationParams,
 ): Promise<{ task: GatewayTask; next: Record<string, string> }> {
  return request<{ task: GatewayTask; next: Record<string, string> }>('/api/v1/videos/generations', {
    body: input,
@@ -33,6 +33,7 @@ import {
  swapFirstLastFrameUploads as sharedSwapFirstLastFrameUploads,
  uploadPlaygroundFiles as sharedUploadPlaygroundFiles,
  UploadAttachmentList as SharedUploadAttachmentList,
+  videoGenerationContentFromPromptAndUploads as sharedVideoGenerationContentFromPromptAndUploads,
  allowedMediaUploadKinds as sharedAllowedMediaUploadKinds,
  type PlaygroundUpload,
  type PlaygroundUploadRole,
@@ -283,19 +284,25 @@ export function PlaygroundPage(props: {
    setMediaMessage('');
    try {
      const requestPrompt = replacePlaygroundResourceTokens(trimmedPrompt, runUploads, runMode);
-      const uploadPayload = sharedMediaUploadRequestPayload(runUploads, runMode, videoMode);
-      const requestPayload = {
-        model: runModel,
-        prompt: requestPrompt,
-        ...mediaRequestPayload(runSettings, runMode),
-        ...videoModeRequestPayload(runMode, videoMode, runUploads, runModelOption),
-        ...uploadPayload,
-      };
-      const response = runMode === 'video'
-        ? await createVideoGenerationTask(credential, requestPayload)
-        : runUploads.some((item) => item.kind === 'image')
+      let response: { task: GatewayTask; next: Record<string, string> };
+      if (runMode === 'video') {
+        response = await createVideoGenerationTask(credential, {
+          model: runModel,
+          content: sharedVideoGenerationContentFromPromptAndUploads(requestPrompt, runUploads, videoMode),
+          ...mediaRequestPayload(runSettings, 'video'),
+        });
+      } else {
+        const uploadPayload = sharedMediaUploadRequestPayload(runUploads, 'image');
+        const requestPayload = {
+          model: runModel,
+          prompt: requestPrompt,
+          ...mediaRequestPayload(runSettings, 'image'),
+          ...uploadPayload,
+        };
+        response = runUploads.some((item) => item.kind === 'image')
          ? await createImageEditTask(credential, requestPayload)
          : await createImageGenerationTask(credential, requestPayload);
+      }
      setMediaRuns((current) => updateMediaRun(current, localId, { status: response.task.status, task: response.task }));
      if (!overrides) {
        setMediaUploads([]);
@@ -674,31 +681,6 @@ function mediaPromptPlaceholder(mode: PlaygroundMode) {
  return placeholderByMode.chat;
 }

-function videoModeRequestPayload(
-  mode: Exclude<PlaygroundMode, 'chat'>,
-  videoMode: VideoCreateMode,
-  uploads: PlaygroundUpload[],
-  modelOption?: ModelOption,
-) {
-  if (mode !== 'video') return {};
-  const modelTypes = new Set(modelOption?.models.flatMap((model) => model.modelType) ?? []);
-  if (videoMode === 'first_last_frame') {
-    const modelType = modelTypes.has('video_first_last_frame') ? 'video_first_last_frame' : 'image_to_video';
-    return { capabilityType: modelType, model_type: modelType };
-  }
-  if (videoMode === 'omni_reference' || uploads.length > 0) {
-    const modelType = modelTypes.has('omni_video')
-      ? 'omni_video'
-      : modelTypes.has('video_reference')
-        ? 'video_reference'
-        : modelTypes.has('image_to_video')
-          ? 'image_to_video'
-          : 'video_generate';
-    return { capabilityType: modelType, model_type: modelType };
-  }
-  return {};
-}
-
 function filterModelsForMode(models: PlatformModel[], mode: PlaygroundMode, hasReference: boolean, videoMode: VideoCreateMode) {
  if (mode === 'chat') {
    return filterWithFallback(models, ['text_generate', 'chat', 'responses', 'text']);
@@ -159,8 +159,6 @@ export function mediaRequestPayload(settings: MediaGenerationSettings, mode: Exc
      aspect_ratio: settings.aspectRatio === 'auto' ? undefined : settings.aspectRatio,
      audio: settings.outputAudio,
      duration: settings.durationSeconds,
-      duration_seconds: settings.durationSeconds,
-      output_audio: settings.outputAudio,
      resolution: settings.resolution,
    };
  }
@@ -11,6 +11,7 @@ import {
  X,
 } from 'lucide-react';
 import { uploadFileToStorage } from '../api';
+import type { VideoGenerationContent } from '../api';
 import type { PlaygroundMode } from '../types';

 export type PlaygroundUploadKind = 'audio' | 'file' | 'image' | 'video';
@@ -522,10 +523,8 @@ function openAIContentPartFromUpload(item: PlaygroundUpload): OpenAIChatContentP
  return { type: 'file_url', file_url: { filename: item.name, url: item.url } };
 }

-export function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>, videoMode: PlaygroundVideoCreateMode) {
+export function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exclude<PlaygroundMode, 'chat'>) {
  const images = uploads.filter((item) => item.kind === 'image').map((item) => item.url);
-  const videos = uploads.filter((item) => item.kind === 'video').map((item) => item.url);
-  const audios = uploads.filter((item) => item.kind === 'audio').map((item) => item.url);
  const payload: Record<string, string | string[]> = {};
  if (mode === 'image') {
    if (images.length) {
@@ -534,27 +533,49 @@ export function mediaUploadRequestPayload(uploads: PlaygroundUpload[], mode: Exc
    }
    return payload;
  }
+  return payload;
+}
+
+export function videoGenerationContentFromPromptAndUploads(
+  prompt: string,
+  uploads: PlaygroundUpload[],
+  videoMode: PlaygroundVideoCreateMode,
+): VideoGenerationContent[] {
+  const content: VideoGenerationContent[] = [];
+  const text = prompt.trim();
+  if (text) {
+    content.push({ type: 'text', text });
+  }
  if (videoMode === 'first_last_frame') {
    const first = frameUploadByRole(uploads, 'first_frame');
    const last = frameUploadByRole(uploads, 'last_frame');
-    if (first) {
-      payload.first_frame = first.url;
+    if (first?.url) {
+      content.push({ type: 'image_url', role: 'first_frame', image_url: { url: first.url } });
    }
-    if (last) {
-      payload.last_frame = last.url;
+    if (last?.url) {
+      content.push({ type: 'image_url', role: 'last_frame', image_url: { url: last.url } });
    }
-    return payload;
+    return content.length ? content : [{ type: 'text', text: '' }];
  }
-  if (images.length) {
-    payload.reference_image = singleOrMany(images);
+  uploads.forEach((item) => {
+    const part = videoGenerationContentFromUpload(item);
+    if (part) content.push(part);
+  });
+  return content.length ? content : [{ type: 'text', text: '' }];
+}
+
+function videoGenerationContentFromUpload(item: PlaygroundUpload): VideoGenerationContent | undefined {
+  if (!item.url) return undefined;
+  if (item.kind === 'image') {
+    return { type: 'image_url', role: 'reference_image', image_url: { url: item.url } };
  }
-  if (videos.length) {
-    payload.reference_video = singleOrMany(videos);
+  if (item.kind === 'video') {
+    return { type: 'video_url', role: 'reference_video', video_url: { url: item.url, refer_type: 'feature' } };
  }
-  if (audios.length) {
-    payload.reference_audio = singleOrMany(audios);
+  if (item.kind === 'audio') {
+    return { type: 'audio_url', role: 'reference_audio', audio_url: { url: item.url } };
  }
-  return payload;
+  return undefined;
 }

 function singleOrMany(values: string[]) {