feat(clients): 添加Keling客户端支持视频生成和多模态功能
- 实现KelingClient结构体及其Run方法，支持视频生成功能 - 添加对image2video、text2video和omni-video任务类型的完整支持 - 集成Keling平台的身份验证机制，包括JWT令牌生成 - 实现视频任务的提交和轮询逻辑，支持异步处理流程 - 添加对多种输入格式的支持，包括图像帧、基础视频和参考素材 - 实现Keling元素管理和清理机制，处理临时创建的素材 - 在服务初始化中注册keling和kling两个提供商标识 - 添加数据库迁移脚本，更新Keling模型的音频功能配置 - 完善错误处理和重试机制，提升服务稳定性 - 编写完整的单元测试，覆盖各种视频生成场景和边界情况
This commit is contained in:
@@ -662,6 +662,266 @@ func TestVolcesClientVideoResumePollsExistingTaskID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestKelingClientVideoSubmitsAndPollsImageTask(t *testing.T) {
|
||||
var submitPath string
|
||||
var pollPath string
|
||||
var gotAuth string
|
||||
var submittedTaskID string
|
||||
var submittedPayload map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
switch r.Method + " " + r.URL.Path {
|
||||
case "POST /videos/image2video":
|
||||
submitPath = r.URL.Path
|
||||
if err := json.NewDecoder(r.Body).Decode(&submittedPayload); err != nil {
|
||||
t.Fatalf("decode keling submit: %v", err)
|
||||
}
|
||||
if _, ok := submittedPayload["aspect_ratio"]; ok {
|
||||
t.Fatalf("image2video payload should not include aspect_ratio: %+v", submittedPayload)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"request_id": "req-submit",
|
||||
"data": map[string]any{"task_id": "keling-task-1"},
|
||||
})
|
||||
case "GET /videos/image2video/keling-task-1":
|
||||
pollPath = r.URL.Path
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"request_id": "req-poll",
|
||||
"data": map[string]any{
|
||||
"task_id": "keling-task-1",
|
||||
"task_status": "succeed",
|
||||
"created_at": 456,
|
||||
"task_result": map[string]any{
|
||||
"videos": []any{map[string]any{"url": "https://example.com/keling.mp4", "duration": 6}},
|
||||
},
|
||||
},
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
response, err := (KelingClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
|
||||
Kind: "videos.generations",
|
||||
ModelType: "image_to_video",
|
||||
Model: "可灵2.6",
|
||||
Body: map[string]any{
|
||||
"model": "可灵2.6",
|
||||
"prompt": "A clean product reveal",
|
||||
"first_frame": "data:image/png;base64,Zmlyc3Q=",
|
||||
"last_frame": "data:image/png;base64,bGFzdA==",
|
||||
"duration": 6,
|
||||
"resolution": "1080p",
|
||||
"aspect_ratio": "16:9",
|
||||
"audio": true,
|
||||
"camera_control": "simple:zoom",
|
||||
"camera_control_strength": 0.6,
|
||||
},
|
||||
Candidate: store.RuntimeModelCandidate{
|
||||
BaseURL: server.URL,
|
||||
Provider: "keling",
|
||||
AuthType: "AccessKey-SecretKey",
|
||||
ModelName: "可灵2.6",
|
||||
ProviderModelName: "kling-v2-6",
|
||||
Credentials: map[string]any{"accessKey": "ak", "secretKey": "sk"},
|
||||
PlatformConfig: map[string]any{
|
||||
"kelingPollIntervalMs": 100,
|
||||
"kelingPollTimeoutSeconds": 1,
|
||||
},
|
||||
},
|
||||
OnRemoteTaskSubmitted: func(remoteTaskID string, payload map[string]any) error {
|
||||
submittedTaskID = remoteTaskID
|
||||
if payload["endpoint"] != "/videos/image2video" || payload["taskType"] != "image2video" {
|
||||
t.Fatalf("unexpected submitted keling payload: %+v", payload)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run keling video: %v", err)
|
||||
}
|
||||
if submitPath != "/videos/image2video" || pollPath != "/videos/image2video/keling-task-1" || !strings.HasPrefix(gotAuth, "Bearer ") {
|
||||
t.Fatalf("unexpected keling paths/auth submit=%s poll=%s auth=%s", submitPath, pollPath, gotAuth)
|
||||
}
|
||||
if submittedTaskID != "keling-task-1" {
|
||||
t.Fatalf("remote task submit callback did not receive task id, got %q", submittedTaskID)
|
||||
}
|
||||
if submittedPayload["model_name"] != "kling-v2-6" ||
|
||||
submittedPayload["prompt"] != "A clean product reveal" ||
|
||||
submittedPayload["duration"] != "6" ||
|
||||
submittedPayload["mode"] != "pro" ||
|
||||
submittedPayload["sound"] != "on" ||
|
||||
submittedPayload["image"] != "Zmlyc3Q=" ||
|
||||
submittedPayload["image_tail"] != "bGFzdA==" {
|
||||
t.Fatalf("unexpected keling submit payload: %+v", submittedPayload)
|
||||
}
|
||||
camera, _ := submittedPayload["camera_control"].(map[string]any)
|
||||
config, _ := camera["config"].(map[string]any)
|
||||
if camera["type"] != "simple" || numericValue(config["zoom"], 0) != 0.6 || numericValue(config["pan"], -1) != 0 {
|
||||
t.Fatalf("unexpected keling camera conversion: %+v", submittedPayload["camera_control"])
|
||||
}
|
||||
data, _ := response.Result["data"].([]any)
|
||||
item, _ := data[0].(map[string]any)
|
||||
if response.Result["upstream_task_id"] != "keling-task-1" || item["url"] != "https://example.com/keling.mp4" || item["video_url"] != "https://example.com/keling.mp4" {
|
||||
t.Fatalf("unexpected keling response: %+v", response.Result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKelingOmniPayloadConvertsGatewayContent(t *testing.T) {
|
||||
payload, cleanupIDs, err := (KelingClient{}).kelingOmniPayload(context.Background(), Request{
|
||||
Kind: "videos.generations",
|
||||
ModelType: "omni_video",
|
||||
Model: "可灵V3多模态",
|
||||
Body: map[string]any{
|
||||
"model": "可灵V3多模态",
|
||||
"duration": 8,
|
||||
"aspect_ratio": "9:16",
|
||||
"resolution": "2160p",
|
||||
"audio": true,
|
||||
"content": []any{
|
||||
map[string]any{"type": "text", "text": "Refine the base video"},
|
||||
map[string]any{"type": "image_url", "role": "first_frame", "image_url": map[string]any{"url": "https://example.com/first.png"}},
|
||||
map[string]any{"type": "image_url", "role": "last_frame", "image_url": map[string]any{"url": "https://example.com/last.png"}},
|
||||
map[string]any{
|
||||
"type": "video_url",
|
||||
"role": "video_base",
|
||||
"video_url": map[string]any{
|
||||
"url": "https://example.com/base.mp4",
|
||||
"keep_original_sound": "yes",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Candidate: store.RuntimeModelCandidate{
|
||||
Provider: "keling",
|
||||
ProviderModelName: "kling-v3-omni",
|
||||
Capabilities: map[string]any{"omni_video": map[string]any{}},
|
||||
},
|
||||
}, "token")
|
||||
if err != nil {
|
||||
t.Fatalf("build keling omni payload: %v", err)
|
||||
}
|
||||
if len(cleanupIDs) != 0 {
|
||||
t.Fatalf("unexpected cleanup ids: %+v", cleanupIDs)
|
||||
}
|
||||
if payload["model_name"] != "kling-v3-omni" || payload["mode"] != "4k" || payload["prompt"] != "Refine the base video" {
|
||||
t.Fatalf("unexpected keling omni base fields: %+v", payload)
|
||||
}
|
||||
if _, ok := payload["sound"]; ok {
|
||||
t.Fatalf("omni payload with base video should not include sound: %+v", payload)
|
||||
}
|
||||
if _, ok := payload["duration"]; ok {
|
||||
t.Fatalf("base video edit should not include duration: %+v", payload)
|
||||
}
|
||||
if _, ok := payload["aspect_ratio"]; ok {
|
||||
t.Fatalf("base video edit should not include aspect_ratio: %+v", payload)
|
||||
}
|
||||
watermark, _ := payload["watermark_info"].(map[string]any)
|
||||
if watermark["enabled"] != false {
|
||||
t.Fatalf("keling watermark should be disabled by default: %+v", payload)
|
||||
}
|
||||
images, _ := payload["image_list"].([]any)
|
||||
if len(images) != 2 {
|
||||
t.Fatalf("unexpected keling image_list: %+v", payload["image_list"])
|
||||
}
|
||||
firstImage, _ := images[0].(map[string]any)
|
||||
lastImage, _ := images[1].(map[string]any)
|
||||
if firstImage["type"] != "first_frame" || lastImage["type"] != "end_frame" {
|
||||
t.Fatalf("frame roles should convert to keling omni types: %+v", images)
|
||||
}
|
||||
videos, _ := payload["video_list"].([]map[string]any)
|
||||
if len(videos) != 1 || videos[0]["refer_type"] != "base" || videos[0]["keep_original_sound"] != "yes" {
|
||||
t.Fatalf("video roles should convert to keling omni refer_type: %+v", payload["video_list"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestKelingClientVideoResumePollsWithoutSubmitting(t *testing.T) {
|
||||
var submitCalled bool
|
||||
var pollPath string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method + " " + r.URL.Path {
|
||||
case "POST /general/custom-elements", "POST /videos/omni-video":
|
||||
submitCalled = true
|
||||
t.Fatalf("resume should not submit or upload temporary elements")
|
||||
case "GET /videos/omni-video/keling-existing":
|
||||
pollPath = r.URL.Path
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"request_id": "req-resume",
|
||||
"data": map[string]any{
|
||||
"task_id": "keling-existing",
|
||||
"task_status": "succeed",
|
||||
"task_result": map[string]any{
|
||||
"videos": []any{map[string]any{"url": "https://example.com/resumed-keling.mp4"}},
|
||||
},
|
||||
},
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
response, err := (KelingClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
|
||||
Kind: "videos.generations",
|
||||
ModelType: "omni_video",
|
||||
Model: "可灵V3多模态",
|
||||
Body: map[string]any{"prompt": "resume", "pollIntervalMs": 100, "pollTimeoutSeconds": 1},
|
||||
RemoteTaskID: "keling-existing",
|
||||
RemoteTaskPayload: map[string]any{
|
||||
"endpoint": "/videos/omni-video",
|
||||
},
|
||||
Candidate: store.RuntimeModelCandidate{
|
||||
BaseURL: server.URL,
|
||||
Provider: "keling",
|
||||
AuthType: "AccessKey-SecretKey",
|
||||
ProviderModelName: "kling-v3-omni",
|
||||
Credentials: map[string]any{"accessKey": "ak", "secretKey": "sk"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resume keling video: %v", err)
|
||||
}
|
||||
if submitCalled || pollPath != "/videos/omni-video/keling-existing" {
|
||||
t.Fatalf("resume should poll existing task only, submit=%v poll=%s", submitCalled, pollPath)
|
||||
}
|
||||
data, _ := response.Result["data"].([]any)
|
||||
item, _ := data[0].(map[string]any)
|
||||
if response.Result["upstream_task_id"] != "keling-existing" || item["url"] != "https://example.com/resumed-keling.mp4" {
|
||||
t.Fatalf("unexpected resumed keling response: %+v", response.Result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKelingElementPayloadMapsTags(t *testing.T) {
|
||||
payload := kelingCreateElementPayload(map[string]any{
|
||||
"name": "subject",
|
||||
"frontal_image_url": "https://example.com/front.png",
|
||||
"tags": []any{"character", "unknown"},
|
||||
"refer_images": []any{
|
||||
map[string]any{"url": "https://example.com/side.png"},
|
||||
},
|
||||
})
|
||||
if payload["element_name"] != "subject" || payload["element_frontal_image"] != "https://example.com/front.png" {
|
||||
t.Fatalf("unexpected element payload base fields: %+v", payload)
|
||||
}
|
||||
tags, _ := payload["tag_list"].([]any)
|
||||
if len(tags) != 2 {
|
||||
t.Fatalf("unexpected tag list: %+v", payload["tag_list"])
|
||||
}
|
||||
firstTag, _ := tags[0].(map[string]any)
|
||||
secondTag, _ := tags[1].(map[string]any)
|
||||
if firstTag["tag_id"] != "o_102" || secondTag["tag_id"] != "o_108" {
|
||||
t.Fatalf("unexpected keling tag conversion: %+v", payload["tag_list"])
|
||||
}
|
||||
refs, _ := payload["element_refer_list"].([]any)
|
||||
if len(refs) != 1 {
|
||||
t.Fatalf("unexpected element references: %+v", payload["element_refer_list"])
|
||||
}
|
||||
}
|
||||
|
||||
func extractText(result map[string]any) string {
|
||||
choices, _ := result["choices"].([]any)
|
||||
choice, _ := choices[0].(map[string]any)
|
||||
|
||||
Reference in New Issue
Block a user