feat(clients): 添加Keling客户端支持视频生成和多模态功能
- 实现KelingClient结构体及其Run方法，支持视频生成功能 - 添加对image2video、text2video和omni-video任务类型的完整支持 - 集成Keling平台的身份验证机制，包括JWT令牌生成 - 实现视频任务的提交和轮询逻辑，支持异步处理流程 - 添加对多种输入格式的支持，包括图像帧、基础视频和参考素材 - 实现Keling元素管理和清理机制，处理临时创建的素材 - 在服务初始化中注册keling和kling两个提供商标识 - 添加数据库迁移脚本，更新Keling模型的音频功能配置 - 完善错误处理和重试机制，提升服务稳定性 - 编写完整的单元测试，覆盖各种视频生成场景和边界情况
This commit is contained in:
parent
90c3315468
commit
d09a4c2e4d
@ -662,6 +662,266 @@ func TestVolcesClientVideoResumePollsExistingTaskID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestKelingClientVideoSubmitsAndPollsImageTask(t *testing.T) {
|
||||||
|
var submitPath string
|
||||||
|
var pollPath string
|
||||||
|
var gotAuth string
|
||||||
|
var submittedTaskID string
|
||||||
|
var submittedPayload map[string]any
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
gotAuth = r.Header.Get("Authorization")
|
||||||
|
switch r.Method + " " + r.URL.Path {
|
||||||
|
case "POST /videos/image2video":
|
||||||
|
submitPath = r.URL.Path
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&submittedPayload); err != nil {
|
||||||
|
t.Fatalf("decode keling submit: %v", err)
|
||||||
|
}
|
||||||
|
if _, ok := submittedPayload["aspect_ratio"]; ok {
|
||||||
|
t.Fatalf("image2video payload should not include aspect_ratio: %+v", submittedPayload)
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"code": 0,
|
||||||
|
"request_id": "req-submit",
|
||||||
|
"data": map[string]any{"task_id": "keling-task-1"},
|
||||||
|
})
|
||||||
|
case "GET /videos/image2video/keling-task-1":
|
||||||
|
pollPath = r.URL.Path
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"code": 0,
|
||||||
|
"request_id": "req-poll",
|
||||||
|
"data": map[string]any{
|
||||||
|
"task_id": "keling-task-1",
|
||||||
|
"task_status": "succeed",
|
||||||
|
"created_at": 456,
|
||||||
|
"task_result": map[string]any{
|
||||||
|
"videos": []any{map[string]any{"url": "https://example.com/keling.mp4", "duration": 6}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
response, err := (KelingClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
|
||||||
|
Kind: "videos.generations",
|
||||||
|
ModelType: "image_to_video",
|
||||||
|
Model: "可灵2.6",
|
||||||
|
Body: map[string]any{
|
||||||
|
"model": "可灵2.6",
|
||||||
|
"prompt": "A clean product reveal",
|
||||||
|
"first_frame": "data:image/png;base64,Zmlyc3Q=",
|
||||||
|
"last_frame": "data:image/png;base64,bGFzdA==",
|
||||||
|
"duration": 6,
|
||||||
|
"resolution": "1080p",
|
||||||
|
"aspect_ratio": "16:9",
|
||||||
|
"audio": true,
|
||||||
|
"camera_control": "simple:zoom",
|
||||||
|
"camera_control_strength": 0.6,
|
||||||
|
},
|
||||||
|
Candidate: store.RuntimeModelCandidate{
|
||||||
|
BaseURL: server.URL,
|
||||||
|
Provider: "keling",
|
||||||
|
AuthType: "AccessKey-SecretKey",
|
||||||
|
ModelName: "可灵2.6",
|
||||||
|
ProviderModelName: "kling-v2-6",
|
||||||
|
Credentials: map[string]any{"accessKey": "ak", "secretKey": "sk"},
|
||||||
|
PlatformConfig: map[string]any{
|
||||||
|
"kelingPollIntervalMs": 100,
|
||||||
|
"kelingPollTimeoutSeconds": 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
OnRemoteTaskSubmitted: func(remoteTaskID string, payload map[string]any) error {
|
||||||
|
submittedTaskID = remoteTaskID
|
||||||
|
if payload["endpoint"] != "/videos/image2video" || payload["taskType"] != "image2video" {
|
||||||
|
t.Fatalf("unexpected submitted keling payload: %+v", payload)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run keling video: %v", err)
|
||||||
|
}
|
||||||
|
if submitPath != "/videos/image2video" || pollPath != "/videos/image2video/keling-task-1" || !strings.HasPrefix(gotAuth, "Bearer ") {
|
||||||
|
t.Fatalf("unexpected keling paths/auth submit=%s poll=%s auth=%s", submitPath, pollPath, gotAuth)
|
||||||
|
}
|
||||||
|
if submittedTaskID != "keling-task-1" {
|
||||||
|
t.Fatalf("remote task submit callback did not receive task id, got %q", submittedTaskID)
|
||||||
|
}
|
||||||
|
if submittedPayload["model_name"] != "kling-v2-6" ||
|
||||||
|
submittedPayload["prompt"] != "A clean product reveal" ||
|
||||||
|
submittedPayload["duration"] != "6" ||
|
||||||
|
submittedPayload["mode"] != "pro" ||
|
||||||
|
submittedPayload["sound"] != "on" ||
|
||||||
|
submittedPayload["image"] != "Zmlyc3Q=" ||
|
||||||
|
submittedPayload["image_tail"] != "bGFzdA==" {
|
||||||
|
t.Fatalf("unexpected keling submit payload: %+v", submittedPayload)
|
||||||
|
}
|
||||||
|
camera, _ := submittedPayload["camera_control"].(map[string]any)
|
||||||
|
config, _ := camera["config"].(map[string]any)
|
||||||
|
if camera["type"] != "simple" || numericValue(config["zoom"], 0) != 0.6 || numericValue(config["pan"], -1) != 0 {
|
||||||
|
t.Fatalf("unexpected keling camera conversion: %+v", submittedPayload["camera_control"])
|
||||||
|
}
|
||||||
|
data, _ := response.Result["data"].([]any)
|
||||||
|
item, _ := data[0].(map[string]any)
|
||||||
|
if response.Result["upstream_task_id"] != "keling-task-1" || item["url"] != "https://example.com/keling.mp4" || item["video_url"] != "https://example.com/keling.mp4" {
|
||||||
|
t.Fatalf("unexpected keling response: %+v", response.Result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKelingOmniPayloadConvertsGatewayContent(t *testing.T) {
|
||||||
|
payload, cleanupIDs, err := (KelingClient{}).kelingOmniPayload(context.Background(), Request{
|
||||||
|
Kind: "videos.generations",
|
||||||
|
ModelType: "omni_video",
|
||||||
|
Model: "可灵V3多模态",
|
||||||
|
Body: map[string]any{
|
||||||
|
"model": "可灵V3多模态",
|
||||||
|
"duration": 8,
|
||||||
|
"aspect_ratio": "9:16",
|
||||||
|
"resolution": "2160p",
|
||||||
|
"audio": true,
|
||||||
|
"content": []any{
|
||||||
|
map[string]any{"type": "text", "text": "Refine the base video"},
|
||||||
|
map[string]any{"type": "image_url", "role": "first_frame", "image_url": map[string]any{"url": "https://example.com/first.png"}},
|
||||||
|
map[string]any{"type": "image_url", "role": "last_frame", "image_url": map[string]any{"url": "https://example.com/last.png"}},
|
||||||
|
map[string]any{
|
||||||
|
"type": "video_url",
|
||||||
|
"role": "video_base",
|
||||||
|
"video_url": map[string]any{
|
||||||
|
"url": "https://example.com/base.mp4",
|
||||||
|
"keep_original_sound": "yes",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Candidate: store.RuntimeModelCandidate{
|
||||||
|
Provider: "keling",
|
||||||
|
ProviderModelName: "kling-v3-omni",
|
||||||
|
Capabilities: map[string]any{"omni_video": map[string]any{}},
|
||||||
|
},
|
||||||
|
}, "token")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("build keling omni payload: %v", err)
|
||||||
|
}
|
||||||
|
if len(cleanupIDs) != 0 {
|
||||||
|
t.Fatalf("unexpected cleanup ids: %+v", cleanupIDs)
|
||||||
|
}
|
||||||
|
if payload["model_name"] != "kling-v3-omni" || payload["mode"] != "4k" || payload["prompt"] != "Refine the base video" {
|
||||||
|
t.Fatalf("unexpected keling omni base fields: %+v", payload)
|
||||||
|
}
|
||||||
|
if _, ok := payload["sound"]; ok {
|
||||||
|
t.Fatalf("omni payload with base video should not include sound: %+v", payload)
|
||||||
|
}
|
||||||
|
if _, ok := payload["duration"]; ok {
|
||||||
|
t.Fatalf("base video edit should not include duration: %+v", payload)
|
||||||
|
}
|
||||||
|
if _, ok := payload["aspect_ratio"]; ok {
|
||||||
|
t.Fatalf("base video edit should not include aspect_ratio: %+v", payload)
|
||||||
|
}
|
||||||
|
watermark, _ := payload["watermark_info"].(map[string]any)
|
||||||
|
if watermark["enabled"] != false {
|
||||||
|
t.Fatalf("keling watermark should be disabled by default: %+v", payload)
|
||||||
|
}
|
||||||
|
images, _ := payload["image_list"].([]any)
|
||||||
|
if len(images) != 2 {
|
||||||
|
t.Fatalf("unexpected keling image_list: %+v", payload["image_list"])
|
||||||
|
}
|
||||||
|
firstImage, _ := images[0].(map[string]any)
|
||||||
|
lastImage, _ := images[1].(map[string]any)
|
||||||
|
if firstImage["type"] != "first_frame" || lastImage["type"] != "end_frame" {
|
||||||
|
t.Fatalf("frame roles should convert to keling omni types: %+v", images)
|
||||||
|
}
|
||||||
|
videos, _ := payload["video_list"].([]map[string]any)
|
||||||
|
if len(videos) != 1 || videos[0]["refer_type"] != "base" || videos[0]["keep_original_sound"] != "yes" {
|
||||||
|
t.Fatalf("video roles should convert to keling omni refer_type: %+v", payload["video_list"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKelingClientVideoResumePollsWithoutSubmitting(t *testing.T) {
|
||||||
|
var submitCalled bool
|
||||||
|
var pollPath string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.Method + " " + r.URL.Path {
|
||||||
|
case "POST /general/custom-elements", "POST /videos/omni-video":
|
||||||
|
submitCalled = true
|
||||||
|
t.Fatalf("resume should not submit or upload temporary elements")
|
||||||
|
case "GET /videos/omni-video/keling-existing":
|
||||||
|
pollPath = r.URL.Path
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"code": 0,
|
||||||
|
"request_id": "req-resume",
|
||||||
|
"data": map[string]any{
|
||||||
|
"task_id": "keling-existing",
|
||||||
|
"task_status": "succeed",
|
||||||
|
"task_result": map[string]any{
|
||||||
|
"videos": []any{map[string]any{"url": "https://example.com/resumed-keling.mp4"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
response, err := (KelingClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
|
||||||
|
Kind: "videos.generations",
|
||||||
|
ModelType: "omni_video",
|
||||||
|
Model: "可灵V3多模态",
|
||||||
|
Body: map[string]any{"prompt": "resume", "pollIntervalMs": 100, "pollTimeoutSeconds": 1},
|
||||||
|
RemoteTaskID: "keling-existing",
|
||||||
|
RemoteTaskPayload: map[string]any{
|
||||||
|
"endpoint": "/videos/omni-video",
|
||||||
|
},
|
||||||
|
Candidate: store.RuntimeModelCandidate{
|
||||||
|
BaseURL: server.URL,
|
||||||
|
Provider: "keling",
|
||||||
|
AuthType: "AccessKey-SecretKey",
|
||||||
|
ProviderModelName: "kling-v3-omni",
|
||||||
|
Credentials: map[string]any{"accessKey": "ak", "secretKey": "sk"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resume keling video: %v", err)
|
||||||
|
}
|
||||||
|
if submitCalled || pollPath != "/videos/omni-video/keling-existing" {
|
||||||
|
t.Fatalf("resume should poll existing task only, submit=%v poll=%s", submitCalled, pollPath)
|
||||||
|
}
|
||||||
|
data, _ := response.Result["data"].([]any)
|
||||||
|
item, _ := data[0].(map[string]any)
|
||||||
|
if response.Result["upstream_task_id"] != "keling-existing" || item["url"] != "https://example.com/resumed-keling.mp4" {
|
||||||
|
t.Fatalf("unexpected resumed keling response: %+v", response.Result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKelingElementPayloadMapsTags(t *testing.T) {
|
||||||
|
payload := kelingCreateElementPayload(map[string]any{
|
||||||
|
"name": "subject",
|
||||||
|
"frontal_image_url": "https://example.com/front.png",
|
||||||
|
"tags": []any{"character", "unknown"},
|
||||||
|
"refer_images": []any{
|
||||||
|
map[string]any{"url": "https://example.com/side.png"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if payload["element_name"] != "subject" || payload["element_frontal_image"] != "https://example.com/front.png" {
|
||||||
|
t.Fatalf("unexpected element payload base fields: %+v", payload)
|
||||||
|
}
|
||||||
|
tags, _ := payload["tag_list"].([]any)
|
||||||
|
if len(tags) != 2 {
|
||||||
|
t.Fatalf("unexpected tag list: %+v", payload["tag_list"])
|
||||||
|
}
|
||||||
|
firstTag, _ := tags[0].(map[string]any)
|
||||||
|
secondTag, _ := tags[1].(map[string]any)
|
||||||
|
if firstTag["tag_id"] != "o_102" || secondTag["tag_id"] != "o_108" {
|
||||||
|
t.Fatalf("unexpected keling tag conversion: %+v", payload["tag_list"])
|
||||||
|
}
|
||||||
|
refs, _ := payload["element_refer_list"].([]any)
|
||||||
|
if len(refs) != 1 {
|
||||||
|
t.Fatalf("unexpected element references: %+v", payload["element_refer_list"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func extractText(result map[string]any) string {
|
func extractText(result map[string]any) string {
|
||||||
choices, _ := result["choices"].([]any)
|
choices, _ := result["choices"].([]any)
|
||||||
choice, _ := choices[0].(map[string]any)
|
choice, _ := choices[0].(map[string]any)
|
||||||
|
|||||||
960
apps/api/internal/clients/keling.go
Normal file
960
apps/api/internal/clients/keling.go
Normal file
@ -0,0 +1,960 @@
|
|||||||
|
package clients
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
"github.com/golang-jwt/jwt/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
type KelingClient struct {
|
||||||
|
HTTPClient *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
type kelingPreparedTask struct {
|
||||||
|
Endpoint string
|
||||||
|
Payload map[string]any
|
||||||
|
RemoteTaskPayload map[string]any
|
||||||
|
CleanupElementIDs []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) Run(ctx context.Context, request Request) (Response, error) {
|
||||||
|
if request.Kind != "videos.generations" {
|
||||||
|
return Response{}, &ClientError{Code: "unsupported_kind", Message: "unsupported keling request kind", Retryable: false}
|
||||||
|
}
|
||||||
|
token, err := kelingAuthToken(request.Candidate)
|
||||||
|
if err != nil {
|
||||||
|
return Response{}, err
|
||||||
|
}
|
||||||
|
return c.runVideo(ctx, request, token)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) runVideo(ctx context.Context, request Request, token string) (Response, error) {
|
||||||
|
submitStartedAt := time.Now()
|
||||||
|
submitRequestID := strings.TrimSpace(request.RemoteTaskID)
|
||||||
|
upstreamTaskID := strings.TrimSpace(request.RemoteTaskID)
|
||||||
|
prepared := kelingResumePreparedTask(request)
|
||||||
|
if upstreamTaskID == "" {
|
||||||
|
var err error
|
||||||
|
prepared, err = c.prepareVideoTask(ctx, request, token)
|
||||||
|
if err != nil {
|
||||||
|
return Response{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if upstreamTaskID == "" {
|
||||||
|
_ = c.cleanupKelingElements(context.WithoutCancel(ctx), request, token, prepared.CleanupElementIDs)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if upstreamTaskID == "" {
|
||||||
|
submitResult, requestID, err := c.postJSON(ctx, request, prepared.Endpoint, token, prepared.Payload)
|
||||||
|
submitRequestID = requestID
|
||||||
|
if err != nil {
|
||||||
|
return Response{}, annotateResponseError(err, submitRequestID, submitStartedAt, time.Now())
|
||||||
|
}
|
||||||
|
upstreamTaskID = strings.TrimSpace(stringFromAny(kelingData(submitResult)["task_id"]))
|
||||||
|
if upstreamTaskID == "" {
|
||||||
|
_ = c.cleanupKelingElements(context.WithoutCancel(ctx), request, token, prepared.CleanupElementIDs)
|
||||||
|
return Response{}, &ClientError{Code: "invalid_response", Message: "keling video task id is missing", RequestID: submitRequestID, Retryable: false}
|
||||||
|
}
|
||||||
|
prepared.RemoteTaskPayload["submit"] = submitResult
|
||||||
|
if request.OnRemoteTaskSubmitted != nil {
|
||||||
|
if err := request.OnRemoteTaskSubmitted(upstreamTaskID, prepared.RemoteTaskPayload); err != nil {
|
||||||
|
return Response{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pollEndpoint := kelingPollEndpoint(request, prepared.Endpoint)
|
||||||
|
interval := kelingPollInterval(request)
|
||||||
|
timeout := kelingPollTimeout(request)
|
||||||
|
deadline := time.NewTimer(timeout)
|
||||||
|
defer deadline.Stop()
|
||||||
|
ticker := time.NewTicker(interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
var lastResult map[string]any
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Response{}, &ClientError{Code: "cancelled", Message: ctx.Err().Error(), RequestID: submitRequestID, Retryable: true}
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
pollStartedAt := time.Now()
|
||||||
|
pollResult, pollRequestID, err := c.getJSON(ctx, request, pollEndpoint+"/"+upstreamTaskID, token)
|
||||||
|
pollFinishedAt := time.Now()
|
||||||
|
requestID := firstNonEmpty(pollRequestID, submitRequestID, upstreamTaskID)
|
||||||
|
if err != nil {
|
||||||
|
return Response{}, annotateResponseError(err, requestID, pollStartedAt, pollFinishedAt)
|
||||||
|
}
|
||||||
|
lastResult = pollResult
|
||||||
|
|
||||||
|
switch kelingTaskStatus(pollResult) {
|
||||||
|
case "succeed":
|
||||||
|
_ = c.cleanupKelingElements(context.WithoutCancel(ctx), request, token, prepared.CleanupElementIDs)
|
||||||
|
prepared.CleanupElementIDs = nil
|
||||||
|
result := kelingVideoSuccessResult(request, upstreamTaskID, pollResult)
|
||||||
|
return Response{
|
||||||
|
Result: result,
|
||||||
|
RequestID: requestID,
|
||||||
|
Progress: kelingVideoProgress(request, upstreamTaskID),
|
||||||
|
ResponseStartedAt: submitStartedAt,
|
||||||
|
ResponseFinishedAt: pollFinishedAt,
|
||||||
|
ResponseDurationMS: responseDurationMS(submitStartedAt, pollFinishedAt),
|
||||||
|
}, nil
|
||||||
|
case "failed":
|
||||||
|
_ = c.cleanupKelingElements(context.WithoutCancel(ctx), request, token, prepared.CleanupElementIDs)
|
||||||
|
prepared.CleanupElementIDs = nil
|
||||||
|
return Response{}, &ClientError{
|
||||||
|
Code: kelingTaskErrorCode(pollResult),
|
||||||
|
Message: kelingTaskErrorMessage(request.Candidate, pollResult),
|
||||||
|
RequestID: requestID,
|
||||||
|
ResponseStartedAt: submitStartedAt,
|
||||||
|
ResponseFinishedAt: pollFinishedAt,
|
||||||
|
ResponseDurationMS: responseDurationMS(submitStartedAt, pollFinishedAt),
|
||||||
|
Retryable: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Response{}, &ClientError{Code: "cancelled", Message: ctx.Err().Error(), RequestID: requestID, Retryable: true}
|
||||||
|
case <-deadline.C:
|
||||||
|
return Response{}, &ClientError{
|
||||||
|
Code: "timeout",
|
||||||
|
Message: fmt.Sprintf("keling video task %s did not finish before timeout; last status: %s", upstreamTaskID, kelingTaskStatus(lastResult)),
|
||||||
|
RequestID: requestID,
|
||||||
|
Retryable: true,
|
||||||
|
}
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) prepareVideoTask(ctx context.Context, request Request, token string) (kelingPreparedTask, error) {
|
||||||
|
if kelingIsOmniRequest(request) {
|
||||||
|
payload, cleanupIDs, err := c.kelingOmniPayload(ctx, request, token)
|
||||||
|
if err != nil {
|
||||||
|
return kelingPreparedTask{}, err
|
||||||
|
}
|
||||||
|
return kelingPreparedTask{
|
||||||
|
Endpoint: "/videos/omni-video",
|
||||||
|
Payload: payload,
|
||||||
|
RemoteTaskPayload: map[string]any{"endpoint": "/videos/omni-video", "mode": "omni_video", "cleanupElementIds": cleanupIDs},
|
||||||
|
CleanupElementIDs: cleanupIDs,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
payload, taskType, err := kelingVideoPayload(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
return kelingPreparedTask{}, err
|
||||||
|
}
|
||||||
|
endpoint := "/videos/" + taskType
|
||||||
|
return kelingPreparedTask{
|
||||||
|
Endpoint: endpoint,
|
||||||
|
Payload: payload,
|
||||||
|
RemoteTaskPayload: map[string]any{"endpoint": endpoint, "taskType": taskType},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingResumePreparedTask(request Request) kelingPreparedTask {
|
||||||
|
endpoint := ""
|
||||||
|
for _, key := range []string{"endpoint", "pollEndpoint"} {
|
||||||
|
if value := strings.TrimSpace(stringFromAny(request.RemoteTaskPayload[key])); value != "" {
|
||||||
|
endpoint = value
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if endpoint == "" {
|
||||||
|
if kelingIsOmniRequest(request) {
|
||||||
|
endpoint = "/videos/omni-video"
|
||||||
|
} else {
|
||||||
|
endpoint = "/videos/" + kelingTaskTypeFromRequest(request)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return kelingPreparedTask{Endpoint: endpoint, RemoteTaskPayload: map[string]any{"endpoint": endpoint}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingVideoPayload(ctx context.Context, request Request) (map[string]any, string, error) {
|
||||||
|
body := cleanProviderBody(request.Body)
|
||||||
|
content := contentItems(body["content"])
|
||||||
|
if len(content) == 0 {
|
||||||
|
content = buildVolcesContentFromBody(body)
|
||||||
|
}
|
||||||
|
prompt := firstKelingPrompt(content)
|
||||||
|
if prompt == "" {
|
||||||
|
return nil, "", &ClientError{Code: "invalid_parameter", Message: "keling video prompt is required", StatusCode: 400, Retryable: false}
|
||||||
|
}
|
||||||
|
firstFrame, lastFrame, referenceImages := kelingImageInputs(content)
|
||||||
|
isImage2Video := firstFrame != "" || lastFrame != "" || len(referenceImages) > 0
|
||||||
|
primaryImage := firstFrame
|
||||||
|
if primaryImage == "" && len(referenceImages) <= 1 && len(referenceImages) > 0 {
|
||||||
|
primaryImage = referenceImages[0]
|
||||||
|
}
|
||||||
|
if primaryImage == "" {
|
||||||
|
primaryImage = lastFrame
|
||||||
|
}
|
||||||
|
|
||||||
|
payload := map[string]any{
|
||||||
|
"prompt": prompt,
|
||||||
|
"model_name": upstreamModelName(request.Candidate),
|
||||||
|
"duration": fmtDuration(body["duration"], 5),
|
||||||
|
}
|
||||||
|
if value := strings.TrimSpace(stringFromAny(body["negative_prompt"])); value != "" {
|
||||||
|
payload["negative_prompt"] = value
|
||||||
|
}
|
||||||
|
if value, ok := body["cfg_scale"]; ok && numericValue(value, 0) > 0 {
|
||||||
|
payload["cfg_scale"] = value
|
||||||
|
}
|
||||||
|
if boolValue(body, "audio") || boolValue(body, "output_audio") {
|
||||||
|
payload["sound"] = "on"
|
||||||
|
}
|
||||||
|
if mode := kelingModeByResolution(firstNonEmptyStringValue(body, "resolution", "size")); mode != "" {
|
||||||
|
payload["mode"] = mode
|
||||||
|
}
|
||||||
|
if ratio := strings.TrimSpace(firstNonEmptyStringValue(body, "aspect_ratio", "aspectRatio", "ratio")); strings.Contains(ratio, ":") {
|
||||||
|
payload["aspect_ratio"] = ratio
|
||||||
|
}
|
||||||
|
if camera := kelingCameraControl(body); camera != nil {
|
||||||
|
payload["camera_control"] = camera
|
||||||
|
}
|
||||||
|
if primaryImage != "" {
|
||||||
|
encoded, err := kelingImageToBase64(ctx, request, primaryImage)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
payload["image"] = encoded
|
||||||
|
}
|
||||||
|
if lastFrame != "" {
|
||||||
|
encoded, err := kelingImageToBase64(ctx, request, lastFrame)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
payload["image_tail"] = encoded
|
||||||
|
}
|
||||||
|
if len(referenceImages) > 0 {
|
||||||
|
imageList := make([]any, 0, len(referenceImages))
|
||||||
|
for _, url := range referenceImages {
|
||||||
|
encoded, err := kelingImageToBase64(ctx, request, url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
imageList = append(imageList, map[string]any{"image": encoded})
|
||||||
|
}
|
||||||
|
payload["image_list"] = imageList
|
||||||
|
}
|
||||||
|
if !strings.Contains(stringFromAny(payload["aspect_ratio"]), ":") || isImage2Video {
|
||||||
|
delete(payload, "aspect_ratio")
|
||||||
|
}
|
||||||
|
|
||||||
|
taskType := "text2video"
|
||||||
|
if primaryImage != "" {
|
||||||
|
taskType = "image2video"
|
||||||
|
} else if len(referenceImages) > 1 {
|
||||||
|
taskType = "multi-image2video"
|
||||||
|
}
|
||||||
|
return payload, taskType, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingTaskTypeFromRequest(request Request) string {
|
||||||
|
body := cleanProviderBody(request.Body)
|
||||||
|
content := contentItems(body["content"])
|
||||||
|
if len(content) == 0 {
|
||||||
|
content = buildVolcesContentFromBody(body)
|
||||||
|
}
|
||||||
|
firstFrame, lastFrame, referenceImages := kelingImageInputs(content)
|
||||||
|
if firstFrame != "" || lastFrame != "" || len(referenceImages) == 1 {
|
||||||
|
return "image2video"
|
||||||
|
}
|
||||||
|
if len(referenceImages) > 1 {
|
||||||
|
return "multi-image2video"
|
||||||
|
}
|
||||||
|
return "text2video"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) kelingOmniPayload(ctx context.Context, request Request, token string) (map[string]any, []string, error) {
|
||||||
|
body := cleanProviderBody(request.Body)
|
||||||
|
content := contentItems(body["content"])
|
||||||
|
if len(content) == 0 {
|
||||||
|
content = buildVolcesContentFromBody(body)
|
||||||
|
}
|
||||||
|
prompt := firstKelingPrompt(content)
|
||||||
|
images := kelingOmniImageList(content)
|
||||||
|
videos := kelingOmniVideoList(content)
|
||||||
|
uploadedElementIDs := make([]string, 0)
|
||||||
|
elements, createdIDs, err := c.kelingOmniElementList(ctx, request, token, content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
uploadedElementIDs = append(uploadedElementIDs, createdIDs...)
|
||||||
|
shots := kelingShotPrompts(content)
|
||||||
|
hasMultiPrompt := len(shots) > 0
|
||||||
|
hasVideo := len(videos) > 0
|
||||||
|
hasVideoEdit := kelingHasBaseVideo(videos)
|
||||||
|
hasFirstFrame := kelingHasFirstFrame(images)
|
||||||
|
|
||||||
|
payload := map[string]any{
|
||||||
|
"model_name": upstreamModelName(request.Candidate),
|
||||||
|
"mode": kelingModeByResolution(firstNonEmptyStringValue(body, "resolution", "size")),
|
||||||
|
"watermark_info": map[string]any{"enabled": false},
|
||||||
|
"negative_prompt": strings.TrimSpace(stringFromAny(body["negative_prompt"])),
|
||||||
|
}
|
||||||
|
if !hasMultiPrompt {
|
||||||
|
payload["prompt"] = prompt
|
||||||
|
if body["duration"] != nil {
|
||||||
|
payload["duration"] = fmtDuration(body["duration"], 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ratio := strings.TrimSpace(firstNonEmptyStringValue(body, "aspect_ratio", "aspectRatio", "ratio")); strings.Contains(ratio, ":") {
|
||||||
|
payload["aspect_ratio"] = ratio
|
||||||
|
}
|
||||||
|
if len(images) > 0 {
|
||||||
|
payload["image_list"] = images
|
||||||
|
}
|
||||||
|
if len(videos) > 0 {
|
||||||
|
payload["video_list"] = videos
|
||||||
|
}
|
||||||
|
if len(elements) > 0 {
|
||||||
|
payload["element_list"] = elements
|
||||||
|
}
|
||||||
|
if (boolValue(body, "audio") || boolValue(body, "output_audio")) && !hasVideo {
|
||||||
|
payload["sound"] = "on"
|
||||||
|
}
|
||||||
|
if hasMultiPrompt {
|
||||||
|
payload["multi_shot"] = true
|
||||||
|
payload["shot_type"] = "customize"
|
||||||
|
total := 0.0
|
||||||
|
multiPrompt := make([]any, 0, len(shots))
|
||||||
|
for index, shot := range shots {
|
||||||
|
duration := shot.duration
|
||||||
|
if duration <= 0 {
|
||||||
|
duration = 5
|
||||||
|
}
|
||||||
|
total += duration
|
||||||
|
multiPrompt = append(multiPrompt, map[string]any{
|
||||||
|
"index": index + 1,
|
||||||
|
"prompt": shot.text,
|
||||||
|
"duration": fmtDuration(duration, 5),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
delete(payload, "prompt")
|
||||||
|
payload["multi_prompt"] = multiPrompt
|
||||||
|
payload["duration"] = fmtDuration(total, 0)
|
||||||
|
}
|
||||||
|
deleteEmptyStringFields(payload)
|
||||||
|
if hasVideoEdit {
|
||||||
|
delete(payload, "duration")
|
||||||
|
delete(payload, "aspect_ratio")
|
||||||
|
}
|
||||||
|
if hasVideo && !hasVideoEdit && !strings.Contains(stringFromAny(payload["aspect_ratio"]), ":") {
|
||||||
|
payload["aspect_ratio"] = "16:9"
|
||||||
|
}
|
||||||
|
if !hasVideoEdit && !hasFirstFrame && !strings.Contains(stringFromAny(payload["aspect_ratio"]), ":") {
|
||||||
|
payload["aspect_ratio"] = "16:9"
|
||||||
|
}
|
||||||
|
return payload, uploadedElementIDs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) kelingOmniElementList(ctx context.Context, request Request, token string, content []map[string]any) ([]any, []string, error) {
|
||||||
|
elements := make([]any, 0)
|
||||||
|
createdIDs := make([]string, 0)
|
||||||
|
for _, item := range content {
|
||||||
|
if stringFromAny(item["type"]) != "element" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
element := mapFromAny(item["element"])
|
||||||
|
if element == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if id := kelingStringFromAny(firstPresent(element["element_id"], element["id"])); id != "" {
|
||||||
|
elements = append(elements, map[string]any{"element_id": id})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
inline := mapFromAny(element["inline_element"])
|
||||||
|
if inline == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
payload := kelingCreateElementPayload(inline)
|
||||||
|
if payload == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
id, err := c.createKelingElement(ctx, request, token, payload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, createdIDs, err
|
||||||
|
}
|
||||||
|
elements = append(elements, map[string]any{"element_id": id})
|
||||||
|
createdIDs = append(createdIDs, id)
|
||||||
|
}
|
||||||
|
return elements, createdIDs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) postJSON(ctx context.Context, request Request, path string, token string, body map[string]any) (map[string]any, string, error) {
|
||||||
|
raw, _ := json.Marshal(body)
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinURL(request.Candidate.BaseURL, path), bytes.NewReader(raw))
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
|
resp, err := httpClient(request.HTTPClient, c.HTTPClient).Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", &ClientError{Code: "network", Message: err.Error(), Retryable: true}
|
||||||
|
}
|
||||||
|
requestID := requestIDFromHTTPResponse(resp)
|
||||||
|
result, err := decodeHTTPResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return result, requestID, err
|
||||||
|
}
|
||||||
|
if code := intFromAny(result["code"]); code != 0 {
|
||||||
|
return result, requestID, &ClientError{Code: kelingEnvelopeErrorCode(result), Message: kelingEnvelopeErrorMessage(result), RequestID: firstNonEmpty(requestID, stringFromAny(result["request_id"])), Retryable: false}
|
||||||
|
}
|
||||||
|
return result, firstNonEmpty(requestID, stringFromAny(result["request_id"])), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) getJSON(ctx context.Context, request Request, path string, token string) (map[string]any, string, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, joinURL(request.Candidate.BaseURL, path), nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
|
resp, err := httpClient(request.HTTPClient, c.HTTPClient).Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", &ClientError{Code: "network", Message: err.Error(), Retryable: true}
|
||||||
|
}
|
||||||
|
requestID := requestIDFromHTTPResponse(resp)
|
||||||
|
result, err := decodeHTTPResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return result, requestID, err
|
||||||
|
}
|
||||||
|
if code := intFromAny(result["code"]); code != 0 {
|
||||||
|
return result, requestID, &ClientError{Code: kelingEnvelopeErrorCode(result), Message: kelingEnvelopeErrorMessage(result), RequestID: firstNonEmpty(requestID, stringFromAny(result["request_id"])), Retryable: false}
|
||||||
|
}
|
||||||
|
return result, firstNonEmpty(requestID, stringFromAny(result["request_id"])), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) createKelingElement(ctx context.Context, request Request, token string, payload map[string]any) (string, error) {
|
||||||
|
raw, _ := json.Marshal(payload)
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinURL(request.Candidate.BaseURL, "/general/custom-elements"), bytes.NewReader(raw))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
|
resp, err := httpClient(request.HTTPClient, c.HTTPClient).Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", &ClientError{Code: "network", Message: err.Error(), Retryable: true}
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 16*1024*1024))
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return "", &ClientError{Code: statusCodeName(resp.StatusCode), Message: errorMessage(body, resp.Status), StatusCode: resp.StatusCode, RequestID: requestIDFromHTTPResponse(resp), Retryable: HTTPRetryable(resp.StatusCode)}
|
||||||
|
}
|
||||||
|
var parsed struct {
|
||||||
|
Code int `json:"code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
RequestID string `json:"request_id"`
|
||||||
|
Data map[string]any `json:"data"`
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(body))
|
||||||
|
decoder.UseNumber()
|
||||||
|
if err := decoder.Decode(&parsed); err != nil {
|
||||||
|
return "", &ClientError{Code: "invalid_response", Message: err.Error(), Retryable: false}
|
||||||
|
}
|
||||||
|
if parsed.Code != 0 {
|
||||||
|
return "", &ClientError{Code: "keling_element_create_failed", Message: parsed.Message, RequestID: parsed.RequestID, Retryable: false}
|
||||||
|
}
|
||||||
|
id := kelingStringFromAny(parsed.Data["element_id"])
|
||||||
|
if id == "" {
|
||||||
|
return "", &ClientError{Code: "invalid_response", Message: "keling element id is missing", RequestID: parsed.RequestID, Retryable: false}
|
||||||
|
}
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KelingClient) cleanupKelingElements(ctx context.Context, request Request, token string, elementIDs []string) error {
|
||||||
|
for _, id := range elementIDs {
|
||||||
|
id = strings.TrimSpace(id)
|
||||||
|
if id == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, _, _ = c.postJSON(ctx, request, "/general/delete-elements", token, map[string]any{"element_id": id})
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingAuthToken(candidate store.RuntimeModelCandidate) (string, error) {
|
||||||
|
apiKey := credential(candidate.Credentials, "apiKey", "api_key", "key", "token")
|
||||||
|
accessKey := credential(candidate.Credentials, "accessKey", "access_key", "ak")
|
||||||
|
secretKey := credential(candidate.Credentials, "secretKey", "secret_key", "sk")
|
||||||
|
if accessKey != "" || secretKey != "" || strings.EqualFold(strings.TrimSpace(candidate.AuthType), "AccessKey-SecretKey") {
|
||||||
|
if accessKey == "" || secretKey == "" {
|
||||||
|
return "", &ClientError{Code: "missing_credentials", Message: "keling accessKey and secretKey are required", Retryable: false}
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
claims := jwt.MapClaims{
|
||||||
|
"iss": accessKey,
|
||||||
|
"exp": now.Add(30 * time.Minute).Unix(),
|
||||||
|
"nbf": now.Add(-5 * time.Second).Unix(),
|
||||||
|
}
|
||||||
|
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
|
||||||
|
signed, err := token.SignedString([]byte(secretKey))
|
||||||
|
if err != nil {
|
||||||
|
return "", &ClientError{Code: "auth_failed", Message: err.Error(), Retryable: false}
|
||||||
|
}
|
||||||
|
return signed, nil
|
||||||
|
}
|
||||||
|
if apiKey == "" {
|
||||||
|
return "", &ClientError{Code: "missing_credentials", Message: "keling api key is required", Retryable: false}
|
||||||
|
}
|
||||||
|
return apiKey, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingImageToBase64(ctx context.Context, request Request, value string) (string, error) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(value, "data:") {
|
||||||
|
parts := strings.SplitN(value, ",", 2)
|
||||||
|
if len(parts) == 2 {
|
||||||
|
return strings.TrimSpace(parts[1]), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, value, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
resp, err := httpClient(request.HTTPClient).Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", &ClientError{Code: "network", Message: err.Error(), Retryable: true}
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||||
|
return "", &ClientError{Code: statusCodeName(resp.StatusCode), Message: errorMessage(raw, resp.Status), StatusCode: resp.StatusCode, RequestID: requestIDFromHTTPResponse(resp), Retryable: HTTPRetryable(resp.StatusCode)}
|
||||||
|
}
|
||||||
|
raw, err := io.ReadAll(io.LimitReader(resp.Body, 16*1024*1024))
|
||||||
|
if err != nil {
|
||||||
|
return "", &ClientError{Code: "network", Message: err.Error(), Retryable: true}
|
||||||
|
}
|
||||||
|
return base64.StdEncoding.EncodeToString(raw), nil
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingIsOmniRequest(request Request) bool {
|
||||||
|
modelType := strings.TrimSpace(request.ModelType)
|
||||||
|
return modelType == "omni_video" || modelType == "omni" ||
|
||||||
|
request.Candidate.Capabilities["omni_video"] != nil ||
|
||||||
|
request.Candidate.Capabilities["omni"] != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstKelingPrompt(content []map[string]any) string {
|
||||||
|
for _, item := range content {
|
||||||
|
if stringFromAny(item["type"]) == "text" && stringFromAny(item["role"]) != "shot_prompt" && item["shot_index"] == nil {
|
||||||
|
if text := strings.TrimSpace(stringFromAny(item["text"])); text != "" {
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingImageInputs(content []map[string]any) (string, string, []string) {
|
||||||
|
firstFrame := ""
|
||||||
|
lastFrame := ""
|
||||||
|
references := make([]string, 0)
|
||||||
|
for _, item := range content {
|
||||||
|
if !isKelingImageContent(item) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
url := kelingNestedURL(item, "image_url")
|
||||||
|
if url == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch stringFromAny(item["role"]) {
|
||||||
|
case "first_frame":
|
||||||
|
if firstFrame == "" {
|
||||||
|
firstFrame = url
|
||||||
|
}
|
||||||
|
case "last_frame":
|
||||||
|
if lastFrame == "" {
|
||||||
|
lastFrame = url
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
references = append(references, url)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return firstFrame, lastFrame, references
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingOmniImageList(content []map[string]any) []any {
|
||||||
|
out := make([]any, 0)
|
||||||
|
for _, item := range content {
|
||||||
|
if !isKelingImageContent(item) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
url := kelingNestedURL(item, "image_url")
|
||||||
|
if url == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
image := map[string]any{"image_url": url}
|
||||||
|
switch stringFromAny(item["role"]) {
|
||||||
|
case "first_frame":
|
||||||
|
image["type"] = "first_frame"
|
||||||
|
case "last_frame":
|
||||||
|
image["type"] = "end_frame"
|
||||||
|
}
|
||||||
|
out = append(out, image)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingOmniVideoList(content []map[string]any) []map[string]any {
|
||||||
|
out := make([]map[string]any, 0)
|
||||||
|
for _, item := range content {
|
||||||
|
if !isKelingVideoContent(item) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
nested := mapFromAny(item["video_url"])
|
||||||
|
url := strings.TrimSpace(stringFromAny(nested["url"]))
|
||||||
|
if url == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
video := map[string]any{"video_url": url}
|
||||||
|
referType := strings.TrimSpace(stringFromAny(nested["refer_type"]))
|
||||||
|
if referType == "" {
|
||||||
|
switch stringFromAny(item["role"]) {
|
||||||
|
case "video_base":
|
||||||
|
referType = "base"
|
||||||
|
case "video_feature", "reference_video":
|
||||||
|
referType = "feature"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if referType == "base" || referType == "feature" {
|
||||||
|
video["refer_type"] = referType
|
||||||
|
}
|
||||||
|
if keep := strings.TrimSpace(stringFromAny(nested["keep_original_sound"])); keep != "" {
|
||||||
|
video["keep_original_sound"] = keep
|
||||||
|
}
|
||||||
|
out = append(out, video)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
type kelingShotPrompt struct {
|
||||||
|
index int
|
||||||
|
text string
|
||||||
|
duration float64
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingShotPrompts(content []map[string]any) []kelingShotPrompt {
|
||||||
|
shots := make([]kelingShotPrompt, 0)
|
||||||
|
for index, item := range content {
|
||||||
|
if stringFromAny(item["type"]) != "text" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if stringFromAny(item["role"]) != "shot_prompt" && item["shot_index"] == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
text := strings.TrimSpace(stringFromAny(item["text"]))
|
||||||
|
if text == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
shotIndex := int(math.Floor(numericValue(item["shot_index"], float64(index))))
|
||||||
|
shots = append(shots, kelingShotPrompt{index: shotIndex, text: text, duration: numericValue(item["duration"], 5)})
|
||||||
|
}
|
||||||
|
sort.SliceStable(shots, func(i, j int) bool { return shots[i].index < shots[j].index })
|
||||||
|
return shots
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingHasBaseVideo(videos []map[string]any) bool {
|
||||||
|
for _, video := range videos {
|
||||||
|
if stringFromAny(video["refer_type"]) == "base" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingHasFirstFrame(images []any) bool {
|
||||||
|
for _, item := range images {
|
||||||
|
image := mapFromAny(item)
|
||||||
|
if stringFromAny(image["type"]) == "first_frame" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingCreateElementPayload(inline map[string]any) map[string]any {
|
||||||
|
frontURL := strings.TrimSpace(firstNonEmptyStringValue(inline, "frontal_image_url", "frontalImageUrl", "element_frontal_image", "image_url", "imageUrl", "url"))
|
||||||
|
if frontURL == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
name := firstNonEmptyStringValue(inline, "name", "element_name", "elementName")
|
||||||
|
if name == "" {
|
||||||
|
name = "temporary element"
|
||||||
|
}
|
||||||
|
payload := map[string]any{
|
||||||
|
"element_name": name,
|
||||||
|
"element_description": firstNonEmpty(firstNonEmptyStringValue(inline, "description"), name),
|
||||||
|
"element_frontal_image": frontURL,
|
||||||
|
}
|
||||||
|
referImages := make([]any, 0)
|
||||||
|
for _, ref := range mapListFromAny(firstPresent(inline["refer_images"], inline["referImages"], inline["element_refer_list"])) {
|
||||||
|
url := strings.TrimSpace(firstNonEmptyStringValue(ref, "url", "image_url", "imageUrl"))
|
||||||
|
if url != "" {
|
||||||
|
referImages = append(referImages, map[string]any{"image_url": url})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(referImages) > 0 {
|
||||||
|
payload["element_refer_list"] = referImages
|
||||||
|
}
|
||||||
|
if tags := kelingElementTagList(inline["tags"]); len(tags) > 0 {
|
||||||
|
payload["tag_list"] = tags
|
||||||
|
}
|
||||||
|
return payload
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingElementTagList(value any) []any {
|
||||||
|
mapping := map[string]string{
|
||||||
|
"hot_meme": "o_101",
|
||||||
|
"character": "o_102",
|
||||||
|
"animal": "o_103",
|
||||||
|
"prop": "o_104",
|
||||||
|
"costume": "o_105",
|
||||||
|
"scene": "o_106",
|
||||||
|
"effect": "o_107",
|
||||||
|
"other": "o_108",
|
||||||
|
}
|
||||||
|
out := make([]any, 0)
|
||||||
|
for _, tag := range stringListFromAny(value) {
|
||||||
|
id := mapping[strings.TrimSpace(tag)]
|
||||||
|
if id == "" {
|
||||||
|
id = mapping["other"]
|
||||||
|
}
|
||||||
|
out = append(out, map[string]any{"tag_id": id})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingNestedURL(item map[string]any, key string) string {
|
||||||
|
nested := mapFromAny(item[key])
|
||||||
|
if nested != nil {
|
||||||
|
if value := strings.TrimSpace(stringFromAny(nested["url"])); value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(stringFromAny(item[key]))
|
||||||
|
}
|
||||||
|
|
||||||
|
func isKelingImageContent(item map[string]any) bool {
|
||||||
|
return stringFromAny(item["type"]) == "image_url" || mapFromAny(item["image_url"]) != nil || strings.TrimSpace(stringFromAny(item["image_url"])) != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func isKelingVideoContent(item map[string]any) bool {
|
||||||
|
return stringFromAny(item["type"]) == "video_url" || mapFromAny(item["video_url"]) != nil || strings.TrimSpace(stringFromAny(item["video_url"])) != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingModeByResolution(resolution string) string {
|
||||||
|
switch strings.TrimSpace(resolution) {
|
||||||
|
case "2160p":
|
||||||
|
return "4k"
|
||||||
|
case "1080p":
|
||||||
|
return "pro"
|
||||||
|
case "480p", "720p", "":
|
||||||
|
return "std"
|
||||||
|
default:
|
||||||
|
if strings.HasSuffix(strings.TrimSpace(resolution), "p") {
|
||||||
|
return "std"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingCameraControl(body map[string]any) map[string]any {
|
||||||
|
cameraControl := strings.TrimSpace(stringFromAny(body["camera_control"]))
|
||||||
|
if cameraControl == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(cameraControl, "simple") {
|
||||||
|
directions := []string{"horizontal", "vertical", "pan", "tilt", "roll", "zoom"}
|
||||||
|
current := ""
|
||||||
|
parts := strings.SplitN(cameraControl, ":", 2)
|
||||||
|
if len(parts) == 2 {
|
||||||
|
current = parts[1]
|
||||||
|
}
|
||||||
|
strength := firstPresent(body["camera_control_strength"], body["cameraControlStrength"])
|
||||||
|
config := map[string]any{}
|
||||||
|
for _, direction := range directions {
|
||||||
|
if direction == current {
|
||||||
|
config[direction] = strength
|
||||||
|
} else {
|
||||||
|
config[direction] = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map[string]any{"type": "simple", "config": config}
|
||||||
|
}
|
||||||
|
return map[string]any{"type": cameraControl}
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingData(result map[string]any) map[string]any {
|
||||||
|
data, _ := result["data"].(map[string]any)
|
||||||
|
if data == nil {
|
||||||
|
return map[string]any{}
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingTaskStatus(result map[string]any) string {
|
||||||
|
return strings.ToLower(strings.TrimSpace(stringFromAny(kelingData(result)["task_status"])))
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingTaskErrorCode(result map[string]any) string {
|
||||||
|
if code := intFromAny(result["code"]); code != 0 {
|
||||||
|
return fmt.Sprintf("keling_%d", code)
|
||||||
|
}
|
||||||
|
return "keling_task_failed"
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingTaskErrorMessage(candidate store.RuntimeModelCandidate, result map[string]any) string {
|
||||||
|
message := strings.TrimSpace(stringFromAny(kelingData(result)["task_status_msg"]))
|
||||||
|
if message == "" {
|
||||||
|
message = strings.TrimSpace(stringFromAny(result["message"]))
|
||||||
|
}
|
||||||
|
if message == "" {
|
||||||
|
message = "keling video task failed"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Platform:%s,Code:%v,requestId:%s,message:%s", candidate.Provider, result["code"], stringFromAny(result["request_id"]), message)
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingEnvelopeErrorCode(result map[string]any) string {
|
||||||
|
if code := intFromAny(result["code"]); code != 0 {
|
||||||
|
return fmt.Sprintf("keling_%d", code)
|
||||||
|
}
|
||||||
|
return "keling_error"
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingEnvelopeErrorMessage(result map[string]any) string {
|
||||||
|
if message := strings.TrimSpace(stringFromAny(result["message"])); message != "" {
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
return "keling request failed"
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingVideoSuccessResult(request Request, upstreamTaskID string, raw map[string]any) map[string]any {
|
||||||
|
data := kelingData(raw)
|
||||||
|
taskResult, _ := data["task_result"].(map[string]any)
|
||||||
|
videos, _ := taskResult["videos"].([]any)
|
||||||
|
items := make([]any, 0, len(videos))
|
||||||
|
for _, rawVideo := range videos {
|
||||||
|
video := mapFromAny(rawVideo)
|
||||||
|
url := strings.TrimSpace(stringFromAny(video["url"]))
|
||||||
|
if url == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
item := map[string]any{"url": url, "video_url": url, "type": "video"}
|
||||||
|
if duration := intFromAny(video["duration"]); duration > 0 {
|
||||||
|
item["duration"] = duration
|
||||||
|
}
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
created := intFromAny(data["created_at"])
|
||||||
|
if created == 0 {
|
||||||
|
created = int(nowUnix())
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"id": upstreamTaskID,
|
||||||
|
"object": "video.generation",
|
||||||
|
"created": created,
|
||||||
|
"model": upstreamModelName(request.Candidate),
|
||||||
|
"status": "succeeded",
|
||||||
|
"upstream_task_id": upstreamTaskID,
|
||||||
|
"data": items,
|
||||||
|
"raw": raw,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingVideoProgress(request Request, upstreamTaskID string) []Progress {
|
||||||
|
progress := providerProgress(request)
|
||||||
|
progress = append(progress, Progress{
|
||||||
|
Phase: "polling_result",
|
||||||
|
Progress: 0.9,
|
||||||
|
Message: "keling video task completed",
|
||||||
|
Payload: map[string]any{"upstreamTaskId": upstreamTaskID},
|
||||||
|
})
|
||||||
|
return progress
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingPollEndpoint(request Request, fallback string) string {
|
||||||
|
for _, key := range []string{"endpoint", "pollEndpoint"} {
|
||||||
|
if value := strings.TrimSpace(stringFromAny(request.RemoteTaskPayload[key])); value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingPollInterval(request Request) time.Duration {
|
||||||
|
ms := numericValue(firstPresent(request.Candidate.PlatformConfig["kelingPollIntervalMs"], request.Candidate.PlatformConfig["klingPollIntervalMs"], request.Body["pollIntervalMs"], request.Body["poll_interval_ms"]), 15000)
|
||||||
|
if ms < 100 {
|
||||||
|
ms = 100
|
||||||
|
}
|
||||||
|
return time.Duration(ms) * time.Millisecond
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingPollTimeout(request Request) time.Duration {
|
||||||
|
seconds := numericValue(firstPresent(request.Candidate.PlatformConfig["kelingPollTimeoutSeconds"], request.Candidate.PlatformConfig["klingPollTimeoutSeconds"], request.Body["pollTimeoutSeconds"], request.Body["poll_timeout_seconds"]), 600)
|
||||||
|
if seconds < 1 {
|
||||||
|
seconds = 600
|
||||||
|
}
|
||||||
|
return time.Duration(seconds) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
func fmtDuration(value any, fallback float64) string {
|
||||||
|
duration := numericValue(value, fallback)
|
||||||
|
if math.Abs(duration-math.Round(duration)) < 1e-9 {
|
||||||
|
return fmt.Sprintf("%d", int(math.Round(duration)))
|
||||||
|
}
|
||||||
|
return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.6f", duration), "0"), ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteEmptyStringFields(payload map[string]any) {
|
||||||
|
for key, value := range payload {
|
||||||
|
if text, ok := value.(string); ok && strings.TrimSpace(text) == "" {
|
||||||
|
delete(payload, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func kelingStringFromAny(value any) string {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case json.Number:
|
||||||
|
return typed.String()
|
||||||
|
case float64:
|
||||||
|
if math.Abs(typed-math.Round(typed)) < 1e-9 {
|
||||||
|
return fmt.Sprintf("%.0f", typed)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%v", typed)
|
||||||
|
case int:
|
||||||
|
return fmt.Sprintf("%d", typed)
|
||||||
|
case int64:
|
||||||
|
return fmt.Sprintf("%d", typed)
|
||||||
|
case string:
|
||||||
|
return strings.TrimSpace(typed)
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -55,6 +55,8 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger) *Service {
|
|||||||
"openai": clients.OpenAIClient{HTTPClient: httpClients.none},
|
"openai": clients.OpenAIClient{HTTPClient: httpClients.none},
|
||||||
"gemini": clients.GeminiClient{HTTPClient: httpClients.none},
|
"gemini": clients.GeminiClient{HTTPClient: httpClients.none},
|
||||||
"volces": clients.VolcesClient{HTTPClient: httpClients.none},
|
"volces": clients.VolcesClient{HTTPClient: httpClients.none},
|
||||||
|
"keling": clients.KelingClient{HTTPClient: httpClients.none},
|
||||||
|
"kling": clients.KelingClient{HTTPClient: httpClients.none},
|
||||||
"simulation": clients.SimulationClient{},
|
"simulation": clients.SimulationClient{},
|
||||||
},
|
},
|
||||||
httpClients: httpClients,
|
httpClients: httpClients,
|
||||||
|
|||||||
25
apps/api/migrations/0038_keling_omni_audio_flags.sql
Normal file
25
apps/api/migrations/0038_keling_omni_audio_flags.sql
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
UPDATE base_model_catalog
|
||||||
|
SET capabilities = jsonb_set(
|
||||||
|
jsonb_set(capabilities, '{omni_video,input_audio}', 'false'::jsonb, true),
|
||||||
|
'{omni_video,max_audios}', '0'::jsonb, true
|
||||||
|
),
|
||||||
|
metadata = jsonb_set(
|
||||||
|
jsonb_set(metadata, '{rawModel,capabilities,omni_video,input_audio}', 'false'::jsonb, true),
|
||||||
|
'{rawModel,capabilities,omni_video,max_audios}', '0'::jsonb, true
|
||||||
|
),
|
||||||
|
updated_at = now()
|
||||||
|
WHERE provider_key = 'keling'
|
||||||
|
AND provider_model_name IN ('kling-video-o1', 'kling-v3-omni')
|
||||||
|
AND capabilities ? 'omni_video';
|
||||||
|
|
||||||
|
UPDATE platform_models m
|
||||||
|
SET capabilities = jsonb_set(
|
||||||
|
jsonb_set(m.capabilities, '{omni_video,input_audio}', 'false'::jsonb, true),
|
||||||
|
'{omni_video,max_audios}', '0'::jsonb, true
|
||||||
|
),
|
||||||
|
updated_at = now()
|
||||||
|
FROM integration_platforms p
|
||||||
|
WHERE m.platform_id = p.id
|
||||||
|
AND p.provider = 'keling'
|
||||||
|
AND COALESCE(NULLIF(m.provider_model_name, ''), m.model_name) IN ('kling-video-o1', 'kling-v3-omni')
|
||||||
|
AND m.capabilities ? 'omni_video';
|
||||||
Loading…
Reference in New Issue
Block a user