From 5a7164309963bbaf7e5f0259917a5319d402fbb1 Mon Sep 17 00:00:00 2001 From: wangbo Date: Tue, 21 Jul 2026 23:53:20 +0800 Subject: [PATCH] =?UTF-8?q?feat(kling):=20=E6=8E=A5=E5=85=A5=20Omni=20?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E5=85=BC=E5=AE=B9=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/internal/clients/keling.go | 86 +- .../internal/clients/keling_compat_test.go | 166 ++++ .../httpapi/keling_compat_handlers.go | 885 ++++++++++++++++++ .../httpapi/keling_compat_handlers_test.go | 276 ++++++ .../httpapi/keling_compat_integration_test.go | 293 ++++++ apps/api/internal/httpapi/openapi_models.go | 3 + apps/api/internal/httpapi/server.go | 2 + apps/api/internal/runner/keling_audio_test.go | 27 + .../internal/runner/param_processor_media.go | 23 + docs/kling-omni-compatible.md | 132 +++ 10 files changed, 1886 insertions(+), 7 deletions(-) create mode 100644 apps/api/internal/clients/keling_compat_test.go create mode 100644 apps/api/internal/httpapi/keling_compat_handlers.go create mode 100644 apps/api/internal/httpapi/keling_compat_handlers_test.go create mode 100644 apps/api/internal/httpapi/keling_compat_integration_test.go create mode 100644 apps/api/internal/runner/keling_audio_test.go create mode 100644 docs/kling-omni-compatible.md diff --git a/apps/api/internal/clients/keling.go b/apps/api/internal/clients/keling.go index 55d5071..d5c82f5 100644 --- a/apps/api/internal/clients/keling.go +++ b/apps/api/internal/clients/keling.go @@ -338,8 +338,11 @@ func kelingVideoPayload(ctx context.Context, request Request) (map[string]any, s if value, ok := body["cfg_scale"]; ok && numericValue(value, 0) > 0 { payload["cfg_scale"] = value } - if boolValue(body, "audio") || boolValue(body, "output_audio") { - payload["sound"] = "on" + if sound, ok := kelingSoundSetting(body); ok { + if sound == "on" && !kelingSupportsGeneratedSound(request.Candidate) { + return nil, "", &ClientError{Code: "invalid_parameter", Message: "kling-video-o1 does not support generated audio", StatusCode: http.StatusBadRequest, Retryable: false} + } + payload["sound"] = sound } if mode := kelingModeByResolution(firstNonEmptyStringValue(body, "resolution", "size")); mode != "" { payload["mode"] = mode @@ -432,7 +435,7 @@ func (c KelingClient) kelingOmniPayload(ctx context.Context, request Request, to watermarkEnabled = boolValue(watermarkInfo, "enabled") } payload := map[string]any{ - "model_name": upstreamModelName(request.Candidate), + "model_name": kelingOmniUpstreamModelName(request.Candidate), "mode": kelingModeByResolution(firstNonEmptyStringValue(body, "resolution", "size")), "watermark_info": map[string]any{"enabled": watermarkEnabled}, "negative_prompt": strings.TrimSpace(stringFromAny(body["negative_prompt"])), @@ -458,8 +461,13 @@ func (c KelingClient) kelingOmniPayload(ctx context.Context, request Request, to if voices := mapListFromAny(body["voice_list"]); len(voices) > 0 { payload["voice_list"] = voices } - if (boolValue(body, "audio") || boolValue(body, "output_audio")) && !hasVideo { - payload["sound"] = "on" + if sound, ok := kelingSoundSetting(body); ok { + if sound == "on" && !kelingSupportsGeneratedSound(request.Candidate) { + return nil, nil, &ClientError{Code: "invalid_parameter", Message: "kling-video-o1 does not support generated audio", StatusCode: http.StatusBadRequest, Retryable: false} + } + if !hasVideo { + payload["sound"] = sound + } } if multiShot { payload["multi_shot"] = true @@ -728,6 +736,18 @@ func kelingIsOmniRequest(request Request) bool { request.Candidate.Capabilities["omni"] != nil } +func kelingOmniUpstreamModelName(candidate store.RuntimeModelCandidate) string { + model := strings.TrimSpace(upstreamModelName(candidate)) + switch strings.ToLower(model) { + case "kling-o1": + return "kling-video-o1" + case "kling-3.0-omni": + return "kling-v3-omni" + default: + return model + } +} + func kelingIs30TurboRequest(request Request) bool { switch strings.ToLower(strings.TrimSpace(upstreamModelName(request.Candidate))) { case "kling-3.0-turbo", "kling-v3-turbo", "kling-3-0-turbo": @@ -1073,6 +1093,54 @@ func kelingModeByResolution(resolution string) string { } } +func kelingSoundSetting(body map[string]any) (string, bool) { + if sound := strings.ToLower(strings.TrimSpace(stringFromAny(body["sound"]))); sound == "on" || sound == "off" { + return sound, true + } + for _, key := range []string{"audio", "output_audio", "generate_audio"} { + if enabled, ok := kelingBoolFieldValue(body, key); ok { + if enabled { + return "on", true + } + return "off", true + } + } + return "", false +} + +func kelingSupportsGeneratedSound(candidate store.RuntimeModelCandidate) bool { + switch strings.ToLower(strings.TrimSpace(upstreamModelName(candidate))) { + case "kling-o1", "kling-video-o1": + return false + default: + return true + } +} + +func kelingWatermarkEnabled(body map[string]any) bool { + if enabled, ok := kelingBoolFieldValue(body, "watermark"); ok { + return enabled + } + info := mapFromAny(body["watermark_info"]) + if info == nil { + return false + } + enabled, _ := kelingBoolFieldValue(info, "enabled") + return enabled +} + +func kelingBoolFieldValue(body map[string]any, key string) (bool, bool) { + if body == nil { + return false, false + } + value, ok := body[key] + if !ok { + return false, false + } + typed, ok := value.(bool) + return typed, ok +} + func kelingCameraControl(body map[string]any) map[string]any { cameraControl := strings.TrimSpace(stringFromAny(body["camera_control"])) if cameraControl == "" { @@ -1182,7 +1250,7 @@ func kelingVideoSuccessResult(request Request, upstreamTaskID string, raw map[st if id := strings.TrimSpace(stringFromAny(video["id"])); id != "" { item["id"] = id } - if duration := intFromAny(video["duration"]); duration > 0 { + if duration := firstPresent(video["duration"]); duration != nil && strings.TrimSpace(stringFromAny(duration)) != "" { item["duration"] = duration } if watermarkURL := strings.TrimSpace(stringFromAny(video["watermark_url"])); watermarkURL != "" { @@ -1194,11 +1262,15 @@ func kelingVideoSuccessResult(request Request, upstreamTaskID string, raw map[st if created == 0 { created = int(nowUnix()) } + modelName := upstreamModelName(request.Candidate) + if kelingIsOmniRequest(request) { + modelName = kelingOmniUpstreamModelName(request.Candidate) + } return map[string]any{ "id": upstreamTaskID, "object": "video.generation", "created": created, - "model": upstreamModelName(request.Candidate), + "model": modelName, "status": "succeeded", "upstream_task_id": upstreamTaskID, "data": items, diff --git a/apps/api/internal/clients/keling_compat_test.go b/apps/api/internal/clients/keling_compat_test.go new file mode 100644 index 0000000..e4fd247 --- /dev/null +++ b/apps/api/internal/clients/keling_compat_test.go @@ -0,0 +1,166 @@ +package clients + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestKelingOmniPayloadPreservesCompatibleSettings(t *testing.T) { + payload, cleanupIDs, err := (KelingClient{}).kelingOmniPayload(context.Background(), Request{ + Kind: "videos.generations", + ModelType: "omni_video", + Body: map[string]any{ + "prompt": "A product reveal", + "duration": 3, + "resolution": "720p", + "aspect_ratio": "16:9", + "audio": false, + "watermark_info": map[string]any{"enabled": true}, + "external_task_id": "external-1", + }, + Candidate: store.RuntimeModelCandidate{ + Provider: "keling", + ProviderModelName: "kling-video-o1", + Capabilities: map[string]any{"omni_video": map[string]any{}}, + }, + }, "token") + if err != nil { + t.Fatalf("build compatible Omni payload: %v", err) + } + if len(cleanupIDs) != 0 || + payload["model_name"] != "kling-video-o1" || + payload["mode"] != "std" || + payload["sound"] != "off" || + payload["duration"] != "3" || + payload["aspect_ratio"] != "16:9" || + payload["external_task_id"] != "external-1" { + t.Fatalf("unexpected compatible Omni payload: %+v", payload) + } + watermark, _ := payload["watermark_info"].(map[string]any) + if watermark["enabled"] != true { + t.Fatalf("watermark setting was not preserved: %+v", payload) + } +} + +func TestKelingOmniUpstreamModelNameSeparatesGatewayAliases(t *testing.T) { + tests := map[string]string{ + "kling-o1": "kling-video-o1", + "kling-video-o1": "kling-video-o1", + "kling-3.0-omni": "kling-v3-omni", + "kling-v3-omni": "kling-v3-omni", + } + for configured, want := range tests { + got := kelingOmniUpstreamModelName(store.RuntimeModelCandidate{ProviderModelName: configured}) + if got != want { + t.Fatalf("configured=%s got=%s want=%s", configured, got, want) + } + } +} + +func TestKelingOmniRejectsGeneratedAudioForO1(t *testing.T) { + _, _, err := (KelingClient{}).kelingOmniPayload(context.Background(), Request{ + Body: map[string]any{ + "prompt": "A beach", + "duration": 5, + "resolution": "1080p", + "aspect_ratio": "9:16", + "audio": true, + }, + Candidate: store.RuntimeModelCandidate{ProviderModelName: "kling-video-o1"}, + }, "token") + if err == nil || ErrorCode(err) != "invalid_parameter" { + t.Fatalf("expected generated-audio rejection for O1, got %v", err) + } +} + +func TestKelingOmniResumeReturnsUpstreamFailureCodeAndModel(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/videos/omni-video/remote-failed" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer upstream-key" { + t.Fatalf("unexpected Authorization header: %q", r.Header.Get("Authorization")) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 0, + "request_id": "failure-request", + "data": map[string]any{ + "task_id": "remote-failed", + "task_status": "failed", + "task_status_msg": "content policy rejection", + }, + }) + })) + defer server.Close() + + _, err := (KelingClient{HTTPClient: server.Client()}).Run(context.Background(), Request{ + Kind: "videos.generations", + ModelType: "omni_video", + RemoteTaskID: "remote-failed", + RemoteTaskPayload: map[string]any{"endpoint": "/videos/omni-video"}, + Candidate: store.RuntimeModelCandidate{ + BaseURL: server.URL, + Provider: "keling", + ProviderModelName: "kling-v3-omni", + Credentials: map[string]any{"apiKey": "upstream-key"}, + PlatformConfig: map[string]any{ + "kelingPollIntervalMs": 10, + "kelingPollTimeoutSeconds": 1, + }, + }, + }) + if err == nil || ErrorCode(err) != "keling_task_failed" || !strings.Contains(err.Error(), "content policy rejection") { + t.Fatalf("expected preserved Keling task failure, got code=%q err=%v", ErrorCode(err), err) + } +} + +func TestKelingOmniPayloadPreservesIntelligentMultiShot(t *testing.T) { + payload, _, err := (KelingClient{}).kelingOmniPayload(context.Background(), Request{ + Kind: "videos.generations", + ModelType: "omni_video", + Body: map[string]any{ + "prompt": "Create three coherent shots", + "duration": 5, + "resolution": "1080p", + "aspect_ratio": "9:16", + "multi_shot": true, + "shot_type": "intelligence", + }, + Candidate: store.RuntimeModelCandidate{ + ProviderModelName: "kling-v3-omni", + Capabilities: map[string]any{"omni_video": map[string]any{}}, + }, + }, "token") + if err != nil { + t.Fatalf("build intelligent multi-shot payload: %v", err) + } + if payload["multi_shot"] != true || payload["shot_type"] != "intelligence" || payload["prompt"] != "Create three coherent shots" { + t.Fatalf("unexpected intelligent multi-shot payload: %+v", payload) + } +} + +func TestKelingVideoSuccessResultPreservesOfficialVideoMetadata(t *testing.T) { + result := kelingVideoSuccessResult(Request{Candidate: store.RuntimeModelCandidate{ProviderModelName: "kling-video-o1"}}, "remote-1", map[string]any{ + "data": map[string]any{ + "task_result": map[string]any{ + "videos": []any{map[string]any{ + "id": "video-1", + "url": "https://example.com/video.mp4", + "watermark_url": "https://example.com/watermarked.mp4", + "duration": "3", + }}, + }, + }, + }) + data, _ := result["data"].([]any) + video, _ := data[0].(map[string]any) + if video["id"] != "video-1" || video["watermark_url"] != "https://example.com/watermarked.mp4" || video["duration"] != "3" { + t.Fatalf("official video metadata was lost: %+v", video) + } +} diff --git a/apps/api/internal/httpapi/keling_compat_handlers.go b/apps/api/internal/httpapi/keling_compat_handlers.go new file mode 100644 index 0000000..56151ad --- /dev/null +++ b/apps/api/internal/httpapi/keling_compat_handlers.go @@ -0,0 +1,885 @@ +package httpapi + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +const kelingOmniCompatibilityMarker = "keling_omni_v1" + +type kelingCompatRequestIDKey struct{} + +type KelingOmniVideoRequest struct { + ModelName string `json:"model_name" example:"kling-v3-omni"` + Prompt string `json:"prompt" example:"A quiet street in the rain with natural ambient sound"` + MultiShot bool `json:"multi_shot" example:"false"` + ShotType string `json:"shot_type,omitempty" example:"customize"` + MultiPrompt []KelingOmniMultiPrompt `json:"multi_prompt,omitempty"` + ImageList []KelingOmniImageInput `json:"image_list,omitempty"` + ElementList []KelingOmniElementInput `json:"element_list,omitempty"` + VideoList []KelingOmniVideoInput `json:"video_list,omitempty"` + Sound string `json:"sound" enums:"on,off" example:"on"` + Mode string `json:"mode" enums:"std,pro,4k" example:"pro"` + AspectRatio string `json:"aspect_ratio" enums:"16:9,9:16,1:1" example:"9:16"` + Duration any `json:"duration" swaggertype:"string" example:"5"` + WatermarkInfo KelingOmniWatermarkInfo `json:"watermark_info,omitempty"` + CallbackURL string `json:"callback_url,omitempty"` + ExternalTask string `json:"external_task_id,omitempty"` +} + +type KelingOmniMultiPrompt struct { + Index int `json:"index" example:"1"` + Prompt string `json:"prompt" example:"A wide establishing shot"` + Duration any `json:"duration" swaggertype:"string" example:"3"` +} + +type KelingOmniImageInput struct { + ImageURL string `json:"image_url"` + Type string `json:"type,omitempty" enums:"first_frame,end_frame"` +} + +type KelingOmniElementInput struct { + ElementID any `json:"element_id"` +} + +type KelingOmniVideoInput struct { + VideoURL string `json:"video_url"` + ReferType string `json:"refer_type,omitempty" enums:"base,feature"` + KeepOriginalSound string `json:"keep_original_sound,omitempty" enums:"yes,no"` +} + +type KelingOmniWatermarkInfo struct { + Enabled bool `json:"enabled" example:"false"` +} + +type KelingCompatibleEnvelope struct { + Code int `json:"code" example:"0"` + Message string `json:"message" example:"SUCCEED"` + RequestID string `json:"request_id"` + Data any `json:"data,omitempty"` +} + +type kelingCompatError struct { + HTTPStatus int + Code int + Message string + RequestID string +} + +func (e *kelingCompatError) Error() string { + if e == nil { + return "keling compatibility error" + } + return e.Message +} + +func newKelingCompatError(status int, code int, message string) *kelingCompatError { + return &kelingCompatError{HTTPStatus: status, Code: code, Message: message} +} + +func (s *Server) requireKelingAPIKey(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestID := newKelingCompatRequestID(r) + r = r.WithContext(context.WithValue(r.Context(), kelingCompatRequestIDKey{}, requestID)) + user, err := s.auth.Authenticate(r) + if err != nil { + code := 1002 + message := "Authorization is invalid" + if strings.TrimSpace(r.Header.Get("Authorization")) == "" { + code = 1001 + message = "Authorization is required" + } + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusUnauthorized, code, message)) + return + } + if user == nil || strings.TrimSpace(user.APIKeyID) == "" { + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusUnauthorized, 1002, "a Gateway API Key is required")) + return + } + if !apiKeyScopeAllowed(user, "videos.generations") { + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusForbidden, 1103, "API Key scope does not allow video generation")) + return + } + next.ServeHTTP(w, r.WithContext(auth.WithUser(r.Context(), user))) + }) +} + +// createKelingOmniVideo godoc +// @Summary 创建 Kling Omni 视频任务 +// @Description 兼容 Kling 旧版 /v1/videos/omni-video；Bearer token 必须为 Gateway API Key。任务固定异步执行，返回的 task_id 为网关任务 ID。 +// @Tags kling-compatible +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param input body KelingOmniVideoRequest true "Kling Omni 官方兼容请求" +// @Success 200 {object} KelingCompatibleEnvelope +// @Failure 400 {object} KelingCompatibleEnvelope +// @Failure 401 {object} KelingCompatibleEnvelope +// @Failure 403 {object} KelingCompatibleEnvelope +// @Failure 429 {object} KelingCompatibleEnvelope +// @Failure 500 {object} KelingCompatibleEnvelope +// @Failure 503 {object} KelingCompatibleEnvelope +// @Router /v1/videos/omni-video [post] +func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) { + requestID := kelingCompatRequestID(r) + user, ok := auth.UserFromContext(r.Context()) + if !ok || user == nil { + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusUnauthorized, 1002, "Authorization is invalid")) + return + } + body, err := s.decodeTaskRequestBody(r.Context(), w, r, "videos.generations") + if err != nil { + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusBadRequest, 1201, err.Error())) + return + } + normalized, compatErr := normalizeKelingOmniRequest(body) + if compatErr != nil { + writeKelingCompatError(w, requestID, compatErr) + return + } + model := strings.TrimSpace(stringFromKelingCompat(normalized["model"])) + if normalized["resolution"] == "2160p" { + candidates, candidateErr := s.store.ListModelCandidates(r.Context(), model, "omni_video", user) + if candidateErr != nil { + writeKelingCompatError(w, requestID, kelingCompatGatewayError(candidateErr)) + return + } + if !kelingCompatCandidatesSupport4K(candidates) { + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusBadRequest, 1201, "mode=4k is not enabled by the selected model capabilities")) + return + } + } + task, createErr := s.prepareAndCreateGatewayTask( + r.Context(), + r, + user, + "videos.generations", + model, + normalized, + true, + ) + if createErr != nil { + var staged *gatewayTaskCreationError + if errors.As(createErr, &staged) && staged.Stage == gatewayTaskCreationPrepare { + writeKelingCompatError(w, requestID, kelingCompatGatewayError(staged.Err)) + return + } + s.logger.Error("create Kling-compatible task failed", "error", createErr) + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusInternalServerError, 5000, "create task failed")) + return + } + if err := s.runner.EnqueueAsyncTask(r.Context(), task); err != nil { + s.logger.Error("enqueue Kling-compatible task failed", "taskId", task.ID, "error", err) + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusServiceUnavailable, 5001, "video task queue is unavailable")) + return + } + writeJSON(w, http.StatusOK, KelingCompatibleEnvelope{ + Code: 0, + Message: "SUCCEED", + RequestID: requestID, + Data: kelingCompatTaskData(task), + }) +} + +// getKelingOmniVideo godoc +// @Summary 查询 Kling Omni 视频任务 +// @Description 按创建接口返回的网关 task_id 查询任务；仅允许创建任务的 Gateway 用户访问。 +// @Tags kling-compatible +// @Produce json +// @Security BearerAuth +// @Param taskID path string true "网关任务 ID" +// @Success 200 {object} KelingCompatibleEnvelope +// @Failure 401 {object} KelingCompatibleEnvelope +// @Failure 403 {object} KelingCompatibleEnvelope +// @Failure 404 {object} KelingCompatibleEnvelope +// @Failure 500 {object} KelingCompatibleEnvelope +// @Router /v1/videos/omni-video/{taskID} [get] +func (s *Server) getKelingOmniVideo(w http.ResponseWriter, r *http.Request) { + requestID := kelingCompatRequestID(r) + user, ok := auth.UserFromContext(r.Context()) + if !ok || user == nil { + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusUnauthorized, 1002, "Authorization is invalid")) + return + } + task, err := s.store.GetTask(r.Context(), strings.TrimSpace(r.PathValue("taskID"))) + if err != nil { + if store.IsNotFound(err) { + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusNotFound, 1203, "task not found")) + return + } + s.logger.Error("get Kling-compatible task failed", "error", err) + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusInternalServerError, 5000, "query task failed")) + return + } + if !kelingCompatTaskOwnedBy(task, user) || !isKelingCompatTask(task) { + writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusNotFound, 1203, "task not found")) + return + } + writeJSON(w, http.StatusOK, KelingCompatibleEnvelope{ + Code: 0, + Message: "SUCCEED", + RequestID: requestID, + Data: kelingCompatTaskData(task), + }) +} + +func normalizeKelingOmniRequest(input map[string]any) (map[string]any, *kelingCompatError) { + if input == nil { + input = map[string]any{} + } + if callbackURL := strings.TrimSpace(stringFromKelingCompat(input["callback_url"])); callbackURL != "" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "callback_url is not supported by this Gateway endpoint") + } + requestedModel := strings.TrimSpace(stringFromKelingCompat(input["model_name"])) + if requestedModel == "" { + requestedModel = "kling-video-o1" + } + model, maxDuration, ok := kelingCompatModel(requestedModel) + if !ok { + return nil, newKelingCompatError(http.StatusNotFound, 1203, "unsupported model_name: "+requestedModel) + } + + mode := strings.ToLower(strings.TrimSpace(stringFromKelingCompat(input["mode"]))) + if mode == "" { + mode = "pro" + } + resolutionByMode := map[string]string{"std": "720p", "pro": "1080p", "4k": "2160p"} + resolution := resolutionByMode[mode] + if resolution == "" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "mode must be std, pro, or 4k") + } + + sound := strings.ToLower(strings.TrimSpace(stringFromKelingCompat(input["sound"]))) + if sound == "" { + sound = "off" + } + if sound != "on" && sound != "off" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "sound must be on or off") + } + if model == "kling-o1" && sound == "on" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "kling-video-o1 does not support generated audio; sound must be off") + } + + content := make([]any, 0) + prompt := strings.TrimSpace(stringFromKelingCompat(input["prompt"])) + images, hasFirstFrame, imageErr := normalizeKelingImageList(input["image_list"]) + if imageErr != nil { + return nil, imageErr + } + content = append(content, images...) + elements, elementErr := normalizeKelingElementList(input["element_list"]) + if elementErr != nil { + return nil, elementErr + } + content = append(content, elements...) + videos, hasBaseVideo, hasVideo, videoErr := normalizeKelingVideoList(input["video_list"]) + if videoErr != nil { + return nil, videoErr + } + content = append(content, videos...) + if hasVideo && sound == "on" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "sound must be off when video_list is provided") + } + + multiShot, multiShotPresent, boolErr := kelingCompatOptionalBool(input, "multi_shot") + if boolErr != nil { + return nil, boolErr + } + shotType := strings.ToLower(strings.TrimSpace(stringFromKelingCompat(input["shot_type"]))) + multiPrompts, shotDuration, multiPromptErr := normalizeKelingMultiPrompts(input["multi_prompt"]) + if multiPromptErr != nil { + return nil, multiPromptErr + } + if !multiShotPresent { + multiShot = false + } + if multiShot { + if shotType != "customize" && shotType != "intelligence" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "shot_type must be customize or intelligence when multi_shot is true") + } + if shotType == "customize" && len(multiPrompts) == 0 { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "multi_prompt is required for customized multi-shot generation") + } + if shotType == "intelligence" && len(multiPrompts) > 0 { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "multi_prompt is only supported when shot_type is customize") + } + } else if len(multiPrompts) > 0 || shotType != "" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "shot_type and multi_prompt require multi_shot=true") + } + if (len(multiPrompts) == 0 || shotType == "intelligence") && prompt == "" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "prompt is required") + } + if prompt != "" { + content = append([]any{map[string]any{"type": "text", "text": prompt}}, content...) + } + content = append(content, multiPrompts...) + + duration, durationProvided, durationErr := kelingCompatOptionalInt(input, "duration") + if durationErr != nil { + return nil, durationErr + } + if hasBaseVideo { + if durationProvided { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "duration is not supported for base video editing") + } + } else { + if len(multiPrompts) > 0 { + if durationProvided && duration != shotDuration { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "duration must equal the sum of multi_prompt durations") + } + duration = shotDuration + } else if !durationProvided { + duration = 5 + } + if duration < 3 || duration > maxDuration { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("duration for %s must be an integer between 3 and %d seconds", requestedModel, maxDuration)) + } + if model == "kling-o1" && (len(images) == 0 || hasFirstFrame) && duration != 5 && duration != 10 { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "kling-video-o1 text-to-video and first-frame generation only support 5 or 10 seconds") + } + } + + aspectRatio := strings.TrimSpace(stringFromKelingCompat(input["aspect_ratio"])) + if aspectRatio != "" && aspectRatio != "16:9" && aspectRatio != "9:16" && aspectRatio != "1:1" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "aspect_ratio must be 16:9, 9:16, or 1:1") + } + if hasFirstFrame || hasBaseVideo { + if aspectRatio != "" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "aspect_ratio is not supported with a first frame or base video") + } + } else if aspectRatio == "" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, "aspect_ratio is required when no first frame or base video is provided") + } + + watermarkEnabled, watermarkErr := kelingCompatWatermarkEnabled(input["watermark_info"]) + if watermarkErr != nil { + return nil, watermarkErr + } + externalTaskID := strings.TrimSpace(stringFromKelingCompat(input["external_task_id"])) + normalized := map[string]any{ + "model": model, + "model_name": requestedModel, + "modelType": "omni_video", + "runMode": "real", + "content": content, + "resolution": resolution, + "mode": mode, + "sound": sound, + "audio": sound == "on", + "multi_shot": multiShot, + "watermark": watermarkEnabled, + "watermark_info": map[string]any{"enabled": watermarkEnabled}, + "external_task_id": externalTaskID, + "_gateway_compatibility": kelingOmniCompatibilityMarker, + } + if prompt != "" { + normalized["prompt"] = prompt + } + if shotType != "" { + normalized["shot_type"] = shotType + } + if !hasBaseVideo { + normalized["duration"] = duration + } + if aspectRatio != "" { + normalized["aspect_ratio"] = aspectRatio + } + return normalized, nil +} + +func normalizeKelingImageList(value any) ([]any, bool, *kelingCompatError) { + items, err := kelingCompatObjectList(value, "image_list") + if err != nil { + return nil, false, err + } + out := make([]any, 0, len(items)) + hasFirstFrame := false + hasEndFrame := false + for index, item := range items { + url := strings.TrimSpace(stringFromKelingCompat(item["image_url"])) + if url == "" { + return nil, false, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("image_list[%d].image_url is required", index)) + } + frameType := strings.TrimSpace(stringFromKelingCompat(item["type"])) + role := "reference_image" + switch frameType { + case "": + case "first_frame": + role = "first_frame" + hasFirstFrame = true + case "end_frame": + role = "last_frame" + hasEndFrame = true + default: + return nil, false, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("image_list[%d].type must be first_frame or end_frame", index)) + } + out = append(out, map[string]any{ + "type": "image_url", + "role": role, + "image_url": map[string]any{"url": url}, + }) + } + if hasEndFrame && !hasFirstFrame { + return nil, false, newKelingCompatError(http.StatusBadRequest, 1201, "end_frame requires first_frame") + } + return out, hasFirstFrame, nil +} + +func normalizeKelingElementList(value any) ([]any, *kelingCompatError) { + items, err := kelingCompatObjectList(value, "element_list") + if err != nil { + return nil, err + } + out := make([]any, 0, len(items)) + for index, item := range items { + id := item["element_id"] + if strings.TrimSpace(stringFromKelingCompat(id)) == "" { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("element_list[%d].element_id is required", index)) + } + out = append(out, map[string]any{"type": "element", "element": map[string]any{"element_id": id}}) + } + return out, nil +} + +func normalizeKelingVideoList(value any) ([]any, bool, bool, *kelingCompatError) { + items, err := kelingCompatObjectList(value, "video_list") + if err != nil { + return nil, false, false, err + } + if len(items) > 1 { + return nil, false, false, newKelingCompatError(http.StatusBadRequest, 1201, "video_list supports at most one video") + } + out := make([]any, 0, len(items)) + hasBase := false + for index, item := range items { + url := strings.TrimSpace(stringFromKelingCompat(item["video_url"])) + if url == "" { + return nil, false, false, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("video_list[%d].video_url is required", index)) + } + referType := strings.ToLower(strings.TrimSpace(stringFromKelingCompat(item["refer_type"]))) + if referType == "" { + referType = "base" + } + if referType != "base" && referType != "feature" { + return nil, false, false, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("video_list[%d].refer_type must be base or feature", index)) + } + keepSound := strings.ToLower(strings.TrimSpace(stringFromKelingCompat(item["keep_original_sound"]))) + if keepSound != "" && keepSound != "yes" && keepSound != "no" { + return nil, false, false, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("video_list[%d].keep_original_sound must be yes or no", index)) + } + nested := map[string]any{"url": url, "refer_type": referType} + if keepSound != "" { + nested["keep_original_sound"] = keepSound + } + role := "video_feature" + if referType == "base" { + role = "video_base" + hasBase = true + } + out = append(out, map[string]any{"type": "video_url", "role": role, "video_url": nested}) + } + return out, hasBase, len(items) > 0, nil +} + +func normalizeKelingMultiPrompts(value any) ([]any, int, *kelingCompatError) { + items, err := kelingCompatObjectList(value, "multi_prompt") + if err != nil { + return nil, 0, err + } + if len(items) > 6 { + return nil, 0, newKelingCompatError(http.StatusBadRequest, 1201, "multi_prompt supports at most six shots") + } + out := make([]any, 0, len(items)) + seen := map[int]bool{} + total := 0 + for index, item := range items { + shotIndex, ok := kelingCompatInt(item["index"]) + if !ok || shotIndex < 1 || shotIndex > 6 || seen[shotIndex] { + return nil, 0, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("multi_prompt[%d].index must be a unique integer from 1 to 6", index)) + } + seen[shotIndex] = true + prompt := strings.TrimSpace(stringFromKelingCompat(item["prompt"])) + if prompt == "" { + return nil, 0, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("multi_prompt[%d].prompt is required", index)) + } + duration, ok := kelingCompatInt(item["duration"]) + if !ok || duration < 1 { + return nil, 0, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("multi_prompt[%d].duration must be an integer of at least 1 second", index)) + } + total += duration + out = append(out, map[string]any{ + "type": "text", + "role": "shot_prompt", + "shot_index": shotIndex, + "text": prompt, + "duration": duration, + }) + } + return out, total, nil +} + +func kelingCompatModel(value string) (string, int, bool) { + switch strings.ToLower(strings.TrimSpace(value)) { + case "kling-video-o1", "kling-o1": + return "kling-o1", 10, true + case "kling-v3-omni", "kling-3.0-omni": + return "kling-3.0-omni", 15, true + default: + return "", 0, false + } +} + +func kelingCompatObjectList(value any, field string) ([]map[string]any, *kelingCompatError) { + if value == nil { + return nil, nil + } + raw, ok := value.([]any) + if !ok { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, field+" must be an array") + } + out := make([]map[string]any, 0, len(raw)) + for index, item := range raw { + object, ok := item.(map[string]any) + if !ok { + return nil, newKelingCompatError(http.StatusBadRequest, 1201, fmt.Sprintf("%s[%d] must be an object", field, index)) + } + out = append(out, object) + } + return out, nil +} + +func kelingCompatOptionalInt(body map[string]any, key string) (int, bool, *kelingCompatError) { + value, present := body[key] + if !present || value == nil || strings.TrimSpace(stringFromKelingCompat(value)) == "" { + return 0, false, nil + } + parsed, ok := kelingCompatInt(value) + if !ok { + return 0, true, newKelingCompatError(http.StatusBadRequest, 1201, key+" must be an integer") + } + return parsed, true, nil +} + +func kelingCompatOptionalBool(body map[string]any, key string) (bool, bool, *kelingCompatError) { + value, present := body[key] + if !present || value == nil { + return false, false, nil + } + parsed, ok := value.(bool) + if !ok { + return false, true, newKelingCompatError(http.StatusBadRequest, 1201, key+" must be a boolean") + } + return parsed, true, nil +} + +func kelingCompatWatermarkEnabled(value any) (bool, *kelingCompatError) { + if value == nil { + return false, nil + } + object, ok := value.(map[string]any) + if !ok { + return false, newKelingCompatError(http.StatusBadRequest, 1201, "watermark_info must be an object") + } + enabled, present := object["enabled"] + if !present { + return false, nil + } + result, ok := enabled.(bool) + if !ok { + return false, newKelingCompatError(http.StatusBadRequest, 1201, "watermark_info.enabled must be a boolean") + } + return result, nil +} + +func kelingCompatInt(value any) (int, bool) { + switch typed := value.(type) { + case int: + return typed, true + case int64: + return int(typed), true + case float64: + if math.Abs(typed-math.Round(typed)) > 1e-9 { + return 0, false + } + return int(math.Round(typed)), true + case json.Number: + parsed, err := strconv.Atoi(typed.String()) + return parsed, err == nil + case string: + parsed, err := strconv.Atoi(strings.TrimSpace(typed)) + return parsed, err == nil + default: + return 0, false + } +} + +func stringFromKelingCompat(value any) string { + switch typed := value.(type) { + case string: + return typed + case json.Number: + return typed.String() + case float64: + if math.Abs(typed-math.Round(typed)) < 1e-9 { + return strconv.FormatInt(int64(math.Round(typed)), 10) + } + return strconv.FormatFloat(typed, 'f', -1, 64) + case int: + return strconv.Itoa(typed) + case int64: + return strconv.FormatInt(typed, 10) + default: + return "" + } +} + +func kelingCompatTaskData(task store.GatewayTask) map[string]any { + data := map[string]any{ + "task_id": task.ID, + "task_status": kelingCompatTaskStatus(task.Status), + "task_info": map[string]any{ + "external_task_id": strings.TrimSpace(stringFromKelingCompat(task.Request["external_task_id"])), + }, + "created_at": task.CreatedAt.UnixMilli(), + "updated_at": task.UpdatedAt.UnixMilli(), + "watermark_info": map[string]any{ + "enabled": kelingCompatTaskWatermark(task.Request), + }, + } + if message := kelingCompatTaskMessage(task); message != "" { + data["task_status_msg"] = message + } + if kelingCompatTaskStatus(task.Status) == "failed" { + data["task_status_code"] = kelingCompatBusinessCode(task.ErrorCode, kelingCompatTaskMessage(task)) + } + if videos := kelingCompatTaskVideos(task.Result); len(videos) > 0 { + data["task_result"] = map[string]any{"videos": videos} + } + if task.FinalChargeAmount > 0 { + data["final_unit_deduction"] = strconv.FormatFloat(task.FinalChargeAmount, 'f', -1, 64) + } + return data +} + +func kelingCompatTaskStatus(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "succeeded", "success", "completed": + return "succeed" + case "failed", "cancelled", "canceled": + return "failed" + case "running", "processing": + return "processing" + default: + return "submitted" + } +} + +func kelingCompatTaskVideos(result map[string]any) []any { + raw, _ := result["data"].([]any) + out := make([]any, 0, len(raw)) + for _, itemValue := range raw { + item, _ := itemValue.(map[string]any) + if item == nil { + continue + } + url := strings.TrimSpace(stringFromKelingCompat(firstKelingCompatValue(item["url"], item["video_url"]))) + if url == "" { + continue + } + video := map[string]any{"url": url} + if id := strings.TrimSpace(stringFromKelingCompat(item["id"])); id != "" { + video["id"] = id + } + if watermarkURL := strings.TrimSpace(stringFromKelingCompat(item["watermark_url"])); watermarkURL != "" { + video["watermark_url"] = watermarkURL + } + if duration := strings.TrimSpace(stringFromKelingCompat(item["duration"])); duration != "" { + video["duration"] = duration + } + out = append(out, video) + } + return out +} + +func firstKelingCompatValue(values ...any) any { + for _, value := range values { + if strings.TrimSpace(stringFromKelingCompat(value)) != "" { + return value + } + } + return nil +} + +func kelingCompatTaskMessage(task store.GatewayTask) string { + return strings.TrimSpace(firstNonEmpty(task.ErrorMessage, task.Error, task.Message)) +} + +func kelingCompatTaskWatermark(request map[string]any) bool { + if value, ok := request["watermark"].(bool); ok { + return value + } + if info, ok := request["watermark_info"].(map[string]any); ok { + value, _ := info["enabled"].(bool) + return value + } + return false +} + +func kelingCompatCandidatesSupport4K(candidates []store.RuntimeModelCandidate) bool { + for _, candidate := range candidates { + if strings.ToLower(strings.TrimSpace(candidate.Provider)) != "keling" { + continue + } + capability, _ := candidate.Capabilities["omni_video"].(map[string]any) + if capability == nil { + capability, _ = candidate.Capabilities["omni"].(map[string]any) + } + for _, resolution := range kelingCompatStringList(capability["output_resolutions"]) { + switch strings.ToLower(strings.TrimSpace(resolution)) { + case "2160p", "4k": + return true + } + } + } + return false +} + +func kelingCompatStringList(value any) []string { + switch typed := value.(type) { + case []string: + return typed + case []any: + result := make([]string, 0, len(typed)) + for _, item := range typed { + if text := strings.TrimSpace(stringFromKelingCompat(item)); text != "" { + result = append(result, text) + } + } + return result + default: + return nil + } +} + +func kelingCompatGatewayError(err error) *kelingCompatError { + if err == nil { + return newKelingCompatError(http.StatusInternalServerError, 5000, "unknown gateway error") + } + codeText := clients.ErrorCode(err) + businessCode := kelingCompatBusinessCode(codeText, err.Error()) + status := http.StatusInternalServerError + switch businessCode { + case 1101: + status = http.StatusPaymentRequired + case 1103: + status = http.StatusForbidden + case 1201: + status = http.StatusBadRequest + case 1203: + status = http.StatusNotFound + case 1302, 1303: + status = http.StatusTooManyRequests + case 5001: + status = http.StatusBadGateway + } + return newKelingCompatError(status, businessCode, err.Error()) +} + +func kelingCompatBusinessCode(errorCode string, message string) int { + combined := strings.ToLower(strings.TrimSpace(errorCode + " " + message)) + containsAny := func(values ...string) bool { + for _, value := range values { + if strings.Contains(combined, value) { + return true + } + } + return false + } + switch { + case containsAny("insufficient_balance", "insufficient balance", "balance_not_enough", "wallet balance", "余额不足", "欠费", "quota exceeded"): + return 1101 + case containsAny("permission_denied", "permission denied", "forbidden", "access denied", "scope does not allow"): + return 1103 + case containsAny("concurrent", "concurrency"): + return 1303 + case containsAny("rate_limit", "rate limit", "too many requests", "rpm", "tpm"): + return 1302 + case containsAny("no_model_candidate", "no model candidate", "model_not_found", "unsupported model", "resource not found"): + return 1203 + case containsAny("invalid_parameter", "invalid parameter", "bad_request", "parameter_preprocessing", "duration", "aspect_ratio"): + return 1201 + case containsAny("upload_", "request_asset_", "network", "timeout", "upstream", "service unavailable", "bad gateway"): + return 5001 + default: + return 5000 + } +} + +func isKelingCompatTask(task store.GatewayTask) bool { + return task.Kind == "videos.generations" && strings.TrimSpace(stringFromKelingCompat(task.Request["_gateway_compatibility"])) == kelingOmniCompatibilityMarker +} + +func kelingCompatTaskOwnedBy(task store.GatewayTask, user *auth.User) bool { + if user == nil { + return false + } + taskOwner := strings.TrimSpace(firstNonEmpty(task.GatewayUserID, task.UserID)) + requestOwner := strings.TrimSpace(firstNonEmpty(user.GatewayUserID, user.ID)) + return taskOwner != "" && requestOwner != "" && taskOwner == requestOwner +} + +func newKelingCompatRequestID(r *http.Request) string { + if r != nil { + if value := strings.TrimSpace(firstNonEmpty(r.Header.Get("X-Request-ID"), r.Header.Get("X-Request-Id"))); value != "" { + return value + } + } + random := make([]byte, 16) + if _, err := rand.Read(random); err == nil { + return hex.EncodeToString(random) + } + return strconv.FormatInt(time.Now().UnixNano(), 36) +} + +func kelingCompatRequestID(r *http.Request) string { + if r != nil { + if value, ok := r.Context().Value(kelingCompatRequestIDKey{}).(string); ok && strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return newKelingCompatRequestID(r) +} + +func writeKelingCompatError(w http.ResponseWriter, requestID string, err *kelingCompatError) { + if err == nil { + err = newKelingCompatError(http.StatusInternalServerError, 5000, "internal error") + } + if err.RequestID != "" { + requestID = err.RequestID + } + if requestID == "" { + requestID = newKelingCompatRequestID(nil) + } + status := err.HTTPStatus + if status == 0 { + status = http.StatusInternalServerError + } + writeJSON(w, status, KelingCompatibleEnvelope{ + Code: err.Code, + Message: err.Message, + RequestID: requestID, + }) +} diff --git a/apps/api/internal/httpapi/keling_compat_handlers_test.go b/apps/api/internal/httpapi/keling_compat_handlers_test.go new file mode 100644 index 0000000..0a59403 --- /dev/null +++ b/apps/api/internal/httpapi/keling_compat_handlers_test.go @@ -0,0 +1,276 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestNormalizeKelingOmniRequestMapsOfficialFields(t *testing.T) { + normalized, err := normalizeKelingOmniRequest(map[string]any{ + "model_name": "kling-v3-omni", + "prompt": "A rainy street with natural ambience", + "mode": "pro", + "sound": "on", + "aspect_ratio": "9:16", + "duration": "5", + "external_task_id": "external-1", + "watermark_info": map[string]any{"enabled": true}, + }) + if err != nil { + t.Fatalf("normalize Kling request: %v", err) + } + if normalized["model"] != "kling-3.0-omni" || + normalized["modelType"] != "omni_video" || + normalized["resolution"] != "1080p" || + normalized["aspect_ratio"] != "9:16" || + normalized["duration"] != 5 || + normalized["audio"] != true || + normalized["sound"] != "on" || + normalized["watermark"] != true || + normalized["external_task_id"] != "external-1" || + normalized["_gateway_compatibility"] != kelingOmniCompatibilityMarker { + t.Fatalf("unexpected normalized request: %+v", normalized) + } + content, _ := normalized["content"].([]any) + if len(content) != 1 { + t.Fatalf("unexpected content: %+v", normalized["content"]) + } + text, _ := content[0].(map[string]any) + if text["type"] != "text" || text["text"] != "A rainy street with natural ambience" { + t.Fatalf("unexpected text content: %+v", text) + } +} + +func TestNormalizeKelingOmniRequestBuildsMultiShotMedia(t *testing.T) { + normalized, err := normalizeKelingOmniRequest(map[string]any{ + "model_name": "kling-3.0-omni", + "multi_shot": true, + "shot_type": "customize", + "aspect_ratio": "16:9", + "duration": 5, + "multi_prompt": []any{ + map[string]any{"index": 1, "prompt": "First shot", "duration": "2"}, + map[string]any{"index": 2, "prompt": "Second shot", "duration": 3}, + }, + "image_list": []any{ + map[string]any{"image_url": "https://example.com/reference.png"}, + }, + "element_list": []any{ + map[string]any{"element_id": float64(123)}, + }, + }) + if err != nil { + t.Fatalf("normalize multi-shot request: %v", err) + } + if normalized["model"] != "kling-3.0-omni" || normalized["duration"] != 5 || normalized["multi_shot"] != true || normalized["shot_type"] != "customize" { + t.Fatalf("unexpected multi-shot fields: %+v", normalized) + } + content, _ := normalized["content"].([]any) + if len(content) != 4 { + t.Fatalf("expected image, element and two shot prompts, got %+v", content) + } +} + +func TestNormalizeKelingOmniRequestRejectsUnsupportedCombinations(t *testing.T) { + tests := []struct { + name string + body map[string]any + }{ + { + name: "callback", + body: map[string]any{"callback_url": "https://example.com/callback"}, + }, + { + name: "unknown model", + body: map[string]any{"model_name": "kling-unknown", "prompt": "x", "aspect_ratio": "16:9"}, + }, + { + name: "o1 duration", + body: map[string]any{"model_name": "kling-video-o1", "prompt": "x", "aspect_ratio": "16:9", "duration": 11}, + }, + { + name: "o1 text to video three seconds", + body: map[string]any{"model_name": "kling-video-o1", "prompt": "x", "aspect_ratio": "16:9", "duration": 3}, + }, + { + name: "o1 generated audio", + body: map[string]any{"model_name": "kling-video-o1", "prompt": "x", "sound": "on", "aspect_ratio": "16:9", "duration": 5}, + }, + { + name: "video sound", + body: map[string]any{ + "model_name": "kling-v3-omni", + "prompt": "edit", + "sound": "on", + "video_list": []any{map[string]any{"video_url": "https://example.com/base.mp4", "refer_type": "base"}}, + }, + }, + { + name: "first frame ratio", + body: map[string]any{ + "prompt": "animate", + "aspect_ratio": "16:9", + "image_list": []any{map[string]any{"image_url": "https://example.com/first.png", "type": "first_frame"}}, + }, + }, + } + for _, item := range tests { + t.Run(item.name, func(t *testing.T) { + _, err := normalizeKelingOmniRequest(item.body) + if err == nil || err.Code != 1201 && err.Code != 1203 { + t.Fatalf("expected official compatibility error, got %v", err) + } + }) + } +} + +func TestNormalizeKelingO1AllowsThreeSecondsWithReferenceImage(t *testing.T) { + normalized, err := normalizeKelingOmniRequest(map[string]any{ + "model_name": "kling-video-o1", + "prompt": "Use the landscape as a visual reference", + "aspect_ratio": "16:9", + "duration": 3, + "image_list": []any{ + map[string]any{"image_url": "https://placehold.co/1024x1024/png"}, + }, + }) + if err != nil { + t.Fatalf("reference-image O1 request should allow three seconds: %v", err) + } + if normalized["duration"] != 3 { + t.Fatalf("unexpected duration: %+v", normalized) + } +} + +func TestKelingCompatTaskDataAndOwnership(t *testing.T) { + createdAt := time.Unix(100, 0) + task := store.GatewayTask{ + ID: "task-1", + Kind: "videos.generations", + GatewayUserID: "user-1", + Status: "succeeded", + FinalChargeAmount: 2.5, + Request: map[string]any{ + "_gateway_compatibility": kelingOmniCompatibilityMarker, + "external_task_id": "external-1", + "watermark": true, + }, + Result: map[string]any{"data": []any{map[string]any{ + "id": "video-1", + "url": "https://example.com/video.mp4", + "watermark_url": "https://example.com/watermarked.mp4", + "duration": "5", + }}}, + CreatedAt: createdAt, + UpdatedAt: createdAt.Add(time.Second), + } + if !isKelingCompatTask(task) || !kelingCompatTaskOwnedBy(task, &auth.User{GatewayUserID: "user-1"}) { + t.Fatalf("expected task ownership and compatibility marker") + } + if kelingCompatTaskOwnedBy(task, &auth.User{GatewayUserID: "user-2"}) { + t.Fatalf("cross-user task access must be rejected") + } + data := kelingCompatTaskData(task) + if data["task_status"] != "succeed" || data["final_unit_deduction"] != "2.5" { + t.Fatalf("unexpected task data: %+v", data) + } + result, _ := data["task_result"].(map[string]any) + videos, _ := result["videos"].([]any) + video, _ := videos[0].(map[string]any) + if video["id"] != "video-1" || video["watermark_url"] != "https://example.com/watermarked.mp4" || video["duration"] != "5" { + t.Fatalf("unexpected compatible video: %+v", video) + } +} + +func TestRequireKelingAPIKeyWritesOfficialAuthEnvelope(t *testing.T) { + server := &Server{auth: auth.New("secret", "", "")} + server.auth.LocalAPIKeyVerifier = func(_ context.Context, key string) (*auth.User, error) { + if key != "sk-gw-valid" { + return nil, auth.ErrUnauthorized + } + return &auth.User{ID: "user-1", GatewayUserID: "user-1", APIKeyID: "key-1", APIKeyScopes: []string{"video"}}, nil + } + handler := server.requireKelingAPIKey(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := auth.UserFromContext(r.Context()); !ok { + t.Fatal("authenticated user is missing") + } + w.WriteHeader(http.StatusNoContent) + })) + + missing := httptest.NewRecorder() + handler.ServeHTTP(missing, httptest.NewRequest(http.MethodPost, "/v1/videos/omni-video", nil)) + if missing.Code != http.StatusUnauthorized { + t.Fatalf("missing auth status=%d body=%s", missing.Code, missing.Body.String()) + } + var missingBody KelingCompatibleEnvelope + if err := json.Unmarshal(missing.Body.Bytes(), &missingBody); err != nil || missingBody.Code != 1001 || missingBody.RequestID == "" { + t.Fatalf("unexpected missing auth envelope: %+v err=%v", missingBody, err) + } + + invalid := httptest.NewRecorder() + invalidRequest := httptest.NewRequest(http.MethodPost, "/v1/videos/omni-video", nil) + invalidRequest.Header.Set("Authorization", "Bearer sk-gw-invalid") + handler.ServeHTTP(invalid, invalidRequest) + var invalidBody KelingCompatibleEnvelope + if err := json.Unmarshal(invalid.Body.Bytes(), &invalidBody); err != nil || invalid.Code != http.StatusUnauthorized || invalidBody.Code != 1002 { + t.Fatalf("unexpected invalid auth envelope: status=%d body=%+v err=%v", invalid.Code, invalidBody, err) + } + + valid := httptest.NewRecorder() + validRequest := httptest.NewRequest(http.MethodPost, "/v1/videos/omni-video", nil) + validRequest.Header.Set("Authorization", "Bearer sk-gw-valid") + handler.ServeHTTP(valid, validRequest) + if valid.Code != http.StatusNoContent { + t.Fatalf("valid API Key status=%d body=%s", valid.Code, valid.Body.String()) + } +} + +func TestKelingCompatErrorImplementsError(t *testing.T) { + err := newKelingCompatError(http.StatusBadRequest, 1201, "invalid") + if !errors.Is(err, err) || err.Error() != "invalid" { + t.Fatalf("unexpected error behavior: %v", err) + } +} + +func TestKelingCompatBusinessCodeMapping(t *testing.T) { + tests := []struct { + code string + message string + want int + }{ + {code: "insufficient_balance", want: 1101}, + {code: "permission_denied", want: 1103}, + {code: "invalid_parameter", want: 1201}, + {code: "no_model_candidate", want: 1203}, + {code: "rate_limit_exceeded", want: 1302}, + {code: "concurrency_limit", want: 1303}, + {code: "network", want: 5001}, + {code: "unknown", want: 5000}, + } + for _, item := range tests { + if got := kelingCompatBusinessCode(item.code, item.message); got != item.want { + t.Fatalf("code=%q message=%q got=%d want=%d", item.code, item.message, got, item.want) + } + } +} + +func TestKelingCompatCandidatesSupport4K(t *testing.T) { + candidates := []store.RuntimeModelCandidate{ + {Provider: "keling", Capabilities: map[string]any{"omni_video": map[string]any{"output_resolutions": []any{"720p", "1080p"}}}}, + {Provider: "keling", Capabilities: map[string]any{"omni_video": map[string]any{"output_resolutions": []any{"2160p"}}}}, + } + if !kelingCompatCandidatesSupport4K(candidates) { + t.Fatal("expected 2160p Keling capability to enable mode=4k") + } + if kelingCompatCandidatesSupport4K(candidates[:1]) { + t.Fatal("mode=4k must stay disabled without an explicit capability") + } +} diff --git a/apps/api/internal/httpapi/keling_compat_integration_test.go b/apps/api/internal/httpapi/keling_compat_integration_test.go new file mode 100644 index 0000000..c48b85c --- /dev/null +++ b/apps/api/internal/httpapi/keling_compat_integration_test.go @@ -0,0 +1,293 @@ +package httpapi + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestKelingOmniCompatibleHTTPFlow(t *testing.T) { + databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run the Kling-compatible HTTP integration flow") + } + + var upstreamTaskSequence atomic.Int64 + var upstreamPayloadMu sync.Mutex + var upstreamPayloads []map[string]any + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer upstream-keling-key" { + t.Fatalf("unexpected upstream Authorization: %q", r.Header.Get("Authorization")) + } + switch { + case r.Method == http.MethodPost && r.URL.Path == "/videos/omni-video": + var payload map[string]any + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatalf("decode upstream request: %v", err) + } + upstreamPayloadMu.Lock() + upstreamPayloads = append(upstreamPayloads, payload) + upstreamPayloadMu.Unlock() + id := "upstream-" + strconv.FormatInt(upstreamTaskSequence.Add(1), 10) + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 0, + "request_id": "submit-" + id, + "data": map[string]any{"task_id": id, "task_status": "submitted"}, + }) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/videos/omni-video/upstream-"): + id := strings.TrimPrefix(r.URL.Path, "/videos/omni-video/") + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 0, + "request_id": "poll-" + id, + "data": map[string]any{ + "task_id": id, + "task_status": "succeed", + "created_at": time.Now().UnixMilli(), + "task_result": map[string]any{"videos": []any{map[string]any{ + "id": "video-" + id, + "url": "https://example.com/" + id + ".mp4", + "watermark_url": "https://example.com/" + id + "-watermark.mp4", + "duration": "3", + }}}, + }, + }) + default: + t.Fatalf("unexpected upstream request %s %s", r.Method, r.URL.Path) + } + })) + defer upstream.Close() + + ctx := context.Background() + applyMigration(t, ctx, databaseURL) + db, err := store.Connect(ctx, databaseURL) + if err != nil { + t.Fatalf("connect store: %v", err) + } + defer db.Close() + + suffix := strconv.FormatInt(time.Now().UnixNano(), 10) + platform, err := db.CreatePlatform(ctx, store.CreatePlatformInput{ + Provider: "keling", + PlatformKey: "keling-compatible-test-" + suffix, + Name: "Kling Compatible Test", + BaseURL: upstream.URL, + AuthType: "APIKey", + Credentials: map[string]any{"apiKey": "upstream-keling-key"}, + Config: map[string]any{ + "kelingPollIntervalMs": 100, + "kelingPollTimeoutSeconds": 5, + }, + Priority: 1, + Status: "enabled", + }) + if err != nil { + t.Fatalf("create test platform: %v", err) + } + _, err = db.CreatePlatformModel(ctx, store.CreatePlatformModelInput{ + PlatformID: platform.ID, + CanonicalModelKey: "keling:kling-video-o1", + ModelName: "kling-o1", + ProviderModelName: "kling-video-o1", + ModelAlias: "kling-o1", + ModelType: store.StringList{"omni_video", "video_generate"}, + DisplayName: "Kling O1 Compatible Test", + Capabilities: map[string]any{ + "omni_video": map[string]any{ + "supported_modes": []any{"text_to_video", "image_reference"}, + "output_resolutions": []any{"720p", "1080p"}, + "aspect_ratio_allowed": []any{"16:9", "9:16", "1:1"}, + "duration_options": []any{3, 4, 5, 6, 7, 8, 9, 10}, + "output_audio": false, + "max_images": 7, + }, + "video_generate": map[string]any{ + "supported_modes": []any{"text_to_video"}, + "output_resolutions": []any{"720p", "1080p"}, + "aspect_ratio_allowed": []any{"16:9", "9:16", "1:1"}, + "duration_options": []any{3, 4, 5, 6, 7, 8, 9, 10}, + "output_audio": true, + }, + }, + Enabled: true, + }) + if err != nil { + t.Fatalf("create test platform model: %v", err) + } + + serverCtx, cancelServer := context.WithCancel(ctx) + defer cancelServer() + gateway := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{ + AppEnv: "test", + HTTPAddr: ":0", + DatabaseURL: databaseURL, + IdentityMode: "hybrid", + JWTSecret: "test-secret", + CORSAllowedOrigin: "*", + }, db, slog.New(slog.NewTextHandler(io.Discard, nil)))) + defer gateway.Close() + + firstUserToken, firstAPIKey := createKelingCompatIntegrationUser(t, ctx, db, gateway.URL, "first", suffix, true) + _ = firstUserToken + _, secondAPIKey := createKelingCompatIntegrationUser(t, ctx, db, gateway.URL, "second", suffix, false) + + var created KelingCompatibleEnvelope + doJSON(t, gateway.URL, http.MethodPost, "/v1/videos/omni-video", firstAPIKey, map[string]any{ + "model_name": "kling-video-o1", + "prompt": "A clean product reveal", + "mode": "std", + "aspect_ratio": "16:9", + "duration": "3", + "sound": "off", + "image_list": []any{map[string]any{"image_url": "https://example.com/reference.png"}}, + "external_task_id": "compat-http-1", + }, http.StatusOK, &created) + createdData, _ := created.Data.(map[string]any) + if created.Code != 0 || created.RequestID == "" || strings.TrimSpace(stringFromKelingCompat(createdData["task_id"])) == "" || createdData["task_status"] != "submitted" { + t.Fatalf("unexpected compatible create response: %+v", created) + } + taskID := stringFromKelingCompat(createdData["task_id"]) + + var hidden KelingCompatibleEnvelope + doJSON(t, gateway.URL, http.MethodGet, "/v1/videos/omni-video/"+taskID, secondAPIKey, nil, http.StatusNotFound, &hidden) + if hidden.Code != 1203 { + t.Fatalf("cross-user task must be hidden: %+v", hidden) + } + + completed := waitForKelingCompatTask(t, gateway.URL, firstAPIKey, taskID, 5*time.Second) + if completed.Code != 0 { + t.Fatalf("compatible task failed: %+v", completed) + } + completedData, _ := completed.Data.(map[string]any) + if completedData["task_status"] != "succeed" { + t.Fatalf("compatible task did not succeed: %+v", completedData) + } + taskResult, _ := completedData["task_result"].(map[string]any) + videos, _ := taskResult["videos"].([]any) + video, _ := videos[0].(map[string]any) + if video["id"] == "" || video["watermark_url"] == "" || video["duration"] != "3" { + t.Fatalf("compatible result lost video metadata: %+v", video) + } + + var standard struct { + TaskID string `json:"taskId"` + } + doJSONWithHeaders(t, gateway.URL, http.MethodPost, "/api/v1/videos/generations", firstAPIKey, map[string]any{ + "model": "kling-o1", + "prompt": "A second product reveal", + "resolution": "720p", + "aspect_ratio": "16:9", + "duration": 3, + "audio": false, + }, map[string]string{"X-Async": "true"}, http.StatusAccepted, &standard) + if standard.TaskID == "" { + t.Fatal("standard video generation did not return taskId") + } + waitForTaskStatus(t, gateway.URL, firstAPIKey, standard.TaskID, []string{"succeeded"}, 5*time.Second) + + var unsupportedAudio struct { + TaskID string `json:"taskId"` + } + doJSONWithHeaders(t, gateway.URL, http.MethodPost, "/api/v1/videos/generations", firstAPIKey, map[string]any{ + "model": "kling-o1", + "prompt": "An O1 request that must not silently ignore audio", + "resolution": "1080p", + "aspect_ratio": "9:16", + "duration": 5, + "audio": true, + }, map[string]string{"X-Async": "true"}, http.StatusAccepted, &unsupportedAudio) + if unsupportedAudio.TaskID == "" { + t.Fatal("unsupported O1 audio request did not return taskId") + } + waitForTaskStatus(t, gateway.URL, firstAPIKey, unsupportedAudio.TaskID, []string{"failed"}, 5*time.Second) + var failedAudioTask store.GatewayTask + doJSON(t, gateway.URL, http.MethodGet, "/api/v1/tasks/"+unsupportedAudio.TaskID, firstAPIKey, nil, http.StatusOK, &failedAudioTask) + if failedAudioTask.ErrorCode != "invalid_parameter" || !strings.Contains(failedAudioTask.ErrorMessage, "does not support generated audio") { + t.Fatalf("O1 audio request must fail visibly before upstream submission: %+v", failedAudioTask) + } + + upstreamPayloadMu.Lock() + defer upstreamPayloadMu.Unlock() + if len(upstreamPayloads) != 2 { + t.Fatalf("expected two upstream submissions, got %d", len(upstreamPayloads)) + } + compatiblePayload := upstreamPayloads[0] + if compatiblePayload["model_name"] != "kling-video-o1" || compatiblePayload["mode"] != "std" || compatiblePayload["sound"] != "off" || compatiblePayload["duration"] != "3" || compatiblePayload["aspect_ratio"] != "16:9" || compatiblePayload["external_task_id"] != "compat-http-1" { + t.Fatalf("unexpected compatible upstream payload: %+v", compatiblePayload) + } +} + +func createKelingCompatIntegrationUser(t *testing.T, ctx context.Context, db *store.Store, baseURL string, prefix string, suffix string, fund bool) (string, string) { + t.Helper() + username := fmt.Sprintf("kling_compat_%s_%s", prefix, suffix) + password := "password123" + var registered struct { + AccessToken string `json:"accessToken"` + } + doJSON(t, baseURL, http.MethodPost, "/api/v1/auth/register", "", map[string]any{ + "username": username, + "email": username + "@example.com", + "password": password, + }, http.StatusCreated, ®istered) + var apiKey struct { + Secret string `json:"secret"` + } + doJSON(t, baseURL, http.MethodPost, "/api/v1/api-keys", registered.AccessToken, map[string]any{ + "name": "Kling compatible integration key", + "scopes": []string{"video"}, + }, http.StatusCreated, &apiKey) + if fund { + if _, err := db.Pool().Exec(ctx, `UPDATE gateway_users SET roles = '["admin"]'::jsonb WHERE username = $1`, username); err != nil { + t.Fatalf("promote integration user: %v", err) + } + var loggedIn struct { + AccessToken string `json:"accessToken"` + } + doJSON(t, baseURL, http.MethodPost, "/api/v1/auth/login", "", map[string]any{ + "account": username, + "password": password, + }, http.StatusOK, &loggedIn) + var gatewayUserID string + if err := db.Pool().QueryRow(ctx, `SELECT id::text FROM gateway_users WHERE username = $1`, username).Scan(&gatewayUserID); err != nil { + t.Fatalf("read integration user id: %v", err) + } + doJSON(t, baseURL, http.MethodPatch, "/api/admin/users/"+gatewayUserID+"/wallet", loggedIn.AccessToken, map[string]any{ + "currency": "resource", + "balance": 1000, + "reason": "seed Kling compatible integration wallet", + }, http.StatusOK, nil) + } + return registered.AccessToken, apiKey.Secret +} + +func waitForKelingCompatTask(t *testing.T, baseURL string, apiKey string, taskID string, timeout time.Duration) KelingCompatibleEnvelope { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + var response KelingCompatibleEnvelope + doJSON(t, baseURL, http.MethodGet, "/v1/videos/omni-video/"+taskID, apiKey, nil, http.StatusOK, &response) + data, _ := response.Data.(map[string]any) + switch data["task_status"] { + case "succeed": + return response + case "failed": + t.Fatalf("Kling-compatible task failed: %+v", response) + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("timed out waiting for Kling-compatible task %s", taskID) + return KelingCompatibleEnvelope{} +} diff --git a/apps/api/internal/httpapi/openapi_models.go b/apps/api/internal/httpapi/openapi_models.go index 19a922d..918ce20 100644 --- a/apps/api/internal/httpapi/openapi_models.go +++ b/apps/api/internal/httpapi/openapi_models.go @@ -229,6 +229,9 @@ type TaskRequest struct { Size string `json:"size,omitempty" example:"1024x1024"` Duration int `json:"duration,omitempty" example:"5"` Resolution string `json:"resolution,omitempty" example:"720p"` + AspectRatio string `json:"aspect_ratio,omitempty" example:"16:9"` + Audio *bool `json:"audio,omitempty" example:"false"` + Watermark *bool `json:"watermark,omitempty" example:"false"` MakeInstrumental bool `json:"makeInstrumental,omitempty" example:"false"` CustomMode bool `json:"customMode,omitempty" example:"false"` Style string `json:"style,omitempty" example:"city pop, bright synth"` diff --git a/apps/api/internal/httpapi/server.go b/apps/api/internal/httpapi/server.go index 6c9f010..04454e8 100644 --- a/apps/api/internal/httpapi/server.go +++ b/apps/api/internal/httpapi/server.go @@ -291,6 +291,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor mux.Handle("POST /v1/images/generations", server.requireUser(auth.PermissionBasic, server.createTask("images.generations", true))) mux.Handle("POST /images/edits", server.requireUser(auth.PermissionBasic, server.createTask("images.edits", true))) mux.Handle("POST /v1/images/edits", server.requireUser(auth.PermissionBasic, server.createTask("images.edits", true))) + mux.Handle("POST /v1/videos/omni-video", server.requireKelingAPIKey(http.HandlerFunc(server.createKelingOmniVideo))) + mux.Handle("GET /v1/videos/omni-video/{taskID}", server.requireKelingAPIKey(http.HandlerFunc(server.getKelingOmniVideo))) mux.Handle("POST /song/generations", server.requireUser(auth.PermissionBasic, server.createTask("song.generations", true))) mux.Handle("POST /v1/song/generations", server.requireUser(auth.PermissionBasic, server.createTask("song.generations", true))) mux.Handle("POST /music/generations", server.requireUser(auth.PermissionBasic, server.createTask("music.generations", true))) diff --git a/apps/api/internal/runner/keling_audio_test.go b/apps/api/internal/runner/keling_audio_test.go new file mode 100644 index 0000000..2538e17 --- /dev/null +++ b/apps/api/internal/runner/keling_audio_test.go @@ -0,0 +1,27 @@ +package runner + +import ( + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestKelingO1GeneratedAudioIsRejectedInsteadOfSilentlyRemoved(t *testing.T) { + result := preprocessRequestWithLog("videos.generations", map[string]any{ + "model": "kling-o1", + "audio": true, + }, store.RuntimeModelCandidate{ + Provider: "keling", + ProviderModelName: "kling-video-o1", + ModelType: "video_generate", + Capabilities: map[string]any{ + "video_generate": map[string]any{"output_audio": false}, + }, + }) + if result.Err == nil { + t.Fatal("Keling O1 audio=true must be rejected") + } + if len(result.Log.Changes) == 0 || result.Log.Changes[len(result.Log.Changes)-1].Action != "reject" { + t.Fatalf("expected an auditable reject change, got %+v", result.Log.Changes) + } +} diff --git a/apps/api/internal/runner/param_processor_media.go b/apps/api/internal/runner/param_processor_media.go index 11dbbda..bf0ffda 100644 --- a/apps/api/internal/runner/param_processor_media.go +++ b/apps/api/internal/runner/param_processor_media.go @@ -4,6 +4,8 @@ import ( "fmt" "math" "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) type resolutionNormalizeProcessor struct{} @@ -691,6 +693,16 @@ func (audioProcessor) ShouldProcess(params map[string]any, modelType string, con } func (audioProcessor) Process(params map[string]any, modelType string, context *paramProcessContext) bool { + if context != nil && kelingO1GeneratedAudioRequested(params, context.candidate) { + return context.reject( + "AudioProcessor", + "audio", + params["audio"], + "kling-video-o1 does not support generated audio", + capabilityPath(modelType, "output_audio"), + capabilityValue(context.modelCapability, modelType, "output_audio"), + ) + } capability := capabilityForType(context.modelCapability, modelType) if capability == nil || !boolFromAny(capability["output_audio"]) { for _, key := range []string{"audio", "output_audio"} { @@ -712,6 +724,17 @@ func (audioProcessor) Process(params map[string]any, modelType string, context * return true } +func kelingO1GeneratedAudioRequested(params map[string]any, candidate store.RuntimeModelCandidate) bool { + if !strings.EqualFold(strings.TrimSpace(candidate.Provider), "keling") { + return false + } + model := strings.ToLower(strings.TrimSpace(candidate.ProviderModelName)) + if model != "kling-o1" && model != "kling-video-o1" { + return false + } + return boolFromAny(params["audio"]) || boolFromAny(params["output_audio"]) +} + type imageCountProcessor struct{} func (imageCountProcessor) Name() string { return "ImageCountProcessor" } diff --git a/docs/kling-omni-compatible.md b/docs/kling-omni-compatible.md new file mode 100644 index 0000000..29acf9f --- /dev/null +++ b/docs/kling-omni-compatible.md @@ -0,0 +1,132 @@ +# Kling Omni 兼容接口 + +EasyAI AI Gateway 提供 Kling 旧版 Omni 协议兼容接口。调用方继续使用 Gateway API Key，任务仍经过网关候选选择、异步队列、审计和计费；响应中的 `task_id` 是网关任务 UUID，不是上游任务 ID。 + +## 模型与参数映射 + +| 请求 `model_name` | 网关模型别名 | TranStreams 原生 `model_name` | 时长范围 | +| --- | --- | --- | --- | +| `kling-video-o1`、`kling-o1` | `kling-o1` | `kling-video-o1` | 3–10 秒 | +| `kling-v3-omni`、`kling-3.0-omni` | `kling-3.0-omni` | `kling-v3-omni` | 3–15 秒 | + +网关别名用于候选匹配，原生模型名用于发往 TranStreams 的 Kling Omni 请求；两类名称不会混用。 + +`kling-video-o1` 的纯文生视频和首帧生视频只接受 5 或 10 秒；3–10 秒中的其他整数需要使用普通参考图等支持该时长的 Omni 输入。`kling-v3-omni` 接受 3–15 秒。 + +真实上游结果表明 `kling-video-o1` 不生成音频，因此该模型的 `sound=on` 会返回 `1201`，标准接口的 `audio=true` 也会在参数预处理阶段失败，避免静默返回无声视频。`kling-v3-omni` 支持 `sound=on/off`。 + +`mode` 映射为网关分辨率：`std` = 720p，`pro` = 1080p，`4k` = 2160p。4K 只有在平台模型能力也声明支持时才能执行。`sound=on/off` 映射为 `audio=true/false`，`duration` 同时接受 JSON 字符串和整数。 + +兼容字段包括：`prompt`、`multi_shot`、`shot_type`、`multi_prompt`、`image_list`、`element_list`、`video_list`、`sound`、`mode`、`aspect_ratio`、`duration`、`watermark_info`、`external_task_id`。`callback_url` 可以省略或传空字符串；非空值会返回业务码 `1201`，本期不投递回调。 + +## 创建任务 + +`POST /v1/videos/omni-video` 固定异步受理，不需要 `X-Async`，成功返回 HTTP 200。 + +```bash +curl -sS -X POST "$GATEWAY_BASE_URL/v1/videos/omni-video" \ + -H "Authorization: Bearer $GATEWAY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model_name": "kling-v3-omni", + "prompt": "A quiet street in the rain with natural ambient sound", + "mode": "pro", + "aspect_ratio": "9:16", + "duration": "5", + "sound": "on", + "watermark_info": {"enabled": false}, + "external_task_id": "client-job-001" + }' +``` + +```json +{ + "code": 0, + "message": "SUCCEED", + "request_id": "...", + "data": { + "task_id": "00000000-0000-0000-0000-000000000000", + "task_info": {"external_task_id": "client-job-001"}, + "task_status": "submitted", + "created_at": 0, + "updated_at": 0 + } +} +``` + +## 查询任务 + +使用创建任务时的同一个 Gateway API Key 轮询。跨用户查询与不存在的任务统一返回 HTTP 404 和业务码 `1203`。 + +```bash +curl -sS \ + -H "Authorization: Bearer $GATEWAY_API_KEY" \ + "$GATEWAY_BASE_URL/v1/videos/omni-video/$TASK_ID" +``` + +`task_status` 为 `submitted`、`processing`、`succeed` 或 `failed`。成功时结果位于 `data.task_result.videos`： + +```json +{ + "code": 0, + "message": "SUCCEED", + "request_id": "...", + "data": { + "task_id": "00000000-0000-0000-0000-000000000000", + "task_status": "succeed", + "task_result": { + "videos": [ + { + "id": "...", + "url": "https://.../video.mp4", + "watermark_url": "https://.../watermark.mp4", + "duration": "5" + } + ] + } + } +} +``` + +## 网关标准视频接口 + +标准接口仍为 `POST /api/v1/videos/generations`。异步调用需要 `X-Async: true`，再通过 `GET /api/v1/tasks/{taskId}` 轮询。 + +```bash +curl -sS -X POST "$GATEWAY_BASE_URL/api/v1/videos/generations" \ + -H "Authorization: Bearer $GATEWAY_API_KEY" \ + -H "Content-Type: application/json" \ + -H "X-Async: true" \ + -d '{ + "model": "kling-o1", + "prompt": "A product reveal in a daylight studio", + "resolution": "1080p", + "aspect_ratio": "9:16", + "duration": 5, + "audio": true, + "watermark": false, + "runMode": "real" + }' +``` + +```bash +curl -sS \ + -H "Authorization: Bearer $GATEWAY_API_KEY" \ + "$GATEWAY_BASE_URL/api/v1/tasks/$TASK_ID" +``` + +## 错误格式 + +所有兼容接口错误都返回同一包络： + +```json +{ + "code": 1201, + "message": "duration must be between 3 and 10 seconds", + "request_id": "..." +} +``` + +业务码分类：`1001/1002` 为鉴权错误，`1101/1103` 为余额或权限错误，`1201/1203` 为参数或资源错误，`1302/1303` 为限流错误，`5000/5001` 为网关或上游服务错误。HTTP 状态码仍反映错误类型。 + +OpenAPI 文档由服务的 `/openapi.json` 和 `/openapi.yaml` 提供。