From ebdb96e7d7b1a91670c258abcd4dc2234ae29e39 Mon Sep 17 00:00:00 2001 From: wangbo Date: Wed, 5 Aug 2026 00:40:42 +0800 Subject: [PATCH] =?UTF-8?q?fix(provider):=20=E4=BF=AE=E6=AD=A3=E5=AA=92?= =?UTF-8?q?=E4=BD=93=E8=AF=B7=E6=B1=82=E8=BD=AC=E6=8D=A2=E4=B8=8E=E4=B8=8A?= =?UTF-8?q?=E6=B8=B8=E9=94=99=E8=AF=AF=E9=80=8F=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按上游协议能力延迟处理媒体资源：OpenAI 兼容平台默认使用 multipart，显式配置后才发送 JSON URL；Gemini 官方协议使用 Files API，兼容协议使用内嵌 Base64，并同步覆盖相关媒体客户端。\n\n安全的上游 400/422 原始错误会作为下游 message 返回，同时保留结构化诊断信息和历史任务兼容。\n\n验证：API 全量无缓存测试、go vet、pnpm lint、pnpm test、pnpm build、pnpm openapi、git diff --check。 --- apps/api/docs/swagger.json | 4 + apps/api/docs/swagger.yaml | 3 + apps/api/internal/clients/clients_test.go | 175 ++++++++++++- apps/api/internal/clients/gemini.go | 241 ++++++++++++++++++ apps/api/internal/clients/gemini_veo.go | 24 +- apps/api/internal/clients/gemini_veo_test.go | 120 ++++++++- apps/api/internal/clients/openai.go | 156 ++++++++++-- .../internal/httpapi/easyai_compat_test.go | 29 +++ apps/api/internal/httpapi/public_errors.go | 43 +++- .../internal/httpapi/public_errors_test.go | 53 ++++ apps/api/internal/publicerror/public_error.go | 83 ++++-- .../internal/publicerror/public_error_test.go | 30 +++ apps/api/internal/runner/request_assets.go | 123 +++++++-- .../internal/runner/request_assets_test.go | 142 +++++++++-- .../model-platforms-and-bindings.md | 2 + .../pages/admin/PlatformManagementPanel.tsx | 16 ++ .../web/src/pages/admin/platform-form.test.ts | 8 + apps/web/src/pages/admin/platform-form.ts | 3 + packages/contracts/src/index.ts | 1 + 19 files changed, 1163 insertions(+), 93 deletions(-) diff --git a/apps/api/docs/swagger.json b/apps/api/docs/swagger.json index c46b0b7..d6ed1b3 100644 --- a/apps/api/docs/swagger.json +++ b/apps/api/docs/swagger.json @@ -13503,6 +13503,10 @@ "code": { "type": "string" }, + "details": { + "type": "object", + "additionalProperties": {} + }, "httpStatus": { "type": "integer" }, diff --git a/apps/api/docs/swagger.yaml b/apps/api/docs/swagger.yaml index db9de7f..51df397 100644 --- a/apps/api/docs/swagger.yaml +++ b/apps/api/docs/swagger.yaml @@ -2043,6 +2043,9 @@ definitions: type: string code: type: string + details: + additionalProperties: {} + type: object httpStatus: type: integer message: diff --git a/apps/api/internal/clients/clients_test.go b/apps/api/internal/clients/clients_test.go index 47e9179..dea08ff 100644 --- a/apps/api/internal/clients/clients_test.go +++ b/apps/api/internal/clients/clients_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -549,7 +550,7 @@ func TestOpenAIClientImageEditUsesMultipartWithMultipleImages(t *testing.T) { if !strings.HasPrefix(contentType, "multipart/form-data; boundary=") { t.Fatalf("image edit should use multipart/form-data, got %q", contentType) } - if receivedImageCount != 2 || len(receivedImageFields) != 1 || receivedImageFields[0] != "image" { + if receivedImageCount != 2 || len(receivedImageFields) != 1 || receivedImageFields[0] != "image[]" { t.Fatalf("unexpected multipart image files: fields=%v count=%d", receivedImageFields, receivedImageCount) } if receivedFields["model"][0] != "gpt-image-2-vip" || @@ -568,6 +569,178 @@ func TestOpenAIClientImageEditUsesMultipartWithMultipleImages(t *testing.T) { } } +func TestOpenAIClientImageEditDownloadsURLWhenBuildingMultipart(t *testing.T) { + imagePayload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4} + fetchCount := 0 + imageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fetchCount++ + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(imagePayload) + })) + defer imageServer.Close() + + var uploaded []byte + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart image edit: %v", err) + } + files := r.MultipartForm.File["image"] + if len(files) != 1 { + t.Fatalf("expected one multipart image, got %+v", r.MultipartForm.File) + } + file, err := files[0].Open() + if err != nil { + t.Fatalf("open multipart image: %v", err) + } + defer file.Close() + uploaded, err = io.ReadAll(file) + if err != nil { + t.Fatalf("read multipart image: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []any{map[string]any{"b64_json": "aW1hZ2U="}}, + }) + })) + defer upstream.Close() + + _, err := (OpenAIClient{HTTPClient: upstream.Client()}).Run(context.Background(), Request{ + Kind: "images.edits", + ModelType: "image_edit", + Body: map[string]any{ + "image": imageServer.URL + "/source.png", + "prompt": "edit it", + }, + Candidate: store.RuntimeModelCandidate{ + BaseURL: upstream.URL, + Provider: "openai", + ProviderModelName: "gpt-image-compatible", + ModelType: "image_edit", + Credentials: map[string]any{"apiKey": "openai-key"}, + }, + }) + if err != nil { + t.Fatalf("run OpenAI URL multipart image edit: %v", err) + } + if fetchCount != 1 || string(uploaded) != string(imagePayload) { + t.Fatalf("image URL should be fetched once during multipart construction: fetches=%d uploaded=%v", fetchCount, uploaded) + } +} + +func TestKelingAndMinimaxMaterializeRemoteMediaInsideClient(t *testing.T) { + payload := []byte("remote media payload") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "audio/mpeg") + _, _ = w.Write(payload) + })) + defer server.Close() + + encoded, err := kelingImageToBase64(context.Background(), Request{HTTPClient: server.Client()}, server.URL+"/image.png") + if err != nil { + t.Fatalf("materialize Keling image URL: %v", err) + } + if encoded != base64.StdEncoding.EncodeToString(payload) { + t.Fatalf("unexpected Keling image base64: %q", encoded) + } + + audio, filename, contentType, err := minimaxVoiceCloneFilePayload(context.Background(), server.Client(), server.URL+"/audio.mp3", "voice_clone") + if err != nil { + t.Fatalf("materialize Minimax audio URL: %v", err) + } + if string(audio) != string(payload) || filename != "voice_clone.mp3" || contentType != "audio/mpeg" { + t.Fatalf("unexpected Minimax audio materialization: filename=%q contentType=%q payload=%q", filename, contentType, audio) + } +} + +func TestOpenAIClientImageEditUsesJSONURLsOnlyWhenExplicitlyConfigured(t *testing.T) { + var contentType string + var received map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + contentType = r.Header.Get("Content-Type") + if err := json.NewDecoder(r.Body).Decode(&received); err != nil { + t.Fatalf("decode image edit JSON: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []any{map[string]any{"url": "https://cdn.example.com/output.png"}}, + }) + })) + defer server.Close() + + _, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{ + Kind: "images.edits", + ModelType: "image_edit", + Model: "compatible-image-edit", + Body: map[string]any{ + "images": []any{ + "https://cdn.example.com/first.png", + map[string]any{"url": "https://cdn.example.com/second.png"}, + }, + "mask_image": map[string]any{"image_url": "https://cdn.example.com/mask.png"}, + "prompt": "combine the references", + "_metadata": map[string]any{"private": true}, + }, + Candidate: store.RuntimeModelCandidate{ + BaseURL: server.URL, + Provider: "compatible-openai", + SpecType: "openai", + ProviderModelName: "compatible-image-edit-v2", + ModelType: "image_edit", + Credentials: map[string]any{"apiKey": "openai-key"}, + PlatformConfig: map[string]any{"imageEditRequestFormat": "json_url"}, + }, + }) + if err != nil { + t.Fatalf("run OpenAI-compatible JSON image edit: %v", err) + } + if !strings.HasPrefix(contentType, "application/json") { + t.Fatalf("explicit JSON URL mode should use application/json, got %q", contentType) + } + images, _ := received["image"].([]any) + if len(images) != 2 || images[0] != "https://cdn.example.com/first.png" || images[1] != "https://cdn.example.com/second.png" { + t.Fatalf("unexpected JSON image URLs: %+v", received) + } + if received["mask"] != "https://cdn.example.com/mask.png" || received["model"] != "compatible-image-edit-v2" { + t.Fatalf("unexpected JSON image edit body: %+v", received) + } + if _, ok := received["images"]; ok { + t.Fatalf("images alias must be normalized to image: %+v", received) + } + if _, ok := received["_metadata"]; ok { + t.Fatalf("internal metadata must not be forwarded: %+v", received) + } +} + +func TestOpenAIClientImageEditJSONModeRejectsInlineImageData(t *testing.T) { + _, _, err := openAIRequestPayload(context.Background(), "images.edits", map[string]any{ + "image": "data:image/png;base64,aW1hZ2U=", + }, store.RuntimeModelCandidate{PlatformConfig: map[string]any{"imageEditRequestFormat": "json_url"}}) + clientErr, ok := err.(*ClientError) + if !ok || clientErr.Code != "invalid_parameter" || clientErr.Param != "image" || clientErr.StatusCode != http.StatusBadRequest { + t.Fatalf("expected non-retryable image URL validation error, got %#v", err) + } +} + +func TestOpenAIImageEditFieldNameUsesArraySyntaxForMultipleImages(t *testing.T) { + tests := []struct { + name string + candidate store.RuntimeModelCandidate + imageCount int + want string + }{ + {name: "compatible single image", candidate: store.RuntimeModelCandidate{BaseURL: "https://compatible.example/v1"}, imageCount: 1, want: "image"}, + {name: "compatible multiple images", candidate: store.RuntimeModelCandidate{BaseURL: "https://compatible.example/v1"}, imageCount: 2, want: "image[]"}, + {name: "official single image", candidate: store.RuntimeModelCandidate{BaseURL: "https://api.openai.com/v1"}, imageCount: 1, want: "image[]"}, + {name: "explicit scalar override", candidate: store.RuntimeModelCandidate{BaseURL: "https://compatible.example/v1", PlatformConfig: map[string]any{"imageEditMultipartFieldName": "image"}}, imageCount: 2, want: "image"}, + {name: "explicit array override", candidate: store.RuntimeModelCandidate{BaseURL: "https://compatible.example/v1", PlatformConfig: map[string]any{"imageEditMultipartFieldName": "image[]"}}, imageCount: 1, want: "image[]"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := openAIImageEditFieldName(test.candidate, test.imageCount); got != test.want { + t.Fatalf("field name = %q, want %q", got, test.want) + } + }) + } +} + func TestOpenAIClientImageGenerationUsesSizeWithoutGenericGeometryFields(t *testing.T) { var received map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/apps/api/internal/clients/gemini.go b/apps/api/internal/clients/gemini.go index ce2505b..102c6b2 100644 --- a/apps/api/internal/clients/gemini.go +++ b/apps/api/internal/clients/gemini.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "mime" @@ -12,6 +13,8 @@ import ( "path" "strings" "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) type GeminiClient struct { @@ -27,6 +30,20 @@ func (c GeminiClient) Run(ctx context.Context, request Request) (Response, error return c.runVeo(ctx, request, apiKey) } body := geminiBody(request) + _, nativeContents := request.Body["contents"] + if !nativeContents && GeminiUsesOfficialAPI(request.Candidate) { + var err error + body, err = c.prepareOfficialGeminiImageFiles(ctx, request, apiKey, body) + if err != nil { + return Response{}, err + } + } else if !nativeContents { + var err error + body, err = prepareCompatibleGeminiInlineImages(ctx, body) + if err != nil { + return Response{}, err + } + } raw, _ := json.Marshal(body) endpoint := geminiURL(request.Candidate.BaseURL, upstreamModelName(request.Candidate), apiKey) if request.Stream { @@ -92,6 +109,230 @@ func (c GeminiClient) Run(ctx context.Context, request Request) (Response, error }, nil } +// GeminiUsesOfficialAPI distinguishes the native Google Gemini endpoint from +// Gemini-shaped compatibility services and Google's OpenAI compatibility path. +func GeminiUsesOfficialAPI(candidate store.RuntimeModelCandidate) bool { + base := strings.TrimSpace(candidate.BaseURL) + if base == "" { + return true + } + parsed, err := url.Parse(base) + if err != nil || !strings.EqualFold(parsed.Hostname(), "generativelanguage.googleapis.com") { + return false + } + pathValue := strings.ToLower(strings.TrimRight(parsed.Path, "/")) + return pathValue != "/openai" && !strings.HasSuffix(pathValue, "/openai") +} + +func (c GeminiClient) prepareOfficialGeminiImageFiles(ctx context.Context, request Request, apiKey string, body map[string]any) (map[string]any, error) { + client := httpClient(request.HTTPClient, c.HTTPClient) + cache := map[string]map[string]any{} + prepared, err := prepareOfficialGeminiImageFileValue(ctx, client, request.Candidate, apiKey, body, cache) + if err != nil { + return nil, err + } + out, _ := prepared.(map[string]any) + if out == nil { + return body, nil + } + return out, nil +} + +func prepareOfficialGeminiImageFileValue(ctx context.Context, client *http.Client, candidate store.RuntimeModelCandidate, apiKey string, value any, cache map[string]map[string]any) (any, error) { + switch typed := value.(type) { + case map[string]any: + next := make(map[string]any, len(typed)) + for key, item := range typed { + prepared, err := prepareOfficialGeminiImageFileValue(ctx, client, candidate, apiKey, item, cache) + if err != nil { + return nil, err + } + next[key] = prepared + } + for _, key := range []string{"fileData", "file_data"} { + fileData, ok := next[key].(map[string]any) + if !ok { + continue + } + fileURI := firstNonEmptyString(fileData["fileUri"], fileData["file_uri"], fileData["uri"]) + mimeType := firstNonEmptyString(fileData["mimeType"], fileData["mime_type"], mimeFromURI(fileURI)) + if !requestAssetStringIsHTTPURL(fileURI) || !strings.HasPrefix(strings.ToLower(mimeType), "image/") || geminiOfficialFileURI(fileURI) { + continue + } + cacheKey := fileURI + "\x00" + mimeType + uploaded := cache[cacheKey] + if uploaded == nil { + var err error + uploaded, err = uploadOfficialGeminiImageFile(ctx, client, candidate, apiKey, fileURI, mimeType) + if err != nil { + fallback, fallbackErr := compatibleGeminiInlineImage(ctx, fileURI, mimeType) + if fallbackErr != nil { + return nil, err + } + delete(next, key) + next["inlineData"] = fallback + continue + } + cache[cacheKey] = uploaded + } + next[key] = cloneBody(uploaded) + } + return next, nil + case []any: + next := make([]any, 0, len(typed)) + for _, item := range typed { + prepared, err := prepareOfficialGeminiImageFileValue(ctx, client, candidate, apiKey, item, cache) + if err != nil { + return nil, err + } + next = append(next, prepared) + } + return next, nil + default: + return value, nil + } +} + +func prepareCompatibleGeminiInlineImages(ctx context.Context, body map[string]any) (map[string]any, error) { + prepared, err := prepareCompatibleGeminiInlineImageValue(ctx, body, map[string]map[string]any{}) + if err != nil { + return nil, err + } + out, _ := prepared.(map[string]any) + if out == nil { + return body, nil + } + return out, nil +} + +func prepareCompatibleGeminiInlineImageValue(ctx context.Context, value any, cache map[string]map[string]any) (any, error) { + switch typed := value.(type) { + case map[string]any: + next := make(map[string]any, len(typed)) + for key, item := range typed { + prepared, err := prepareCompatibleGeminiInlineImageValue(ctx, item, cache) + if err != nil { + return nil, err + } + next[key] = prepared + } + for _, key := range []string{"fileData", "file_data"} { + fileData, ok := next[key].(map[string]any) + if !ok { + continue + } + fileURI := firstNonEmptyString(fileData["fileUri"], fileData["file_uri"], fileData["uri"]) + mimeType := firstNonEmptyString(fileData["mimeType"], fileData["mime_type"], mimeFromURI(fileURI)) + if !requestAssetStringIsHTTPURL(fileURI) || !strings.HasPrefix(strings.ToLower(mimeType), "image/") { + continue + } + cacheKey := fileURI + "\x00" + mimeType + inline := cache[cacheKey] + if inline == nil { + var err error + inline, err = compatibleGeminiInlineImage(ctx, fileURI, mimeType) + if err != nil { + return nil, err + } + cache[cacheKey] = inline + } + delete(next, key) + next["inlineData"] = cloneBody(inline) + } + return next, nil + case []any: + next := make([]any, 0, len(typed)) + for _, item := range typed { + prepared, err := prepareCompatibleGeminiInlineImageValue(ctx, item, cache) + if err != nil { + return nil, err + } + next = append(next, prepared) + } + return next, nil + default: + return value, nil + } +} + +func compatibleGeminiInlineImage(ctx context.Context, sourceURL string, declaredMimeType string) (map[string]any, error) { + fetchedMimeType, payload, err := fetchRemoteMediaInputPayload(ctx, sourceURL, 256<<20) + if err != nil { + return nil, err + } + return map[string]any{ + "mimeType": geminiMediaMime(firstNonEmptyString(declaredMimeType, fetchedMimeType), "image"), + "data": base64.StdEncoding.EncodeToString(payload), + }, nil +} + +func uploadOfficialGeminiImageFile(ctx context.Context, client *http.Client, candidate store.RuntimeModelCandidate, apiKey string, sourceURL string, declaredMimeType string) (map[string]any, error) { + fetchedMimeType, payload, err := fetchRemoteMediaInputPayload(ctx, sourceURL, 256<<20) + if err != nil { + return nil, err + } + mimeType := geminiMediaMime(firstNonEmptyString(declaredMimeType, fetchedMimeType), "image") + base := strings.TrimRight(strings.TrimSpace(candidate.BaseURL), "/") + if base == "" { + base = "https://generativelanguage.googleapis.com" + } + parsed, err := url.Parse(base) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return nil, &ClientError{Code: "invalid_parameter", Message: "invalid official Gemini base URL", Retryable: false} + } + uploadStartURL := parsed.Scheme + "://" + parsed.Host + "/upload/v1beta/files" + metadata, _ := json.Marshal(map[string]any{"file": map[string]any{"display_name": fmt.Sprintf("easyai-image-%d", time.Now().UnixMilli())}}) + startRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadStartURL, bytes.NewReader(metadata)) + if err != nil { + return nil, err + } + startRequest.Header.Set("x-goog-api-key", apiKey) + startRequest.Header.Set("X-Goog-Upload-Protocol", "resumable") + startRequest.Header.Set("X-Goog-Upload-Command", "start") + startRequest.Header.Set("X-Goog-Upload-Header-Content-Length", fmt.Sprintf("%d", len(payload))) + startRequest.Header.Set("X-Goog-Upload-Header-Content-Type", mimeType) + startRequest.Header.Set("Content-Type", "application/json") + startResponse, err := client.Do(startRequest) + if err != nil { + return nil, &ClientError{Code: "network", Message: err.Error(), Retryable: true} + } + if startResponse.StatusCode < http.StatusOK || startResponse.StatusCode >= http.StatusMultipleChoices { + _, responseErr := decodeHTTPResponse(startResponse) + return nil, responseErr + } + uploadURL := strings.TrimSpace(startResponse.Header.Get("X-Goog-Upload-URL")) + _ = startResponse.Body.Close() + if uploadURL == "" { + return nil, &ClientError{Code: "invalid_response", Message: "Gemini Files API did not return an upload URL", Retryable: false} + } + uploadRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadURL, bytes.NewReader(payload)) + if err != nil { + return nil, err + } + uploadRequest.Header.Set("X-Goog-Upload-Offset", "0") + uploadRequest.Header.Set("X-Goog-Upload-Command", "upload, finalize") + uploadRequest.Header.Set("Content-Type", mimeType) + uploadResponse, err := client.Do(uploadRequest) + if err != nil { + return nil, &ClientError{Code: "network", Message: err.Error(), Retryable: true} + } + result, err := decodeHTTPResponse(uploadResponse) + if err != nil { + return nil, err + } + file := mapFromAny(result["file"]) + fileURI := firstNonEmptyString(file["uri"], result["uri"]) + if fileURI == "" { + return nil, &ClientError{Code: "invalid_response", Message: "Gemini Files API response did not include file.uri", Retryable: false} + } + return map[string]any{"fileUri": fileURI, "mimeType": firstNonEmptyString(file["mimeType"], file["mime_type"], mimeType)}, nil +} + +func geminiOfficialFileURI(value string) bool { + parsed, err := url.Parse(strings.TrimSpace(value)) + return err == nil && strings.EqualFold(parsed.Hostname(), "generativelanguage.googleapis.com") && strings.Contains(strings.ToLower(parsed.Path), "/files/") +} + func geminiURL(baseURL string, model string, apiKey string) string { return geminiActionURL(baseURL, model, "generateContent", apiKey, false) } diff --git a/apps/api/internal/clients/gemini_veo.go b/apps/api/internal/clients/gemini_veo.go index 74d4b23..1df36ec 100644 --- a/apps/api/internal/clients/gemini_veo.go +++ b/apps/api/internal/clients/gemini_veo.go @@ -33,7 +33,7 @@ func (c GeminiClient) runVeo(ctx context.Context, request Request, apiKey string var wire *WireResponse if operationName == "" { - body, err := geminiVeoBody(request) + body, err := geminiVeoBody(ctx, request) if err != nil { return Response{}, err } @@ -107,7 +107,7 @@ func (c GeminiClient) runVeo(ctx context.Context, request Request, apiKey string } } -func geminiVeoBody(request Request) (map[string]any, error) { +func geminiVeoBody(ctx context.Context, request Request) (map[string]any, error) { prompt := firstNonEmptyPrompt(request.Body, "") if prompt == "" { return nil, &ClientError{Code: "invalid_parameter", Message: "gemini Veo prompt is required", Param: "prompt", StatusCode: http.StatusBadRequest, Retryable: false} @@ -116,14 +116,14 @@ func geminiVeoBody(request Request) (map[string]any, error) { firstFrame, lastFrame, references := geminiVeoImageInputs(request.Body) instance := map[string]any{"prompt": prompt} if firstFrame.URI != "" { - image, err := geminiVeoInlineImage(firstFrame) + image, err := geminiVeoInlineImage(ctx, firstFrame) if err != nil { return nil, err } instance["image"] = image } if lastFrame.URI != "" { - image, err := geminiVeoInlineImage(lastFrame) + image, err := geminiVeoInlineImage(ctx, lastFrame) if err != nil { return nil, err } @@ -135,7 +135,7 @@ func geminiVeoBody(request Request) (map[string]any, error) { if len(references) > 0 { items := make([]any, 0, len(references)) for _, reference := range references { - image, err := geminiVeoInlineImage(reference) + image, err := geminiVeoInlineImage(ctx, reference) if err != nil { return nil, err } @@ -340,13 +340,20 @@ func deduplicateGeminiVeoImages(images []geminiVeoImage, exclusions ...string) [ return out } -func geminiVeoInlineImage(image geminiVeoImage) (map[string]any, error) { +func geminiVeoInlineImage(ctx context.Context, image geminiVeoImage) (map[string]any, error) { parsed := geminiDataURL(image.URI) mimeType := strings.TrimSpace(image.MimeType) data := "" if parsed != nil { data = parsed.data mimeType = firstNonEmptyString(mimeType, parsed.mimeType) + } else if requestAssetStringIsHTTPURL(image.URI) { + fetchedMimeType, payload, err := fetchRemoteMediaInputPayload(ctx, image.URI, 256<<20) + if err != nil { + return nil, err + } + data = base64.StdEncoding.EncodeToString(payload) + mimeType = firstNonEmptyString(mimeType, fetchedMimeType) } else if !requestLikeURL(image.URI) { data = strings.TrimSpace(image.URI) } @@ -368,6 +375,11 @@ func geminiVeoInlineImage(image geminiVeoImage) (map[string]any, error) { return map[string]any{"bytesBase64Encoded": data, "mimeType": geminiMediaMime(mimeType, "image")}, nil } +func requestAssetStringIsHTTPURL(value string) bool { + parsed, err := url.Parse(strings.TrimSpace(value)) + return err == nil && parsed.Host != "" && (strings.EqualFold(parsed.Scheme, "http") || strings.EqualFold(parsed.Scheme, "https")) +} + func requestLikeURL(value string) bool { normalized := strings.ToLower(strings.TrimSpace(value)) return strings.HasPrefix(normalized, "http://") || strings.HasPrefix(normalized, "https://") || strings.HasPrefix(normalized, "/static/") diff --git a/apps/api/internal/clients/gemini_veo_test.go b/apps/api/internal/clients/gemini_veo_test.go index 9387d14..bd4a1c2 100644 --- a/apps/api/internal/clients/gemini_veo_test.go +++ b/apps/api/internal/clients/gemini_veo_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "io" "net/http" "net/http/httptest" "os" @@ -144,7 +145,7 @@ func TestGeminiVeoBodyMapsImageAndParameters(t *testing.T) { first := "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("first")) last := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("last")) reference := "data:image/webp;base64," + base64.StdEncoding.EncodeToString([]byte("reference")) - body, err := geminiVeoBody(Request{Body: map[string]any{ + body, err := geminiVeoBody(context.Background(), Request{Body: map[string]any{ "prompt": "animate", "first_frame": first, "last_frame": last, @@ -175,8 +176,123 @@ func TestGeminiVeoBodyMapsImageAndParameters(t *testing.T) { } } +func TestGeminiVeoDownloadsImageURLWhenBuildingOfficialPayload(t *testing.T) { + payload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4} + fetchCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fetchCount++ + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(payload) + })) + defer server.Close() + + body, err := geminiVeoBody(context.Background(), Request{Body: map[string]any{ + "prompt": "animate", + "first_frame": server.URL + "/input.png", + "duration": 8, + }}) + if err != nil { + t.Fatalf("build Gemini Veo body from URL: %v", err) + } + instance := body["instances"].([]any)[0].(map[string]any) + image := instance["image"].(map[string]any) + if fetchCount != 1 || image["bytesBase64Encoded"] != base64.StdEncoding.EncodeToString(payload) || image["mimeType"] != "image/png" { + t.Fatalf("Veo URL should be fetched during official payload construction: fetches=%d image=%+v", fetchCount, image) + } +} + +func TestGeminiOfficialAPIClassification(t *testing.T) { + tests := []struct { + baseURL string + want bool + }{ + {baseURL: "", want: true}, + {baseURL: "https://generativelanguage.googleapis.com", want: true}, + {baseURL: "https://generativelanguage.googleapis.com/v1beta", want: true}, + {baseURL: "https://generativelanguage.googleapis.com/v1beta/openai", want: false}, + {baseURL: "https://gemini-compatible.example.com/v1beta", want: false}, + } + for _, test := range tests { + if got := GeminiUsesOfficialAPI(store.RuntimeModelCandidate{BaseURL: test.baseURL}); got != test.want { + t.Fatalf("GeminiUsesOfficialAPI(%q) = %t, want %t", test.baseURL, got, test.want) + } + } +} + +func TestGeminiCompatibleConvertsFileDataURLToInlineBase64(t *testing.T) { + payload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(payload) + })) + defer server.Close() + body := map[string]any{"contents": []any{map[string]any{"parts": []any{map[string]any{ + "fileData": map[string]any{"fileUri": server.URL + "/source.png", "mimeType": "image/png"}, + }}}}} + + prepared, err := prepareCompatibleGeminiInlineImages(context.Background(), body) + if err != nil { + t.Fatalf("prepare compatible Gemini inline image: %v", err) + } + contents := prepared["contents"].([]any) + parts := contents[0].(map[string]any)["parts"].([]any) + part := parts[0].(map[string]any) + inline := part["inlineData"].(map[string]any) + if _, exists := part["fileData"]; exists || inline["mimeType"] != "image/png" || inline["data"] != base64.StdEncoding.EncodeToString(payload) { + t.Fatalf("compatible Gemini should use inlineData: %+v", part) + } +} + +func TestGeminiOfficialImageUploadUsesFilesAPI(t *testing.T) { + imagePayload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4} + imageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(imagePayload) + })) + defer imageServer.Close() + + var uploadServer *httptest.Server + startCount := 0 + uploadCount := 0 + uploadServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/upload/v1beta/files": + startCount++ + if r.Header.Get("x-goog-api-key") != "gemini-key" || r.Header.Get("X-Goog-Upload-Protocol") != "resumable" || r.Header.Get("X-Goog-Upload-Header-Content-Type") != "image/png" { + t.Fatalf("unexpected Gemini Files start headers: %+v", r.Header) + } + w.Header().Set("X-Goog-Upload-URL", uploadServer.URL+"/upload/session") + w.WriteHeader(http.StatusOK) + case "/upload/session": + uploadCount++ + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read Gemini Files upload: %v", err) + } + if string(raw) != string(imagePayload) || r.Header.Get("X-Goog-Upload-Command") != "upload, finalize" { + t.Fatalf("unexpected Gemini Files upload body=%v headers=%+v", raw, r.Header) + } + _ = json.NewEncoder(w).Encode(map[string]any{"file": map[string]any{ + "uri": "https://generativelanguage.googleapis.com/v1beta/files/easyai-image", + "mimeType": "image/png", + }}) + default: + http.NotFound(w, r) + } + })) + defer uploadServer.Close() + + fileData, err := uploadOfficialGeminiImageFile(context.Background(), uploadServer.Client(), store.RuntimeModelCandidate{BaseURL: uploadServer.URL + "/v1beta"}, "gemini-key", imageServer.URL+"/source.png", "image/png") + if err != nil { + t.Fatalf("upload official Gemini image: %v", err) + } + if startCount != 1 || uploadCount != 1 || fileData["fileUri"] != "https://generativelanguage.googleapis.com/v1beta/files/easyai-image" || fileData["mimeType"] != "image/png" { + t.Fatalf("unexpected Gemini Files result: starts=%d uploads=%d fileData=%+v", startCount, uploadCount, fileData) + } +} + func TestGeminiVeoRejectsInvalidDurationBeforeSubmission(t *testing.T) { - _, err := geminiVeoBody(Request{Body: map[string]any{"prompt": "test", "duration": 5}}) + _, err := geminiVeoBody(context.Background(), Request{Body: map[string]any{"prompt": "test", "duration": 5}}) if err == nil || ErrorCode(err) != "invalid_parameter" || ErrorParam(err) != "duration" || !strings.Contains(err.Error(), "4, 6, or 8") { t.Fatalf("unexpected invalid duration error: %v", err) } diff --git a/apps/api/internal/clients/openai.go b/apps/api/internal/clients/openai.go index d4adc77..a67ed24 100644 --- a/apps/api/internal/clients/openai.go +++ b/apps/api/internal/clients/openai.go @@ -90,8 +90,9 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error provisionalRules := make([]parameterCorrectionRule, 0, 2) seenCorrectionErrors := make(map[string]struct{}) var resp *http.Response + requestClient := httpClient(request.HTTPClient, c.HTTPClient) for correctionAttempt := 0; ; correctionAttempt++ { - raw, contentType, payloadErr := openAIRequestPayload(endpointKind, body, request.Candidate) + raw, contentType, payloadErr := openAIRequestPayload(ctx, endpointKind, body, request.Candidate) if payloadErr != nil { return Response{}, payloadErr } @@ -105,7 +106,7 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error if err := notifySubmissionStarted(request); err != nil { return Response{}, err } - resp, requestErr = httpClient(request.HTTPClient, c.HTTPClient).Do(req) + resp, requestErr = requestClient.Do(req) if requestErr != nil { return Response{}, &ClientError{Code: "network", Message: requestErr.Error(), Retryable: true} } @@ -254,14 +255,16 @@ func normalizeOpenAIImageRequestBody(endpointKind string, body map[string]any, o } } -func openAIRequestPayload(endpointKind string, body map[string]any, candidate store.RuntimeModelCandidate) ([]byte, string, error) { +func openAIRequestPayload(ctx context.Context, endpointKind string, body map[string]any, candidate store.RuntimeModelCandidate) ([]byte, string, error) { if endpointKind != "images.edits" { raw, err := json.Marshal(body) return raw, "application/json", err } + if OpenAIImageEditUsesJSONURL(candidate) { + return openAIImageEditJSONPayload(body) + } var payload bytes.Buffer writer := multipart.NewWriter(&payload) - imageFieldName := openAIImageEditFieldName(candidate) images := openAIImageEditValues(firstPresent(body["images"], body["image"])) if len(images) == 0 { return nil, "", &ClientError{ @@ -272,8 +275,9 @@ func openAIRequestPayload(endpointKind string, body map[string]any, candidate st Retryable: false, } } + imageFieldName := openAIImageEditFieldName(candidate, len(images)) for index, value := range images { - contentType, image, err := openAIImageEditPayload(value) + contentType, image, err := openAIImageEditPayload(ctx, value) if err != nil { return nil, "", err } @@ -282,7 +286,7 @@ func openAIRequestPayload(endpointKind string, body map[string]any, candidate st } } if mask := firstPresent(body["mask"], body["mask_image"], body["maskImage"]); mask != nil { - contentType, image, err := openAIImageEditPayload(mask) + contentType, image, err := openAIImageEditPayload(ctx, mask) if err != nil { return nil, "", err } @@ -315,10 +319,93 @@ func openAIRequestPayload(endpointKind string, body map[string]any, candidate st return payload.Bytes(), writer.FormDataContentType(), nil } -func openAIImageEditFieldName(candidate store.RuntimeModelCandidate) string { +// OpenAIImageEditUsesJSONURL reports whether the platform explicitly opts in to +// JSON URL requests for image edits. Multipart remains the compatibility-safe +// default when the setting is absent or unrecognized. +func OpenAIImageEditUsesJSONURL(candidate store.RuntimeModelCandidate) bool { + format := strings.ToLower(strings.TrimSpace(stringFromAny(candidate.PlatformConfig["imageEditRequestFormat"]))) + format = strings.ReplaceAll(format, "-", "_") + format = strings.ReplaceAll(format, " ", "_") + return format == "json" || format == "json_url" +} + +func openAIImageEditJSONPayload(body map[string]any) ([]byte, string, error) { + images := openAIImageEditValues(firstPresent(body["images"], body["image"])) + if len(images) == 0 { + return nil, "", &ClientError{ + Code: "invalid_parameter", + Message: "image is required", + Param: "image", + StatusCode: http.StatusBadRequest, + Retryable: false, + } + } + imageURLs := make([]any, 0, len(images)) + for _, image := range images { + imageURL, err := openAIImageEditURLValue(image, "image") + if err != nil { + return nil, "", err + } + imageURLs = append(imageURLs, imageURL) + } + payload := make(map[string]any, len(body)) + for key, value := range body { + switch key { + case "image", "images", "mask", "mask_image", "maskImage": + continue + } + if strings.HasPrefix(key, "_") || value == nil { + continue + } + payload[key] = value + } + if len(imageURLs) == 1 { + payload["image"] = imageURLs[0] + } else { + payload["image"] = imageURLs + } + if mask := firstPresent(body["mask"], body["mask_image"], body["maskImage"]); mask != nil { + maskURL, err := openAIImageEditURLValue(mask, "mask") + if err != nil { + return nil, "", err + } + payload["mask"] = maskURL + } + raw, err := json.Marshal(payload) + return raw, "application/json", err +} + +func openAIImageEditURLValue(value any, param string) (string, error) { + switch typed := value.(type) { + case map[string]any: + for _, key := range []string{"url", "image_url", "imageUrl"} { + if nested := typed[key]; nested != nil { + return openAIImageEditURLValue(nested, param) + } + } + case string: + raw := strings.TrimSpace(typed) + parsed, err := url.Parse(raw) + if err == nil && parsed.Host != "" && (strings.EqualFold(parsed.Scheme, "http") || strings.EqualFold(parsed.Scheme, "https")) { + return raw, nil + } + } + return "", &ClientError{ + Code: "invalid_parameter", + Message: "OpenAI image edit JSON mode requires " + param + " to be an HTTP(S) URL", + Param: param, + StatusCode: http.StatusBadRequest, + Retryable: false, + } +} + +func openAIImageEditFieldName(candidate store.RuntimeModelCandidate, imageCount int) string { if configured := strings.TrimSpace(stringFromAny(candidate.PlatformConfig["imageEditMultipartFieldName"])); configured == "image" || configured == "image[]" { return configured } + if imageCount > 1 { + return "image[]" + } parsed, err := url.Parse(strings.TrimSpace(candidate.BaseURL)) if err == nil && strings.EqualFold(parsed.Hostname(), "api.openai.com") { return "image[]" @@ -343,12 +430,12 @@ func openAIImageEditValues(value any) []any { } } -func openAIImageEditPayload(value any) (string, []byte, error) { +func openAIImageEditPayload(ctx context.Context, value any) (string, []byte, error) { switch typed := value.(type) { case map[string]any: for _, key := range []string{"data", "b64_json", "base64", "url"} { if nested := typed[key]; nested != nil { - return openAIImageEditPayload(nested) + return openAIImageEditPayload(ctx, nested) } } case string: @@ -365,14 +452,8 @@ func openAIImageEditPayload(value any) (string, []byte, error) { } contentType = strings.TrimSpace(strings.Split(strings.TrimPrefix(prefix, "data:"), ";")[0]) encoded = payload - } else if strings.Contains(raw, "://") { - return "", nil, &ClientError{ - Code: "invalid_parameter", - Message: "OpenAI image edit input must be hydrated before multipart submission", - Param: "image", - StatusCode: http.StatusBadRequest, - Retryable: false, - } + } else if parsed, parseErr := url.Parse(raw); parseErr == nil && parsed.Host != "" && (strings.EqualFold(parsed.Scheme, "http") || strings.EqualFold(parsed.Scheme, "https")) { + return fetchRemoteMediaInputPayload(ctx, raw, 256<<20) } image, err := decodeOpenAIImageEditBase64(encoded) if err != nil { @@ -396,6 +477,47 @@ func openAIImageEditPayload(value any) (string, []byte, error) { } } +func fetchRemoteMediaInputPayload(ctx context.Context, sourceURL string, maxBytes int64) (string, []byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) + if err != nil { + return "", nil, &ClientError{Code: "request_asset_fetch_failed", Message: err.Error(), Retryable: false} + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", nil, &ClientError{Code: "request_asset_fetch_failed", Message: err.Error(), Retryable: true} + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return "", nil, &ClientError{ + Code: "request_asset_fetch_failed", + Message: resp.Status, + StatusCode: resp.StatusCode, + Retryable: HTTPRetryable(resp.StatusCode), + } + } + if maxBytes <= 0 { + maxBytes = 256 << 20 + } + payload, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return "", nil, &ClientError{Code: "request_asset_fetch_failed", Message: err.Error(), Retryable: true} + } + if int64(len(payload)) > maxBytes { + return "", nil, &ClientError{ + Code: "invalid_parameter", + Message: "remote media input exceeds the download limit", + Param: "image", + StatusCode: http.StatusBadRequest, + Retryable: false, + } + } + contentType := strings.TrimSpace(strings.Split(resp.Header.Get("Content-Type"), ";")[0]) + if contentType == "" || contentType == "application/octet-stream" { + contentType = strings.TrimSpace(strings.Split(http.DetectContentType(payload), ";")[0]) + } + return contentType, payload, nil +} + func decodeOpenAIImageEditBase64(value string) ([]byte, error) { normalized := strings.Map(func(char rune) rune { switch char { diff --git a/apps/api/internal/httpapi/easyai_compat_test.go b/apps/api/internal/httpapi/easyai_compat_test.go index 5a64eee..4555c4f 100644 --- a/apps/api/internal/httpapi/easyai_compat_test.go +++ b/apps/api/internal/httpapi/easyai_compat_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -191,6 +192,34 @@ func TestEasyAITaskResultResponseMovesImageDataURLToB64JSON(t *testing.T) { } } +func TestEasyAITaskResultResponseForwardsSafeUpstreamParameterMessage(t *testing.T) { + raw := "Duplicate parameter: 'image'. Use image[]= for multiple values." + legacy := publicerror.Error{ + Code: "upstream_invalid_request", + Message: "The upstream service rejected the request parameters.", + Category: "upstream", + Source: "upstream", + HTTPStatus: http.StatusBadRequest, + Version: "v1", + } + response := easyAITaskResultResponse(store.GatewayTask{ + ID: "image-error-1", + Kind: "images.edits", + Status: "failed", + ErrorCode: "http_400", + ErrorMessage: raw, + PublicError: &legacy, + }) + standard, ok := response["error"].(publicerror.Error) + if !ok { + t.Fatalf("unexpected EasyAI error payload: %+v", response) + } + upstream, _ := standard.Details["upstreamError"].(map[string]any) + if response["message"] != raw || standard.Message != raw || upstream["message"] != raw { + t.Fatalf("EasyAI result did not forward safe upstream diagnostics: %+v", response) + } +} + func TestEasyAITaskResultResponsePreservesAudioDataURLAsContent(t *testing.T) { base64Payload := "SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjYwLjMuMTAwAAAAAAAAAAAAAAD/" task := store.GatewayTask{ diff --git a/apps/api/internal/httpapi/public_errors.go b/apps/api/internal/httpapi/public_errors.go index b154250..fe81a92 100644 --- a/apps/api/internal/httpapi/public_errors.go +++ b/apps/api/internal/httpapi/public_errors.go @@ -11,9 +11,7 @@ import ( func publicGatewayTask(task store.GatewayTask) store.GatewayTask { if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" { value := publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error), taskErrorHTTPStatus(task), false) - if task.PublicError != nil && task.PublicError.Code != "" && task.PublicError.Source != "" { - value = *task.PublicError - } + value = mergeStoredPublicError(task.PublicError, value) value = publicerror.WithIDs(value, task.RequestID, task.ID) publicerror.Observe(value) task.PublicError = &value @@ -32,9 +30,7 @@ func publicGatewayTask(task store.GatewayTask) store.GatewayTask { status = http.StatusBadGateway } value := publicerror.FromFields(attempt.ErrorCode, attempt.ErrorMessage, status, attempt.Retryable) - if attempt.PublicError != nil && attempt.PublicError.Code != "" && attempt.PublicError.Source != "" { - value = *attempt.PublicError - } + value = mergeStoredPublicError(attempt.PublicError, value) value = publicerror.WithIDs(value, attempt.RequestID, task.ID) attempt.PublicError = &value attempt.ErrorCode = value.Code @@ -44,14 +40,33 @@ func publicGatewayTask(task store.GatewayTask) store.GatewayTask { } func publicTaskError(task store.GatewayTask) publicerror.Error { - if task.PublicError != nil && task.PublicError.Code != "" && task.PublicError.Source != "" { - return publicerror.WithIDs(*task.PublicError, task.RequestID, task.ID) - } status := taskErrorHTTPStatus(task) if status <= 0 { status = http.StatusBadGateway } - return publicerror.WithIDs(publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error, task.Message), status, false), task.RequestID, task.ID) + value := publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error, task.Message), status, false) + value = mergeStoredPublicError(task.PublicError, value) + return publicerror.WithIDs(value, task.RequestID, task.ID) +} + +func mergeStoredPublicError(stored *publicerror.Error, derived publicerror.Error) publicerror.Error { + if stored == nil || stored.Code == "" || stored.Source == "" { + return derived + } + value := *stored + if len(value.Details) == 0 && len(derived.Details) > 0 { + value.Details = derived.Details + } + if message := safeDerivedUpstreamMessage(derived); message != "" { + value.Message = message + } + return value +} + +func safeDerivedUpstreamMessage(value publicerror.Error) string { + upstream, _ := value.Details["upstreamError"].(map[string]any) + message, _ := upstream["message"].(string) + return strings.TrimSpace(message) } func taskErrorHTTPStatus(task store.GatewayTask) int { @@ -109,6 +124,9 @@ func publicErrorMap(value publicerror.Error) map[string]any { if taskID := strings.TrimSpace(value.TaskID); taskID != "" { out["taskId"] = taskID } + if len(value.Details) > 0 { + out["details"] = value.Details + } return out } @@ -146,6 +164,11 @@ func safePublicErrorDetails(details map[string]any, value publicerror.Error, inc } } } + for key, item := range value.Details { + if item != nil { + out[key] = item + } + } if includePublicError { out["publicError"] = value } diff --git a/apps/api/internal/httpapi/public_errors_test.go b/apps/api/internal/httpapi/public_errors_test.go index ab8aed1..3512788 100644 --- a/apps/api/internal/httpapi/public_errors_test.go +++ b/apps/api/internal/httpapi/public_errors_test.go @@ -47,6 +47,25 @@ func TestPublicHTTPErrorReportsSourceAndPreservesSafeUpstreamStatus(t *testing.T } } +func TestPublicHTTPErrorIncludesSafeUpstreamParameterMessage(t *testing.T) { + raw := "Duplicate parameter: 'image'. Use image[]= for multiple values." + recorder := httptest.NewRecorder() + writeProtocolError(recorder, clients.ProtocolOpenAIImages, http.StatusBadRequest, raw, nil, "http_400") + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", recorder.Code, recorder.Body.String()) + } + var body map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + errorPayload := requireObject(t, body["error"]) + details := requireObject(t, errorPayload["details"]) + upstream := requireObject(t, details["upstreamError"]) + if errorPayload["message"] != raw || upstream["message"] != raw || upstream["code"] != "http_400" || upstream["statusCode"] != float64(http.StatusBadRequest) { + t.Fatalf("unexpected upstream error details: %+v", body) + } +} + func TestPublicGatewayTaskSanitizesCopyAndKeepsRawAuditFields(t *testing.T) { rawMessage := `404 page not found: {"privateProject":"secret"}` task := store.GatewayTask{ @@ -75,6 +94,40 @@ func TestPublicGatewayTaskSanitizesCopyAndKeepsRawAuditFields(t *testing.T) { } } +func TestPublicGatewayTaskForwardsSafeUpstreamParameterMessage(t *testing.T) { + raw := "Duplicate parameter: 'image'. Use image[]= for multiple values." + legacy := publicerror.Error{ + Code: "upstream_invalid_request", + Message: "The upstream service rejected the request parameters.", + Category: "upstream", + Source: "upstream", + HTTPStatus: http.StatusBadRequest, + Version: "v1", + } + task := store.GatewayTask{ + ID: "task-parameter-error", + Status: "failed", + ErrorCode: "http_400", + ErrorMessage: raw, + PublicError: &legacy, + Attempts: []store.TaskAttempt{{ + AttemptNo: 1, + Status: "failed", + StatusCode: http.StatusBadRequest, + ErrorCode: "http_400", + ErrorMessage: raw, + PublicError: &legacy, + }}, + } + + public := publicGatewayTask(task) + upstream := requireObject(t, public.PublicError.Details["upstreamError"]) + attemptUpstream := requireObject(t, public.Attempts[0].PublicError.Details["upstreamError"]) + if public.ErrorMessage != raw || public.Attempts[0].ErrorMessage != raw || upstream["message"] != raw || attemptUpstream["message"] != raw { + t.Fatalf("task response did not forward safe upstream diagnostics: %+v", public) + } +} + func TestTaskErrorHTTPStatusRebuildsLegacySnapshotFromAttempt(t *testing.T) { legacy := publicerror.Error{ Code: "upstream_request_rejected", diff --git a/apps/api/internal/publicerror/public_error.go b/apps/api/internal/publicerror/public_error.go index bba65be..57fa1a5 100644 --- a/apps/api/internal/publicerror/public_error.go +++ b/apps/api/internal/publicerror/public_error.go @@ -1,6 +1,7 @@ package publicerror import ( + "encoding/json" "net/http" "strconv" "strings" @@ -9,17 +10,18 @@ import ( const Version = "v1" type Error struct { - Code string `json:"code"` - Message string `json:"message"` - Category string `json:"category"` - Source string `json:"source"` - HTTPStatus int `json:"httpStatus"` - Retryable bool `json:"retryable"` - Action string `json:"action"` - RetryAfterSeconds int `json:"retryAfterSeconds,omitempty"` - RequestID string `json:"requestId,omitempty"` - TaskID string `json:"taskId,omitempty"` - Version string `json:"version"` + Code string `json:"code"` + Message string `json:"message"` + Category string `json:"category"` + Source string `json:"source"` + HTTPStatus int `json:"httpStatus"` + Retryable bool `json:"retryable"` + Action string `json:"action"` + RetryAfterSeconds int `json:"retryAfterSeconds,omitempty"` + RequestID string `json:"requestId,omitempty"` + TaskID string `json:"taskId,omitempty"` + Details map[string]any `json:"details,omitempty"` + Version string `json:"version"` } func FromFields(code string, message string, status int, retryable bool) Error { @@ -115,7 +117,7 @@ func mappedError(code string, message string, status int, retryable bool) (Error return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true } if strings.HasPrefix(code, "http_") || (code == "provider_failed" && status > 0 && status < 500) { - return upstreamHTTPError(status), true + return upstreamHTTPError(status, originalUpstreamErrorDetail(code, message, status)), true } if status >= 500 && code != "upstream_submission_unknown" && (strings.Contains(code, "upstream") || strings.Contains(code, "provider")) { return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true @@ -152,27 +154,66 @@ func newError(code string, message string, category string, status int, retryabl return Error{Code: code, Message: message, Category: category, Source: errorSource(code, category), HTTPStatus: status, Retryable: retryable, Action: action, Version: Version} } -func upstreamHTTPError(status int) Error { +func upstreamHTTPError(status int, details map[string]any) Error { if status < 400 || status >= 500 { status = http.StatusBadRequest } + var value Error switch status { case http.StatusBadRequest: - return newError("upstream_invalid_request", "The upstream service rejected the request parameters.", "upstream", status, false, "fix_request") + value = newError("upstream_invalid_request", "The upstream service rejected the request parameters.", "upstream", status, false, "fix_request") case http.StatusNotFound: - return newError("upstream_not_found", "The upstream service could not find the requested resource.", "upstream", status, false, "contact_support") + value = newError("upstream_not_found", "The upstream service could not find the requested resource.", "upstream", status, false, "contact_support") case http.StatusMethodNotAllowed: - return newError("upstream_method_not_allowed", "The upstream service did not accept the configured request method.", "upstream", status, false, "contact_support") + value = newError("upstream_method_not_allowed", "The upstream service did not accept the configured request method.", "upstream", status, false, "contact_support") case http.StatusConflict: - return newError("upstream_conflict", "The upstream service reported a request conflict.", "upstream", status, false, "fix_request") + value = newError("upstream_conflict", "The upstream service reported a request conflict.", "upstream", status, false, "fix_request") case http.StatusRequestEntityTooLarge: - return newError("upstream_payload_too_large", "The upstream service rejected the request because its payload was too large.", "upstream", status, false, "fix_request") + value = newError("upstream_payload_too_large", "The upstream service rejected the request because its payload was too large.", "upstream", status, false, "fix_request") case http.StatusUnsupportedMediaType: - return newError("upstream_unsupported_media_type", "The upstream service rejected the request media type.", "upstream", status, false, "fix_request") + value = newError("upstream_unsupported_media_type", "The upstream service rejected the request media type.", "upstream", status, false, "fix_request") case http.StatusUnprocessableEntity: - return newError("upstream_unprocessable_request", "The upstream service could not process the request parameters.", "upstream", status, false, "fix_request") + value = newError("upstream_unprocessable_request", "The upstream service could not process the request parameters.", "upstream", status, false, "fix_request") default: - return newError("upstream_request_rejected", "The upstream service rejected the request.", "upstream", status, false, "fix_request") + value = newError("upstream_request_rejected", "The upstream service rejected the request.", "upstream", status, false, "fix_request") + } + if message := originalUpstreamMessage(details); message != "" { + value.Message = message + } + value.Details = details + return value +} + +func originalUpstreamMessage(details map[string]any) string { + upstream, _ := details["upstreamError"].(map[string]any) + message, _ := upstream["message"].(string) + return strings.TrimSpace(message) +} + +func originalUpstreamErrorDetail(code string, message string, status int) map[string]any { + if status != http.StatusBadRequest && status != http.StatusUnprocessableEntity { + return nil + } + message = strings.TrimSpace(strings.ToValidUTF8(message, "")) + if message == "" || sensitiveTransportMessage(message) || json.Valid([]byte(message)) { + return nil + } + lower := strings.ToLower(message) + for _, marker := range []string{"bearer ", "password=", "secret=", "api_key=", "apikey=", "access_token=", "refresh_token=", "sk-"} { + if strings.Contains(lower, marker) { + return nil + } + } + runes := []rune(message) + if len(runes) > 2048 { + message = string(runes[:2048]) + } + return map[string]any{ + "upstreamError": map[string]any{ + "code": strings.TrimSpace(code), + "message": message, + "statusCode": status, + }, } } diff --git a/apps/api/internal/publicerror/public_error_test.go b/apps/api/internal/publicerror/public_error_test.go index f290dc1..4c296eb 100644 --- a/apps/api/internal/publicerror/public_error_test.go +++ b/apps/api/internal/publicerror/public_error_test.go @@ -68,6 +68,36 @@ func TestProviderHTTPErrorDoesNotExposeProviderBody(t *testing.T) { } } +func TestUpstreamParameterErrorPreservesSafeOriginalMessage(t *testing.T) { + raw := "Duplicate parameter: 'image'. Use image[]= for multiple values." + for _, test := range []struct { + code string + status int + }{ + {code: "http_400", status: http.StatusBadRequest}, + {code: "http_422", status: http.StatusUnprocessableEntity}, + } { + got := FromFields(test.code, raw, test.status, false) + upstream, _ := got.Details["upstreamError"].(map[string]any) + if got.Message != raw || upstream["message"] != raw || upstream["code"] != test.code || upstream["statusCode"] != test.status { + t.Fatalf("%s: safe upstream message was not forwarded: %+v", test.code, got) + } + } +} + +func TestUpstreamParameterErrorRejectsOpaqueOrSensitiveDetails(t *testing.T) { + for _, raw := range []string{ + `{"error":{"message":"bucket private-a rejected secret-project"}}`, + "invalid api_key=sk-private-value", + "read tcp 10.42.0.1:1234: connection reset by peer", + } { + got := FromFields("http_400", raw, http.StatusBadRequest, false) + if len(got.Details) != 0 || got.Message == raw { + t.Fatalf("unsafe upstream message was exposed for %q: %+v", raw, got) + } + } +} + func TestGatewayRateLimitIsDistinctFromUpstreamRateLimit(t *testing.T) { gateway := FromFields("gateway_rate_limited", "concurrency limit is saturated and queueing is disabled", http.StatusTooManyRequests, true) if gateway.Code != "gateway_rate_limited" || gateway.Source != "gateway" || gateway.HTTPStatus != http.StatusTooManyRequests { diff --git a/apps/api/internal/runner/request_assets.go b/apps/api/internal/runner/request_assets.go index 40c728c..4e0cade 100644 --- a/apps/api/internal/runner/request_assets.go +++ b/apps/api/internal/runner/request_assets.go @@ -128,6 +128,15 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st if err != nil { return nil, err } + if requestAssetClientMaterializesRemoteMedia(candidate, path, asset) { + assetURL, accessErr := s.requestAssetAccessURL(ctx, asset) + if accessErr != nil { + return nil, accessErr + } + if requestAssetStringIsHTTPURL(assetURL) { + return assetURL, nil + } + } switch requestAssetHydrationForField(path, asset, candidate) { case requestAssetHydrateUnsupported: return nil, requestAssetUnsupportedInputFormatError(path, asset) @@ -155,6 +164,9 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st if strings.TrimSpace(assetURL) == "" { return nil, requestAssetExpiredError(asset) } + if openAIImageEditRequiresJSONURL(candidate) && imageInputFieldNeedsHydration(path) && !requestAssetURLIsPublic(asset.StorageProvider, assetURL) { + return nil, requestAssetPublicURLRequiredError(path) + } return assetURL, nil } @@ -163,8 +175,17 @@ func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value s if raw == "" || !imageInputFieldNeedsHydration(path) { return value, nil } + if openAIImageEditRequiresJSONURL(candidate) { + if !requestAssetURLIsPublic("", raw) { + return nil, requestAssetPublicURLRequiredError(path) + } + return value, nil + } + if requestAssetClientMaterializesRemoteMedia(candidate, path, store.RequestAsset{URL: raw}) && requestAssetStringIsHTTPURL(raw) { + return value, nil + } style, ok := requestAssetHydrateUnsupported, false - if geminiVeoRequiresInlineImage(candidate) || openAIImageEditRequiresMultipartBytes(candidate) { + if openAIImageEditRequiresMultipartBytes(candidate) { style, ok = requestAssetHydrateDataURL, true } else { style, ok = requestAssetCapabilityHydrationForMedia("image", candidate, raw, "") @@ -392,13 +413,22 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand return requestAssetHydrateRawBase64 } if candidate.ModelType == "voice_clone" && voiceCloneAudioFieldNeedsHydration(path, asset) { + if requestAssetStringIsHTTPURL(asset.URL) { + return requestAssetHydrateURL + } return requestAssetHydrateDataURL } if requestAssetMediaKindForHydration(path, asset) == "image" { - if geminiVeoRequiresInlineImage(candidate) { - return requestAssetHydrateDataURL + if requestAssetClientMaterializesRemoteMedia(candidate, path, asset) && requestAssetStringIsHTTPURL(asset.URL) { + return requestAssetHydrateURL + } + if openAIImageEditRequiresJSONURL(candidate) { + return requestAssetHydrateURL } if openAIImageEditRequiresMultipartBytes(candidate) { + if requestAssetStringIsHTTPURL(asset.URL) { + return requestAssetHydrateURL + } return requestAssetHydrateDataURL } if style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, asset.URL, asset.StorageProvider); ok { @@ -409,21 +439,63 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand if style := configuredRequestAssetMediaURLHydration(candidate, requestAssetMediaURLKind(path)); style != "" { return style } - if providerMediaURLNeedsDataURL(candidate) { - return requestAssetHydrateDataURL - } + } + if requestAssetMediaKindForHydration(path, asset) != "" && strings.EqualFold(strings.TrimSpace(asset.StorageProvider), "local_static") { + return requestAssetHydrateDataURL } return requestAssetHydrateURL } func openAIImageEditRequiresMultipartBytes(candidate store.RuntimeModelCandidate) bool { - return normalizeProviderKey(candidate.Provider) == "openai" && - strings.TrimSpace(candidate.ModelType) == "image_edit" + return openAIImageEditCandidate(candidate) && !clients.OpenAIImageEditUsesJSONURL(candidate) } -func geminiVeoRequiresInlineImage(candidate store.RuntimeModelCandidate) bool { - return normalizeProviderKey(candidate.Provider) == "gemini" && - strings.Contains(strings.ToLower(strings.TrimSpace(firstNonEmptyString(candidate.ProviderModelName, candidate.ModelName))), "veo") +func openAIImageEditRequiresJSONURL(candidate store.RuntimeModelCandidate) bool { + return openAIImageEditCandidate(candidate) && clients.OpenAIImageEditUsesJSONURL(candidate) +} + +func openAIImageEditCandidate(candidate store.RuntimeModelCandidate) bool { + if strings.TrimSpace(candidate.ModelType) != "image_edit" { + return false + } + return normalizeProviderKey(candidate.Provider) == "openai" || normalizeProviderKey(candidate.SpecType) == "openai" +} + +func requestAssetClientMaterializesRemoteMedia(candidate store.RuntimeModelCandidate, path []string, asset store.RequestAsset) bool { + mediaKind := requestAssetMediaKindForHydration(path, asset) + if mediaKind == "image" { + if openAIImageEditRequiresMultipartBytes(candidate) || geminiClientMaterializesImages(candidate) || kelingClientMaterializesImages(candidate) { + return true + } + } + return mediaKind == "audio" && strings.TrimSpace(candidate.ModelType) == "voice_clone" && candidateUsesClient(candidate, "minimax") +} + +func geminiClientMaterializesImages(candidate store.RuntimeModelCandidate) bool { + return candidateUsesClient(candidate, "gemini", "google_gemini") +} + +func kelingClientMaterializesImages(candidate store.RuntimeModelCandidate) bool { + return strings.TrimSpace(candidate.ModelType) != "" && candidateUsesClient(candidate, "keling", "kling") +} + +func candidateUsesClient(candidate store.RuntimeModelCandidate, names ...string) bool { + wanted := make(map[string]bool, len(names)) + for _, name := range names { + wanted[normalizeProviderKey(name)] = true + } + for _, value := range []string{candidate.SpecType, candidate.Provider} { + if wanted[normalizeProviderKey(value)] { + return true + } + } + if wanted["gemini"] { + provider := normalizeProviderKey(candidate.Provider) + if provider == "gemini_openai" || strings.Contains(strings.ToLower(strings.TrimSpace(candidate.BaseURL)), "generativelanguage.googleapis.com") { + return true + } + } + return false } func requestAssetMediaKindForHydration(path []string, asset store.RequestAsset) string { @@ -594,6 +666,12 @@ func requestAssetStringLooksURL(value string) bool { strings.HasPrefix(lower, "/static/uploaded/") } +func requestAssetStringIsHTTPURL(value string) bool { + raw := strings.TrimSpace(value) + parsed, err := url.Parse(raw) + return err == nil && parsed.Host != "" && (strings.EqualFold(parsed.Scheme, "http") || strings.EqualFold(parsed.Scheme, "https")) +} + func requestAssetURLIsPublic(storageProvider string, value string) bool { if strings.EqualFold(strings.TrimSpace(storageProvider), "local_static") { return false @@ -701,16 +779,6 @@ func requestAssetHydrationStyleFromString(value string) requestAssetHydrationSty } } -func providerMediaURLNeedsDataURL(candidate store.RuntimeModelCandidate) bool { - for _, name := range []string{candidate.Provider, candidate.SpecType, candidate.PlatformKey} { - switch normalizeProviderKey(name) { - case "openai", "volces", "volces_openai", "gemini", "vidu": - return true - } - } - return false -} - func normalizeProviderKey(value string) string { normalized := strings.ToLower(strings.TrimSpace(value)) normalized = strings.ReplaceAll(normalized, "-", "_") @@ -782,3 +850,16 @@ func requestAssetUnsupportedInputFormatError(path []string, asset store.RequestA } return &clients.ClientError{Code: "request_asset_input_format_unsupported", Message: message, Retryable: false} } + +func requestAssetPublicURLRequiredError(path []string) error { + field := strings.Join(path, ".") + if field == "" { + field = "image" + } + return &clients.ClientError{ + Code: "request_asset_public_url_required", + Message: "OpenAI image edit JSON mode requires a public HTTP(S) URL for " + field, + Param: field, + Retryable: false, + } +} diff --git a/apps/api/internal/runner/request_assets_test.go b/apps/api/internal/runner/request_assets_test.go index 3888428..db0429d 100644 --- a/apps/api/internal/runner/request_assets_test.go +++ b/apps/api/internal/runner/request_assets_test.go @@ -172,7 +172,7 @@ func TestHydrateProviderRequestAssetsConvertsGeminiInlineDataAssetToRawBase64(t } } -func TestGeminiVeoForcesOfficialInlineImageFormat(t *testing.T) { +func TestGeminiVeoPreservesURLUntilOfficialPayloadConstruction(t *testing.T) { candidate := store.RuntimeModelCandidate{ Provider: "gemini", ProviderModelName: "veo-3.1-generate-preview", @@ -185,8 +185,77 @@ func TestGeminiVeoForcesOfficialInlineImageFormat(t *testing.T) { }, } asset := store.RequestAsset{URL: "https://cdn.example.com/input.png", StorageProvider: "remote", ContentType: "image/png"} - if got := requestAssetHydrationForField([]string{"image"}, asset, candidate); got != requestAssetHydrateDataURL { - t.Fatalf("Gemini Veo must hydrate public image URLs as data URLs, got %q", got) + if got := requestAssetHydrationForField([]string{"image"}, asset, candidate); got != requestAssetHydrateURL { + t.Fatalf("Gemini Veo should preserve public URLs until client payload construction, got %q", got) + } +} + +func TestRemoteMediaMaterializationBoundaryByClient(t *testing.T) { + imageAsset := store.RequestAsset{URL: "https://cdn.example.com/input.png", StorageProvider: "remote", ContentType: "image/png"} + audioAsset := store.RequestAsset{URL: "https://cdn.example.com/input.mp3", StorageProvider: "remote", ContentType: "audio/mpeg"} + tests := []struct { + name string + candidate store.RuntimeModelCandidate + path []string + asset store.RequestAsset + want bool + }{ + {name: "OpenAI multipart image edit", candidate: store.RuntimeModelCandidate{SpecType: "openai", ModelType: "image_edit"}, path: []string{"image"}, asset: imageAsset, want: true}, + {name: "OpenAI JSON URL image edit", candidate: store.RuntimeModelCandidate{SpecType: "openai", ModelType: "image_edit", PlatformConfig: map[string]any{"imageEditRequestFormat": "json_url"}}, path: []string{"image"}, asset: imageAsset, want: false}, + {name: "Gemini Veo", candidate: store.RuntimeModelCandidate{SpecType: "gemini", ModelType: "image_to_video", ProviderModelName: "veo-3.1-generate-preview"}, path: []string{"image"}, asset: imageAsset, want: true}, + {name: "Keling video", candidate: store.RuntimeModelCandidate{SpecType: "keling", ModelType: "image_to_video"}, path: []string{"image"}, asset: imageAsset, want: true}, + {name: "Minimax voice clone", candidate: store.RuntimeModelCandidate{SpecType: "minimax", ModelType: "voice_clone"}, path: []string{"audio"}, asset: audioAsset, want: true}, + {name: "Volces JSON image", candidate: store.RuntimeModelCandidate{SpecType: "volces", ModelType: "image_edit"}, path: []string{"image"}, asset: imageAsset, want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := requestAssetClientMaterializesRemoteMedia(test.candidate, test.path, test.asset); got != test.want { + t.Fatalf("client materializes remote media = %t, want %t", got, test.want) + } + }) + } +} + +func TestJSONTaskClientDoesNotForcePublicURLToBase64WithoutCapability(t *testing.T) { + asset := store.RequestAsset{URL: "https://cdn.example.com/input.png", StorageProvider: "remote", ContentType: "image/png"} + candidate := store.RuntimeModelCandidate{SpecType: "volces", ModelType: "image_edit"} + if got := requestAssetHydrationForField([]string{"image"}, asset, candidate); got != requestAssetHydrateURL { + t.Fatalf("JSON task client should preserve a public URL when no capability requires base64, got %q", got) + } +} + +func TestGeminiURLsArePreservedUntilClientProtocolConstruction(t *testing.T) { + service := &Service{} + compatibleURL := "https://cdn.example.com/compatible.png" + body := map[string]any{"image": compatibleURL} + + compatible, err := service.hydrateProviderRequestAssets(context.Background(), body, store.RuntimeModelCandidate{ + SpecType: "gemini", + BaseURL: "https://gemini-compatible.example.com/v1beta", + ModelType: "image_edit", + PlatformConfig: map[string]any{ + "supportUrlInput": true, + "supportBase64Input": true, + }, + }) + if err != nil { + t.Fatalf("hydrate compatible Gemini image: %v", err) + } + if compatible["image"] != compatibleURL { + t.Fatalf("compatible Gemini URL should be preserved until client inline conversion, got %q", compatible["image"]) + } + + officialURL := "https://cdn.example.com/source.png" + official, err := service.hydrateProviderRequestAssets(context.Background(), map[string]any{"image": officialURL}, store.RuntimeModelCandidate{ + SpecType: "gemini", + BaseURL: "https://generativelanguage.googleapis.com/v1beta", + ModelType: "image_edit", + }) + if err != nil { + t.Fatalf("hydrate official Gemini image: %v", err) + } + if official["image"] != officialURL { + t.Fatalf("official Gemini should preserve URL for Files API upload, got %q", official["image"]) } } @@ -273,18 +342,12 @@ func TestHydrateProviderRequestAssetsUsesImageCapabilityBase64ForTopLevelImageAs } } -func TestHydrateProviderRequestAssetsConvertsOpenAIEditImagesForMultipart(t *testing.T) { - payload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4} - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "image/png") - _, _ = w.Write(payload) - })) - defer server.Close() +func TestHydrateProviderRequestAssetsPreservesOpenAIEditURLsUntilMultipartSubmission(t *testing.T) { service := &Service{} body := map[string]any{ "images": []any{ - server.URL + "/first.png", - server.URL + "/second.png", + "https://cdn.example.com/first.png", + "https://cdn.example.com/second.png", }, } @@ -302,9 +365,58 @@ func TestHydrateProviderRequestAssetsConvertsOpenAIEditImagesForMultipart(t *tes t.Fatalf("hydrate OpenAI edit images: %v", err) } images := hydrated["images"].([]any) - want := "data:image/png;base64," + base64.StdEncoding.EncodeToString(payload) - if len(images) != 2 || stringFromAny(images[0]) != want || stringFromAny(images[1]) != want { - t.Fatalf("OpenAI edit images should be hydrated for multipart: %+v", images) + if len(images) != 2 || images[0] != "https://cdn.example.com/first.png" || images[1] != "https://cdn.example.com/second.png" { + t.Fatalf("OpenAI edit URLs should be preserved until multipart construction: %+v", images) + } +} + +func TestHydrateProviderRequestAssetsKeepsURLsForExplicitOpenAIJSONMode(t *testing.T) { + service := &Service{} + body := map[string]any{ + "images": []any{ + "https://cdn.example.com/first.png", + "https://cdn.example.com/second.png", + }, + } + candidate := store.RuntimeModelCandidate{ + Provider: "compatible-openai", + SpecType: "openai", + ModelType: "image_edit", + Capabilities: map[string]any{ + "image_edit": map[string]any{ + "support_url_input": false, + "support_base64_input": true, + }, + }, + PlatformConfig: map[string]any{"imageEditRequestFormat": "json_url"}, + } + + hydrated, err := service.hydrateProviderRequestAssets(context.Background(), body, candidate) + if err != nil { + t.Fatalf("hydrate OpenAI JSON edit images: %v", err) + } + images := hydrated["images"].([]any) + if len(images) != 2 || images[0] != "https://cdn.example.com/first.png" || images[1] != "https://cdn.example.com/second.png" { + t.Fatalf("OpenAI JSON mode should preserve public URLs: %+v", images) + } + asset := store.RequestAsset{URL: "https://cdn.example.com/asset.png", StorageProvider: "remote", ContentType: "image/png"} + if got := requestAssetHydrationForField([]string{"image"}, asset, candidate); got != requestAssetHydrateURL { + t.Fatalf("explicit JSON mode must override base64 capability hydration, got %q", got) + } +} + +func TestHydrateProviderRequestAssetsRejectsInlineDataForExplicitOpenAIJSONMode(t *testing.T) { + service := &Service{} + _, err := service.hydrateProviderRequestAssets(context.Background(), map[string]any{ + "image": "data:image/png;base64,aW1hZ2U=", + }, store.RuntimeModelCandidate{ + Provider: "openai", + ModelType: "image_edit", + PlatformConfig: map[string]any{"imageEditRequestFormat": "json_url"}, + }) + clientErr, ok := err.(*clients.ClientError) + if !ok || clientErr.Code != "request_asset_public_url_required" || clientErr.Param != "image" || clientErr.Retryable { + t.Fatalf("expected public URL validation error, got %#v", err) } } diff --git a/apps/api/internal/skillbundle/content/skills/ai-gateway-ops-management/references/model-platforms-and-bindings.md b/apps/api/internal/skillbundle/content/skills/ai-gateway-ops-management/references/model-platforms-and-bindings.md index c4f6b6e..ffd734a 100644 --- a/apps/api/internal/skillbundle/content/skills/ai-gateway-ops-management/references/model-platforms-and-bindings.md +++ b/apps/api/internal/skillbundle/content/skills/ai-gateway-ops-management/references/model-platforms-and-bindings.md @@ -53,6 +53,8 @@ curl --fail-with-body \ Use `PATCH /api/admin/platforms/{platformID}` with the complete platform body. Omit `credentials` to preserve existing secrets. A non-empty credentials object is merged into stored credentials. An empty object clears credentials and requires explicit confirmation. +OpenAI-compatible image edits use `multipart/form-data` by default. Only set `config.imageEditRequestFormat` to `json_url` when the upstream explicitly documents JSON URL input support; in that mode every image and mask must resolve to a public or presigned HTTP(S) URL. Omitting the setting, or setting it to `multipart`, keeps multipart file upload behavior. + ## Upsert One Platform Model `POST /api/admin/platforms/{platformID}/models` inserts or updates the `(platformID, modelName)` binding: diff --git a/apps/web/src/pages/admin/PlatformManagementPanel.tsx b/apps/web/src/pages/admin/PlatformManagementPanel.tsx index 68911cc..a53eb4d 100644 --- a/apps/web/src/pages/admin/PlatformManagementPanel.tsx +++ b/apps/web/src/pages/admin/PlatformManagementPanel.tsx @@ -504,6 +504,16 @@ export function PlatformManagementPanel(props: { setForm({ ...form, supportBase64Input: checked })} /> setForm({ ...form, supportUrlInput: checked })} /> + } title={`模型绑定 · ${selectedModels.length}/${availableModels.length}`}> @@ -1192,6 +1202,7 @@ function platformToForm( httpProxy: networkProxy.httpProxy, supportBase64Input: readBoolean(config, 'supportBase64Input', true), supportUrlInput: readBoolean(config, 'supportUrlInput', true), + imageEditRequestFormat: readImageEditRequestFormat(config), selectedModelIds: platformModelBaseIds(platform, baseModels, currentModels), modelDiscountFactors: platformModelDiscountFactors(platform, baseModels, currentModels), modelNameMappings: platformModelNameMappings(platform, baseModels, currentModels), @@ -1210,6 +1221,11 @@ function platformToForm( }; } +function readImageEditRequestFormat(config: Record): PlatformWizardForm['imageEditRequestFormat'] { + const value = String(config.imageEditRequestFormat ?? '').trim().toLowerCase().replaceAll('-', '_').replaceAll(' ', '_'); + return value === 'json' || value === 'json_url' ? 'json_url' : 'multipart'; +} + function defaultSelectedModelIds(models: BaseModelCatalogItem[], provider: string) { return modelsForProvider(models, provider).map((model) => model.id); } diff --git a/apps/web/src/pages/admin/platform-form.test.ts b/apps/web/src/pages/admin/platform-form.test.ts index dc6a954..f2708ce 100644 --- a/apps/web/src/pages/admin/platform-form.test.ts +++ b/apps/web/src/pages/admin/platform-form.test.ts @@ -26,6 +26,14 @@ function selectedForm() { } describe('platform model rate limit payload', () => { + it('uses multipart unless JSON URL image edits are explicitly configured', () => { + const defaultConfig = platformPayload(selectedForm()).config ?? {}; + expect(defaultConfig.imageEditRequestFormat).toBeUndefined(); + + const jsonConfig = platformPayload({ ...selectedForm(), imageEditRequestFormat: 'json_url' }).config ?? {}; + expect(jsonConfig.imageEditRequestFormat).toBe('json_url'); + }); + it('preserves an existing model policy when the edit form was not touched', () => { const [payload] = platformModelPayloads([baseModel], selectedForm()); diff --git a/apps/web/src/pages/admin/platform-form.ts b/apps/web/src/pages/admin/platform-form.ts index 801c55c..f453fd7 100644 --- a/apps/web/src/pages/admin/platform-form.ts +++ b/apps/web/src/pages/admin/platform-form.ts @@ -39,6 +39,7 @@ export interface PlatformWizardForm { httpProxy: string; supportBase64Input: boolean; supportUrlInput: boolean; + imageEditRequestFormat: 'multipart' | 'json_url'; modelDiscountFactor: string; modelDiscountFactors: Record; modelNameMappings: Record; @@ -109,6 +110,7 @@ export function createEmptyPlatformForm(provider = '', defaults?: ProviderConnec httpProxy: '', supportBase64Input: true, supportUrlInput: true, + imageEditRequestFormat: 'multipart', modelDiscountFactor: '', modelDiscountFactors: {}, modelNameMappings: {}, @@ -171,6 +173,7 @@ export function platformPayload(form: PlatformWizardForm, options: { preserveEmp networkProxy: networkProxyPayload(form), supportBase64Input: form.supportBase64Input, supportUrlInput: form.supportUrlInput, + ...(form.imageEditRequestFormat === 'json_url' ? { imageEditRequestFormat: 'json_url' } : {}), source: 'gateway-admin', }, retryPolicy: { diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index e46e43f..4853115 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -1264,6 +1264,7 @@ export interface PublicErrorV1 { retryAfterSeconds?: number; requestId?: string; taskId?: string; + details?: Record; version: 'v1' | string; }