diff --git a/apps/api/internal/clients/clients_test.go b/apps/api/internal/clients/clients_test.go index 9f80763..1e4c947 100644 --- a/apps/api/internal/clients/clients_test.go +++ b/apps/api/internal/clients/clients_test.go @@ -344,6 +344,70 @@ func TestOpenAIClientChatContract(t *testing.T) { } } +func TestOpenAIClientImageEditUsesMultipartWithMultipleImages(t *testing.T) { + var contentType string + var receivedFields map[string][]string + var receivedImageCount int + var receivedImageFields []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + contentType = r.Header.Get("Content-Type") + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart image edit: %v", err) + } + receivedFields = r.MultipartForm.Value + for field, files := range r.MultipartForm.File { + receivedImageFields = append(receivedImageFields, field) + receivedImageCount += len(files) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []any{map[string]any{"b64_json": "aW1hZ2U="}}, + }) + })) + defer server.Close() + + _, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{ + Kind: "images.edits", + ModelType: "image_edit", + Model: "gpt-image-2", + Body: map[string]any{ + "model": "gpt-image-2", + "prompt": "combine the references", + "size": "1024x1536", + "quality": "medium", + "images": []any{ + "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("image one")), + "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("image two")), + }, + "_metadata": map[string]any{"private": true}, + }, + Candidate: store.RuntimeModelCandidate{ + BaseURL: server.URL, + Provider: "openai", + ProviderModelName: "gpt-image-2-vip", + ModelType: "image_edit", + Credentials: map[string]any{"apiKey": "openai-key"}, + }, + }) + if err != nil { + t.Fatalf("run OpenAI image edit: %v", err) + } + if !strings.HasPrefix(contentType, "multipart/form-data; boundary=") { + t.Fatalf("image edit should use multipart/form-data, got %q", contentType) + } + if receivedImageCount != 2 || len(receivedImageFields) != 1 || receivedImageFields[0] != "image" { + t.Fatalf("unexpected multipart image files: fields=%v count=%d", receivedImageFields, receivedImageCount) + } + if receivedFields["model"][0] != "gpt-image-2-vip" || + receivedFields["prompt"][0] != "combine the references" || + receivedFields["size"][0] != "1024x1536" || + receivedFields["quality"][0] != "medium" { + t.Fatalf("unexpected multipart fields: %+v", receivedFields) + } + if _, ok := receivedFields["_metadata"]; ok { + t.Fatalf("internal metadata must not be forwarded: %+v", receivedFields) + } +} + func TestOpenAIClientChatReasoningParamsByProvider(t *testing.T) { cases := []struct { name string diff --git a/apps/api/internal/clients/openai.go b/apps/api/internal/clients/openai.go index 5e4404e..3c97272 100644 --- a/apps/api/internal/clients/openai.go +++ b/apps/api/internal/clients/openai.go @@ -3,8 +3,14 @@ package clients import ( "bytes" "context" + "encoding/base64" "encoding/json" + "fmt" + "io" + "mime/multipart" "net/http" + "net/textproto" + "net/url" "strings" "time" @@ -61,13 +67,16 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error body["model"] = upstreamModelName(request.Candidate) stream := openAIEndpointSupportsStream(endpointKind) && (request.Stream || boolValue(body, "stream")) ensureOpenAIStreamUsage(body, endpointKind, stream) - raw, _ := json.Marshal(body) + raw, contentType, err := openAIRequestPayload(endpointKind, body, request.Candidate) + if err != nil { + return Response{}, err + } upstreamEndpoint := joinURL(openAIBaseURL(endpointKind, request.Candidate), endpoint) req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamEndpoint, bytes.NewReader(raw)) if err != nil { return Response{}, err } - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", contentType) req.Header.Set("Authorization", "Bearer "+apiKey) responseStartedAt := time.Now() if err := notifySubmissionStarted(request); err != nil { @@ -162,6 +171,222 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error }, nil } +func openAIRequestPayload(endpointKind string, body map[string]any, candidate store.RuntimeModelCandidate) ([]byte, string, error) { + if endpointKind != "images.edits" { + raw, err := json.Marshal(body) + return raw, "application/json", err + } + var payload bytes.Buffer + writer := multipart.NewWriter(&payload) + imageFieldName := openAIImageEditFieldName(candidate) + images := openAIImageEditValues(firstPresent(body["images"], body["image"])) + if len(images) == 0 { + return nil, "", &ClientError{ + Code: "invalid_parameter", + Message: "image is required", + Param: "image", + StatusCode: http.StatusBadRequest, + Retryable: false, + } + } + for index, value := range images { + contentType, image, err := openAIImageEditPayload(value) + if err != nil { + return nil, "", err + } + if err := writeOpenAIImageEditFile(writer, imageFieldName, fmt.Sprintf("image-%d%s", index+1, openAIImageFileExtension(contentType)), contentType, image); err != nil { + return nil, "", err + } + } + if mask := firstPresent(body["mask"], body["mask_image"], body["maskImage"]); mask != nil { + contentType, image, err := openAIImageEditPayload(mask) + if err != nil { + return nil, "", err + } + if err := writeOpenAIImageEditFile(writer, "mask", "mask"+openAIImageFileExtension(contentType), contentType, image); err != nil { + return nil, "", err + } + } + for key, value := range body { + switch key { + case "image", "images", "mask", "mask_image", "maskImage": + continue + } + if strings.HasPrefix(key, "_") || value == nil { + continue + } + fieldValue, err := openAIFormFieldValue(value) + if err != nil { + return nil, "", err + } + if fieldValue == "" { + continue + } + if err := writer.WriteField(key, fieldValue); err != nil { + return nil, "", err + } + } + if err := writer.Close(); err != nil { + return nil, "", err + } + return payload.Bytes(), writer.FormDataContentType(), nil +} + +func openAIImageEditFieldName(candidate store.RuntimeModelCandidate) string { + if configured := strings.TrimSpace(stringFromAny(candidate.PlatformConfig["imageEditMultipartFieldName"])); configured == "image" || configured == "image[]" { + return configured + } + parsed, err := url.Parse(strings.TrimSpace(candidate.BaseURL)) + if err == nil && strings.EqualFold(parsed.Hostname(), "api.openai.com") { + return "image[]" + } + return "image" +} + +func openAIImageEditValues(value any) []any { + switch typed := value.(type) { + case []any: + return typed + case []string: + out := make([]any, 0, len(typed)) + for _, item := range typed { + out = append(out, item) + } + return out + case nil: + return nil + default: + return []any{typed} + } +} + +func openAIImageEditPayload(value any) (string, []byte, error) { + switch typed := value.(type) { + case map[string]any: + for _, key := range []string{"data", "b64_json", "base64", "url"} { + if nested := typed[key]; nested != nil { + return openAIImageEditPayload(nested) + } + } + case string: + raw := strings.TrimSpace(typed) + if raw == "" { + break + } + contentType := "" + encoded := raw + if strings.HasPrefix(strings.ToLower(raw), "data:") { + prefix, payload, ok := strings.Cut(raw, ",") + if !ok || !strings.Contains(strings.ToLower(prefix), ";base64") { + break + } + contentType = strings.TrimSpace(strings.Split(strings.TrimPrefix(prefix, "data:"), ";")[0]) + encoded = payload + } else if strings.Contains(raw, "://") { + return "", nil, &ClientError{ + Code: "invalid_parameter", + Message: "OpenAI image edit input must be hydrated before multipart submission", + Param: "image", + StatusCode: http.StatusBadRequest, + Retryable: false, + } + } + image, err := decodeOpenAIImageEditBase64(encoded) + if err != nil { + break + } + if contentType == "" { + contentType = strings.TrimSpace(strings.Split(http.DetectContentType(image), ";")[0]) + } + return contentType, image, nil + case []byte: + if len(typed) > 0 { + return strings.TrimSpace(strings.Split(http.DetectContentType(typed), ";")[0]), typed, nil + } + } + return "", nil, &ClientError{ + Code: "invalid_parameter", + Message: "image must be a base64 or data URL payload", + Param: "image", + StatusCode: http.StatusBadRequest, + Retryable: false, + } +} + +func decodeOpenAIImageEditBase64(value string) ([]byte, error) { + normalized := strings.Map(func(char rune) rune { + switch char { + case '\n', '\r', '\t', ' ': + return -1 + default: + return char + } + }, value) + var lastErr error + for _, encoding := range []*base64.Encoding{ + base64.StdEncoding, + base64.RawStdEncoding, + base64.URLEncoding, + base64.RawURLEncoding, + } { + payload, err := encoding.DecodeString(normalized) + if err == nil && len(payload) > 0 { + return payload, nil + } + lastErr = err + } + return nil, lastErr +} + +func writeOpenAIImageEditFile(writer *multipart.Writer, fieldName string, fileName string, contentType string, payload []byte) error { + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, fieldName, escapeMultipartFilename(fileName))) + if contentType != "" { + header.Set("Content-Type", contentType) + } + part, err := writer.CreatePart(header) + if err != nil { + return err + } + _, err = io.Copy(part, bytes.NewReader(payload)) + return err +} + +func openAIImageFileExtension(contentType string) string { + switch strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) { + case "image/jpeg": + return ".jpg" + case "image/webp": + return ".webp" + case "image/gif": + return ".gif" + default: + return ".png" + } +} + +func openAIFormFieldValue(value any) (string, error) { + switch typed := value.(type) { + case string: + return typed, nil + case bool: + return fmt.Sprintf("%t", typed), nil + case float64: + return fmt.Sprintf("%v", typed), nil + case float32: + return fmt.Sprintf("%v", typed), nil + case int: + return fmt.Sprintf("%d", typed), nil + case int64: + return fmt.Sprintf("%d", typed), nil + case json.Number: + return typed.String(), nil + default: + encoded, err := json.Marshal(value) + return string(encoded), err + } +} + func openAIWireStreamDelta(next StreamDelta, protocol string, response *http.Response) StreamDelta { if next == nil { return nil diff --git a/apps/api/internal/runner/request_assets.go b/apps/api/internal/runner/request_assets.go index b3efcdc..4b128ef 100644 --- a/apps/api/internal/runner/request_assets.go +++ b/apps/api/internal/runner/request_assets.go @@ -160,6 +160,9 @@ func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value s return value, nil } style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, raw, "") + if !ok && openAIImageEditRequiresMultipartBytes(candidate) { + style, ok = requestAssetHydrateDataURL, true + } if !ok { return value, nil } @@ -320,6 +323,9 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand if style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, asset.URL, asset.StorageProvider); ok { return style } + if openAIImageEditRequiresMultipartBytes(candidate) { + return requestAssetHydrateDataURL + } } if mediaURLFieldNeedsHydration(path) { if style := configuredRequestAssetMediaURLHydration(candidate, requestAssetMediaURLKind(path)); style != "" { @@ -332,6 +338,11 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand return requestAssetHydrateURL } +func openAIImageEditRequiresMultipartBytes(candidate store.RuntimeModelCandidate) bool { + return normalizeProviderKey(candidate.Provider) == "openai" && + strings.TrimSpace(candidate.ModelType) == "image_edit" +} + func requestAssetMediaKindForHydration(path []string, asset store.RequestAsset) string { if mediaURLFieldNeedsHydration(path) { return requestAssetMediaURLKind(path) diff --git a/apps/api/internal/runner/request_assets_test.go b/apps/api/internal/runner/request_assets_test.go index 2a8659f..b85a009 100644 --- a/apps/api/internal/runner/request_assets_test.go +++ b/apps/api/internal/runner/request_assets_test.go @@ -205,6 +205,35 @@ func TestHydrateProviderRequestAssetsUsesImageCapabilityBase64ForTopLevelImageAs } } +func TestHydrateProviderRequestAssetsConvertsOpenAIEditImagesForMultipart(t *testing.T) { + payload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(payload) + })) + defer server.Close() + service := &Service{} + body := map[string]any{ + "images": []any{ + server.URL + "/first.png", + server.URL + "/second.png", + }, + } + + hydrated, err := service.hydrateProviderRequestAssets(context.Background(), body, store.RuntimeModelCandidate{ + Provider: "openai", + ModelType: "image_edit", + }) + if err != nil { + t.Fatalf("hydrate OpenAI edit images: %v", err) + } + images := hydrated["images"].([]any) + want := "data:image/png;base64," + base64.StdEncoding.EncodeToString(payload) + if len(images) != 2 || stringFromAny(images[0]) != want || stringFromAny(images[1]) != want { + t.Fatalf("OpenAI edit images should be hydrated for multipart: %+v", images) + } +} + func TestHydrateProviderRequestAssetsImageCapabilityOverridesProviderDataURLDefault(t *testing.T) { service := &Service{} body := map[string]any{