fix(images): 兼容 OpenAI 多图编辑 multipart
- 将 OpenAI images/edits 请求构造成 multipart/form-data - 在运行时安全转存 URL 图片为 data URL 后提交二进制文件 - 覆盖多图字段、真实上游模型名和内部元数据过滤测试
This commit is contained in:
@@ -344,6 +344,70 @@ func TestOpenAIClientChatContract(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClientImageEditUsesMultipartWithMultipleImages(t *testing.T) {
|
||||
var contentType string
|
||||
var receivedFields map[string][]string
|
||||
var receivedImageCount int
|
||||
var receivedImageFields []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
contentType = r.Header.Get("Content-Type")
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart image edit: %v", err)
|
||||
}
|
||||
receivedFields = r.MultipartForm.Value
|
||||
for field, files := range r.MultipartForm.File {
|
||||
receivedImageFields = append(receivedImageFields, field)
|
||||
receivedImageCount += len(files)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"data": []any{map[string]any{"b64_json": "aW1hZ2U="}},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
|
||||
Kind: "images.edits",
|
||||
ModelType: "image_edit",
|
||||
Model: "gpt-image-2",
|
||||
Body: map[string]any{
|
||||
"model": "gpt-image-2",
|
||||
"prompt": "combine the references",
|
||||
"size": "1024x1536",
|
||||
"quality": "medium",
|
||||
"images": []any{
|
||||
"data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("image one")),
|
||||
"data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("image two")),
|
||||
},
|
||||
"_metadata": map[string]any{"private": true},
|
||||
},
|
||||
Candidate: store.RuntimeModelCandidate{
|
||||
BaseURL: server.URL,
|
||||
Provider: "openai",
|
||||
ProviderModelName: "gpt-image-2-vip",
|
||||
ModelType: "image_edit",
|
||||
Credentials: map[string]any{"apiKey": "openai-key"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run OpenAI image edit: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(contentType, "multipart/form-data; boundary=") {
|
||||
t.Fatalf("image edit should use multipart/form-data, got %q", contentType)
|
||||
}
|
||||
if receivedImageCount != 2 || len(receivedImageFields) != 1 || receivedImageFields[0] != "image" {
|
||||
t.Fatalf("unexpected multipart image files: fields=%v count=%d", receivedImageFields, receivedImageCount)
|
||||
}
|
||||
if receivedFields["model"][0] != "gpt-image-2-vip" ||
|
||||
receivedFields["prompt"][0] != "combine the references" ||
|
||||
receivedFields["size"][0] != "1024x1536" ||
|
||||
receivedFields["quality"][0] != "medium" {
|
||||
t.Fatalf("unexpected multipart fields: %+v", receivedFields)
|
||||
}
|
||||
if _, ok := receivedFields["_metadata"]; ok {
|
||||
t.Fatalf("internal metadata must not be forwarded: %+v", receivedFields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClientChatReasoningParamsByProvider(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
||||
@@ -3,8 +3,14 @@ package clients
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -61,13 +67,16 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error
|
||||
body["model"] = upstreamModelName(request.Candidate)
|
||||
stream := openAIEndpointSupportsStream(endpointKind) && (request.Stream || boolValue(body, "stream"))
|
||||
ensureOpenAIStreamUsage(body, endpointKind, stream)
|
||||
raw, _ := json.Marshal(body)
|
||||
raw, contentType, err := openAIRequestPayload(endpointKind, body, request.Candidate)
|
||||
if err != nil {
|
||||
return Response{}, err
|
||||
}
|
||||
upstreamEndpoint := joinURL(openAIBaseURL(endpointKind, request.Candidate), endpoint)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamEndpoint, bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return Response{}, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
responseStartedAt := time.Now()
|
||||
if err := notifySubmissionStarted(request); err != nil {
|
||||
@@ -162,6 +171,222 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error
|
||||
}, nil
|
||||
}
|
||||
|
||||
func openAIRequestPayload(endpointKind string, body map[string]any, candidate store.RuntimeModelCandidate) ([]byte, string, error) {
|
||||
if endpointKind != "images.edits" {
|
||||
raw, err := json.Marshal(body)
|
||||
return raw, "application/json", err
|
||||
}
|
||||
var payload bytes.Buffer
|
||||
writer := multipart.NewWriter(&payload)
|
||||
imageFieldName := openAIImageEditFieldName(candidate)
|
||||
images := openAIImageEditValues(firstPresent(body["images"], body["image"]))
|
||||
if len(images) == 0 {
|
||||
return nil, "", &ClientError{
|
||||
Code: "invalid_parameter",
|
||||
Message: "image is required",
|
||||
Param: "image",
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
for index, value := range images {
|
||||
contentType, image, err := openAIImageEditPayload(value)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := writeOpenAIImageEditFile(writer, imageFieldName, fmt.Sprintf("image-%d%s", index+1, openAIImageFileExtension(contentType)), contentType, image); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
if mask := firstPresent(body["mask"], body["mask_image"], body["maskImage"]); mask != nil {
|
||||
contentType, image, err := openAIImageEditPayload(mask)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := writeOpenAIImageEditFile(writer, "mask", "mask"+openAIImageFileExtension(contentType), contentType, image); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
for key, value := range body {
|
||||
switch key {
|
||||
case "image", "images", "mask", "mask_image", "maskImage":
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(key, "_") || value == nil {
|
||||
continue
|
||||
}
|
||||
fieldValue, err := openAIFormFieldValue(value)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if fieldValue == "" {
|
||||
continue
|
||||
}
|
||||
if err := writer.WriteField(key, fieldValue); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return payload.Bytes(), writer.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
func openAIImageEditFieldName(candidate store.RuntimeModelCandidate) string {
|
||||
if configured := strings.TrimSpace(stringFromAny(candidate.PlatformConfig["imageEditMultipartFieldName"])); configured == "image" || configured == "image[]" {
|
||||
return configured
|
||||
}
|
||||
parsed, err := url.Parse(strings.TrimSpace(candidate.BaseURL))
|
||||
if err == nil && strings.EqualFold(parsed.Hostname(), "api.openai.com") {
|
||||
return "image[]"
|
||||
}
|
||||
return "image"
|
||||
}
|
||||
|
||||
func openAIImageEditValues(value any) []any {
|
||||
switch typed := value.(type) {
|
||||
case []any:
|
||||
return typed
|
||||
case []string:
|
||||
out := make([]any, 0, len(typed))
|
||||
for _, item := range typed {
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
case nil:
|
||||
return nil
|
||||
default:
|
||||
return []any{typed}
|
||||
}
|
||||
}
|
||||
|
||||
func openAIImageEditPayload(value any) (string, []byte, error) {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
for _, key := range []string{"data", "b64_json", "base64", "url"} {
|
||||
if nested := typed[key]; nested != nil {
|
||||
return openAIImageEditPayload(nested)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
raw := strings.TrimSpace(typed)
|
||||
if raw == "" {
|
||||
break
|
||||
}
|
||||
contentType := ""
|
||||
encoded := raw
|
||||
if strings.HasPrefix(strings.ToLower(raw), "data:") {
|
||||
prefix, payload, ok := strings.Cut(raw, ",")
|
||||
if !ok || !strings.Contains(strings.ToLower(prefix), ";base64") {
|
||||
break
|
||||
}
|
||||
contentType = strings.TrimSpace(strings.Split(strings.TrimPrefix(prefix, "data:"), ";")[0])
|
||||
encoded = payload
|
||||
} else if strings.Contains(raw, "://") {
|
||||
return "", nil, &ClientError{
|
||||
Code: "invalid_parameter",
|
||||
Message: "OpenAI image edit input must be hydrated before multipart submission",
|
||||
Param: "image",
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
image, err := decodeOpenAIImageEditBase64(encoded)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if contentType == "" {
|
||||
contentType = strings.TrimSpace(strings.Split(http.DetectContentType(image), ";")[0])
|
||||
}
|
||||
return contentType, image, nil
|
||||
case []byte:
|
||||
if len(typed) > 0 {
|
||||
return strings.TrimSpace(strings.Split(http.DetectContentType(typed), ";")[0]), typed, nil
|
||||
}
|
||||
}
|
||||
return "", nil, &ClientError{
|
||||
Code: "invalid_parameter",
|
||||
Message: "image must be a base64 or data URL payload",
|
||||
Param: "image",
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
func decodeOpenAIImageEditBase64(value string) ([]byte, error) {
|
||||
normalized := strings.Map(func(char rune) rune {
|
||||
switch char {
|
||||
case '\n', '\r', '\t', ' ':
|
||||
return -1
|
||||
default:
|
||||
return char
|
||||
}
|
||||
}, value)
|
||||
var lastErr error
|
||||
for _, encoding := range []*base64.Encoding{
|
||||
base64.StdEncoding,
|
||||
base64.RawStdEncoding,
|
||||
base64.URLEncoding,
|
||||
base64.RawURLEncoding,
|
||||
} {
|
||||
payload, err := encoding.DecodeString(normalized)
|
||||
if err == nil && len(payload) > 0 {
|
||||
return payload, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func writeOpenAIImageEditFile(writer *multipart.Writer, fieldName string, fileName string, contentType string, payload []byte) error {
|
||||
header := make(textproto.MIMEHeader)
|
||||
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, fieldName, escapeMultipartFilename(fileName)))
|
||||
if contentType != "" {
|
||||
header.Set("Content-Type", contentType)
|
||||
}
|
||||
part, err := writer.CreatePart(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(part, bytes.NewReader(payload))
|
||||
return err
|
||||
}
|
||||
|
||||
func openAIImageFileExtension(contentType string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) {
|
||||
case "image/jpeg":
|
||||
return ".jpg"
|
||||
case "image/webp":
|
||||
return ".webp"
|
||||
case "image/gif":
|
||||
return ".gif"
|
||||
default:
|
||||
return ".png"
|
||||
}
|
||||
}
|
||||
|
||||
func openAIFormFieldValue(value any) (string, error) {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return typed, nil
|
||||
case bool:
|
||||
return fmt.Sprintf("%t", typed), nil
|
||||
case float64:
|
||||
return fmt.Sprintf("%v", typed), nil
|
||||
case float32:
|
||||
return fmt.Sprintf("%v", typed), nil
|
||||
case int:
|
||||
return fmt.Sprintf("%d", typed), nil
|
||||
case int64:
|
||||
return fmt.Sprintf("%d", typed), nil
|
||||
case json.Number:
|
||||
return typed.String(), nil
|
||||
default:
|
||||
encoded, err := json.Marshal(value)
|
||||
return string(encoded), err
|
||||
}
|
||||
}
|
||||
|
||||
func openAIWireStreamDelta(next StreamDelta, protocol string, response *http.Response) StreamDelta {
|
||||
if next == nil {
|
||||
return nil
|
||||
|
||||
@@ -160,6 +160,9 @@ func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value s
|
||||
return value, nil
|
||||
}
|
||||
style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, raw, "")
|
||||
if !ok && openAIImageEditRequiresMultipartBytes(candidate) {
|
||||
style, ok = requestAssetHydrateDataURL, true
|
||||
}
|
||||
if !ok {
|
||||
return value, nil
|
||||
}
|
||||
@@ -320,6 +323,9 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand
|
||||
if style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, asset.URL, asset.StorageProvider); ok {
|
||||
return style
|
||||
}
|
||||
if openAIImageEditRequiresMultipartBytes(candidate) {
|
||||
return requestAssetHydrateDataURL
|
||||
}
|
||||
}
|
||||
if mediaURLFieldNeedsHydration(path) {
|
||||
if style := configuredRequestAssetMediaURLHydration(candidate, requestAssetMediaURLKind(path)); style != "" {
|
||||
@@ -332,6 +338,11 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand
|
||||
return requestAssetHydrateURL
|
||||
}
|
||||
|
||||
func openAIImageEditRequiresMultipartBytes(candidate store.RuntimeModelCandidate) bool {
|
||||
return normalizeProviderKey(candidate.Provider) == "openai" &&
|
||||
strings.TrimSpace(candidate.ModelType) == "image_edit"
|
||||
}
|
||||
|
||||
func requestAssetMediaKindForHydration(path []string, asset store.RequestAsset) string {
|
||||
if mediaURLFieldNeedsHydration(path) {
|
||||
return requestAssetMediaURLKind(path)
|
||||
|
||||
@@ -205,6 +205,35 @@ func TestHydrateProviderRequestAssetsUsesImageCapabilityBase64ForTopLevelImageAs
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrateProviderRequestAssetsConvertsOpenAIEditImagesForMultipart(t *testing.T) {
|
||||
payload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
service := &Service{}
|
||||
body := map[string]any{
|
||||
"images": []any{
|
||||
server.URL + "/first.png",
|
||||
server.URL + "/second.png",
|
||||
},
|
||||
}
|
||||
|
||||
hydrated, err := service.hydrateProviderRequestAssets(context.Background(), body, store.RuntimeModelCandidate{
|
||||
Provider: "openai",
|
||||
ModelType: "image_edit",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("hydrate OpenAI edit images: %v", err)
|
||||
}
|
||||
images := hydrated["images"].([]any)
|
||||
want := "data:image/png;base64," + base64.StdEncoding.EncodeToString(payload)
|
||||
if len(images) != 2 || stringFromAny(images[0]) != want || stringFromAny(images[1]) != want {
|
||||
t.Fatalf("OpenAI edit images should be hydrated for multipart: %+v", images)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrateProviderRequestAssetsImageCapabilityOverridesProviderDataURLDefault(t *testing.T) {
|
||||
service := &Service{}
|
||||
body := map[string]any{
|
||||
|
||||
Reference in New Issue
Block a user