fix(images): 兼容 OpenAI 多图编辑 multipart

- 将 OpenAI images/edits 请求构造成 multipart/form-data
- 在运行时安全转存 URL 图片为 data URL 后提交二进制文件
- 覆盖多图字段、真实上游模型名和内部元数据过滤测试
This commit is contained in:
2026-07-24 08:59:42 +08:00
parent 1da712854f
commit db640c1211
4 changed files with 331 additions and 2 deletions
+64
View File
@@ -344,6 +344,70 @@ func TestOpenAIClientChatContract(t *testing.T) {
}
}
func TestOpenAIClientImageEditUsesMultipartWithMultipleImages(t *testing.T) {
var contentType string
var receivedFields map[string][]string
var receivedImageCount int
var receivedImageFields []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
contentType = r.Header.Get("Content-Type")
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart image edit: %v", err)
}
receivedFields = r.MultipartForm.Value
for field, files := range r.MultipartForm.File {
receivedImageFields = append(receivedImageFields, field)
receivedImageCount += len(files)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"data": []any{map[string]any{"b64_json": "aW1hZ2U="}},
})
}))
defer server.Close()
_, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
Kind: "images.edits",
ModelType: "image_edit",
Model: "gpt-image-2",
Body: map[string]any{
"model": "gpt-image-2",
"prompt": "combine the references",
"size": "1024x1536",
"quality": "medium",
"images": []any{
"data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("image one")),
"data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("image two")),
},
"_metadata": map[string]any{"private": true},
},
Candidate: store.RuntimeModelCandidate{
BaseURL: server.URL,
Provider: "openai",
ProviderModelName: "gpt-image-2-vip",
ModelType: "image_edit",
Credentials: map[string]any{"apiKey": "openai-key"},
},
})
if err != nil {
t.Fatalf("run OpenAI image edit: %v", err)
}
if !strings.HasPrefix(contentType, "multipart/form-data; boundary=") {
t.Fatalf("image edit should use multipart/form-data, got %q", contentType)
}
if receivedImageCount != 2 || len(receivedImageFields) != 1 || receivedImageFields[0] != "image" {
t.Fatalf("unexpected multipart image files: fields=%v count=%d", receivedImageFields, receivedImageCount)
}
if receivedFields["model"][0] != "gpt-image-2-vip" ||
receivedFields["prompt"][0] != "combine the references" ||
receivedFields["size"][0] != "1024x1536" ||
receivedFields["quality"][0] != "medium" {
t.Fatalf("unexpected multipart fields: %+v", receivedFields)
}
if _, ok := receivedFields["_metadata"]; ok {
t.Fatalf("internal metadata must not be forwarded: %+v", receivedFields)
}
}
func TestOpenAIClientChatReasoningParamsByProvider(t *testing.T) {
cases := []struct {
name string
+227 -2
View File
@@ -3,8 +3,14 @@ package clients
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"strings"
"time"
@@ -61,13 +67,16 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error
body["model"] = upstreamModelName(request.Candidate)
stream := openAIEndpointSupportsStream(endpointKind) && (request.Stream || boolValue(body, "stream"))
ensureOpenAIStreamUsage(body, endpointKind, stream)
raw, _ := json.Marshal(body)
raw, contentType, err := openAIRequestPayload(endpointKind, body, request.Candidate)
if err != nil {
return Response{}, err
}
upstreamEndpoint := joinURL(openAIBaseURL(endpointKind, request.Candidate), endpoint)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamEndpoint, bytes.NewReader(raw))
if err != nil {
return Response{}, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Type", contentType)
req.Header.Set("Authorization", "Bearer "+apiKey)
responseStartedAt := time.Now()
if err := notifySubmissionStarted(request); err != nil {
@@ -162,6 +171,222 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error
}, nil
}
func openAIRequestPayload(endpointKind string, body map[string]any, candidate store.RuntimeModelCandidate) ([]byte, string, error) {
if endpointKind != "images.edits" {
raw, err := json.Marshal(body)
return raw, "application/json", err
}
var payload bytes.Buffer
writer := multipart.NewWriter(&payload)
imageFieldName := openAIImageEditFieldName(candidate)
images := openAIImageEditValues(firstPresent(body["images"], body["image"]))
if len(images) == 0 {
return nil, "", &ClientError{
Code: "invalid_parameter",
Message: "image is required",
Param: "image",
StatusCode: http.StatusBadRequest,
Retryable: false,
}
}
for index, value := range images {
contentType, image, err := openAIImageEditPayload(value)
if err != nil {
return nil, "", err
}
if err := writeOpenAIImageEditFile(writer, imageFieldName, fmt.Sprintf("image-%d%s", index+1, openAIImageFileExtension(contentType)), contentType, image); err != nil {
return nil, "", err
}
}
if mask := firstPresent(body["mask"], body["mask_image"], body["maskImage"]); mask != nil {
contentType, image, err := openAIImageEditPayload(mask)
if err != nil {
return nil, "", err
}
if err := writeOpenAIImageEditFile(writer, "mask", "mask"+openAIImageFileExtension(contentType), contentType, image); err != nil {
return nil, "", err
}
}
for key, value := range body {
switch key {
case "image", "images", "mask", "mask_image", "maskImage":
continue
}
if strings.HasPrefix(key, "_") || value == nil {
continue
}
fieldValue, err := openAIFormFieldValue(value)
if err != nil {
return nil, "", err
}
if fieldValue == "" {
continue
}
if err := writer.WriteField(key, fieldValue); err != nil {
return nil, "", err
}
}
if err := writer.Close(); err != nil {
return nil, "", err
}
return payload.Bytes(), writer.FormDataContentType(), nil
}
func openAIImageEditFieldName(candidate store.RuntimeModelCandidate) string {
if configured := strings.TrimSpace(stringFromAny(candidate.PlatformConfig["imageEditMultipartFieldName"])); configured == "image" || configured == "image[]" {
return configured
}
parsed, err := url.Parse(strings.TrimSpace(candidate.BaseURL))
if err == nil && strings.EqualFold(parsed.Hostname(), "api.openai.com") {
return "image[]"
}
return "image"
}
func openAIImageEditValues(value any) []any {
switch typed := value.(type) {
case []any:
return typed
case []string:
out := make([]any, 0, len(typed))
for _, item := range typed {
out = append(out, item)
}
return out
case nil:
return nil
default:
return []any{typed}
}
}
func openAIImageEditPayload(value any) (string, []byte, error) {
switch typed := value.(type) {
case map[string]any:
for _, key := range []string{"data", "b64_json", "base64", "url"} {
if nested := typed[key]; nested != nil {
return openAIImageEditPayload(nested)
}
}
case string:
raw := strings.TrimSpace(typed)
if raw == "" {
break
}
contentType := ""
encoded := raw
if strings.HasPrefix(strings.ToLower(raw), "data:") {
prefix, payload, ok := strings.Cut(raw, ",")
if !ok || !strings.Contains(strings.ToLower(prefix), ";base64") {
break
}
contentType = strings.TrimSpace(strings.Split(strings.TrimPrefix(prefix, "data:"), ";")[0])
encoded = payload
} else if strings.Contains(raw, "://") {
return "", nil, &ClientError{
Code: "invalid_parameter",
Message: "OpenAI image edit input must be hydrated before multipart submission",
Param: "image",
StatusCode: http.StatusBadRequest,
Retryable: false,
}
}
image, err := decodeOpenAIImageEditBase64(encoded)
if err != nil {
break
}
if contentType == "" {
contentType = strings.TrimSpace(strings.Split(http.DetectContentType(image), ";")[0])
}
return contentType, image, nil
case []byte:
if len(typed) > 0 {
return strings.TrimSpace(strings.Split(http.DetectContentType(typed), ";")[0]), typed, nil
}
}
return "", nil, &ClientError{
Code: "invalid_parameter",
Message: "image must be a base64 or data URL payload",
Param: "image",
StatusCode: http.StatusBadRequest,
Retryable: false,
}
}
func decodeOpenAIImageEditBase64(value string) ([]byte, error) {
normalized := strings.Map(func(char rune) rune {
switch char {
case '\n', '\r', '\t', ' ':
return -1
default:
return char
}
}, value)
var lastErr error
for _, encoding := range []*base64.Encoding{
base64.StdEncoding,
base64.RawStdEncoding,
base64.URLEncoding,
base64.RawURLEncoding,
} {
payload, err := encoding.DecodeString(normalized)
if err == nil && len(payload) > 0 {
return payload, nil
}
lastErr = err
}
return nil, lastErr
}
func writeOpenAIImageEditFile(writer *multipart.Writer, fieldName string, fileName string, contentType string, payload []byte) error {
header := make(textproto.MIMEHeader)
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, fieldName, escapeMultipartFilename(fileName)))
if contentType != "" {
header.Set("Content-Type", contentType)
}
part, err := writer.CreatePart(header)
if err != nil {
return err
}
_, err = io.Copy(part, bytes.NewReader(payload))
return err
}
func openAIImageFileExtension(contentType string) string {
switch strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) {
case "image/jpeg":
return ".jpg"
case "image/webp":
return ".webp"
case "image/gif":
return ".gif"
default:
return ".png"
}
}
func openAIFormFieldValue(value any) (string, error) {
switch typed := value.(type) {
case string:
return typed, nil
case bool:
return fmt.Sprintf("%t", typed), nil
case float64:
return fmt.Sprintf("%v", typed), nil
case float32:
return fmt.Sprintf("%v", typed), nil
case int:
return fmt.Sprintf("%d", typed), nil
case int64:
return fmt.Sprintf("%d", typed), nil
case json.Number:
return typed.String(), nil
default:
encoded, err := json.Marshal(value)
return string(encoded), err
}
}
func openAIWireStreamDelta(next StreamDelta, protocol string, response *http.Response) StreamDelta {
if next == nil {
return nil
@@ -160,6 +160,9 @@ func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value s
return value, nil
}
style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, raw, "")
if !ok && openAIImageEditRequiresMultipartBytes(candidate) {
style, ok = requestAssetHydrateDataURL, true
}
if !ok {
return value, nil
}
@@ -320,6 +323,9 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand
if style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, asset.URL, asset.StorageProvider); ok {
return style
}
if openAIImageEditRequiresMultipartBytes(candidate) {
return requestAssetHydrateDataURL
}
}
if mediaURLFieldNeedsHydration(path) {
if style := configuredRequestAssetMediaURLHydration(candidate, requestAssetMediaURLKind(path)); style != "" {
@@ -332,6 +338,11 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand
return requestAssetHydrateURL
}
func openAIImageEditRequiresMultipartBytes(candidate store.RuntimeModelCandidate) bool {
return normalizeProviderKey(candidate.Provider) == "openai" &&
strings.TrimSpace(candidate.ModelType) == "image_edit"
}
func requestAssetMediaKindForHydration(path []string, asset store.RequestAsset) string {
if mediaURLFieldNeedsHydration(path) {
return requestAssetMediaURLKind(path)
@@ -205,6 +205,35 @@ func TestHydrateProviderRequestAssetsUsesImageCapabilityBase64ForTopLevelImageAs
}
}
func TestHydrateProviderRequestAssetsConvertsOpenAIEditImagesForMultipart(t *testing.T) {
payload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/png")
_, _ = w.Write(payload)
}))
defer server.Close()
service := &Service{}
body := map[string]any{
"images": []any{
server.URL + "/first.png",
server.URL + "/second.png",
},
}
hydrated, err := service.hydrateProviderRequestAssets(context.Background(), body, store.RuntimeModelCandidate{
Provider: "openai",
ModelType: "image_edit",
})
if err != nil {
t.Fatalf("hydrate OpenAI edit images: %v", err)
}
images := hydrated["images"].([]any)
want := "data:image/png;base64," + base64.StdEncoding.EncodeToString(payload)
if len(images) != 2 || stringFromAny(images[0]) != want || stringFromAny(images[1]) != want {
t.Fatalf("OpenAI edit images should be hydrated for multipart: %+v", images)
}
}
func TestHydrateProviderRequestAssetsImageCapabilityOverridesProviderDataURLDefault(t *testing.T) {
service := &Service{}
body := map[string]any{