fix(images): 兼容 OpenAI 多图编辑 multipart
- 将 OpenAI images/edits 请求构造成 multipart/form-data - 在运行时安全转存 URL 图片为 data URL 后提交二进制文件 - 覆盖多图字段、真实上游模型名和内部元数据过滤测试
This commit is contained in:
@@ -344,6 +344,70 @@ func TestOpenAIClientChatContract(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAIClientImageEditUsesMultipartWithMultipleImages(t *testing.T) {
|
||||||
|
var contentType string
|
||||||
|
var receivedFields map[string][]string
|
||||||
|
var receivedImageCount int
|
||||||
|
var receivedImageFields []string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
contentType = r.Header.Get("Content-Type")
|
||||||
|
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||||
|
t.Fatalf("parse multipart image edit: %v", err)
|
||||||
|
}
|
||||||
|
receivedFields = r.MultipartForm.Value
|
||||||
|
for field, files := range r.MultipartForm.File {
|
||||||
|
receivedImageFields = append(receivedImageFields, field)
|
||||||
|
receivedImageCount += len(files)
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"data": []any{map[string]any{"b64_json": "aW1hZ2U="}},
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
_, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
|
||||||
|
Kind: "images.edits",
|
||||||
|
ModelType: "image_edit",
|
||||||
|
Model: "gpt-image-2",
|
||||||
|
Body: map[string]any{
|
||||||
|
"model": "gpt-image-2",
|
||||||
|
"prompt": "combine the references",
|
||||||
|
"size": "1024x1536",
|
||||||
|
"quality": "medium",
|
||||||
|
"images": []any{
|
||||||
|
"data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("image one")),
|
||||||
|
"data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("image two")),
|
||||||
|
},
|
||||||
|
"_metadata": map[string]any{"private": true},
|
||||||
|
},
|
||||||
|
Candidate: store.RuntimeModelCandidate{
|
||||||
|
BaseURL: server.URL,
|
||||||
|
Provider: "openai",
|
||||||
|
ProviderModelName: "gpt-image-2-vip",
|
||||||
|
ModelType: "image_edit",
|
||||||
|
Credentials: map[string]any{"apiKey": "openai-key"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run OpenAI image edit: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(contentType, "multipart/form-data; boundary=") {
|
||||||
|
t.Fatalf("image edit should use multipart/form-data, got %q", contentType)
|
||||||
|
}
|
||||||
|
if receivedImageCount != 2 || len(receivedImageFields) != 1 || receivedImageFields[0] != "image" {
|
||||||
|
t.Fatalf("unexpected multipart image files: fields=%v count=%d", receivedImageFields, receivedImageCount)
|
||||||
|
}
|
||||||
|
if receivedFields["model"][0] != "gpt-image-2-vip" ||
|
||||||
|
receivedFields["prompt"][0] != "combine the references" ||
|
||||||
|
receivedFields["size"][0] != "1024x1536" ||
|
||||||
|
receivedFields["quality"][0] != "medium" {
|
||||||
|
t.Fatalf("unexpected multipart fields: %+v", receivedFields)
|
||||||
|
}
|
||||||
|
if _, ok := receivedFields["_metadata"]; ok {
|
||||||
|
t.Fatalf("internal metadata must not be forwarded: %+v", receivedFields)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOpenAIClientChatReasoningParamsByProvider(t *testing.T) {
|
func TestOpenAIClientChatReasoningParamsByProvider(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -3,8 +3,14 @@ package clients
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/textproto"
|
||||||
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -61,13 +67,16 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error
|
|||||||
body["model"] = upstreamModelName(request.Candidate)
|
body["model"] = upstreamModelName(request.Candidate)
|
||||||
stream := openAIEndpointSupportsStream(endpointKind) && (request.Stream || boolValue(body, "stream"))
|
stream := openAIEndpointSupportsStream(endpointKind) && (request.Stream || boolValue(body, "stream"))
|
||||||
ensureOpenAIStreamUsage(body, endpointKind, stream)
|
ensureOpenAIStreamUsage(body, endpointKind, stream)
|
||||||
raw, _ := json.Marshal(body)
|
raw, contentType, err := openAIRequestPayload(endpointKind, body, request.Candidate)
|
||||||
|
if err != nil {
|
||||||
|
return Response{}, err
|
||||||
|
}
|
||||||
upstreamEndpoint := joinURL(openAIBaseURL(endpointKind, request.Candidate), endpoint)
|
upstreamEndpoint := joinURL(openAIBaseURL(endpointKind, request.Candidate), endpoint)
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamEndpoint, bytes.NewReader(raw))
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamEndpoint, bytes.NewReader(raw))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Response{}, err
|
return Response{}, err
|
||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", contentType)
|
||||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
responseStartedAt := time.Now()
|
responseStartedAt := time.Now()
|
||||||
if err := notifySubmissionStarted(request); err != nil {
|
if err := notifySubmissionStarted(request); err != nil {
|
||||||
@@ -162,6 +171,222 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func openAIRequestPayload(endpointKind string, body map[string]any, candidate store.RuntimeModelCandidate) ([]byte, string, error) {
|
||||||
|
if endpointKind != "images.edits" {
|
||||||
|
raw, err := json.Marshal(body)
|
||||||
|
return raw, "application/json", err
|
||||||
|
}
|
||||||
|
var payload bytes.Buffer
|
||||||
|
writer := multipart.NewWriter(&payload)
|
||||||
|
imageFieldName := openAIImageEditFieldName(candidate)
|
||||||
|
images := openAIImageEditValues(firstPresent(body["images"], body["image"]))
|
||||||
|
if len(images) == 0 {
|
||||||
|
return nil, "", &ClientError{
|
||||||
|
Code: "invalid_parameter",
|
||||||
|
Message: "image is required",
|
||||||
|
Param: "image",
|
||||||
|
StatusCode: http.StatusBadRequest,
|
||||||
|
Retryable: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for index, value := range images {
|
||||||
|
contentType, image, err := openAIImageEditPayload(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if err := writeOpenAIImageEditFile(writer, imageFieldName, fmt.Sprintf("image-%d%s", index+1, openAIImageFileExtension(contentType)), contentType, image); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mask := firstPresent(body["mask"], body["mask_image"], body["maskImage"]); mask != nil {
|
||||||
|
contentType, image, err := openAIImageEditPayload(mask)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if err := writeOpenAIImageEditFile(writer, "mask", "mask"+openAIImageFileExtension(contentType), contentType, image); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key, value := range body {
|
||||||
|
switch key {
|
||||||
|
case "image", "images", "mask", "mask_image", "maskImage":
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(key, "_") || value == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fieldValue, err := openAIFormFieldValue(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if fieldValue == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := writer.WriteField(key, fieldValue); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := writer.Close(); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
return payload.Bytes(), writer.FormDataContentType(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIImageEditFieldName(candidate store.RuntimeModelCandidate) string {
|
||||||
|
if configured := strings.TrimSpace(stringFromAny(candidate.PlatformConfig["imageEditMultipartFieldName"])); configured == "image" || configured == "image[]" {
|
||||||
|
return configured
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(strings.TrimSpace(candidate.BaseURL))
|
||||||
|
if err == nil && strings.EqualFold(parsed.Hostname(), "api.openai.com") {
|
||||||
|
return "image[]"
|
||||||
|
}
|
||||||
|
return "image"
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIImageEditValues(value any) []any {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case []any:
|
||||||
|
return typed
|
||||||
|
case []string:
|
||||||
|
out := make([]any, 0, len(typed))
|
||||||
|
for _, item := range typed {
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case nil:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return []any{typed}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIImageEditPayload(value any) (string, []byte, error) {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
for _, key := range []string{"data", "b64_json", "base64", "url"} {
|
||||||
|
if nested := typed[key]; nested != nil {
|
||||||
|
return openAIImageEditPayload(nested)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
raw := strings.TrimSpace(typed)
|
||||||
|
if raw == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
contentType := ""
|
||||||
|
encoded := raw
|
||||||
|
if strings.HasPrefix(strings.ToLower(raw), "data:") {
|
||||||
|
prefix, payload, ok := strings.Cut(raw, ",")
|
||||||
|
if !ok || !strings.Contains(strings.ToLower(prefix), ";base64") {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
contentType = strings.TrimSpace(strings.Split(strings.TrimPrefix(prefix, "data:"), ";")[0])
|
||||||
|
encoded = payload
|
||||||
|
} else if strings.Contains(raw, "://") {
|
||||||
|
return "", nil, &ClientError{
|
||||||
|
Code: "invalid_parameter",
|
||||||
|
Message: "OpenAI image edit input must be hydrated before multipart submission",
|
||||||
|
Param: "image",
|
||||||
|
StatusCode: http.StatusBadRequest,
|
||||||
|
Retryable: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
image, err := decodeOpenAIImageEditBase64(encoded)
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = strings.TrimSpace(strings.Split(http.DetectContentType(image), ";")[0])
|
||||||
|
}
|
||||||
|
return contentType, image, nil
|
||||||
|
case []byte:
|
||||||
|
if len(typed) > 0 {
|
||||||
|
return strings.TrimSpace(strings.Split(http.DetectContentType(typed), ";")[0]), typed, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", nil, &ClientError{
|
||||||
|
Code: "invalid_parameter",
|
||||||
|
Message: "image must be a base64 or data URL payload",
|
||||||
|
Param: "image",
|
||||||
|
StatusCode: http.StatusBadRequest,
|
||||||
|
Retryable: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeOpenAIImageEditBase64(value string) ([]byte, error) {
|
||||||
|
normalized := strings.Map(func(char rune) rune {
|
||||||
|
switch char {
|
||||||
|
case '\n', '\r', '\t', ' ':
|
||||||
|
return -1
|
||||||
|
default:
|
||||||
|
return char
|
||||||
|
}
|
||||||
|
}, value)
|
||||||
|
var lastErr error
|
||||||
|
for _, encoding := range []*base64.Encoding{
|
||||||
|
base64.StdEncoding,
|
||||||
|
base64.RawStdEncoding,
|
||||||
|
base64.URLEncoding,
|
||||||
|
base64.RawURLEncoding,
|
||||||
|
} {
|
||||||
|
payload, err := encoding.DecodeString(normalized)
|
||||||
|
if err == nil && len(payload) > 0 {
|
||||||
|
return payload, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
}
|
||||||
|
return nil, lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeOpenAIImageEditFile(writer *multipart.Writer, fieldName string, fileName string, contentType string, payload []byte) error {
|
||||||
|
header := make(textproto.MIMEHeader)
|
||||||
|
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, fieldName, escapeMultipartFilename(fileName)))
|
||||||
|
if contentType != "" {
|
||||||
|
header.Set("Content-Type", contentType)
|
||||||
|
}
|
||||||
|
part, err := writer.CreatePart(header)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(part, bytes.NewReader(payload))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIImageFileExtension(contentType string) string {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) {
|
||||||
|
case "image/jpeg":
|
||||||
|
return ".jpg"
|
||||||
|
case "image/webp":
|
||||||
|
return ".webp"
|
||||||
|
case "image/gif":
|
||||||
|
return ".gif"
|
||||||
|
default:
|
||||||
|
return ".png"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIFormFieldValue(value any) (string, error) {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case string:
|
||||||
|
return typed, nil
|
||||||
|
case bool:
|
||||||
|
return fmt.Sprintf("%t", typed), nil
|
||||||
|
case float64:
|
||||||
|
return fmt.Sprintf("%v", typed), nil
|
||||||
|
case float32:
|
||||||
|
return fmt.Sprintf("%v", typed), nil
|
||||||
|
case int:
|
||||||
|
return fmt.Sprintf("%d", typed), nil
|
||||||
|
case int64:
|
||||||
|
return fmt.Sprintf("%d", typed), nil
|
||||||
|
case json.Number:
|
||||||
|
return typed.String(), nil
|
||||||
|
default:
|
||||||
|
encoded, err := json.Marshal(value)
|
||||||
|
return string(encoded), err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func openAIWireStreamDelta(next StreamDelta, protocol string, response *http.Response) StreamDelta {
|
func openAIWireStreamDelta(next StreamDelta, protocol string, response *http.Response) StreamDelta {
|
||||||
if next == nil {
|
if next == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -160,6 +160,9 @@ func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value s
|
|||||||
return value, nil
|
return value, nil
|
||||||
}
|
}
|
||||||
style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, raw, "")
|
style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, raw, "")
|
||||||
|
if !ok && openAIImageEditRequiresMultipartBytes(candidate) {
|
||||||
|
style, ok = requestAssetHydrateDataURL, true
|
||||||
|
}
|
||||||
if !ok {
|
if !ok {
|
||||||
return value, nil
|
return value, nil
|
||||||
}
|
}
|
||||||
@@ -320,6 +323,9 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand
|
|||||||
if style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, asset.URL, asset.StorageProvider); ok {
|
if style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, asset.URL, asset.StorageProvider); ok {
|
||||||
return style
|
return style
|
||||||
}
|
}
|
||||||
|
if openAIImageEditRequiresMultipartBytes(candidate) {
|
||||||
|
return requestAssetHydrateDataURL
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if mediaURLFieldNeedsHydration(path) {
|
if mediaURLFieldNeedsHydration(path) {
|
||||||
if style := configuredRequestAssetMediaURLHydration(candidate, requestAssetMediaURLKind(path)); style != "" {
|
if style := configuredRequestAssetMediaURLHydration(candidate, requestAssetMediaURLKind(path)); style != "" {
|
||||||
@@ -332,6 +338,11 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand
|
|||||||
return requestAssetHydrateURL
|
return requestAssetHydrateURL
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func openAIImageEditRequiresMultipartBytes(candidate store.RuntimeModelCandidate) bool {
|
||||||
|
return normalizeProviderKey(candidate.Provider) == "openai" &&
|
||||||
|
strings.TrimSpace(candidate.ModelType) == "image_edit"
|
||||||
|
}
|
||||||
|
|
||||||
func requestAssetMediaKindForHydration(path []string, asset store.RequestAsset) string {
|
func requestAssetMediaKindForHydration(path []string, asset store.RequestAsset) string {
|
||||||
if mediaURLFieldNeedsHydration(path) {
|
if mediaURLFieldNeedsHydration(path) {
|
||||||
return requestAssetMediaURLKind(path)
|
return requestAssetMediaURLKind(path)
|
||||||
|
|||||||
@@ -205,6 +205,35 @@ func TestHydrateProviderRequestAssetsUsesImageCapabilityBase64ForTopLevelImageAs
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHydrateProviderRequestAssetsConvertsOpenAIEditImagesForMultipart(t *testing.T) {
|
||||||
|
payload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "image/png")
|
||||||
|
_, _ = w.Write(payload)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
service := &Service{}
|
||||||
|
body := map[string]any{
|
||||||
|
"images": []any{
|
||||||
|
server.URL + "/first.png",
|
||||||
|
server.URL + "/second.png",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
hydrated, err := service.hydrateProviderRequestAssets(context.Background(), body, store.RuntimeModelCandidate{
|
||||||
|
Provider: "openai",
|
||||||
|
ModelType: "image_edit",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("hydrate OpenAI edit images: %v", err)
|
||||||
|
}
|
||||||
|
images := hydrated["images"].([]any)
|
||||||
|
want := "data:image/png;base64," + base64.StdEncoding.EncodeToString(payload)
|
||||||
|
if len(images) != 2 || stringFromAny(images[0]) != want || stringFromAny(images[1]) != want {
|
||||||
|
t.Fatalf("OpenAI edit images should be hydrated for multipart: %+v", images)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHydrateProviderRequestAssetsImageCapabilityOverridesProviderDataURLDefault(t *testing.T) {
|
func TestHydrateProviderRequestAssetsImageCapabilityOverridesProviderDataURLDefault(t *testing.T) {
|
||||||
service := &Service{}
|
service := &Service{}
|
||||||
body := map[string]any{
|
body := map[string]any{
|
||||||
|
|||||||
Reference in New Issue
Block a user