BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(provider): 修正媒体请求转换与上游错误透传

Browse Source 
按上游协议能力延迟处理媒体资源：OpenAI 兼容平台默认使用 multipart，显式配置后才发送 JSON URL；Gemini 官方协议使用 Files API，兼容协议使用内嵌 Base64，并同步覆盖相关媒体客户端。\n\n安全的上游 400/422 原始错误会作为下游 message 返回，同时保留结构化诊断信息和历史任务兼容。\n\n验证：API 全量无缓存测试、go vet、pnpm lint、pnpm test、pnpm build、pnpm openapi、git diff --check。
This commit is contained in:
wangbo
2026-08-05 00:40:42 +08:00
parent c79c2a7b44
commit ebdb96e7d7
19 changed files with 1163 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/runner/request_assets.go
+102 -21
View File
@@ -128,6 +128,15 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st
if err != nil {
return nil, err
}
if requestAssetClientMaterializesRemoteMedia(candidate, path, asset) {
assetURL, accessErr := s.requestAssetAccessURL(ctx, asset)
if accessErr != nil {
return nil, accessErr
}
if requestAssetStringIsHTTPURL(assetURL) {
return assetURL, nil
}
}
switch requestAssetHydrationForField(path, asset, candidate) {
case requestAssetHydrateUnsupported:
return nil, requestAssetUnsupportedInputFormatError(path, asset)
@@ -155,6 +164,9 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st
if strings.TrimSpace(assetURL) == "" {
return nil, requestAssetExpiredError(asset)
}
if openAIImageEditRequiresJSONURL(candidate) && imageInputFieldNeedsHydration(path) && !requestAssetURLIsPublic(asset.StorageProvider, assetURL) {
return nil, requestAssetPublicURLRequiredError(path)
}
return assetURL, nil
}
@@ -163,8 +175,17 @@ func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value s
if raw == "" || !imageInputFieldNeedsHydration(path) {
return value, nil
}
if openAIImageEditRequiresJSONURL(candidate) {
if !requestAssetURLIsPublic("", raw) {
return nil, requestAssetPublicURLRequiredError(path)
}
return value, nil
}
if requestAssetClientMaterializesRemoteMedia(candidate, path, store.RequestAsset{URL: raw}) && requestAssetStringIsHTTPURL(raw) {
return value, nil
}
style, ok := requestAssetHydrateUnsupported, false
if geminiVeoRequiresInlineImage(candidate) || openAIImageEditRequiresMultipartBytes(candidate) {
if openAIImageEditRequiresMultipartBytes(candidate) {
style, ok = requestAssetHydrateDataURL, true
} else {
style, ok = requestAssetCapabilityHydrationForMedia("image", candidate, raw, "")
@@ -392,13 +413,22 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand
return requestAssetHydrateRawBase64
}
if candidate.ModelType == "voice_clone" && voiceCloneAudioFieldNeedsHydration(path, asset) {
if requestAssetStringIsHTTPURL(asset.URL) {
return requestAssetHydrateURL
}
return requestAssetHydrateDataURL
}
if requestAssetMediaKindForHydration(path, asset) == "image" {
if geminiVeoRequiresInlineImage(candidate) {
return requestAssetHydrateDataURL
if requestAssetClientMaterializesRemoteMedia(candidate, path, asset) && requestAssetStringIsHTTPURL(asset.URL) {
return requestAssetHydrateURL
}
if openAIImageEditRequiresJSONURL(candidate) {
return requestAssetHydrateURL
}
if openAIImageEditRequiresMultipartBytes(candidate) {
if requestAssetStringIsHTTPURL(asset.URL) {
return requestAssetHydrateURL
}
return requestAssetHydrateDataURL
}
if style, ok := requestAssetCapabilityHydrationForMedia("image", candidate, asset.URL, asset.StorageProvider); ok {
@@ -409,21 +439,63 @@ func requestAssetHydrationForField(path []string, asset store.RequestAsset, cand
if style := configuredRequestAssetMediaURLHydration(candidate, requestAssetMediaURLKind(path)); style != "" {
return style
}
if providerMediaURLNeedsDataURL(candidate) {
return requestAssetHydrateDataURL
}
}
if requestAssetMediaKindForHydration(path, asset) != "" && strings.EqualFold(strings.TrimSpace(asset.StorageProvider), "local_static") {
return requestAssetHydrateDataURL
}
return requestAssetHydrateURL
}
func openAIImageEditRequiresMultipartBytes(candidate store.RuntimeModelCandidate) bool {
return normalizeProviderKey(candidate.Provider) == "openai" &&
strings.TrimSpace(candidate.ModelType) == "image_edit"
return openAIImageEditCandidate(candidate) && !clients.OpenAIImageEditUsesJSONURL(candidate)
}
func geminiVeoRequiresInlineImage(candidate store.RuntimeModelCandidate) bool {
return normalizeProviderKey(candidate.Provider) == "gemini" &&
strings.Contains(strings.ToLower(strings.TrimSpace(firstNonEmptyString(candidate.ProviderModelName, candidate.ModelName))), "veo")
func openAIImageEditRequiresJSONURL(candidate store.RuntimeModelCandidate) bool {
return openAIImageEditCandidate(candidate) && clients.OpenAIImageEditUsesJSONURL(candidate)
}
func openAIImageEditCandidate(candidate store.RuntimeModelCandidate) bool {
if strings.TrimSpace(candidate.ModelType) != "image_edit" {
return false
}
return normalizeProviderKey(candidate.Provider) == "openai" || normalizeProviderKey(candidate.SpecType) == "openai"
}
func requestAssetClientMaterializesRemoteMedia(candidate store.RuntimeModelCandidate, path []string, asset store.RequestAsset) bool {
mediaKind := requestAssetMediaKindForHydration(path, asset)
if mediaKind == "image" {
if openAIImageEditRequiresMultipartBytes(candidate) || geminiClientMaterializesImages(candidate) || kelingClientMaterializesImages(candidate) {
return true
}
}
return mediaKind == "audio" && strings.TrimSpace(candidate.ModelType) == "voice_clone" && candidateUsesClient(candidate, "minimax")
}
func geminiClientMaterializesImages(candidate store.RuntimeModelCandidate) bool {
return candidateUsesClient(candidate, "gemini", "google_gemini")
}
func kelingClientMaterializesImages(candidate store.RuntimeModelCandidate) bool {
return strings.TrimSpace(candidate.ModelType) != "" && candidateUsesClient(candidate, "keling", "kling")
}
func candidateUsesClient(candidate store.RuntimeModelCandidate, names ...string) bool {
wanted := make(map[string]bool, len(names))
for _, name := range names {
wanted[normalizeProviderKey(name)] = true
}
for _, value := range []string{candidate.SpecType, candidate.Provider} {
if wanted[normalizeProviderKey(value)] {
return true
}
}
if wanted["gemini"] {
provider := normalizeProviderKey(candidate.Provider)
if provider == "gemini_openai" || strings.Contains(strings.ToLower(strings.TrimSpace(candidate.BaseURL)), "generativelanguage.googleapis.com") {
return true
}
}
return false
}
func requestAssetMediaKindForHydration(path []string, asset store.RequestAsset) string {
@@ -594,6 +666,12 @@ func requestAssetStringLooksURL(value string) bool {
strings.HasPrefix(lower, "/static/uploaded/")
}
func requestAssetStringIsHTTPURL(value string) bool {
raw := strings.TrimSpace(value)
parsed, err := url.Parse(raw)
return err == nil && parsed.Host != "" && (strings.EqualFold(parsed.Scheme, "http") || strings.EqualFold(parsed.Scheme, "https"))
}
func requestAssetURLIsPublic(storageProvider string, value string) bool {
if strings.EqualFold(strings.TrimSpace(storageProvider), "local_static") {
return false
@@ -701,16 +779,6 @@ func requestAssetHydrationStyleFromString(value string) requestAssetHydrationSty
}
}
func providerMediaURLNeedsDataURL(candidate store.RuntimeModelCandidate) bool {
for _, name := range []string{candidate.Provider, candidate.SpecType, candidate.PlatformKey} {
switch normalizeProviderKey(name) {
case "openai", "volces", "volces_openai", "gemini", "vidu":
return true
}
}
return false
}
func normalizeProviderKey(value string) string {
normalized := strings.ToLower(strings.TrimSpace(value))
normalized = strings.ReplaceAll(normalized, "-", "_")
@@ -782,3 +850,16 @@ func requestAssetUnsupportedInputFormatError(path []string, asset store.RequestA
}
return &clients.ClientError{Code: "request_asset_input_format_unsupported", Message: message, Retryable: false}
}
func requestAssetPublicURLRequiredError(path []string) error {
field := strings.Join(path, ".")
if field == "" {
field = "image"
}
return &clients.ClientError{
Code: "request_asset_public_url_required",
Message: "OpenAI image edit JSON mode requires a public HTTP(S) URL for " + field,
Param: field,
Retryable: false,
}
}
apps/api/internal/runner/request_assets_test.go
+127 -15
View File
@@ -172,7 +172,7 @@ func TestHydrateProviderRequestAssetsConvertsGeminiInlineDataAssetToRawBase64(t
}
}
func TestGeminiVeoForcesOfficialInlineImageFormat(t *testing.T) {
func TestGeminiVeoPreservesURLUntilOfficialPayloadConstruction(t *testing.T) {
candidate := store.RuntimeModelCandidate{
Provider: "gemini",
ProviderModelName: "veo-3.1-generate-preview",
@@ -185,8 +185,77 @@ func TestGeminiVeoForcesOfficialInlineImageFormat(t *testing.T) {
},
}
asset := store.RequestAsset{URL: "https://cdn.example.com/input.png", StorageProvider: "remote", ContentType: "image/png"}
if got := requestAssetHydrationForField([]string{"image"}, asset, candidate); got != requestAssetHydrateDataURL {
t.Fatalf("Gemini Veo must hydrate public image URLs as data URLs, got %q", got)
if got := requestAssetHydrationForField([]string{"image"}, asset, candidate); got != requestAssetHydrateURL {
t.Fatalf("Gemini Veo should preserve public URLs until client payload construction, got %q", got)
}
}
func TestRemoteMediaMaterializationBoundaryByClient(t *testing.T) {
imageAsset := store.RequestAsset{URL: "https://cdn.example.com/input.png", StorageProvider: "remote", ContentType: "image/png"}
audioAsset := store.RequestAsset{URL: "https://cdn.example.com/input.mp3", StorageProvider: "remote", ContentType: "audio/mpeg"}
tests := []struct {
name string
candidate store.RuntimeModelCandidate
path []string
asset store.RequestAsset
want bool
}{
{name: "OpenAI multipart image edit", candidate: store.RuntimeModelCandidate{SpecType: "openai", ModelType: "image_edit"}, path: []string{"image"}, asset: imageAsset, want: true},
{name: "OpenAI JSON URL image edit", candidate: store.RuntimeModelCandidate{SpecType: "openai", ModelType: "image_edit", PlatformConfig: map[string]any{"imageEditRequestFormat": "json_url"}}, path: []string{"image"}, asset: imageAsset, want: false},
{name: "Gemini Veo", candidate: store.RuntimeModelCandidate{SpecType: "gemini", ModelType: "image_to_video", ProviderModelName: "veo-3.1-generate-preview"}, path: []string{"image"}, asset: imageAsset, want: true},
{name: "Keling video", candidate: store.RuntimeModelCandidate{SpecType: "keling", ModelType: "image_to_video"}, path: []string{"image"}, asset: imageAsset, want: true},
{name: "Minimax voice clone", candidate: store.RuntimeModelCandidate{SpecType: "minimax", ModelType: "voice_clone"}, path: []string{"audio"}, asset: audioAsset, want: true},
{name: "Volces JSON image", candidate: store.RuntimeModelCandidate{SpecType: "volces", ModelType: "image_edit"}, path: []string{"image"}, asset: imageAsset, want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := requestAssetClientMaterializesRemoteMedia(test.candidate, test.path, test.asset); got != test.want {
t.Fatalf("client materializes remote media = %t, want %t", got, test.want)
}
})
}
}
func TestJSONTaskClientDoesNotForcePublicURLToBase64WithoutCapability(t *testing.T) {
asset := store.RequestAsset{URL: "https://cdn.example.com/input.png", StorageProvider: "remote", ContentType: "image/png"}
candidate := store.RuntimeModelCandidate{SpecType: "volces", ModelType: "image_edit"}
if got := requestAssetHydrationForField([]string{"image"}, asset, candidate); got != requestAssetHydrateURL {
t.Fatalf("JSON task client should preserve a public URL when no capability requires base64, got %q", got)
}
}
func TestGeminiURLsArePreservedUntilClientProtocolConstruction(t *testing.T) {
service := &Service{}
compatibleURL := "https://cdn.example.com/compatible.png"
body := map[string]any{"image": compatibleURL}
compatible, err := service.hydrateProviderRequestAssets(context.Background(), body, store.RuntimeModelCandidate{
SpecType: "gemini",
BaseURL: "https://gemini-compatible.example.com/v1beta",
ModelType: "image_edit",
PlatformConfig: map[string]any{
"supportUrlInput": true,
"supportBase64Input": true,
},
})
if err != nil {
t.Fatalf("hydrate compatible Gemini image: %v", err)
}
if compatible["image"] != compatibleURL {
t.Fatalf("compatible Gemini URL should be preserved until client inline conversion, got %q", compatible["image"])
}
officialURL := "https://cdn.example.com/source.png"
official, err := service.hydrateProviderRequestAssets(context.Background(), map[string]any{"image": officialURL}, store.RuntimeModelCandidate{
SpecType: "gemini",
BaseURL: "https://generativelanguage.googleapis.com/v1beta",
ModelType: "image_edit",
})
if err != nil {
t.Fatalf("hydrate official Gemini image: %v", err)
}
if official["image"] != officialURL {
t.Fatalf("official Gemini should preserve URL for Files API upload, got %q", official["image"])
}
}
@@ -273,18 +342,12 @@ func TestHydrateProviderRequestAssetsUsesImageCapabilityBase64ForTopLevelImageAs
}
}
func TestHydrateProviderRequestAssetsConvertsOpenAIEditImagesForMultipart(t *testing.T) {
payload := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 1, 2, 3, 4}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/png")
_, _ = w.Write(payload)
}))
defer server.Close()
func TestHydrateProviderRequestAssetsPreservesOpenAIEditURLsUntilMultipartSubmission(t *testing.T) {
service := &Service{}
body := map[string]any{
"images": []any{
server.URL + "/first.png",
server.URL + "/second.png",
"https://cdn.example.com/first.png",
"https://cdn.example.com/second.png",
},
}
@@ -302,9 +365,58 @@ func TestHydrateProviderRequestAssetsConvertsOpenAIEditImagesForMultipart(t *tes
t.Fatalf("hydrate OpenAI edit images: %v", err)
}
images := hydrated["images"].([]any)
want := "data:image/png;base64," + base64.StdEncoding.EncodeToString(payload)
if len(images) != 2 || stringFromAny(images[0]) != want || stringFromAny(images[1]) != want {
t.Fatalf("OpenAI edit images should be hydrated for multipart: %+v", images)
if len(images) != 2 || images[0] != "https://cdn.example.com/first.png" || images[1] != "https://cdn.example.com/second.png" {
t.Fatalf("OpenAI edit URLs should be preserved until multipart construction: %+v", images)
}
}
func TestHydrateProviderRequestAssetsKeepsURLsForExplicitOpenAIJSONMode(t *testing.T) {
service := &Service{}
body := map[string]any{
"images": []any{
"https://cdn.example.com/first.png",
"https://cdn.example.com/second.png",
},
}
candidate := store.RuntimeModelCandidate{
Provider: "compatible-openai",
SpecType: "openai",
ModelType: "image_edit",
Capabilities: map[string]any{
"image_edit": map[string]any{
"support_url_input": false,
"support_base64_input": true,
},
},
PlatformConfig: map[string]any{"imageEditRequestFormat": "json_url"},
}
hydrated, err := service.hydrateProviderRequestAssets(context.Background(), body, candidate)
if err != nil {
t.Fatalf("hydrate OpenAI JSON edit images: %v", err)
}
images := hydrated["images"].([]any)
if len(images) != 2 || images[0] != "https://cdn.example.com/first.png" || images[1] != "https://cdn.example.com/second.png" {
t.Fatalf("OpenAI JSON mode should preserve public URLs: %+v", images)
}
asset := store.RequestAsset{URL: "https://cdn.example.com/asset.png", StorageProvider: "remote", ContentType: "image/png"}
if got := requestAssetHydrationForField([]string{"image"}, asset, candidate); got != requestAssetHydrateURL {
t.Fatalf("explicit JSON mode must override base64 capability hydration, got %q", got)
}
}
func TestHydrateProviderRequestAssetsRejectsInlineDataForExplicitOpenAIJSONMode(t *testing.T) {
service := &Service{}
_, err := service.hydrateProviderRequestAssets(context.Background(), map[string]any{
"image": "data:image/png;base64,aW1hZ2U=",
}, store.RuntimeModelCandidate{
Provider: "openai",
ModelType: "image_edit",
PlatformConfig: map[string]any{"imageEditRequestFormat": "json_url"},
})
clientErr, ok := err.(*clients.ClientError)
if !ok || clientErr.Code != "request_asset_public_url_required" || clientErr.Param != "image" || clientErr.Retryable {
t.Fatalf("expected public URL validation error, got %#v", err)
}
}