fix(provider): 修正媒体请求转换与上游错误透传

按上游协议能力延迟处理媒体资源：OpenAI 兼容平台默认使用 multipart，显式配置后才发送 JSON URL；Gemini 官方协议使用 Files API，兼容协议使用内嵌 Base64，并同步覆盖相关媒体客户端。\n\n安全的上游 400/422 原始错误会作为下游 message 返回，同时保留结构化诊断信息和历史任务兼容。\n\n验证：API 全量无缓存测试、go vet、pnpm lint、pnpm test、pnpm build、pnpm openapi、git diff --check。
2026-08-05 00:40:42 +08:00
parent c79c2a7b44
commit ebdb96e7d7
19 changed files with 1163 additions and 93 deletions
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"mime"
@@ -12,6 +13,8 @@ import (
 	"path"
 	"strings"
 	"time"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
 )

 type GeminiClient struct {
@@ -27,6 +30,20 @@ func (c GeminiClient) Run(ctx context.Context, request Request) (Response, error
 		return c.runVeo(ctx, request, apiKey)
 	}
 	body := geminiBody(request)
+	_, nativeContents := request.Body["contents"]
+	if !nativeContents && GeminiUsesOfficialAPI(request.Candidate) {
+		var err error
+		body, err = c.prepareOfficialGeminiImageFiles(ctx, request, apiKey, body)
+		if err != nil {
+			return Response{}, err
+		}
+	} else if !nativeContents {
+		var err error
+		body, err = prepareCompatibleGeminiInlineImages(ctx, body)
+		if err != nil {
+			return Response{}, err
+		}
+	}
 	raw, _ := json.Marshal(body)
 	endpoint := geminiURL(request.Candidate.BaseURL, upstreamModelName(request.Candidate), apiKey)
 	if request.Stream {
@@ -92,6 +109,230 @@ func (c GeminiClient) Run(ctx context.Context, request Request) (Response, error
 	}, nil
 }

+// GeminiUsesOfficialAPI distinguishes the native Google Gemini endpoint from
+// Gemini-shaped compatibility services and Google's OpenAI compatibility path.
+func GeminiUsesOfficialAPI(candidate store.RuntimeModelCandidate) bool {
+	base := strings.TrimSpace(candidate.BaseURL)
+	if base == "" {
+		return true
+	}
+	parsed, err := url.Parse(base)
+	if err != nil || !strings.EqualFold(parsed.Hostname(), "generativelanguage.googleapis.com") {
+		return false
+	}
+	pathValue := strings.ToLower(strings.TrimRight(parsed.Path, "/"))
+	return pathValue != "/openai" && !strings.HasSuffix(pathValue, "/openai")
+}
+
+func (c GeminiClient) prepareOfficialGeminiImageFiles(ctx context.Context, request Request, apiKey string, body map[string]any) (map[string]any, error) {
+	client := httpClient(request.HTTPClient, c.HTTPClient)
+	cache := map[string]map[string]any{}
+	prepared, err := prepareOfficialGeminiImageFileValue(ctx, client, request.Candidate, apiKey, body, cache)
+	if err != nil {
+		return nil, err
+	}
+	out, _ := prepared.(map[string]any)
+	if out == nil {
+		return body, nil
+	}
+	return out, nil
+}
+
+func prepareOfficialGeminiImageFileValue(ctx context.Context, client *http.Client, candidate store.RuntimeModelCandidate, apiKey string, value any, cache map[string]map[string]any) (any, error) {
+	switch typed := value.(type) {
+	case map[string]any:
+		next := make(map[string]any, len(typed))
+		for key, item := range typed {
+			prepared, err := prepareOfficialGeminiImageFileValue(ctx, client, candidate, apiKey, item, cache)
+			if err != nil {
+				return nil, err
+			}
+			next[key] = prepared
+		}
+		for _, key := range []string{"fileData", "file_data"} {
+			fileData, ok := next[key].(map[string]any)
+			if !ok {
+				continue
+			}
+			fileURI := firstNonEmptyString(fileData["fileUri"], fileData["file_uri"], fileData["uri"])
+			mimeType := firstNonEmptyString(fileData["mimeType"], fileData["mime_type"], mimeFromURI(fileURI))
+			if !requestAssetStringIsHTTPURL(fileURI) || !strings.HasPrefix(strings.ToLower(mimeType), "image/") || geminiOfficialFileURI(fileURI) {
+				continue
+			}
+			cacheKey := fileURI + "\x00" + mimeType
+			uploaded := cache[cacheKey]
+			if uploaded == nil {
+				var err error
+				uploaded, err = uploadOfficialGeminiImageFile(ctx, client, candidate, apiKey, fileURI, mimeType)
+				if err != nil {
+					fallback, fallbackErr := compatibleGeminiInlineImage(ctx, fileURI, mimeType)
+					if fallbackErr != nil {
+						return nil, err
+					}
+					delete(next, key)
+					next["inlineData"] = fallback
+					continue
+				}
+				cache[cacheKey] = uploaded
+			}
+			next[key] = cloneBody(uploaded)
+		}
+		return next, nil
+	case []any:
+		next := make([]any, 0, len(typed))
+		for _, item := range typed {
+			prepared, err := prepareOfficialGeminiImageFileValue(ctx, client, candidate, apiKey, item, cache)
+			if err != nil {
+				return nil, err
+			}
+			next = append(next, prepared)
+		}
+		return next, nil
+	default:
+		return value, nil
+	}
+}
+
+func prepareCompatibleGeminiInlineImages(ctx context.Context, body map[string]any) (map[string]any, error) {
+	prepared, err := prepareCompatibleGeminiInlineImageValue(ctx, body, map[string]map[string]any{})
+	if err != nil {
+		return nil, err
+	}
+	out, _ := prepared.(map[string]any)
+	if out == nil {
+		return body, nil
+	}
+	return out, nil
+}
+
+func prepareCompatibleGeminiInlineImageValue(ctx context.Context, value any, cache map[string]map[string]any) (any, error) {
+	switch typed := value.(type) {
+	case map[string]any:
+		next := make(map[string]any, len(typed))
+		for key, item := range typed {
+			prepared, err := prepareCompatibleGeminiInlineImageValue(ctx, item, cache)
+			if err != nil {
+				return nil, err
+			}
+			next[key] = prepared
+		}
+		for _, key := range []string{"fileData", "file_data"} {
+			fileData, ok := next[key].(map[string]any)
+			if !ok {
+				continue
+			}
+			fileURI := firstNonEmptyString(fileData["fileUri"], fileData["file_uri"], fileData["uri"])
+			mimeType := firstNonEmptyString(fileData["mimeType"], fileData["mime_type"], mimeFromURI(fileURI))
+			if !requestAssetStringIsHTTPURL(fileURI) || !strings.HasPrefix(strings.ToLower(mimeType), "image/") {
+				continue
+			}
+			cacheKey := fileURI + "\x00" + mimeType
+			inline := cache[cacheKey]
+			if inline == nil {
+				var err error
+				inline, err = compatibleGeminiInlineImage(ctx, fileURI, mimeType)
+				if err != nil {
+					return nil, err
+				}
+				cache[cacheKey] = inline
+			}
+			delete(next, key)
+			next["inlineData"] = cloneBody(inline)
+		}
+		return next, nil
+	case []any:
+		next := make([]any, 0, len(typed))
+		for _, item := range typed {
+			prepared, err := prepareCompatibleGeminiInlineImageValue(ctx, item, cache)
+			if err != nil {
+				return nil, err
+			}
+			next = append(next, prepared)
+		}
+		return next, nil
+	default:
+		return value, nil
+	}
+}
+
+func compatibleGeminiInlineImage(ctx context.Context, sourceURL string, declaredMimeType string) (map[string]any, error) {
+	fetchedMimeType, payload, err := fetchRemoteMediaInputPayload(ctx, sourceURL, 256<<20)
+	if err != nil {
+		return nil, err
+	}
+	return map[string]any{
+		"mimeType": geminiMediaMime(firstNonEmptyString(declaredMimeType, fetchedMimeType), "image"),
+		"data":     base64.StdEncoding.EncodeToString(payload),
+	}, nil
+}
+
+func uploadOfficialGeminiImageFile(ctx context.Context, client *http.Client, candidate store.RuntimeModelCandidate, apiKey string, sourceURL string, declaredMimeType string) (map[string]any, error) {
+	fetchedMimeType, payload, err := fetchRemoteMediaInputPayload(ctx, sourceURL, 256<<20)
+	if err != nil {
+		return nil, err
+	}
+	mimeType := geminiMediaMime(firstNonEmptyString(declaredMimeType, fetchedMimeType), "image")
+	base := strings.TrimRight(strings.TrimSpace(candidate.BaseURL), "/")
+	if base == "" {
+		base = "https://generativelanguage.googleapis.com"
+	}
+	parsed, err := url.Parse(base)
+	if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+		return nil, &ClientError{Code: "invalid_parameter", Message: "invalid official Gemini base URL", Retryable: false}
+	}
+	uploadStartURL := parsed.Scheme + "://" + parsed.Host + "/upload/v1beta/files"
+	metadata, _ := json.Marshal(map[string]any{"file": map[string]any{"display_name": fmt.Sprintf("easyai-image-%d", time.Now().UnixMilli())}})
+	startRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadStartURL, bytes.NewReader(metadata))
+	if err != nil {
+		return nil, err
+	}
+	startRequest.Header.Set("x-goog-api-key", apiKey)
+	startRequest.Header.Set("X-Goog-Upload-Protocol", "resumable")
+	startRequest.Header.Set("X-Goog-Upload-Command", "start")
+	startRequest.Header.Set("X-Goog-Upload-Header-Content-Length", fmt.Sprintf("%d", len(payload)))
+	startRequest.Header.Set("X-Goog-Upload-Header-Content-Type", mimeType)
+	startRequest.Header.Set("Content-Type", "application/json")
+	startResponse, err := client.Do(startRequest)
+	if err != nil {
+		return nil, &ClientError{Code: "network", Message: err.Error(), Retryable: true}
+	}
+	if startResponse.StatusCode < http.StatusOK || startResponse.StatusCode >= http.StatusMultipleChoices {
+		_, responseErr := decodeHTTPResponse(startResponse)
+		return nil, responseErr
+	}
+	uploadURL := strings.TrimSpace(startResponse.Header.Get("X-Goog-Upload-URL"))
+	_ = startResponse.Body.Close()
+	if uploadURL == "" {
+		return nil, &ClientError{Code: "invalid_response", Message: "Gemini Files API did not return an upload URL", Retryable: false}
+	}
+	uploadRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadURL, bytes.NewReader(payload))
+	if err != nil {
+		return nil, err
+	}
+	uploadRequest.Header.Set("X-Goog-Upload-Offset", "0")
+	uploadRequest.Header.Set("X-Goog-Upload-Command", "upload, finalize")
+	uploadRequest.Header.Set("Content-Type", mimeType)
+	uploadResponse, err := client.Do(uploadRequest)
+	if err != nil {
+		return nil, &ClientError{Code: "network", Message: err.Error(), Retryable: true}
+	}
+	result, err := decodeHTTPResponse(uploadResponse)
+	if err != nil {
+		return nil, err
+	}
+	file := mapFromAny(result["file"])
+	fileURI := firstNonEmptyString(file["uri"], result["uri"])
+	if fileURI == "" {
+		return nil, &ClientError{Code: "invalid_response", Message: "Gemini Files API response did not include file.uri", Retryable: false}
+	}
+	return map[string]any{"fileUri": fileURI, "mimeType": firstNonEmptyString(file["mimeType"], file["mime_type"], mimeType)}, nil
+}
+
+func geminiOfficialFileURI(value string) bool {
+	parsed, err := url.Parse(strings.TrimSpace(value))
+	return err == nil && strings.EqualFold(parsed.Hostname(), "generativelanguage.googleapis.com") && strings.Contains(strings.ToLower(parsed.Path), "/files/")
+}
+
 func geminiURL(baseURL string, model string, apiKey string) string {
 	return geminiActionURL(baseURL, model, "generateContent", apiKey, false)
 }