feat(storage): 统一二进制对象存储与公开错误

新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
@@ -12,7 +12,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -38,12 +37,6 @@ type decodedRequestAsset struct {
 	ContentType string
 }

-type requestAssetOptions struct {
-	RequirePublicURL bool
-	UploadScene      string
-	Source           string
-}
-
 type requestAssetLock struct {
 	mu   sync.Mutex
 	refs int
@@ -115,6 +108,19 @@ func (s *Server) prepareRequestAssetRefs(ctx context.Context, body map[string]an
 }

 func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path []string, siblings map[string]any) (any, error) {
+	if decoded, ok, err := requestAssetFromBinaryValue(requestAssetPathKey(path), path, value, siblings); err != nil {
+		return nil, err
+	} else if ok {
+		if err := s.acquireMediaRequestSlot(ctx); err != nil {
+			return nil, err
+		}
+		defer s.releaseMediaRequestSlot()
+		ref, err := s.ensureRequestAsset(ctx, decoded)
+		if err != nil {
+			return nil, err
+		}
+		return requestAssetWrapper(ref), nil
+	}
 	switch typed := value.(type) {
 	case map[string]any:
 		if typed["assetRef"] != nil {
@@ -156,6 +162,83 @@ func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path [
 	}
 }

+func requestAssetFromBinaryValue(key string, path []string, value any, siblings map[string]any) (decodedRequestAsset, bool, error) {
+	var payload []byte
+	contentType := ""
+	switch typed := value.(type) {
+	case []byte:
+		payload = append([]byte(nil), typed...)
+	case map[string]any:
+		if !strings.EqualFold(strings.TrimSpace(stringFromRequestAny(typed["type"])), "buffer") {
+			return decodedRequestAsset{}, false, nil
+		}
+		contentType = firstNonEmptyRequestString(typed, "contentType", "mimeType", "mime_type")
+		switch data := typed["data"].(type) {
+		case []byte:
+			payload = append([]byte(nil), data...)
+		case []any:
+			var ok bool
+			payload, ok = requestBytesFromNumberArray(data)
+			if !ok {
+				return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data must contain byte values"))
+			}
+		default:
+			return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data is required"))
+		}
+	case []any:
+		if !strictRequestBinaryArrayField(key, path) {
+			return decodedRequestAsset{}, false, nil
+		}
+		var ok bool
+		payload, ok = requestBytesFromNumberArray(typed)
+		if !ok {
+			return decodedRequestAsset{}, false, nil
+		}
+	default:
+		return decodedRequestAsset{}, false, nil
+	}
+	if len(payload) == 0 {
+		return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("binary media payload is empty"))
+	}
+	return decodedRequestAsset{
+		Bytes:       payload,
+		ContentType: requestAssetContentType(contentType, payload, key, path, siblings),
+	}, true, nil
+}
+
+func requestAssetPathKey(path []string) string {
+	for index := len(path) - 1; index >= 0; index-- {
+		value := strings.TrimSpace(path[index])
+		if value != "" && !strings.HasPrefix(value, "[") {
+			return value
+		}
+	}
+	return ""
+}
+
+func strictRequestBinaryArrayField(key string, path []string) bool {
+	value := strings.ToLower(strings.TrimSpace(key))
+	if value == "bytes" || value == "buffer" || strings.Contains(value, "binary") || strings.Contains(value, "buffer") {
+		return true
+	}
+	return requestMediaKind(key, path, nil) != "" && (value == "data" || value == "content")
+}
+
+func requestBytesFromNumberArray(values []any) ([]byte, bool) {
+	if len(values) == 0 {
+		return nil, false
+	}
+	payload := make([]byte, len(values))
+	for index, value := range values {
+		number, ok := value.(float64)
+		if !ok || number < 0 || number > 255 || number != float64(byte(number)) {
+			return nil, false
+		}
+		payload[index] = byte(number)
+	}
+	return payload, true
+}
+
 func (s *Server) prepareRequestAssetField(ctx context.Context, key string, path []string, value any, siblings map[string]any) (map[string]any, bool, error) {
 	text, ok := value.(string)
 	if !ok {
@@ -291,68 +374,43 @@ func requestAssetFromValue(key string, path []string, value any, siblings map[st
 }

 func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
-	return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
-		UploadScene: store.FileStorageSceneRequestAsset,
-		Source:      "ai-gateway-request",
-	})
-}
-
-func (s *Server) ensurePublicRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
-	return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
-		RequirePublicURL: true,
-		UploadScene:      store.FileStorageSceneUpload,
-		Source:           "ai-gateway-form-data",
-	})
-}
-
-func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded decodedRequestAsset, options requestAssetOptions) (map[string]any, error) {
 	sum := sha256.Sum256(decoded.Bytes)
 	sha := hex.EncodeToString(sum[:])
 	contentType := strings.TrimSpace(decoded.ContentType)
 	if contentType == "" {
 		contentType = "application/octet-stream"
 	}
-	release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + options.UploadScene + "\x00" + strconv.FormatBool(options.RequirePublicURL))
+	release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + store.FileStorageSceneRequestAsset)
 	defer release()
 	now := time.Now()
 	if existing, ok, err := s.store.FindRequestAsset(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
 		return nil, err
 	} else if ok && requestAssetStillUsable(existing, now) {
 		ref := requestAssetRef(existing)
-		if !options.RequirePublicURL || requestAssetRefHasPublicURL(ref) {
-			if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
-				return nil, err
-			}
-			return ref, nil
+		if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
+			return nil, err
 		}
+		return ref, nil
 	}

-	uploadScene := strings.TrimSpace(options.UploadScene)
-	if uploadScene == "" {
-		uploadScene = store.FileStorageSceneRequestAsset
-	}
-	source := strings.TrimSpace(options.Source)
-	if source == "" {
-		source = "ai-gateway-request"
-	}
 	upload, err := s.runner.UploadFile(ctx, runner.FileUploadPayload{
 		Bytes:       decoded.Bytes,
 		ContentType: contentType,
 		FileName:    requestAssetFileName(sha, contentType),
-		Scene:       uploadScene,
-		Source:      source,
+		Scene:       store.FileStorageSceneRequestAsset,
+		Source:      "ai-gateway-request",
 	})
 	if err != nil {
 		return nil, err
 	}
 	storageProvider := requestAssetStorageProvider(upload)
+	storageChannelID, storageChannelKey := requestAssetStorageChannel(upload)
+	objectKey := stringFromRequestAny(upload["objectKey"])
+	accessScope := firstNonEmpty(stringFromRequestAny(upload["accessScope"]), "private")
 	url := stringFromRequestAny(upload["url"])
 	if url == "" {
 		return nil, &clients.ClientError{Code: "request_asset_upload_failed", Message: "file storage response did not include url", Retryable: false}
 	}
-	if options.RequirePublicURL && !requestAssetURLIsPublic(storageProvider, url) {
-		return nil, &clients.ClientError{Code: "request_asset_public_url_required", Message: "multipart image assets require a public file storage URL; enable a non-local file storage channel for uploads", Retryable: false}
-	}
 	var expiresAt *time.Time
 	localPath := ""
 	if storageProvider == "local_static" {
@@ -361,23 +419,31 @@ func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded deco
 		localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"]))
 	}
 	asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{
-		SHA256:          sha,
-		ContentType:     contentType,
-		ByteSize:        int64(len(decoded.Bytes)),
-		URL:             url,
-		StorageProvider: storageProvider,
-		LocalPath:       localPath,
-		ExpiresAt:       expiresAt,
+		SHA256:            sha,
+		ContentType:       contentType,
+		ByteSize:          int64(len(decoded.Bytes)),
+		URL:               url,
+		StorageProvider:   storageProvider,
+		StorageChannelID:  storageChannelID,
+		StorageChannelKey: storageChannelKey,
+		ObjectKey:         objectKey,
+		AccessScope:       accessScope,
+		LocalPath:         localPath,
+		ExpiresAt:         expiresAt,
 	})
 	if err != nil {
 		if store.IsUndefinedDatabaseObject(err) {
 			return map[string]any{
-				"sha256":          sha,
-				"url":             url,
-				"contentType":     contentType,
-				"size":            len(decoded.Bytes),
-				"storageProvider": storageProvider,
-				"expiresAt":       timePtrToRFC3339(expiresAt),
+				"sha256":            sha,
+				"url":               url,
+				"contentType":       contentType,
+				"size":              len(decoded.Bytes),
+				"storageProvider":   storageProvider,
+				"storageChannelId":  storageChannelID,
+				"storageChannelKey": storageChannelKey,
+				"objectKey":         objectKey,
+				"accessScope":       accessScope,
+				"expiresAt":         timePtrToRFC3339(expiresAt),
 			}, nil
 		}
 		return nil, err
@@ -493,27 +559,19 @@ func requestAssetWrapper(ref map[string]any) map[string]any {

 func requestAssetRef(asset store.RequestAsset) map[string]any {
 	return map[string]any{
-		"sha256":          asset.SHA256,
-		"url":             asset.URL,
-		"contentType":     asset.ContentType,
-		"size":            asset.ByteSize,
-		"storageProvider": asset.StorageProvider,
-		"expiresAt":       timePtrToRFC3339(asset.ExpiresAt),
+		"sha256":            asset.SHA256,
+		"url":               asset.URL,
+		"contentType":       asset.ContentType,
+		"size":              asset.ByteSize,
+		"storageProvider":   asset.StorageProvider,
+		"storageChannelId":  asset.StorageChannelID,
+		"storageChannelKey": asset.StorageChannelKey,
+		"objectKey":         asset.ObjectKey,
+		"accessScope":       asset.AccessScope,
+		"expiresAt":         timePtrToRFC3339(asset.ExpiresAt),
 	}
 }

-func requestAssetRefHasPublicURL(ref map[string]any) bool {
-	return requestAssetURLIsPublic(stringFromRequestAny(ref["storageProvider"]), stringFromRequestAny(ref["url"]))
-}
-
-func requestAssetURLIsPublic(storageProvider string, url string) bool {
-	if strings.EqualFold(strings.TrimSpace(storageProvider), "local_static") {
-		return false
-	}
-	lower := strings.ToLower(strings.TrimSpace(url))
-	return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
-}
-
 func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
 	if asset.ExpiredAt != nil {
 		return false
@@ -543,6 +601,11 @@ func requestAssetStorageProvider(upload map[string]any) string {
 	return "unknown"
 }

+func requestAssetStorageChannel(upload map[string]any) (string, string) {
+	channel, _ := upload["storageChannel"].(map[string]any)
+	return stringFromRequestAny(channel["id"]), stringFromRequestAny(channel["channelKey"])
+}
+
 func requestAssetLocalPath(storageDir string, fileName string) string {
 	if strings.TrimSpace(storageDir) == "" {
 		storageDir = config.DefaultLocalUploadedStorageDir