feat(storage): 统一二进制对象存储与公开错误

新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
@@ -17,25 +17,18 @@ import (
 	"net/http"
 	"net/textproto"
 	"net/url"
-	"os"
-	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 	"time"

 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
-	"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
 )

 const defaultServerMainOpenAPIUploadURL = "http://127.0.0.1:3001/v1/files/upload"
 const maxGeneratedAssetFetchBytes = 256 << 20

-const (
-	localStaticGeneratedPathPrefix = "/static/generated/"
-	localStaticUploadedPathPrefix  = "/static/uploaded/"
-)
-
 type FileUploadPayload struct {
 	ContentType string
 	FileName    string
@@ -136,14 +129,6 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
 			return nil, &clients.ClientError{Code: "acceptance_run_inactive", Message: err.Error(), Retryable: false}
 		}
 	}
-	if policy.LocalizeInlineMedia {
-		next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result)
-		if err != nil {
-			return nil, err
-		}
-		redactGeneratedResultRawData(next)
-		return next, nil
-	}
 	if len(data) == 0 && !rawNeedsUpload {
 		return s.finalizeGeneratedAssets(ctx, taskID, taskKind, result, policy, nil, false, 0)
 	}
@@ -179,11 +164,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
 	var channels []store.FileStorageChannel
 	channelsLoaded := false
 	if needsUpload || rawNeedsUpload {
-		if s.directOSS == nil {
-			channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
-			if err != nil {
-				return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
-			}
+		channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
+		if err != nil {
+			return nil, &clients.ClientError{Code: "storage_write_failed", Message: "object storage configuration is unavailable", Retryable: true}
 		}
 		channelsLoaded = true
 	}
@@ -323,12 +306,12 @@ func (s *Service) finalizeGeneratedAssets(
 	if !TaskResultHasInlineBinary(next) {
 		return next, nil
 	}
-	persistent, _, err := s.materializeLocalBinaryResult(ctx, taskID, next)
-	if err != nil {
-		return nil, err
+	return nil, &clients.ClientError{
+		Code:       "storage_write_failed",
+		Message:    "generated binary result could not be written to object storage",
+		StatusCode: http.StatusServiceUnavailable,
+		Retryable:  true,
 	}
-	redactGeneratedResultRawData(persistent)
-	return persistent, nil
 }

 func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool {
@@ -355,6 +338,9 @@ func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]
 func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID string, taskKind string, value any, key string, siblings map[string]any, policy generatedAssetUploadPolicy, channels []store.FileStorageChannel, index *int) (any, bool, error) {
 	switch typed := value.(type) {
 	case map[string]any:
+		if payload, contentType, ok := localBufferObjectBytes(typed); ok {
+			return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, contentType, firstNonEmptyString(key, "buffer"), siblings, channels, index)
+		}
 		next := make(map[string]any, len(typed))
 		changed := false
 		for childKey, childValue := range typed {
@@ -372,6 +358,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
 		}
 		return value, false, nil
 	case []any:
+		if localBinaryKey(key) {
+			if payload, ok := bytesFromNumberArray(typed); ok {
+				return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
+			}
+		}
 		next := make([]any, len(typed))
 		changed := false
 		for itemIndex, item := range typed {
@@ -388,6 +379,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
 			return next, true, nil
 		}
 		return value, false, nil
+	case []byte:
+		if len(typed) == 0 {
+			return value, false, nil
+		}
+		return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, append([]byte(nil), typed...), mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
 	case string:
 		asset, ok := generatedRawInlineMediaAsset(key, typed, siblings, taskKind)
 		if !ok {
@@ -404,19 +400,36 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
 	}
 }

+func (s *Service) uploadGeneratedBinaryValue(ctx context.Context, taskID string, taskKind string, payload []byte, contentType string, sourceKey string, siblings map[string]any, channels []store.FileStorageChannel, index *int) (any, bool, error) {
+	contentType = firstNonEmptyString(contentType, defaultContentTypeForRawMediaKey(sourceKey))
+	asset := &generatedInlineAsset{
+		Bytes:       payload,
+		ContentType: contentType,
+		Kind:        mediaKindForAsset(taskKind, siblings, sourceKey, contentType),
+		SourceKey:   sourceKey,
+	}
+	upload, resolvedContentType, kind, strategy, err := s.uploadGeneratedAsset(ctx, taskID, asset, *index, channels)
+	if err != nil {
+		return nil, false, err
+	}
+	*index = *index + 1
+	return generatedRawMediaReference(asset, upload, resolvedContentType, kind, strategy), true, nil
+}
+
 func generatedRawInlineMediaAsset(key string, value string, siblings map[string]any, taskKind string) (*generatedInlineAsset, bool) {
 	raw := strings.TrimSpace(value)
 	if raw == "" {
 		return nil, false
 	}
+	keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key)
 	contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
-	if !generatedRawDataMediaPayloadKey(key) && !generatedContentTypeIsMedia(contentType) {
+	if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) {
 		return nil, false
 	}
-	if !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
+	if !keyLooksLikeMediaPayload && !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
 		return nil, false
 	}
-	payload, payloadContentType, ok, err := inlineMediaPayload(raw, generatedRawDataMediaPayloadKey(key))
+	payload, payloadContentType, ok, err := inlineMediaPayload(raw, keyLooksLikeMediaPayload)
 	if err != nil || !ok || len(payload) == 0 {
 		return nil, false
 	}
@@ -454,6 +467,18 @@ func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]a
 		if provider := stringFromAny(channel["provider"]); provider != "" {
 			ref["storageProvider"] = provider
 		}
+		if id := stringFromAny(channel["id"]); id != "" {
+			ref["storageChannelId"] = id
+		}
+		if key := stringFromAny(channel["channelKey"]); key != "" {
+			ref["storageChannelKey"] = key
+		}
+	}
+	if objectKey := stringFromAny(upload["objectKey"]); objectKey != "" {
+		ref["objectKey"] = objectKey
+	}
+	if accessScope := stringFromAny(upload["accessScope"]); accessScope != "" {
+		ref["accessScope"] = accessScope
 	}
 	out := map[string]any{
 		"assetRef": ref,
@@ -650,6 +675,9 @@ func removeASCIIWhitespace(value string) string {
 }

 func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) {
+	if s.store == nil {
+		return defaultGeneratedAssetUploadPolicy(), nil
+	}
 	settings, err := s.store.GetFileStorageSettings(ctx)
 	if err != nil {
 		if store.IsUndefinedDatabaseObject(err) {
@@ -665,8 +693,6 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP
 	switch policyName {
 	case store.FileStorageResultUploadPolicyUploadAll:
 		return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
-	case store.FileStorageResultUploadPolicyUploadNone:
-		return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}
 	default:
 		return defaultGeneratedAssetUploadPolicy()
 	}
@@ -682,13 +708,8 @@ func (s *Service) uploadGeneratedAsset(ctx context.Context, taskID string, asset
 		Scene:       store.FileStorageSceneImageResult,
 		Source:      "ai-gateway",
 	}
-	if s.directOSS != nil {
-		upload, err := s.directOSS.upload(ctx, payload)
-		return upload, contentType, kind, "direct_aliyun_oss", err
-	}
 	if len(channels) == 0 {
-		upload, err := s.storeFileLocally(payload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
-		return upload, contentType, kind, "local_static_inline_media", err
+		return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
 	}
 	upload, err := s.uploadFileWithFailover(ctx, payload, channels)
 	return upload, contentType, kind, "upload_inline_media", err
@@ -708,95 +729,13 @@ func (s *Service) uploadGeneratedURLAsset(ctx context.Context, taskID string, as
 		Scene:       store.FileStorageSceneImageResult,
 		Source:      "ai-gateway",
 	}
-	if s.directOSS != nil {
-		upload, err := s.directOSS.upload(ctx, uploadPayload)
-		return upload, contentType, kind, "direct_aliyun_oss", err
-	}
 	if len(channels) == 0 {
-		upload, err := s.storeFileLocally(uploadPayload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
-		return upload, contentType, kind, "local_static_url_media", err
+		return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
 	}
 	upload, err := s.uploadFileWithFailover(ctx, uploadPayload, channels)
 	return upload, contentType, kind, "upload_url_media", err
 }

-func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string, fallbackStorageDir string, pathPrefix string) (map[string]any, error) {
-	storageDir = strings.TrimSpace(storageDir)
-	if storageDir == "" {
-		storageDir = fallbackStorageDir
-	}
-	if err := os.MkdirAll(storageDir, 0o755); err != nil {
-		return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
-	}
-	fileName := filepath.Base(strings.TrimSpace(payload.FileName))
-	if fileName == "" || fileName == "." || fileName == ".." || fileName == string(filepath.Separator) {
-		kind := generatedAssetKindFromContentType("", payload.ContentType)
-		fileName = generatedAssetFileName("generated", 0, payload.ContentType, kind)
-	}
-	targetPath := filepath.Join(storageDir, fileName)
-	file, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
-	if err != nil {
-		return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
-	}
-	_, writeErr := file.Write(payload.Bytes)
-	closeErr := file.Close()
-	if writeErr != nil {
-		_ = os.Remove(targetPath)
-		return nil, &clients.ClientError{Code: "local_static_store_failed", Message: writeErr.Error(), Retryable: true}
-	}
-	if closeErr != nil {
-		_ = os.Remove(targetPath)
-		return nil, &clients.ClientError{Code: "local_static_store_failed", Message: closeErr.Error(), Retryable: true}
-	}
-	expiresAt := time.Now().Add(time.Duration(s.localStaticAssetTTLHours()) * time.Hour).UTC().Format(time.RFC3339)
-	return map[string]any{
-		"url":         s.localStaticFileURL(fileName, pathPrefix),
-		"fileName":    fileName,
-		"contentType": payload.ContentType,
-		"size":        len(payload.Bytes),
-		"expiresAt":   expiresAt,
-		"storageChannel": map[string]any{
-			"id":         "local-static",
-			"channelKey": "local-static",
-			"name":       "AI Gateway local static storage",
-			"provider":   "local_static",
-		},
-	}, nil
-}
-
-func (s *Service) localStaticAssetTTLHours() int {
-	if s.cfg.LocalTempAssetTTLHours <= 0 {
-		return 24
-	}
-	return s.cfg.LocalTempAssetTTLHours
-}
-
-func (s *Service) localStaticFileURL(fileName string, pathPrefix string) string {
-	if strings.TrimSpace(pathPrefix) == "" {
-		pathPrefix = localStaticUploadedPathPrefix
-	}
-	path := pathPrefix + url.PathEscape(filepath.Base(fileName))
-	baseURL := strings.TrimRight(strings.TrimSpace(s.cfg.PublicBaseURL), "/")
-	if baseURL == "" {
-		return path
-	}
-	return baseURL + path
-}
-
-func localStaticUploadFileName(originalName string, contentType string) string {
-	baseName := filepath.Base(strings.TrimSpace(originalName))
-	originalExt := strings.ToLower(filepath.Ext(baseName))
-	namePart := strings.TrimSuffix(baseName, originalExt)
-	namePart = sanitizeGeneratedAssetNamePart(namePart)
-	if namePart == "" {
-		namePart = "gateway-upload"
-	}
-	if len(namePart) > 48 {
-		namePart = namePart[:48]
-	}
-	return fmt.Sprintf("%s-%s%s", namePart, randomHexSuffix(6), uploadFileExtension(contentType, originalExt))
-}
-
 func uploadFileExtension(contentType string, fallbackExt string) string {
 	normalized := normalizeGeneratedContentType(contentType)
 	if generatedContentTypeIsMedia(normalized) {
@@ -981,46 +920,41 @@ func (s *Service) UploadFile(ctx context.Context, payload FileUploadPayload) (ma
 	if strings.TrimSpace(payload.Scene) == "" {
 		payload.Scene = store.FileStorageSceneUpload
 	}
-	if s.directOSS != nil && directOSSScene(payload.Scene) {
-		return s.directOSS.upload(ctx, payload)
-	}
 	channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
 	if err != nil {
 		return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
 	}
 	if len(channels) == 0 {
-		payload.FileName = localStaticUploadFileName(payload.FileName, payload.ContentType)
-		upload, err := s.storeFileLocally(payload, s.cfg.LocalUploadedStorageDir, config.DefaultLocalUploadedStorageDir, localStaticUploadedPathPrefix)
-		if err != nil {
-			return nil, err
-		}
-		upload["assetStorage"] = map[string]any{
-			"scene":    payload.Scene,
-			"source":   firstNonEmptyString(payload.Source, "ai-gateway-openapi"),
-			"strategy": "local_static_upload",
-		}
-		return upload, nil
+		return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
 	}
 	return s.uploadFileWithFailover(ctx, payload, channels)
 }

 func (s *Service) activeFileStorageChannels(ctx context.Context, scene string) ([]store.FileStorageChannel, error) {
-	if s.store == nil {
-		return nil, nil
+	channels := make([]store.FileStorageChannel, 0)
+	if s.directOSS != nil && directOSSScene(scene) {
+		channels = append(channels, s.directOSS.fileStorageChannel())
 	}
-	channels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
+	if s.store == nil {
+		return channels, nil
+	}
+	storedChannels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
 	if err != nil && !store.IsUndefinedDatabaseObject(err) {
 		return nil, err
 	}
-	if len(channels) > 0 {
-		return channels, nil
-	}
-	return nil, nil
+	channels = append(channels, storedChannels...)
+	sort.SliceStable(channels, func(i, j int) bool {
+		if channels[i].Priority != channels[j].Priority {
+			return channels[i].Priority < channels[j].Priority
+		}
+		return channels[i].ChannelKey < channels[j].ChannelKey
+	})
+	return channels, nil
 }

 func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUploadPayload, channels []store.FileStorageChannel) (map[string]any, error) {
 	var lastErr error
-	for _, channel := range channels {
+	for index, channel := range channels {
 		upload, err := s.uploadWithChannelRetries(ctx, payload, channel)
 		if err == nil {
 			if s.store != nil {
@@ -1032,25 +966,59 @@ func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUpload
 		if s.store != nil {
 			_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(ctx), channel.ID, err.Error())
 		}
+		if !storageFailureAllowsFailover(channel, err) {
+			return nil, err
+		}
+		if index+1 < len(channels) {
+			s.observeObjectStorage("failover", channel.Provider, 0, 0)
+		}
 	}
 	if lastErr != nil {
-		return nil, lastErr
+		s.observeObjectStorage("all_failed", "", 0, 0)
+		return nil, &clients.ClientError{Code: "storage_write_failed", Message: "all configured object storage channels failed", StatusCode: http.StatusServiceUnavailable, Retryable: true}
+	}
+	return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
+}
+
+func storageFailureAllowsFailover(channel store.FileStorageChannel, err error) bool {
+	code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err)))
+	switch code {
+	case "upload_source_too_large", "upload_decode_failed", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio":
+		return false
+	}
+	var clientErr *clients.ClientError
+	if !errors.As(err, &clientErr) {
+		return true
+	}
+	switch clientErr.StatusCode {
+	case http.StatusRequestEntityTooLarge, http.StatusUnsupportedMediaType, http.StatusUnprocessableEntity:
+		return false
+	case http.StatusBadRequest:
+		// Object-storage 400 responses commonly indicate endpoint/signature
+		// configuration and should move to the next channel. server-main 400 is
+		// the existing upload API's request validation result.
+		return !strings.EqualFold(strings.TrimSpace(channel.Provider), "server_main_openapi")
+	default:
+		return true
 	}
-	return nil, &clients.ClientError{Code: "upload_no_channel", Message: "no enabled file storage channel", Retryable: false}
 }

 func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
-	maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy)
+	maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy, channel.Provider)
 	var lastErr error
 	for attempt := 0; attempt <= maxRetries; attempt++ {
+		startedAt := time.Now()
 		upload, err := s.uploadOnce(ctx, payload, channel)
 		if err == nil {
+			s.observeObjectStorage("write_success", channel.Provider, len(payload.Bytes), time.Since(startedAt))
 			return upload, nil
 		}
+		s.observeObjectStorage("write_failure", channel.Provider, 0, time.Since(startedAt))
 		lastErr = err
 		if attempt >= maxRetries || !clients.IsRetryable(err) {
 			break
 		}
+		s.observeObjectStorage("retry", channel.Provider, 0, 0)
 		delay := retryDelayForAttempt(attempt, delays)
 		if err := sleepWithContext(ctx, delay); err != nil {
 			return nil, err
@@ -1060,9 +1028,21 @@ func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUplo
 }

 func (s *Service) uploadOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
-	if strings.ToLower(strings.TrimSpace(channel.Provider)) != "server_main_openapi" {
-		return nil, &clients.ClientError{Code: "upload_unsupported_channel", Message: "unsupported file storage channel: " + channel.Provider, Retryable: false}
+	switch strings.ToLower(strings.TrimSpace(channel.Provider)) {
+	case "aliyun_oss", "s3":
+		adapter, err := newObjectStorageAdapter(channel)
+		if err != nil {
+			return nil, err
+		}
+		return adapter.put(ctx, payload)
+	case "server_main_openapi":
+		return s.uploadServerMainOnce(ctx, payload, channel)
+	default:
+		return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported file storage channel", Retryable: false}
 	}
+}
+
+func (s *Service) uploadServerMainOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
 	uploadURL := strings.TrimSpace(channel.UploadURL)
 	if uploadURL == "" {
 		uploadURL = defaultServerMainOpenAPIUploadURL
@@ -1700,9 +1680,17 @@ func uniqueStringList(values []string) []string {
 	return out
 }

-func uploadRetrySchedule(policy map[string]any) (int, []time.Duration) {
+func uploadRetrySchedule(policy map[string]any, providers ...string) (int, []time.Duration) {
+	provider := ""
+	if len(providers) > 0 {
+		provider = providers[0]
+	}
 	if policy == nil {
-		policy = defaultUploadRetryPolicy()
+		if strings.EqualFold(provider, "aliyun_oss") || strings.EqualFold(provider, "s3") {
+			policy = map[string]any{"enabled": true, "maxRetries": 2, "backoffSeconds": []any{0.25, 1.0}}
+		} else {
+			policy = defaultUploadRetryPolicy()
+		}
 	}
 	if enabled, ok := policy["enabled"].(bool); ok && !enabled {
 		return 0, nil
@@ -1725,9 +1713,9 @@ func uploadRetryDelays(value any) []time.Duration {
 	}
 	delays := make([]time.Duration, 0, len(items))
 	for _, item := range items {
-		seconds := int(floatFromAny(item))
+		seconds := floatFromAny(item)
 		if seconds > 0 {
-			delays = append(delays, time.Duration(seconds)*time.Second)
+			delays = append(delays, time.Duration(seconds*float64(time.Second)))
 		}
 	}
 	return delays