feat(storage): 统一二进制对象存储与公开错误

新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
 	"time"

 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
@@ -26,7 +25,6 @@ const (
 	localBinaryGenericBase64MinLength = 4096
 	localBinaryMaxDepth               = 64
 	defaultLocalResultTTLHours        = 24
-	defaultLocalResultMinFreeBytes    = int64(10 * 1024 * 1024 * 1024)
 	defaultLocalResultMaxBytes        = int64(256 * 1024 * 1024)
 	defaultLocalResultMaxTaskBytes    = int64(512 * 1024 * 1024)
 )
@@ -41,41 +39,38 @@ type localBinaryDescriptor struct {

 type localBinaryMaterializer struct {
 	service       *Service
-	taskDir       string
-	writeFiles    bool
 	enforceLimits bool
 	totalBytes    int64
 	seen          map[string]struct{}
-	createdFiles  []string
 }

-// materializeLocalBinaryResult replaces every inline binary value with a
-// bounded placeholder after atomically writing and verifying the bytes locally.
+// materializeLocalBinaryResult is retained as an internal compatibility seam,
+// but new materialization always targets object storage. Historical local
+// placeholders remain readable through HydrateTaskResult.
 func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
-	if _, _, err := s.transformLocalBinaryResult(ctx, taskID, result, false, true); err != nil {
+	hadInline := TaskResultHasInlineBinary(result)
+	if !hadInline {
+		return result, false, nil
+	}
+	next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
+	if err != nil {
 		return nil, false, err
 	}
-	return s.transformLocalBinaryResult(ctx, taskID, result, true, true)
+	return next, !TaskResultHasInlineBinary(next), nil
 }

-func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string, result map[string]any, writeFiles bool, enforceLimits bool) (map[string]any, bool, error) {
-	root := s.localBinaryResultRoot()
-	taskDir := filepath.Join(root, safeLocalBinaryTaskDir(taskID))
+func (s *Service) transformLocalBinaryResult(ctx context.Context, _ string, result map[string]any, _ bool, enforceLimits bool) (map[string]any, bool, error) {
 	materializer := &localBinaryMaterializer{
 		service:       s,
-		taskDir:       taskDir,
-		writeFiles:    writeFiles,
 		enforceLimits: enforceLimits,
 		seen:          map[string]struct{}{},
 	}
 	next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0)
 	if err != nil {
-		materializer.rollbackCreatedFiles()
 		return nil, false, err
 	}
 	mapped, ok := next.(map[string]any)
 	if !ok {
-		materializer.rollbackCreatedFiles()
 		return nil, false, &clients.ClientError{
 			Code:       "result_binary_not_materialized",
 			Message:    "generated result is not a JSON object",
@@ -83,28 +78,28 @@ func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string,
 			Retryable:  false,
 		}
 	}
-	if changed && enforceLimits && !writeFiles {
-		if err := os.MkdirAll(root, 0o750); err != nil {
-			return nil, false, localBinaryStorageError(err)
-		}
-		if err := ensureLocalBinaryDiskHeadroom(root, materializer.totalBytes, s.localResultMinFreeBytes()); err != nil {
-			return nil, false, err
-		}
-	}
 	return mapped, changed, nil
 }

 // MaterializeTaskResultForStorage exposes the same verified materialization
 // path to the explicit historical maintenance command.
 func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
-	return s.materializeLocalBinaryResult(ctx, taskID, result)
+	hadInline := TaskResultHasInlineBinary(result)
+	if !hadInline {
+		return result, false, nil
+	}
+	next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
+	if err != nil {
+		return nil, false, err
+	}
+	return next, hadInline && !TaskResultHasInlineBinary(next), nil
 }

-// CompactExpiredTaskResultForStorage computes the same deterministic
-// placeholders without creating files that are already outside the recovery
-// window.
+// CompactExpiredTaskResultForStorage keeps the historical maintenance API but
+// now uses the same object-storage path as live results. New GatewayBinary
+// placeholders are never created; their parser remains read-only compatibility.
 func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
-	return s.transformLocalBinaryResult(ctx, taskID, result, false, false)
+	return s.MaterializeTaskResultForStorage(ctx, taskID, result)
 }

 func TaskResultHasInlineBinary(result map[string]any) bool {
@@ -239,15 +234,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
 				Retryable:  false,
 			}
 		}
-		if m.writeFiles {
-			created, err := m.service.writeLocalBinaryResult(m.taskDir, digestHex, payload)
-			if err != nil {
-				return nil, false, err
-			}
-			if created {
-				m.createdFiles = append(m.createdFiles, filepath.Join(m.taskDir, digestHex+".bin"))
-			}
-		}
 		m.seen[digestHex] = struct{}{}
 		m.totalBytes += size
 	}
@@ -261,94 +247,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
 	return localBinaryPlaceholder(descriptor), true, nil
 }

-func (m *localBinaryMaterializer) rollbackCreatedFiles() {
-	for _, path := range m.createdFiles {
-		_ = os.Remove(path)
-	}
-	_ = os.Remove(m.taskDir)
-	m.createdFiles = nil
-}
-
-func (s *Service) writeLocalBinaryResult(taskDir string, digestHex string, payload []byte) (bool, error) {
-	if err := os.MkdirAll(taskDir, 0o750); err != nil {
-		return false, localBinaryStorageError(err)
-	}
-	targetPath := filepath.Join(taskDir, digestHex+".bin")
-	if info, err := os.Stat(targetPath); err == nil {
-		if !info.IsDir() && info.Size() == int64(len(payload)) {
-			if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err == nil {
-				now := time.Now()
-				if err := os.Chtimes(targetPath, now, now); err != nil {
-					return false, localBinaryStorageError(err)
-				}
-				return false, nil
-			}
-		}
-		return false, &clients.ClientError{
-			Code:       "binary_result_corrupted",
-			Message:    "existing local result file does not match its content hash",
-			StatusCode: 500,
-			Retryable:  false,
-		}
-	} else if !errors.Is(err, os.ErrNotExist) {
-		return false, localBinaryStorageError(err)
-	}
-	if err := ensureLocalBinaryDiskHeadroom(taskDir, int64(len(payload)), s.localResultMinFreeBytes()); err != nil {
-		return false, err
-	}
-	tempFile, err := os.CreateTemp(taskDir, ".gateway-result-*")
-	if err != nil {
-		return false, localBinaryStorageError(err)
-	}
-	tempPath := tempFile.Name()
-	cleanup := func() {
-		_ = tempFile.Close()
-		_ = os.Remove(tempPath)
-	}
-	if err := tempFile.Chmod(0o640); err != nil {
-		cleanup()
-		return false, localBinaryStorageError(err)
-	}
-	if _, err := tempFile.Write(payload); err != nil {
-		cleanup()
-		return false, localBinaryStorageError(err)
-	}
-	if err := tempFile.Sync(); err != nil {
-		cleanup()
-		return false, localBinaryStorageError(err)
-	}
-	if err := tempFile.Close(); err != nil {
-		_ = os.Remove(tempPath)
-		return false, localBinaryStorageError(err)
-	}
-	if err := os.Rename(tempPath, targetPath); err != nil {
-		_ = os.Remove(tempPath)
-		return false, localBinaryStorageError(err)
-	}
-	if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err != nil {
-		_ = os.Remove(targetPath)
-		return false, err
-	}
-	return true, nil
-}
-
-func ensureLocalBinaryDiskHeadroom(path string, incomingBytes int64, minFreeBytes int64) error {
-	var stat syscall.Statfs_t
-	if err := syscall.Statfs(path, &stat); err != nil {
-		return localBinaryStorageError(err)
-	}
-	freeBytes := int64(stat.Bavail) * int64(stat.Bsize)
-	if freeBytes-incomingBytes < minFreeBytes {
-		return &clients.ClientError{
-			Code:       "local_result_storage_unavailable",
-			Message:    "local result storage does not have enough free space",
-			StatusCode: 503,
-			Retryable:  false,
-		}
-	}
-	return nil
-}
-
 func localBinaryStorageError(err error) error {
 	return &clients.ClientError{
 		Code:       "local_result_storage_unavailable",
@@ -406,6 +304,30 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
 	}
 	switch typed := value.(type) {
 	case map[string]any:
+		refreshedAccessURL := false
+		if asset, upload, ok := generatedResultUploadReference(typed); ok {
+			accessURL, err := s.requestAssetAccessURL(ctx, asset)
+			if err != nil {
+				return nil, false, err
+			}
+			next := make(map[string]any, len(typed))
+			for key, item := range typed {
+				next[key] = item
+			}
+			nextUpload := make(map[string]any, len(upload))
+			for key, item := range upload {
+				nextUpload[key] = item
+			}
+			nextUpload["url"] = accessURL
+			next["upload"] = nextUpload
+			for _, key := range []string{"url", "image_url", "video_url", "audio_url"} {
+				if _, exists := next[key]; exists {
+					next[key] = accessURL
+				}
+			}
+			typed = next
+			refreshedAccessURL = true
+		}
 		if ref, ok := generatedResultAssetReference(typed); ok {
 			payload, contentType, err := s.readGeneratedResultAsset(ctx, ref)
 			if err != nil {
@@ -418,7 +340,7 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
 			return encoded, true, nil
 		}
 		next := make(map[string]any, len(typed))
-		changed := false
+		changed := refreshedAccessURL
 		for key, childValue := range typed {
 			child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
 			if err != nil {
@@ -465,6 +387,27 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
 	}
 }

+func generatedResultUploadReference(value map[string]any) (store.RequestAsset, map[string]any, bool) {
+	upload, ok := value["upload"].(map[string]any)
+	if !ok {
+		return store.RequestAsset{}, nil, false
+	}
+	objectKey := strings.TrimSpace(stringFromAny(upload["objectKey"]))
+	channel, _ := upload["storageChannel"].(map[string]any)
+	channelKey := strings.TrimSpace(stringFromAny(channel["channelKey"]))
+	if objectKey == "" || channelKey == "" {
+		return store.RequestAsset{}, nil, false
+	}
+	return store.RequestAsset{
+		URL:               stringFromAny(upload["url"]),
+		StorageProvider:   stringFromAny(channel["provider"]),
+		StorageChannelID:  stringFromAny(channel["id"]),
+		StorageChannelKey: channelKey,
+		ObjectKey:         objectKey,
+		AccessScope:       stringFromAny(upload["accessScope"]),
+	}, upload, true
+}
+
 func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bool) {
 	ref, ok := value["assetRef"].(map[string]any)
 	if !ok {
@@ -475,10 +418,14 @@ func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bo
 		return store.RequestAsset{}, false
 	}
 	asset := store.RequestAsset{
-		SHA256:          strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
-		ContentType:     firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
-		URL:             firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
-		StorageProvider: stringFromAny(ref["storageProvider"]),
+		SHA256:            strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
+		ContentType:       firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
+		URL:               firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
+		StorageProvider:   stringFromAny(ref["storageProvider"]),
+		StorageChannelID:  stringFromAny(ref["storageChannelId"]),
+		StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
+		ObjectKey:         stringFromAny(ref["objectKey"]),
+		AccessScope:       stringFromAny(ref["accessScope"]),
 	}
 	if size := floatFromAny(ref["size"]); size > 0 {
 		asset.ByteSize = int64(size)
@@ -767,13 +714,6 @@ func (s *Service) localResultTTLHours() int {
 	return s.cfg.LocalResultTTLHours
 }

-func (s *Service) localResultMinFreeBytes() int64 {
-	if s.cfg.LocalResultMinFreeBytes <= 0 {
-		return defaultLocalResultMinFreeBytes
-	}
-	return s.cfg.LocalResultMinFreeBytes
-}
-
 func (s *Service) localResultMaxBytes() int64 {
 	if s.cfg.LocalResultMaxBytes <= 0 {
 		return defaultLocalResultMaxBytes