feat(storage): 统一二进制对象存储与公开错误

新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
@@ -7,11 +7,11 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"

+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
 )
@@ -227,38 +227,6 @@ func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) {
 	}
 }

-func TestGeneratedAssetDecisionPreservesInlineWhenPolicyUploadNone(t *testing.T) {
-	item := map[string]any{
-		"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
-	}
-
-	decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	if decision.Inline != nil || decision.URL != nil {
-		t.Fatalf("upload_none should not transfer inline payloads: %+v", decision)
-	}
-	if len(decision.StripKeys) != 0 {
-		t.Fatalf("upload_none should preserve b64_json for the caller: %+v", decision.StripKeys)
-	}
-}
-
-func TestGeneratedAssetDecisionPreservesInlineAlongsideURLWhenPolicyUploadNone(t *testing.T) {
-	item := map[string]any{
-		"url":      "https://cdn.example.com/generated.png",
-		"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
-	}
-
-	decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	if decision.Inline != nil || decision.URL != nil || len(decision.StripKeys) != 0 {
-		t.Fatalf("upload_none should preserve the upstream URL and base64 fields unchanged: %+v", decision)
-	}
-}
-
 func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
 	tests := []struct {
 		name       string
@@ -276,9 +244,9 @@ func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
 			want:       generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true, PreserveInlineMedia: false},
 		},
 		{
-			name:       "upload none",
+			name:       "legacy upload none becomes default",
 			policyName: store.FileStorageResultUploadPolicyUploadNone,
-			want:       generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true},
+			want:       generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: false, PreserveInlineMedia: false},
 		},
 	}

@@ -330,8 +298,10 @@ func TestAcceptanceGeneratedMediaAllowsOnlyExactEmulatorOrigin(t *testing.T) {
 }

 func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *testing.T) {
-	storageDir := t.TempDir()
-	service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	service := &Service{}
+	channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
 	payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
 	encoded := base64.StdEncoding.EncodeToString(payload)
 	result := map[string]any{
@@ -348,7 +318,7 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
 		"images.edits",
 		result,
 		defaultGeneratedAssetUploadPolicy(),
-		nil,
+		channels,
 		true,
 		0,
 	)
@@ -363,15 +333,8 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
 	if !ok {
 		t.Fatalf("nested binary should be replaced by an asset reference: %+v", nested)
 	}
-	if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-nested-binary-01-") {
-		t.Fatalf("unexpected local static URL: %s", urlValue)
-	}
-	entries, err := os.ReadDir(storageDir)
-	if err != nil {
-		t.Fatalf("read generated storage: %v", err)
-	}
-	if len(entries) != 1 {
-		t.Fatalf("expected one localized result file, got %d", len(entries))
+	if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") {
+		t.Fatalf("unexpected object storage URL: %s", urlValue)
 	}
 }

@@ -405,9 +368,8 @@ func TestGeneratedAssetFileNameIsUniqueAndTyped(t *testing.T) {
 	}
 }

-func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
-	storageDir := t.TempDir()
-	service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
+func TestUploadGeneratedAssetFailsWithoutObjectStorage(t *testing.T) {
+	service := &Service{}
 	payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}
 	asset := &generatedInlineAsset{
 		Bytes:       payload,
@@ -416,44 +378,14 @@ func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
 		SourceKey:   "b64_json",
 	}

-	upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
-	if err != nil {
+	_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
+	if clients.ErrorCode(err) != "storage_write_failed" {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if contentType != "image/png" || kind != "image" || strategy != "local_static_inline_media" {
-		t.Fatalf("unexpected local upload metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
-	}
-	urlValue := stringFromAny(upload["url"])
-	if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-123-01-") || !strings.HasSuffix(urlValue, ".png") {
-		t.Fatalf("unexpected local static URL: %s", urlValue)
-	}
-	expiresAt, err := time.Parse(time.RFC3339, stringFromAny(upload["expiresAt"]))
-	if err != nil {
-		t.Fatalf("local static upload should expose expiresAt: %+v", upload)
-	}
-	remaining := time.Until(expiresAt)
-	if remaining < 23*time.Hour+59*time.Minute || remaining > 24*time.Hour+time.Minute {
-		t.Fatalf("local static upload should expire after one day, remaining=%s", remaining)
-	}
-	entries, err := os.ReadDir(storageDir)
-	if err != nil {
-		t.Fatalf("failed to read local static dir: %v", err)
-	}
-	if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
-		t.Fatalf("expected one PNG file in local static dir, got %+v", entries)
-	}
-	stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
-	if err != nil {
-		t.Fatalf("failed to read local static file: %v", err)
-	}
-	if !bytes.Equal(stored, payload) {
-		t.Fatalf("stored payload does not match source payload")
-	}
 }

-func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
-	storageDir := t.TempDir()
-	service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
+func TestUploadGeneratedAudioFailsWithoutObjectStorage(t *testing.T) {
+	service := &Service{}
 	asset := &generatedInlineAsset{
 		Bytes:       []byte("inline audio payload"),
 		ContentType: "audio/mpeg",
@@ -461,32 +393,17 @@ func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
 		SourceKey:   "content",
 	}

-	upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
-	if err != nil {
+	_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
+	if clients.ErrorCode(err) != "storage_write_failed" {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if contentType != "audio/mpeg" || kind != "audio" || strategy != "local_static_inline_media" {
-		t.Fatalf("unexpected local audio metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
-	}
-	urlValue := stringFromAny(upload["url"])
-	if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-tts-01-") || !strings.HasSuffix(urlValue, ".mp3") {
-		t.Fatalf("unexpected local audio URL: %s", urlValue)
-	}
-	if stringFromAny(upload["expiresAt"]) == "" {
-		t.Fatalf("local audio static upload should expose expiresAt: %+v", upload)
-	}
-	entries, err := os.ReadDir(storageDir)
-	if err != nil {
-		t.Fatalf("failed to read local static dir: %v", err)
-	}
-	if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".mp3") {
-		t.Fatalf("expected one MP3 file in local static dir, got %+v", entries)
-	}
 }

 func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *testing.T) {
-	storageDir := t.TempDir()
-	service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	service := &Service{}
+	channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
 	payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
 	raw := map[string]any{
 		"candidates": []any{
@@ -506,7 +423,7 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
 	}
 	index := 0

-	uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), nil, &index)
+	uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
 	if err != nil {
 		t.Fatalf("upload raw media: %v", err)
 	}
@@ -528,22 +445,47 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
 	if ref["sha256"] == "" || ref["contentType"] != "image/png" || ref["size"] != len(payload) {
 		t.Fatalf("unexpected asset ref: %+v", ref)
 	}
-	if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-raw-01-") || !strings.HasSuffix(urlValue, ".png") {
+	if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") || !strings.HasSuffix(urlValue, ".png") {
 		t.Fatalf("unexpected raw media URL: %s", urlValue)
 	}
 	if inlineData["data"] == base64.StdEncoding.EncodeToString(payload) {
 		t.Fatal("raw inlineData still contains base64 payload")
 	}
-	entries, err := os.ReadDir(storageDir)
-	if err != nil {
-		t.Fatalf("read generated storage: %v", err)
+}
+
+func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithAssetRefs(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	service := &Service{}
+	channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-buffer-result")}
+	raw := map[string]any{
+		"buffer": map[string]any{
+			"type": "Buffer", "mimeType": "image/png", "data": []any{float64(0x89), float64('P'), float64('N'), float64('G')},
+		},
+		"audio_bytes": []any{float64('I'), float64('D'), float64('3')},
+		"direct":      []byte("direct bytes"),
 	}
-	if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
-		t.Fatalf("expected one generated PNG, got %+v", entries)
+	index := 0
+	uploaded, changed, err := service.uploadGeneratedRawMediaValue(t.Context(), "task-buffer", "images.generations", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !changed || index != 3 {
+		t.Fatalf("changed=%v uploads=%d", changed, index)
+	}
+	next := uploaded.(map[string]any)
+	for _, key := range []string{"buffer", "audio_bytes", "direct"} {
+		item, ok := next[key].(map[string]any)
+		if !ok || item["assetRef"] == nil || item["upload"] == nil {
+			t.Fatalf("%s was not objectified: %#v", key, next[key])
+		}
+	}
+	if TaskResultHasInlineBinary(next) {
+		t.Fatalf("objectified result still contains inline binary: %#v", next)
 	}
 }

-func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
+func TestUploadFileFailsWithoutObjectStorageAndCreatesNoLocalFile(t *testing.T) {
 	storageDir := t.TempDir()
 	service := &Service{cfg: config.Config{
 		LocalUploadedStorageDir: storageDir,
@@ -552,43 +494,21 @@ func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
 	}}
 	payload := []byte("%PDF-1.4")

-	upload, err := service.UploadFile(context.Background(), FileUploadPayload{
+	_, err := service.UploadFile(context.Background(), FileUploadPayload{
 		Bytes:       payload,
 		ContentType: "application/pdf",
 		FileName:    "用户文件.png",
 		Source:      "playground",
 	})
-	if err != nil {
+	if clients.ErrorCode(err) != "storage_write_failed" {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	urlValue := stringFromAny(upload["url"])
-	if !strings.HasPrefix(urlValue, "/static/uploaded/") || !strings.HasSuffix(urlValue, ".pdf") {
-		t.Fatalf("unexpected uploaded local static URL: %s", urlValue)
-	}
-	if stringFromAny(upload["expiresAt"]) == "" {
-		t.Fatalf("local uploaded static file should expose expiresAt: %+v", upload)
-	}
-	storageChannel, _ := upload["storageChannel"].(map[string]any)
-	if stringFromAny(storageChannel["provider"]) != "local_static" {
-		t.Fatalf("expected local static provider metadata, got %+v", upload["storageChannel"])
-	}
-	assetStorage, _ := upload["assetStorage"].(map[string]any)
-	if stringFromAny(assetStorage["strategy"]) != "local_static_upload" || stringFromAny(assetStorage["scene"]) != store.FileStorageSceneUpload {
-		t.Fatalf("unexpected upload asset storage metadata: %+v", assetStorage)
-	}
 	entries, err := os.ReadDir(storageDir)
 	if err != nil {
 		t.Fatalf("failed to read uploaded static dir: %v", err)
 	}
-	if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".pdf") {
-		t.Fatalf("expected one PDF file in uploaded static dir, got %+v", entries)
-	}
-	stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
-	if err != nil {
-		t.Fatalf("failed to read uploaded static file: %v", err)
-	}
-	if !bytes.Equal(stored, payload) {
-		t.Fatalf("stored uploaded payload does not match source payload")
+	if len(entries) != 0 {
+		t.Fatalf("unexpected local files: %+v", entries)
 	}
 }