feat(storage): 统一二进制对象存储与公开错误

新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
@@ -31,7 +31,21 @@ func newLocalBinaryTestService(t *testing.T) *Service {
 	}}
 }

-func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
+func writeHistoricalLocalBinaryFixture(t *testing.T, service *Service, taskID string, payload []byte) string {
+	t.Helper()
+	digest := sha256.Sum256(payload)
+	taskDir := filepath.Join(service.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID))
+	if err := os.MkdirAll(taskDir, 0o750); err != nil {
+		t.Fatalf("create historical result fixture directory: %v", err)
+	}
+	path := filepath.Join(taskDir, hex.EncodeToString(digest[:])+".bin")
+	if err := os.WriteFile(path, payload, 0o640); err != nil {
+		t.Fatalf("write historical result fixture: %v", err)
+	}
+	return path
+}
+
+func TestHydrateHistoricalLocalBinaryResult(t *testing.T) {
 	service := newLocalBinaryTestService(t)
 	payload := []byte("one binary result shared across representations")
 	encoded := base64.StdEncoding.EncodeToString(payload)
@@ -49,7 +63,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
 		},
 	}

-	persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-123", input)
+	persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-123", input, false, true)
 	if err != nil {
 		t.Fatalf("materialize local binary result: %v", err)
 	}
@@ -76,6 +90,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
 	if len(persistentJSON) > 32*1024 {
 		t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
 	}
+	writeHistoricalLocalBinaryFixture(t, service, "task-123", payload)

 	taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
 	entries, err := os.ReadDir(taskDir)
@@ -155,19 +170,53 @@ func TestHydrateGeneratedResultAssetReference(t *testing.T) {
 	assertClientErrorCode(t, err, "binary_result_corrupted")
 }

+func TestHydrateGeneratedResultRefreshesPrivateObjectURL(t *testing.T) {
+	cfg := config.Config{
+		MediaOSSDirectEnabled:   true,
+		MediaOSSEndpoint:        "https://oss.example.com",
+		MediaOSSBucket:          "media-bucket",
+		MediaOSSAccessKeyID:     "access-id",
+		MediaOSSAccessKeySecret: "access-secret",
+		MediaOSSObjectPrefix:    "media",
+	}
+	service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
+	result := map[string]any{
+		"data": []any{map[string]any{
+			"url":       "https://expired.example.com/result.png",
+			"image_url": "https://expired.example.com/result.png",
+			"upload": map[string]any{
+				"url":         "https://expired.example.com/result.png",
+				"objectKey":   "media/image_result/2026/08/04/hash.png",
+				"accessScope": "private",
+				"storageChannel": map[string]any{
+					"channelKey": "environment-direct-oss",
+					"provider":   "aliyun_oss",
+				},
+			},
+		}},
+	}
+	hydrated, err := service.HydrateTaskResult(t.Context(), "task-private-url", result)
+	if err != nil {
+		t.Fatal(err)
+	}
+	item := hydrated["data"].([]any)[0].(map[string]any)
+	for _, key := range []string{"url", "image_url"} {
+		value := stringFromAny(item[key])
+		if !strings.Contains(value, "OSSAccessKeyId=access-id") || strings.Contains(value, "expired.example.com") {
+			t.Fatalf("%s was not refreshed: %q", key, value)
+		}
+	}
+}
+
 func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
 	service := newLocalBinaryTestService(t)
 	service.cfg.LocalResultTTLHours = 1
 	encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
-	persistent, _, err := service.materializeLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded})
+	persistent, _, err := service.transformLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded}, false, true)
 	if err != nil {
 		t.Fatalf("materialize fixture: %v", err)
 	}
-	entries, err := os.ReadDir(filepath.Join(service.localBinaryResultRoot(), "task-expired"))
-	if err != nil || len(entries) != 1 {
-		t.Fatalf("read fixture result: entries=%v err=%v", entries, err)
-	}
-	path := filepath.Join(service.localBinaryResultRoot(), "task-expired", entries[0].Name())
+	path := writeHistoricalLocalBinaryFixture(t, service, "task-expired", []byte("expiring result"))
 	old := time.Now().Add(-2 * time.Hour)
 	if err := os.Chtimes(path, old, old); err != nil {
 		t.Fatalf("age fixture: %v", err)
@@ -186,17 +235,17 @@ func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T)
 	assertClientErrorCode(t, err, "binary_result_corrupted")
 }

-func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
+func TestHistoricalLocalBinaryFixtureEnforcesLimitsAndKeepsText(t *testing.T) {
 	service := newLocalBinaryTestService(t)
 	service.cfg.LocalResultMaxBytes = 4
-	_, _, err := service.materializeLocalBinaryResult(context.Background(), "task-large", map[string]any{
+	_, _, err := service.transformLocalBinaryResult(context.Background(), "task-large", map[string]any{
 		"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
-	})
+	}, false, true)
 	assertClientErrorCode(t, err, "binary_result_too_large")

-	persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-text", map[string]any{
+	persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-text", map[string]any{
 		"message": "YWJjZA==",
-	})
+	}, false, true)
 	if err != nil || changed || persistent["message"] != "YWJjZA==" {
 		t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
 	}
@@ -204,18 +253,30 @@ func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
 	service = newLocalBinaryTestService(t)
 	service.cfg.LocalResultMaxBytes = 1024
 	service.cfg.LocalResultMaxTaskBytes = 8
-	_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
+	_, _, err = service.transformLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
 		"first_base64":  base64.StdEncoding.EncodeToString([]byte("12345")),
 		"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
-	})
+	}, false, true)
 	assertClientErrorCode(t, err, "binary_result_too_large")
 	if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
 		t.Fatalf("preflight task limit must not create partial files: %v", statErr)
 	}
 }

-func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
+func TestCompactExpiredTaskResultUsesObjectStorageAndDoesNotWriteFiles(t *testing.T) {
+	var uploads int
+	storageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			t.Fatalf("method=%s, want PUT", r.Method)
+		}
+		uploads++
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer storageServer.Close()
 	service := newLocalBinaryTestService(t)
+	service.directOSS = &directOSSUploader{
+		endpoint: storageServer.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media",
+	}
 	service.cfg.LocalResultMaxBytes = 1
 	encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))

@@ -225,17 +286,18 @@ func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
 	if err != nil {
 		t.Fatalf("compact expired result: %v", err)
 	}
-	if !changed || !localBinaryResultHasPlaceholders(persistent) {
-		t.Fatalf("expired result was not compacted: %+v", persistent)
+	if !changed || localBinaryResultHasPlaceholders(persistent) || TaskResultHasInlineBinary(persistent) {
+		t.Fatalf("expired result was not objectified: %+v", persistent)
+	}
+	if uploads != 1 {
+		t.Fatalf("object storage uploads=%d, want 1", uploads)
 	}
 	if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
 		t.Fatalf("expired compaction must not create a task directory: %v", err)
 	}
-	_, err = service.HydrateTaskResult(context.Background(), "task-old", persistent)
-	assertClientErrorCode(t, err, "binary_result_expired")
 }

-func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
+func TestHistoricalLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
 	err := localBinaryStorageError(errors.New("write failed"))
 	assertClientErrorCode(t, err, "local_result_storage_unavailable")
 	if clients.IsRetryable(err) {
@@ -247,16 +309,6 @@ func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
 	if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
 		t.Fatal("local result storage failure must not fail over to another provider")
 	}
-
-	service := newLocalBinaryTestService(t)
-	service.cfg.LocalResultMinFreeBytes = 1 << 62
-	_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-low-disk", map[string]any{
-		"b64_json": base64.StdEncoding.EncodeToString([]byte("disk preflight")),
-	})
-	assertClientErrorCode(t, err, "local_result_storage_unavailable")
-	if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-low-disk")); !errors.Is(statErr, os.ErrNotExist) {
-		t.Fatalf("disk preflight must not create partial task files: %v", statErr)
-	}
 }

 func bytesToAny(payload []byte) []any {