feat(storage): 统一二进制对象存储与公开错误

新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
@@ -51,6 +51,8 @@ type Server struct {
 	callbacks          map[string]map[int64]int
 	videoByIdempotency map[string]string
 	geminiIdempotency  map[string]struct{}
+	storageObjects     map[string]fixture
+	storageAttempts    map[string]int
 	pngSmall           string
 	pngLarge           string
 	pngPeak            string
@@ -75,6 +77,11 @@ type Report struct {
 	DuplicateCallbacks   int64            `json:"duplicateCallbacks"`
 	MissingIdempotency   int64            `json:"missingIdempotencyKeys"`
 	DuplicateSubmissions int64            `json:"duplicateSubmissionAttempts"`
+	StoragePuts          int64            `json:"storagePuts"`
+	StorageGets          int64            `json:"storageGets"`
+	StorageHeads         int64            `json:"storageHeads"`
+	StorageDeletes       int64            `json:"storageDeletes"`
+	StorageFailures      int64            `json:"storageFailures"`
 }

 type videoTask struct {
@@ -117,6 +124,8 @@ func New(config Config) *Server {
 		callbacks:          map[string]map[int64]int{},
 		videoByIdempotency: map[string]string{},
 		geminiIdempotency:  map[string]struct{}{},
+		storageObjects:     map[string]fixture{},
+		storageAttempts:    map[string]int{},
 		report: Report{
 			VideoReferenceCounts: map[string]int64{},
 			VideoRoleCounts:      map[string]int64{},
@@ -139,9 +148,107 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("GET /media/{asset}", s.getMedia)
 	mux.HandleFunc("GET /fixtures/{asset}", s.getFixture)
 	mux.HandleFunc("POST /callbacks", s.collectCallback)
+	mux.HandleFunc("PUT /storage/{profile}/{object...}", s.objectStorage)
+	mux.HandleFunc("GET /storage/{profile}/{object...}", s.objectStorage)
+	mux.HandleFunc("HEAD /storage/{profile}/{object...}", s.objectStorage)
+	mux.HandleFunc("DELETE /storage/{profile}/{object...}", s.objectStorage)
 	return mux
 }

+func (s *Server) objectStorage(w http.ResponseWriter, r *http.Request) {
+	profile := strings.ToLower(strings.TrimSpace(r.PathValue("profile")))
+	object := strings.TrimLeft(strings.TrimSpace(r.PathValue("object")), "/")
+	if object == "" {
+		http.NotFound(w, r)
+		return
+	}
+	baseProfile := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(profile, "-transient"), "-auth"), "-fail")
+	if baseProfile != "oss" && baseProfile != "s3" {
+		http.NotFound(w, r)
+		return
+	}
+	key := profile + "/" + object
+	s.mu.Lock()
+	s.storageAttempts[key]++
+	attempt := s.storageAttempts[key]
+	if strings.HasSuffix(profile, "-auth") {
+		s.report.StorageFailures++
+		s.mu.Unlock()
+		http.Error(w, "forbidden", http.StatusForbidden)
+		return
+	}
+	if strings.HasSuffix(profile, "-fail") || (strings.HasSuffix(profile, "-transient") && attempt == 1) {
+		s.report.StorageFailures++
+		s.mu.Unlock()
+		http.Error(w, "temporary storage failure", http.StatusServiceUnavailable)
+		return
+	}
+	switch r.Method {
+	case http.MethodPut:
+		s.mu.Unlock()
+		payload, err := io.ReadAll(io.LimitReader(r.Body, maxProtocolBodyBytes+1))
+		if err != nil || len(payload) > maxProtocolBodyBytes {
+			http.Error(w, "invalid object", http.StatusBadRequest)
+			return
+		}
+		s.mu.Lock()
+		s.storageObjects[key] = fixture{ContentType: firstNonEmpty(r.Header.Get("Content-Type"), "application/octet-stream"), Payload: payload}
+		s.report.StoragePuts++
+		s.mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	case http.MethodGet:
+		item, ok := s.storageObjects[key]
+		if ok {
+			s.report.StorageGets++
+		}
+		s.mu.Unlock()
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", item.ContentType)
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write(item.Payload)
+	case http.MethodHead:
+		item, ok := s.storageObjects[key]
+		if ok {
+			s.report.StorageHeads++
+		}
+		s.mu.Unlock()
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", item.ContentType)
+		w.Header().Set("Content-Length", strconv.Itoa(len(item.Payload)))
+		w.WriteHeader(http.StatusOK)
+	case http.MethodDelete:
+		_, ok := s.storageObjects[key]
+		delete(s.storageObjects, key)
+		if ok {
+			s.report.StorageDeletes++
+		}
+		s.mu.Unlock()
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	default:
+		s.mu.Unlock()
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if value = strings.TrimSpace(value); value != "" {
+			return value
+		}
+	}
+	return ""
+}
+
 func (s *Server) health(w http.ResponseWriter, _ *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]any{"ok": true})
 }
@@ -315,6 +315,86 @@ func TestVolcesProtocolAcceptsThreeSixAndNineReferenceImages(t *testing.T) {
 	}
 }

+func TestObjectStorageEmulationSupportsRetryLifecycleAndFaults(t *testing.T) {
+	server := httptest.NewServer(New(Config{}).Handler())
+	defer server.Close()
+	objectURL := server.URL + "/storage/s3-transient/bucket/media/result.png"
+
+	request, _ := http.NewRequest(http.MethodPut, objectURL, strings.NewReader("image-bytes"))
+	request.Header.Set("Content-Type", "image/png")
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = response.Body.Close()
+	if response.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("first transient PUT status=%d", response.StatusCode)
+	}
+	request, _ = http.NewRequest(http.MethodPut, objectURL, strings.NewReader("image-bytes"))
+	request.Header.Set("Content-Type", "image/png")
+	response, err = http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("retried PUT status=%d", response.StatusCode)
+	}
+
+	response, err = http.Get(objectURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	payload, _ := io.ReadAll(response.Body)
+	_ = response.Body.Close()
+	if response.StatusCode != http.StatusOK || string(payload) != "image-bytes" {
+		t.Fatalf("GET status=%d payload=%q", response.StatusCode, payload)
+	}
+	request, _ = http.NewRequest(http.MethodHead, objectURL, nil)
+	response, err = http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("HEAD status=%d", response.StatusCode)
+	}
+	request, _ = http.NewRequest(http.MethodDelete, objectURL, nil)
+	response, err = http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = response.Body.Close()
+	if response.StatusCode != http.StatusNoContent {
+		t.Fatalf("DELETE status=%d", response.StatusCode)
+	}
+
+	for profile, wantStatus := range map[string]int{"oss-auth": http.StatusForbidden, "s3-fail": http.StatusServiceUnavailable} {
+		request, _ = http.NewRequest(http.MethodPut, server.URL+"/storage/"+profile+"/probe.bin", strings.NewReader("probe"))
+		response, err = http.DefaultClient.Do(request)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = response.Body.Close()
+		if response.StatusCode != wantStatus {
+			t.Fatalf("%s status=%d, want %d", profile, response.StatusCode, wantStatus)
+		}
+	}
+
+	response, err = http.Get(server.URL + "/report")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer response.Body.Close()
+	var report Report
+	if err := json.NewDecoder(response.Body).Decode(&report); err != nil {
+		t.Fatal(err)
+	}
+	if report.StoragePuts != 1 || report.StorageGets != 1 || report.StorageHeads != 1 || report.StorageDeletes != 1 || report.StorageFailures != 3 {
+		t.Fatalf("unexpected storage report: %+v", report)
+	}
+}
+
 func postIdempotent(url string, body []byte, key string) (*http.Response, error) {
 	request, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
 	if err != nil {