perf(storage): 拦截任务二进制并本地暂存结果

原因：任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON，导致 TOAST 与备份体积快速增长。影响：新增统一 JSON 持久化关口；upload_none 将二进制原子写入本地结果目录，数据库仅保存带 SHA-256 的有界占位符；任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。风险：本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500；空间不足时返回 503 且不重试上游。未自动执行历史治理。验证：三种真实图片模型同步/异步与幂等重放通过；Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
2026-07-24 21:13:09 +08:00
parent 2457de6a56
commit 4f163ea6d7
31 changed files with 2067 additions and 30 deletions
@@ -0,0 +1,113 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"log/slog"
+	"os"
+	"time"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+)
+
+func main() {
+	apply := flag.Bool("apply", false, "persist compacted results; default is dry-run")
+	batchSize := flag.Int("batch-size", 100, "rows per batch, maximum 100")
+	maxBatches := flag.Int("max-batches", 10, "maximum batches for one invocation")
+	afterID := flag.String("after-id", "", "resume after this task UUID")
+	flag.Parse()
+
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+	cfg := config.Load()
+	if err := cfg.Validate(); err != nil {
+		logger.Error("invalid gateway configuration", "error", err)
+		os.Exit(1)
+	}
+	if *batchSize < 1 || *batchSize > 100 || *maxBatches < 1 {
+		logger.Error("invalid backfill bounds", "batchSize", *batchSize, "maxBatches", *maxBatches)
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+	db, err := store.Connect(ctx, cfg.DatabaseURL)
+	if err != nil {
+		logger.Error("connect postgres failed", "error", err)
+		os.Exit(1)
+	}
+	defer db.Close()
+
+	service := runner.New(cfg, db, logger)
+	cursor := *afterID
+	scanned := 0
+	matched := 0
+	updated := 0
+	expired := 0
+	for batch := 0; batch < *maxBatches; batch++ {
+		items, err := db.ListTaskBinaryResultBackfillBatch(ctx, cursor, *batchSize)
+		if err != nil {
+			logger.Error("list binary result backfill batch failed", "afterId", cursor, "error", err)
+			os.Exit(1)
+		}
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			cursor = item.ID
+			scanned++
+			if !runner.TaskResultHasInlineBinary(item.Result) {
+				continue
+			}
+			matched++
+			if !*apply {
+				continue
+			}
+			isExpired := item.FinishedAt.Before(time.Now().Add(-time.Duration(localResultTTLHours(cfg)) * time.Hour))
+			var persistent map[string]any
+			var changed bool
+			if isExpired {
+				persistent, changed, err = service.CompactExpiredTaskResultForStorage(ctx, item.ID, item.Result)
+			} else {
+				persistent, changed, err = service.MaterializeTaskResultForStorage(ctx, item.ID, item.Result)
+			}
+			if err != nil {
+				logger.Error("materialize historical binary result failed", "taskId", item.ID, "error", err)
+				os.Exit(1)
+			}
+			if !changed {
+				continue
+			}
+			ok, err := db.UpdateTaskBinaryResultBackfill(ctx, item.ID, persistent)
+			if err != nil {
+				logger.Error("update historical binary result failed", "taskId", item.ID, "error", err)
+				os.Exit(1)
+			}
+			if !ok {
+				continue
+			}
+			updated++
+			if isExpired {
+				expired++
+			}
+		}
+		if len(items) < *batchSize {
+			break
+		}
+	}
+	logger.Info("binary result backfill completed",
+		"apply", *apply,
+		"scanned", scanned,
+		"matched", matched,
+		"updated", updated,
+		"expired", expired,
+		"resumeAfterId", cursor,
+	)
+}
+
+func localResultTTLHours(cfg config.Config) int {
+	if cfg.LocalResultTTLHours <= 0 {
+		return 24
+	}
+	return cfg.LocalResultTTLHours
+}