wangbo
4f163ea6d7
原因：任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON，导致 TOAST 与备份体积快速增长。 影响：新增统一 JSON 持久化关口；upload_none 将二进制原子写入本地结果目录，数据库仅保存带 SHA-256 的有界占位符；任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。 风险：本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500；空间不足时返回 503 且不重试上游。未自动执行历史治理。 验证：三种真实图片模型同步/异步与幂等重放通过；Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
114 lines
2.9 KiB
Go
114 lines
2.9 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"log/slog"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
func main() {
|
|
apply := flag.Bool("apply", false, "persist compacted results; default is dry-run")
|
|
batchSize := flag.Int("batch-size", 100, "rows per batch, maximum 100")
|
|
maxBatches := flag.Int("max-batches", 10, "maximum batches for one invocation")
|
|
afterID := flag.String("after-id", "", "resume after this task UUID")
|
|
flag.Parse()
|
|
|
|
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
|
cfg := config.Load()
|
|
if err := cfg.Validate(); err != nil {
|
|
logger.Error("invalid gateway configuration", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
if *batchSize < 1 || *batchSize > 100 || *maxBatches < 1 {
|
|
logger.Error("invalid backfill bounds", "batchSize", *batchSize, "maxBatches", *maxBatches)
|
|
os.Exit(1)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
db, err := store.Connect(ctx, cfg.DatabaseURL)
|
|
if err != nil {
|
|
logger.Error("connect postgres failed", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
defer db.Close()
|
|
|
|
service := runner.New(cfg, db, logger)
|
|
cursor := *afterID
|
|
scanned := 0
|
|
matched := 0
|
|
updated := 0
|
|
expired := 0
|
|
for batch := 0; batch < *maxBatches; batch++ {
|
|
items, err := db.ListTaskBinaryResultBackfillBatch(ctx, cursor, *batchSize)
|
|
if err != nil {
|
|
logger.Error("list binary result backfill batch failed", "afterId", cursor, "error", err)
|
|
os.Exit(1)
|
|
}
|
|
if len(items) == 0 {
|
|
break
|
|
}
|
|
for _, item := range items {
|
|
cursor = item.ID
|
|
scanned++
|
|
if !runner.TaskResultHasInlineBinary(item.Result) {
|
|
continue
|
|
}
|
|
matched++
|
|
if !*apply {
|
|
continue
|
|
}
|
|
isExpired := item.FinishedAt.Before(time.Now().Add(-time.Duration(localResultTTLHours(cfg)) * time.Hour))
|
|
var persistent map[string]any
|
|
var changed bool
|
|
if isExpired {
|
|
persistent, changed, err = service.CompactExpiredTaskResultForStorage(ctx, item.ID, item.Result)
|
|
} else {
|
|
persistent, changed, err = service.MaterializeTaskResultForStorage(ctx, item.ID, item.Result)
|
|
}
|
|
if err != nil {
|
|
logger.Error("materialize historical binary result failed", "taskId", item.ID, "error", err)
|
|
os.Exit(1)
|
|
}
|
|
if !changed {
|
|
continue
|
|
}
|
|
ok, err := db.UpdateTaskBinaryResultBackfill(ctx, item.ID, persistent)
|
|
if err != nil {
|
|
logger.Error("update historical binary result failed", "taskId", item.ID, "error", err)
|
|
os.Exit(1)
|
|
}
|
|
if !ok {
|
|
continue
|
|
}
|
|
updated++
|
|
if isExpired {
|
|
expired++
|
|
}
|
|
}
|
|
if len(items) < *batchSize {
|
|
break
|
|
}
|
|
}
|
|
logger.Info("binary result backfill completed",
|
|
"apply", *apply,
|
|
"scanned", scanned,
|
|
"matched", matched,
|
|
"updated", updated,
|
|
"expired", expired,
|
|
"resumeAfterId", cursor,
|
|
)
|
|
}
|
|
|
|
func localResultTTLHours(cfg config.Config) int {
|
|
if cfg.LocalResultTTLHours <= 0 {
|
|
return 24
|
|
}
|
|
return cfg.LocalResultTTLHours
|
|
}
|