fix(media): 统一图片结果 URL 化并限制同步 Base64
将上游 URL 直接持久化,内联媒体经对象存储后仅保留 URL 与内部定位元数据;异步轮询、任务详情和幂等重放统一使用零对象读取的 URL 投影,并增加 64KiB 响应门禁。 OpenAI 图片接口接受 url 与 b64_json,同步 Base64 限制为 20MiB 和每 Pod 2 并发;新增历史结果迁移清零门禁、结果指标和 API GOMEMLIMIT。 验证:API go test ./...、go vet、聚焦 race、pnpm openapi、pnpm lint/test/build、迁移安全检查与 docker compose config 均通过。
This commit is contained in:
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
func main() {
|
||||
apply := flag.Bool("apply", false, "persist compacted results; default is dry-run")
|
||||
requireClean := flag.Bool("require-clean", false, "exit non-zero unless a complete dry-run finds no results requiring URL migration")
|
||||
batchSize := flag.Int("batch-size", 100, "rows per batch, maximum 100")
|
||||
maxBatches := flag.Int("max-batches", 10, "maximum batches for one invocation")
|
||||
afterID := flag.String("after-id", "", "resume after this task UUID")
|
||||
@@ -29,6 +30,10 @@ func main() {
|
||||
logger.Error("invalid backfill bounds", "batchSize", *batchSize, "maxBatches", *maxBatches)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *apply && *requireClean {
|
||||
logger.Error("--apply and --require-clean cannot be combined")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
db, err := store.Connect(ctx, cfg.DatabaseURL)
|
||||
@@ -42,8 +47,11 @@ func main() {
|
||||
cursor := *afterID
|
||||
scanned := 0
|
||||
matched := 0
|
||||
blockingMatched := 0
|
||||
updated := 0
|
||||
expired := 0
|
||||
expiredLocalPlaceholders := 0
|
||||
complete := false
|
||||
for batch := 0; batch < *maxBatches; batch++ {
|
||||
items, err := db.ListTaskBinaryResultBackfillBatch(ctx, cursor, *batchSize)
|
||||
if err != nil {
|
||||
@@ -51,26 +59,30 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(items) == 0 {
|
||||
complete = true
|
||||
break
|
||||
}
|
||||
for _, item := range items {
|
||||
cursor = item.ID
|
||||
scanned++
|
||||
if !runner.TaskResultHasInlineBinary(item.Result) {
|
||||
if !runner.TaskResultNeedsURLMigration(item.Result) {
|
||||
continue
|
||||
}
|
||||
matched++
|
||||
isExpired := item.FinishedAt.Before(time.Now().Add(-time.Duration(localResultTTLHours(cfg)) * time.Hour))
|
||||
hasLocalPlaceholder := runner.TaskResultHasLocalPlaceholder(item.Result)
|
||||
if isExpired && hasLocalPlaceholder {
|
||||
expiredLocalPlaceholders++
|
||||
if *apply {
|
||||
logger.Warn("skip expired local result placeholder without overwriting stored result", "taskId", item.ID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
blockingMatched++
|
||||
if !*apply {
|
||||
continue
|
||||
}
|
||||
isExpired := item.FinishedAt.Before(time.Now().Add(-time.Duration(localResultTTLHours(cfg)) * time.Hour))
|
||||
var persistent map[string]any
|
||||
var changed bool
|
||||
if isExpired {
|
||||
persistent, changed, err = service.CompactExpiredTaskResultForStorage(ctx, item.ID, item.Result)
|
||||
} else {
|
||||
persistent, changed, err = service.MaterializeTaskResultForStorage(ctx, item.ID, item.Result)
|
||||
}
|
||||
persistent, changed, err := service.MigrateTaskResultToURLs(ctx, item.ID, item.Result)
|
||||
if err != nil {
|
||||
logger.Error("materialize historical binary result failed", "taskId", item.ID, "error", err)
|
||||
os.Exit(1)
|
||||
@@ -92,6 +104,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
if len(items) < *batchSize {
|
||||
complete = true
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -99,10 +112,17 @@ func main() {
|
||||
"apply", *apply,
|
||||
"scanned", scanned,
|
||||
"matched", matched,
|
||||
"blockingMatched", blockingMatched,
|
||||
"updated", updated,
|
||||
"expired", expired,
|
||||
"expiredLocalPlaceholders", expiredLocalPlaceholders,
|
||||
"complete", complete,
|
||||
"resumeAfterId", cursor,
|
||||
)
|
||||
if *requireClean && (!complete || blockingMatched > 0) {
|
||||
logger.Error("binary result URL migration gate failed", "complete", complete, "blockingMatched", blockingMatched, "expiredLocalPlaceholders", expiredLocalPlaceholders, "resumeAfterId", cursor)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func localResultTTLHours(cfg config.Config) int {
|
||||
|
||||
Reference in New Issue
Block a user