perf(storage): 拦截任务二进制并本地暂存结果

原因:任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON,导致 TOAST 与备份体积快速增长。

影响:新增统一 JSON 持久化关口;upload_none 将二进制原子写入本地结果目录,数据库仅保存带 SHA-256 的有界占位符;任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。

风险:本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500;空间不足时返回 503 且不重试上游。未自动执行历史治理。

验证:三种真实图片模型同步/异步与幂等重放通过;Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
This commit is contained in:
2026-07-24 21:13:09 +08:00
parent 2457de6a56
commit 4f163ea6d7
31 changed files with 2067 additions and 30 deletions
+16 -2
View File
@@ -47,6 +47,7 @@ type generatedAssetUploadPolicy struct {
UploadInlineMedia bool
UploadURLMedia bool
PreserveInlineMedia bool
LocalizeInlineMedia bool
}
type generatedAssetDecision struct {
@@ -79,7 +80,8 @@ func defaultGeneratedAssetUploadPolicy() generatedAssetUploadPolicy {
func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, taskKind string, result map[string]any) (map[string]any, error) {
data, _ := result["data"].([]any)
rawNeedsUpload := generatedRawValueHasInlineMedia(result["raw"], "", nil)
if len(data) == 0 && !rawNeedsUpload {
hasInlineBinary := TaskResultHasInlineBinary(result)
if len(data) == 0 && !rawNeedsUpload && !hasInlineBinary {
redactGeneratedResultRawData(result)
return result, nil
}
@@ -87,6 +89,18 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
if err != nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
}
if policy.LocalizeInlineMedia {
next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result)
if err != nil {
return nil, err
}
redactGeneratedResultRawData(next)
return next, nil
}
if len(data) == 0 && !rawNeedsUpload {
redactGeneratedResultRawData(result)
return result, nil
}
// Topaz download URLs are short-lived. Persist them before the task can be
// marked succeeded even when the global policy normally keeps URL media.
if taskKind == "videos.upscales" {
@@ -542,7 +556,7 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP
case store.FileStorageResultUploadPolicyUploadAll:
return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
case store.FileStorageResultUploadPolicyUploadNone:
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true}
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}
default:
return defaultGeneratedAssetUploadPolicy()
}