perf(storage): 拦截任务二进制并本地暂存结果
原因:任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON,导致 TOAST 与备份体积快速增长。 影响:新增统一 JSON 持久化关口;upload_none 将二进制原子写入本地结果目录,数据库仅保存带 SHA-256 的有界占位符;任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。 风险:本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500;空间不足时返回 503 且不重试上游。未自动执行历史治理。 验证:三种真实图片模型同步/异步与幂等重放通过;Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
This commit is contained in:
@@ -47,6 +47,7 @@ type generatedAssetUploadPolicy struct {
|
||||
UploadInlineMedia bool
|
||||
UploadURLMedia bool
|
||||
PreserveInlineMedia bool
|
||||
LocalizeInlineMedia bool
|
||||
}
|
||||
|
||||
type generatedAssetDecision struct {
|
||||
@@ -79,7 +80,8 @@ func defaultGeneratedAssetUploadPolicy() generatedAssetUploadPolicy {
|
||||
func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, taskKind string, result map[string]any) (map[string]any, error) {
|
||||
data, _ := result["data"].([]any)
|
||||
rawNeedsUpload := generatedRawValueHasInlineMedia(result["raw"], "", nil)
|
||||
if len(data) == 0 && !rawNeedsUpload {
|
||||
hasInlineBinary := TaskResultHasInlineBinary(result)
|
||||
if len(data) == 0 && !rawNeedsUpload && !hasInlineBinary {
|
||||
redactGeneratedResultRawData(result)
|
||||
return result, nil
|
||||
}
|
||||
@@ -87,6 +89,18 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
||||
if err != nil {
|
||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
||||
}
|
||||
if policy.LocalizeInlineMedia {
|
||||
next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
redactGeneratedResultRawData(next)
|
||||
return next, nil
|
||||
}
|
||||
if len(data) == 0 && !rawNeedsUpload {
|
||||
redactGeneratedResultRawData(result)
|
||||
return result, nil
|
||||
}
|
||||
// Topaz download URLs are short-lived. Persist them before the task can be
|
||||
// marked succeeded even when the global policy normally keeps URL media.
|
||||
if taskKind == "videos.upscales" {
|
||||
@@ -542,7 +556,7 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP
|
||||
case store.FileStorageResultUploadPolicyUploadAll:
|
||||
return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
|
||||
case store.FileStorageResultUploadPolicyUploadNone:
|
||||
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true}
|
||||
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}
|
||||
default:
|
||||
return defaultGeneratedAssetUploadPolicy()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user