perf(storage): 拦截任务二进制并本地暂存结果

原因:任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON,导致 TOAST 与备份体积快速增长。

影响:新增统一 JSON 持久化关口;upload_none 将二进制原子写入本地结果目录,数据库仅保存带 SHA-256 的有界占位符;任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。

风险:本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500;空间不足时返回 503 且不重试上游。未自动执行历史治理。

验证:三种真实图片模型同步/异步与幂等重放通过;Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
This commit is contained in:
2026-07-24 21:13:09 +08:00
parent 2457de6a56
commit 4f163ea6d7
31 changed files with 2067 additions and 30 deletions
+29 -13
View File
@@ -503,7 +503,7 @@ WHERE id = $1::uuid`, taskID, riverJobID)
}
func (s *Store) SetTaskRemoteTask(ctx context.Context, taskID string, executionToken string, attemptID string, remoteTaskID string, payload map[string]any) error {
payloadJSON, _ := json.Marshal(emptyObjectIfNil(payload))
payloadJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(payload)))
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
tag, err := tx.Exec(ctx, `
UPDATE gateway_tasks
@@ -732,7 +732,7 @@ func (s *Store) CreateTaskParamPreprocessingLog(ctx context.Context, input Creat
if !input.Changed {
return "", nil
}
changesJSON, _ := json.Marshal(input.Changes)
changesJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Changes))
if input.Changes == nil {
changesJSON = []byte("[]")
}
@@ -1029,12 +1029,20 @@ WHERE id = $1::uuid`,
}
func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessInput) (GatewayTask, error) {
resultJSON, _ := json.Marshal(minimalTaskResult(input.Result))
billingsJSON, _ := json.Marshal(input.Billings)
usageJSON, _ := json.Marshal(emptyObjectIfNil(input.Usage))
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
billingSummaryJSON, _ := json.Marshal(emptyObjectIfNil(input.BillingSummary))
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(input.Result))
if resultReport.BinaryCount > 0 {
return GatewayTask{}, &taskPayloadBinaryError{
target: ErrTaskResultBinaryNotMaterialized,
code: "result_binary_not_materialized",
count: resultReport.BinaryCount,
}
}
resultJSON, _ := json.Marshal(resultReport.Value)
billingsJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Billings))
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Usage)))
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
billingSummaryJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.BillingSummary)))
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
finalChargeAmount := strings.TrimSpace(input.FinalChargeAmountText)
if finalChargeAmount == "" {
finalChargeAmount = strconv.FormatFloat(input.FinalChargeAmount, 'f', 9, 64)
@@ -1158,8 +1166,16 @@ func (s *Store) FinishTaskManualReview(ctx context.Context, input FinishTaskManu
if status == "failed" {
result = nil
}
resultJSON, _ := json.Marshal(minimalTaskResult(result))
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(result))
if resultReport.BinaryCount > 0 {
return GatewayTask{}, &taskPayloadBinaryError{
target: ErrTaskResultBinaryNotMaterialized,
code: "result_binary_not_materialized",
count: resultReport.BinaryCount,
}
}
resultJSON, _ := json.Marshal(resultReport.Value)
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if strings.TrimSpace(input.AttemptID) != "" {
attemptStatus := "failed"
@@ -1261,7 +1277,7 @@ func (s *Store) SettleTaskBilling(ctx context.Context, task GatewayTask) error {
"billings": task.Billings,
"billingSummary": task.BillingSummary,
}
metadata, _ := json.Marshal(metadataMap)
metadata, _ := json.Marshal(sanitizeJSONForStorage(metadataMap))
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_wallet_accounts (
@@ -1353,7 +1369,7 @@ ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO N
"frozenBefore": roundMoney(frozenBefore),
"frozenAfter": frozenAfter,
})
metadata, _ = json.Marshal(billingMetadata)
metadata, _ = json.Marshal(sanitizeJSONForStorage(billingMetadata))
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_wallet_transactions (
account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type,
@@ -1389,7 +1405,7 @@ func taskBillingString(value any) string {
}
func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureInput) (GatewayTask, error) {
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
resultJSON, _ := json.Marshal(minimalTaskResult(nil))
message := truncateUTF8Bytes(input.Message, 2048)
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {