perf(storage): 拦截任务二进制并本地暂存结果
原因:任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON,导致 TOAST 与备份体积快速增长。 影响:新增统一 JSON 持久化关口;upload_none 将二进制原子写入本地结果目录,数据库仅保存带 SHA-256 的有界占位符;任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。 风险:本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500;空间不足时返回 503 且不重试上游。未自动执行历史治理。 验证:三种真实图片模型同步/异步与幂等重放通过;Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
This commit is contained in:
@@ -503,7 +503,7 @@ WHERE id = $1::uuid`, taskID, riverJobID)
|
||||
}
|
||||
|
||||
func (s *Store) SetTaskRemoteTask(ctx context.Context, taskID string, executionToken string, attemptID string, remoteTaskID string, payload map[string]any) error {
|
||||
payloadJSON, _ := json.Marshal(emptyObjectIfNil(payload))
|
||||
payloadJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(payload)))
|
||||
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
@@ -732,7 +732,7 @@ func (s *Store) CreateTaskParamPreprocessingLog(ctx context.Context, input Creat
|
||||
if !input.Changed {
|
||||
return "", nil
|
||||
}
|
||||
changesJSON, _ := json.Marshal(input.Changes)
|
||||
changesJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Changes))
|
||||
if input.Changes == nil {
|
||||
changesJSON = []byte("[]")
|
||||
}
|
||||
@@ -1029,12 +1029,20 @@ WHERE id = $1::uuid`,
|
||||
}
|
||||
|
||||
func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessInput) (GatewayTask, error) {
|
||||
resultJSON, _ := json.Marshal(minimalTaskResult(input.Result))
|
||||
billingsJSON, _ := json.Marshal(input.Billings)
|
||||
usageJSON, _ := json.Marshal(emptyObjectIfNil(input.Usage))
|
||||
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
|
||||
billingSummaryJSON, _ := json.Marshal(emptyObjectIfNil(input.BillingSummary))
|
||||
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
|
||||
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(input.Result))
|
||||
if resultReport.BinaryCount > 0 {
|
||||
return GatewayTask{}, &taskPayloadBinaryError{
|
||||
target: ErrTaskResultBinaryNotMaterialized,
|
||||
code: "result_binary_not_materialized",
|
||||
count: resultReport.BinaryCount,
|
||||
}
|
||||
}
|
||||
resultJSON, _ := json.Marshal(resultReport.Value)
|
||||
billingsJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Billings))
|
||||
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Usage)))
|
||||
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
|
||||
billingSummaryJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.BillingSummary)))
|
||||
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
|
||||
finalChargeAmount := strings.TrimSpace(input.FinalChargeAmountText)
|
||||
if finalChargeAmount == "" {
|
||||
finalChargeAmount = strconv.FormatFloat(input.FinalChargeAmount, 'f', 9, 64)
|
||||
@@ -1158,8 +1166,16 @@ func (s *Store) FinishTaskManualReview(ctx context.Context, input FinishTaskManu
|
||||
if status == "failed" {
|
||||
result = nil
|
||||
}
|
||||
resultJSON, _ := json.Marshal(minimalTaskResult(result))
|
||||
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
|
||||
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(result))
|
||||
if resultReport.BinaryCount > 0 {
|
||||
return GatewayTask{}, &taskPayloadBinaryError{
|
||||
target: ErrTaskResultBinaryNotMaterialized,
|
||||
code: "result_binary_not_materialized",
|
||||
count: resultReport.BinaryCount,
|
||||
}
|
||||
}
|
||||
resultJSON, _ := json.Marshal(resultReport.Value)
|
||||
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if strings.TrimSpace(input.AttemptID) != "" {
|
||||
attemptStatus := "failed"
|
||||
@@ -1261,7 +1277,7 @@ func (s *Store) SettleTaskBilling(ctx context.Context, task GatewayTask) error {
|
||||
"billings": task.Billings,
|
||||
"billingSummary": task.BillingSummary,
|
||||
}
|
||||
metadata, _ := json.Marshal(metadataMap)
|
||||
metadata, _ := json.Marshal(sanitizeJSONForStorage(metadataMap))
|
||||
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_wallet_accounts (
|
||||
@@ -1353,7 +1369,7 @@ ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO N
|
||||
"frozenBefore": roundMoney(frozenBefore),
|
||||
"frozenAfter": frozenAfter,
|
||||
})
|
||||
metadata, _ = json.Marshal(billingMetadata)
|
||||
metadata, _ = json.Marshal(sanitizeJSONForStorage(billingMetadata))
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_wallet_transactions (
|
||||
account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type,
|
||||
@@ -1389,7 +1405,7 @@ func taskBillingString(value any) string {
|
||||
}
|
||||
|
||||
func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureInput) (GatewayTask, error) {
|
||||
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
|
||||
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
|
||||
resultJSON, _ := json.Marshal(minimalTaskResult(nil))
|
||||
message := truncateUTF8Bytes(input.Message, 2048)
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
|
||||
Reference in New Issue
Block a user