perf(storage): 拦截任务二进制并本地暂存结果
原因:任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON,导致 TOAST 与备份体积快速增长。 影响:新增统一 JSON 持久化关口;upload_none 将二进制原子写入本地结果目录,数据库仅保存带 SHA-256 的有界占位符;任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。 风险:本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500;空间不足时返回 503 且不重试上游。未自动执行历史治理。 验证:三种真实图片模型同步/异步与幂等重放通过;Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type TaskBinaryResultBackfillItem struct {
|
||||
ID string
|
||||
Result map[string]any
|
||||
FinishedAt time.Time
|
||||
}
|
||||
|
||||
func (s *Store) ListTaskBinaryResultBackfillBatch(ctx context.Context, afterID string, batchSize int) ([]TaskBinaryResultBackfillItem, error) {
|
||||
if batchSize < 1 || batchSize > 100 {
|
||||
batchSize = 100
|
||||
}
|
||||
var items []TaskBinaryResultBackfillItem
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT id::text, result, COALESCE(finished_at, updated_at)
|
||||
FROM gateway_tasks
|
||||
WHERE status = 'succeeded'
|
||||
AND result <> '{}'::jsonb
|
||||
AND (NULLIF($1::text, '') IS NULL OR id > NULLIF($1::text, '')::uuid)
|
||||
ORDER BY id
|
||||
LIMIT $2`, afterID, batchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
items = make([]TaskBinaryResultBackfillItem, 0, batchSize)
|
||||
for rows.Next() {
|
||||
var item TaskBinaryResultBackfillItem
|
||||
var resultJSON []byte
|
||||
if err := rows.Scan(&item.ID, &resultJSON, &item.FinishedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
item.Result = decodeObject(resultJSON)
|
||||
items = append(items, item)
|
||||
}
|
||||
return rows.Err()
|
||||
})
|
||||
return items, err
|
||||
}
|
||||
|
||||
func (s *Store) UpdateTaskBinaryResultBackfill(ctx context.Context, taskID string, result map[string]any) (bool, error) {
|
||||
report := sanitizeJSONForStorageWithReport(minimalTaskResult(result))
|
||||
if report.BinaryCount > 0 {
|
||||
return false, &taskPayloadBinaryError{
|
||||
target: ErrTaskResultBinaryNotMaterialized,
|
||||
code: "result_binary_not_materialized",
|
||||
count: report.BinaryCount,
|
||||
}
|
||||
}
|
||||
resultJSON, err := json.Marshal(report.Value)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
updated := false
|
||||
err = pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET result = $2::jsonb,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'succeeded'`,
|
||||
taskID,
|
||||
string(resultJSON),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
updated = tag.RowsAffected() == 1
|
||||
return nil
|
||||
})
|
||||
return updated, err
|
||||
}
|
||||
Reference in New Issue
Block a user