wangbo
4f163ea6d7
原因：任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON，导致 TOAST 与备份体积快速增长。 影响：新增统一 JSON 持久化关口；upload_none 将二进制原子写入本地结果目录，数据库仅保存带 SHA-256 的有界占位符；任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。 风险：本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500；空间不足时返回 503 且不重试上游。未自动执行历史治理。 验证：三种真实图片模型同步/异步与幂等重放通过；Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
88 lines
2.3 KiB
Go
88 lines
2.3 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
type TaskBinaryResultBackfillItem struct {
|
|
ID string
|
|
Result map[string]any
|
|
FinishedAt time.Time
|
|
}
|
|
|
|
func (s *Store) ListTaskBinaryResultBackfillBatch(ctx context.Context, afterID string, batchSize int) ([]TaskBinaryResultBackfillItem, error) {
|
|
if batchSize < 1 || batchSize > 100 {
|
|
batchSize = 100
|
|
}
|
|
var items []TaskBinaryResultBackfillItem
|
|
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
|
|
return err
|
|
}
|
|
rows, err := tx.Query(ctx, `
|
|
SELECT id::text, result, COALESCE(finished_at, updated_at)
|
|
FROM gateway_tasks
|
|
WHERE status = 'succeeded'
|
|
AND result <> '{}'::jsonb
|
|
AND (NULLIF($1::text, '') IS NULL OR id > NULLIF($1::text, '')::uuid)
|
|
ORDER BY id
|
|
LIMIT $2`, afterID, batchSize)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
items = make([]TaskBinaryResultBackfillItem, 0, batchSize)
|
|
for rows.Next() {
|
|
var item TaskBinaryResultBackfillItem
|
|
var resultJSON []byte
|
|
if err := rows.Scan(&item.ID, &resultJSON, &item.FinishedAt); err != nil {
|
|
return err
|
|
}
|
|
item.Result = decodeObject(resultJSON)
|
|
items = append(items, item)
|
|
}
|
|
return rows.Err()
|
|
})
|
|
return items, err
|
|
}
|
|
|
|
func (s *Store) UpdateTaskBinaryResultBackfill(ctx context.Context, taskID string, result map[string]any) (bool, error) {
|
|
report := sanitizeJSONForStorageWithReport(minimalTaskResult(result))
|
|
if report.BinaryCount > 0 {
|
|
return false, &taskPayloadBinaryError{
|
|
target: ErrTaskResultBinaryNotMaterialized,
|
|
code: "result_binary_not_materialized",
|
|
count: report.BinaryCount,
|
|
}
|
|
}
|
|
resultJSON, err := json.Marshal(report.Value)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
updated := false
|
|
err = pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
|
|
return err
|
|
}
|
|
tag, err := tx.Exec(ctx, `
|
|
UPDATE gateway_tasks
|
|
SET result = $2::jsonb,
|
|
updated_at = now()
|
|
WHERE id = $1::uuid
|
|
AND status = 'succeeded'`,
|
|
taskID,
|
|
string(resultJSON),
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
updated = tag.RowsAffected() == 1
|
|
return nil
|
|
})
|
|
return updated, err
|
|
}
|