perf(storage): 拦截任务二进制并本地暂存结果
原因:任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON,导致 TOAST 与备份体积快速增长。 影响:新增统一 JSON 持久化关口;upload_none 将二进制原子写入本地结果目录,数据库仅保存带 SHA-256 的有界占位符;任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。 风险:本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500;空间不足时返回 503 且不重试上游。未自动执行历史治理。 验证:三种真实图片模型同步/异步与幂等重放通过;Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type TaskBinaryResultBackfillItem struct {
|
||||
ID string
|
||||
Result map[string]any
|
||||
FinishedAt time.Time
|
||||
}
|
||||
|
||||
func (s *Store) ListTaskBinaryResultBackfillBatch(ctx context.Context, afterID string, batchSize int) ([]TaskBinaryResultBackfillItem, error) {
|
||||
if batchSize < 1 || batchSize > 100 {
|
||||
batchSize = 100
|
||||
}
|
||||
var items []TaskBinaryResultBackfillItem
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT id::text, result, COALESCE(finished_at, updated_at)
|
||||
FROM gateway_tasks
|
||||
WHERE status = 'succeeded'
|
||||
AND result <> '{}'::jsonb
|
||||
AND (NULLIF($1::text, '') IS NULL OR id > NULLIF($1::text, '')::uuid)
|
||||
ORDER BY id
|
||||
LIMIT $2`, afterID, batchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
items = make([]TaskBinaryResultBackfillItem, 0, batchSize)
|
||||
for rows.Next() {
|
||||
var item TaskBinaryResultBackfillItem
|
||||
var resultJSON []byte
|
||||
if err := rows.Scan(&item.ID, &resultJSON, &item.FinishedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
item.Result = decodeObject(resultJSON)
|
||||
items = append(items, item)
|
||||
}
|
||||
return rows.Err()
|
||||
})
|
||||
return items, err
|
||||
}
|
||||
|
||||
func (s *Store) UpdateTaskBinaryResultBackfill(ctx context.Context, taskID string, result map[string]any) (bool, error) {
|
||||
report := sanitizeJSONForStorageWithReport(minimalTaskResult(result))
|
||||
if report.BinaryCount > 0 {
|
||||
return false, &taskPayloadBinaryError{
|
||||
target: ErrTaskResultBinaryNotMaterialized,
|
||||
code: "result_binary_not_materialized",
|
||||
count: report.BinaryCount,
|
||||
}
|
||||
}
|
||||
resultJSON, err := json.Marshal(report.Value)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
updated := false
|
||||
err = pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET result = $2::jsonb,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'succeeded'`,
|
||||
taskID,
|
||||
string(resultJSON),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
updated = tag.RowsAffected() == 1
|
||||
return nil
|
||||
})
|
||||
return updated, err
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func (s *Store) EnsureConversation(ctx context.Context, user *auth.User, convers
|
||||
if userID == "" {
|
||||
userID = "anonymous"
|
||||
}
|
||||
metadataJSON, _ := json.Marshal(emptyObjectIfNil(metadata))
|
||||
metadataJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(metadata)))
|
||||
var conversationID string
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
INSERT INTO gateway_conversations (user_id, gateway_user_id, conversation_key, metadata)
|
||||
@@ -67,7 +67,15 @@ func (s *Store) UpsertConversationMessages(ctx context.Context, conversationID s
|
||||
refs := make([]TaskMessageRefInput, 0, len(messages))
|
||||
newCount := 0
|
||||
for index, message := range messages {
|
||||
snapshotJSON, _ := json.Marshal(emptyObjectIfNil(message.Snapshot))
|
||||
snapshotReport := sanitizeJSONForStorageWithReport(emptyObjectIfNil(message.Snapshot))
|
||||
if snapshotReport.BinaryCount > 0 {
|
||||
return nil, 0, &taskPayloadBinaryError{
|
||||
target: ErrTaskRequestBinaryNotMaterialized,
|
||||
code: "request_binary_not_materialized",
|
||||
count: snapshotReport.BinaryCount,
|
||||
}
|
||||
}
|
||||
snapshotJSON, _ := json.Marshal(snapshotReport.Value)
|
||||
var messageID string
|
||||
var inserted bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
storageBinaryPrefixChars = 16
|
||||
storageGenericBase64MinLength = 4096
|
||||
storageInvalidBinaryMinLength = 512
|
||||
storageJSONSanitizerMaxDepth = 64
|
||||
storagePlaceholderPrefix = "[GatewayBinary:v1;"
|
||||
storageBufferObjectType = "buffer"
|
||||
storageDataURLPrefix = "data:"
|
||||
storageDataURLBase64Marker = ";base64"
|
||||
storageDataURLMaxContentType = 64
|
||||
)
|
||||
|
||||
var (
|
||||
ErrTaskRequestBinaryNotMaterialized = errors.New("task request binary payload was not materialized")
|
||||
ErrTaskResultBinaryNotMaterialized = errors.New("task result binary payload was not materialized")
|
||||
)
|
||||
|
||||
type taskPayloadBinaryError struct {
|
||||
target error
|
||||
code string
|
||||
count int
|
||||
}
|
||||
|
||||
func (e *taskPayloadBinaryError) Error() string {
|
||||
return fmt.Sprintf("%s: detected %d inline binary value(s)", e.code, e.count)
|
||||
}
|
||||
|
||||
func (e *taskPayloadBinaryError) ErrorCode() string {
|
||||
return e.code
|
||||
}
|
||||
|
||||
func (e *taskPayloadBinaryError) Is(target error) bool {
|
||||
return target == e.target
|
||||
}
|
||||
|
||||
type storageSanitizeReport struct {
|
||||
Value any
|
||||
BinaryCount int
|
||||
}
|
||||
|
||||
// sanitizeJSONForStorage is the final task-domain persistence guard. It always
|
||||
// returns a detached JSON-compatible value and replaces inline binary payloads
|
||||
// with a bounded, deterministic placeholder.
|
||||
func sanitizeJSONForStorage(value any) any {
|
||||
return sanitizeJSONForStorageWithReport(value).Value
|
||||
}
|
||||
|
||||
func sanitizeJSONForStorageWithReport(value any) storageSanitizeReport {
|
||||
next, count := sanitizeJSONStorageValue(value, nil, 0)
|
||||
return storageSanitizeReport{Value: next, BinaryCount: count}
|
||||
}
|
||||
|
||||
func sanitizeJSONStorageValue(value any, path []string, depth int) (any, int) {
|
||||
if depth >= storageJSONSanitizerMaxDepth {
|
||||
return "[JSON,max-depth]", 0
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
if payload, contentType, ok := storageBufferObjectBytes(typed); ok {
|
||||
return storageBinaryPlaceholder(payload, contentType, "buffer"), 1
|
||||
}
|
||||
next := make(map[string]any, len(typed))
|
||||
count := 0
|
||||
for key, child := range typed {
|
||||
sanitized, childCount := sanitizeJSONStorageValue(child, appendStoragePath(path, key), depth+1)
|
||||
next[key] = sanitized
|
||||
count += childCount
|
||||
}
|
||||
return next, count
|
||||
case []any:
|
||||
if storagePathIsBinary(path) {
|
||||
if payload, ok := storageNumberArrayBytes(typed); ok {
|
||||
return storageBinaryPlaceholder(payload, "", "buffer"), 1
|
||||
}
|
||||
}
|
||||
next := make([]any, len(typed))
|
||||
count := 0
|
||||
for index, child := range typed {
|
||||
sanitized, childCount := sanitizeJSONStorageValue(child, path, depth+1)
|
||||
next[index] = sanitized
|
||||
count += childCount
|
||||
}
|
||||
return next, count
|
||||
case []byte:
|
||||
return storageBinaryPlaceholder(typed, "", "buffer"), 1
|
||||
case string:
|
||||
if placeholder, ok := storageStringPlaceholder(typed, path); ok {
|
||||
return placeholder, 1
|
||||
}
|
||||
return typed, 0
|
||||
default:
|
||||
return value, 0
|
||||
}
|
||||
}
|
||||
|
||||
func appendStoragePath(path []string, key string) []string {
|
||||
next := make([]string, len(path)+1)
|
||||
copy(next, path)
|
||||
next[len(path)] = key
|
||||
return next
|
||||
}
|
||||
|
||||
func storageStringPlaceholder(value string, path []string) (string, bool) {
|
||||
raw := strings.TrimSpace(value)
|
||||
if raw == "" || strings.HasPrefix(raw, storagePlaceholderPrefix) {
|
||||
return "", false
|
||||
}
|
||||
encoded, contentType, dataURL := storageBase64StringParts(raw)
|
||||
strict := dataURL || storagePathIsBinary(path)
|
||||
if !strict && len(encoded) < storageGenericBase64MinLength {
|
||||
return "", false
|
||||
}
|
||||
payload, ok := storageDecodeBase64(encoded)
|
||||
if ok {
|
||||
encoding := "raw"
|
||||
if dataURL {
|
||||
encoding = "data-uri"
|
||||
}
|
||||
return storageBinaryPlaceholder(payload, contentType, encoding), true
|
||||
}
|
||||
if strict && len(raw) >= storageInvalidBinaryMinLength {
|
||||
return storageBinaryPlaceholder([]byte(raw), contentType, "raw"), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func storageBase64StringParts(value string) (encoded string, contentType string, dataURL bool) {
|
||||
if !strings.HasPrefix(strings.ToLower(value), storageDataURLPrefix) {
|
||||
return value, "", false
|
||||
}
|
||||
prefix, payload, ok := strings.Cut(value, ",")
|
||||
if !ok || !strings.Contains(strings.ToLower(prefix), storageDataURLBase64Marker) {
|
||||
return value, "", false
|
||||
}
|
||||
mediaType := strings.TrimSpace(prefix[len(storageDataURLPrefix):])
|
||||
if before, _, found := strings.Cut(mediaType, ";"); found {
|
||||
mediaType = before
|
||||
}
|
||||
if len(mediaType) > storageDataURLMaxContentType || !storageSafeContentType(mediaType) {
|
||||
mediaType = ""
|
||||
}
|
||||
return payload, strings.ToLower(mediaType), true
|
||||
}
|
||||
|
||||
func storageSafeContentType(value string) bool {
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
for _, char := range value {
|
||||
switch {
|
||||
case char >= 'a' && char <= 'z':
|
||||
case char >= 'A' && char <= 'Z':
|
||||
case char >= '0' && char <= '9':
|
||||
case char == '/', char == '.', char == '+', char == '-':
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func storageDecodeBase64(value string) ([]byte, bool) {
|
||||
normalized := removeStorageASCIIWhitespace(value)
|
||||
if normalized == "" {
|
||||
return nil, false
|
||||
}
|
||||
for _, encoding := range []*base64.Encoding{
|
||||
base64.StdEncoding,
|
||||
base64.RawStdEncoding,
|
||||
base64.URLEncoding,
|
||||
base64.RawURLEncoding,
|
||||
} {
|
||||
payload, err := encoding.DecodeString(normalized)
|
||||
if err != nil || len(payload) == 0 {
|
||||
continue
|
||||
}
|
||||
canonical := encoding.EncodeToString(payload)
|
||||
if strings.TrimRight(normalized, "=") == strings.TrimRight(canonical, "=") {
|
||||
return payload, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func removeStorageASCIIWhitespace(value string) string {
|
||||
return strings.Map(func(char rune) rune {
|
||||
switch char {
|
||||
case ' ', '\n', '\r', '\t':
|
||||
return -1
|
||||
default:
|
||||
return char
|
||||
}
|
||||
}, value)
|
||||
}
|
||||
|
||||
func storagePathIsBinary(path []string) bool {
|
||||
if len(path) == 0 {
|
||||
return false
|
||||
}
|
||||
key := normalizeStorageBinaryKey(path[len(path)-1])
|
||||
if storageBinaryKey(key) {
|
||||
return true
|
||||
}
|
||||
if len(path) < 2 {
|
||||
return false
|
||||
}
|
||||
parent := normalizeStorageBinaryKey(path[len(path)-2])
|
||||
return (parent == "inlinedata" || parent == "binary" || parent == "media") &&
|
||||
(key == "data" || key == "content")
|
||||
}
|
||||
|
||||
func normalizeStorageBinaryKey(value string) string {
|
||||
return strings.Map(func(char rune) rune {
|
||||
switch {
|
||||
case char >= 'a' && char <= 'z':
|
||||
return char
|
||||
case char >= 'A' && char <= 'Z':
|
||||
return char + ('a' - 'A')
|
||||
case char >= '0' && char <= '9':
|
||||
return char
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}, value)
|
||||
}
|
||||
|
||||
func storageBinaryKey(key string) bool {
|
||||
return key == "b64" ||
|
||||
key == "b64json" ||
|
||||
key == "base64" ||
|
||||
key == "buffer" ||
|
||||
key == "bytes" ||
|
||||
strings.Contains(key, "base64") ||
|
||||
strings.Contains(key, "buffer") ||
|
||||
strings.Contains(key, "binary") ||
|
||||
strings.HasSuffix(key, "b64") ||
|
||||
strings.HasSuffix(key, "bytes")
|
||||
}
|
||||
|
||||
func storageBufferObjectBytes(value map[string]any) ([]byte, string, bool) {
|
||||
kind, _ := value["type"].(string)
|
||||
if normalizeStorageBinaryKey(kind) != storageBufferObjectType {
|
||||
return nil, "", false
|
||||
}
|
||||
contentType := firstNonEmpty(
|
||||
stringFromAny(value["mime_type"]),
|
||||
stringFromAny(value["mimeType"]),
|
||||
stringFromAny(value["contentType"]),
|
||||
)
|
||||
switch data := value["data"].(type) {
|
||||
case []byte:
|
||||
if len(data) == 0 {
|
||||
return nil, "", false
|
||||
}
|
||||
return append([]byte(nil), data...), contentType, true
|
||||
case []any:
|
||||
payload, ok := storageNumberArrayBytes(data)
|
||||
return payload, contentType, ok
|
||||
default:
|
||||
return nil, "", false
|
||||
}
|
||||
}
|
||||
|
||||
func storageNumberArrayBytes(values []any) ([]byte, bool) {
|
||||
if len(values) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
payload := make([]byte, len(values))
|
||||
for index, value := range values {
|
||||
next, ok := storageByteFromAny(value)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
payload[index] = next
|
||||
}
|
||||
return payload, true
|
||||
}
|
||||
|
||||
func storageByteFromAny(value any) (byte, bool) {
|
||||
switch typed := value.(type) {
|
||||
case byte:
|
||||
return typed, true
|
||||
case int:
|
||||
if typed >= 0 && typed <= 255 {
|
||||
return byte(typed), true
|
||||
}
|
||||
case int32:
|
||||
if typed >= 0 && typed <= 255 {
|
||||
return byte(typed), true
|
||||
}
|
||||
case int64:
|
||||
if typed >= 0 && typed <= 255 {
|
||||
return byte(typed), true
|
||||
}
|
||||
case float64:
|
||||
asInt := int(typed)
|
||||
if typed == float64(asInt) && asInt >= 0 && asInt <= 255 {
|
||||
return byte(asInt), true
|
||||
}
|
||||
case json.Number:
|
||||
asInt, err := strconv.ParseInt(string(typed), 10, 16)
|
||||
if err == nil && asInt >= 0 && asInt <= 255 {
|
||||
return byte(asInt), true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func storageBinaryPlaceholder(payload []byte, contentType string, encoding string) string {
|
||||
digest := sha256.Sum256(payload)
|
||||
prefix := base64.StdEncoding.EncodeToString(payload)
|
||||
if len(prefix) > storageBinaryPrefixChars {
|
||||
prefix = prefix[:storageBinaryPrefixChars]
|
||||
}
|
||||
contentType = strings.ToLower(strings.TrimSpace(contentType))
|
||||
if contentType == "" || len(contentType) > 32 || !storageSafeContentType(contentType) {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
switch encoding {
|
||||
case "data-uri", "buffer":
|
||||
default:
|
||||
encoding = "raw"
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"[GatewayBinary:v1;prefix=%s;sha256=%x;bytes=%d;mime=%s;encoding=%s]",
|
||||
prefix,
|
||||
digest,
|
||||
len(payload),
|
||||
contentType,
|
||||
encoding,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSanitizeJSONForStorageReplacesBinaryWithoutMutatingInput(t *testing.T) {
|
||||
payload := []byte("shared binary payload")
|
||||
encoded := base64.StdEncoding.EncodeToString(payload)
|
||||
input := map[string]any{
|
||||
"data": []any{
|
||||
map[string]any{"b64_json": encoded},
|
||||
},
|
||||
"buffer": map[string]any{
|
||||
"type": "Buffer",
|
||||
"data": []any{float64(1), float64(2), float64(3)},
|
||||
},
|
||||
"text": "ordinary text",
|
||||
}
|
||||
|
||||
report := sanitizeJSONForStorageWithReport(input)
|
||||
if report.BinaryCount != 2 {
|
||||
t.Fatalf("binary count = %d, want 2", report.BinaryCount)
|
||||
}
|
||||
next := report.Value.(map[string]any)
|
||||
data := next["data"].([]any)
|
||||
placeholder := data[0].(map[string]any)["b64_json"].(string)
|
||||
if !strings.HasPrefix(placeholder, storagePlaceholderPrefix) {
|
||||
t.Fatalf("unexpected placeholder: %q", placeholder)
|
||||
}
|
||||
if len(placeholder) > 200 {
|
||||
t.Fatalf("placeholder exceeds 200 bytes: %d", len(placeholder))
|
||||
}
|
||||
if !strings.Contains(placeholder, ";prefix="+encoded[:16]+";") {
|
||||
t.Fatalf("placeholder should retain the bounded Base64 prefix: %q", placeholder)
|
||||
}
|
||||
if got := input["data"].([]any)[0].(map[string]any)["b64_json"]; got != encoded {
|
||||
t.Fatalf("sanitizer mutated input: %v", got)
|
||||
}
|
||||
if next["text"] != "ordinary text" {
|
||||
t.Fatalf("ordinary text changed: %+v", next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeJSONForStorageUsesDecodedBytesForEquivalentRepresentations(t *testing.T) {
|
||||
payload := []byte("equivalent payload")
|
||||
encoded := base64.StdEncoding.EncodeToString(payload)
|
||||
report := sanitizeJSONForStorageWithReport(map[string]any{
|
||||
"rawBase64": encoded,
|
||||
"dataURL": "data:image/png;base64," + encoded,
|
||||
"bytes": []any{float64('e'), float64('q'), float64('u'), float64('i'), float64('v'), float64('a'), float64('l'), float64('e'), float64('n'), float64('t'), float64(' '), float64('p'), float64('a'), float64('y'), float64('l'), float64('o'), float64('a'), float64('d')},
|
||||
})
|
||||
if report.BinaryCount != 3 {
|
||||
t.Fatalf("binary count = %d, want 3", report.BinaryCount)
|
||||
}
|
||||
next := report.Value.(map[string]any)
|
||||
hashes := map[string]struct{}{}
|
||||
for _, key := range []string{"rawBase64", "dataURL", "bytes"} {
|
||||
value := next[key].(string)
|
||||
hashStart := strings.Index(value, ";sha256=")
|
||||
hashEnd := strings.Index(value[hashStart+1:], ";bytes=")
|
||||
if hashStart < 0 || hashEnd < 0 {
|
||||
t.Fatalf("missing hash in %s placeholder: %q", key, value)
|
||||
}
|
||||
hashes[value[hashStart+8:hashStart+1+hashEnd]] = struct{}{}
|
||||
}
|
||||
if len(hashes) != 1 {
|
||||
t.Fatalf("equivalent binary values produced different hashes: %+v", next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeJSONForStorageAvoidsShortGenericTextFalsePositive(t *testing.T) {
|
||||
input := map[string]any{"message": "YWJjZA==", "count": json.Number("12")}
|
||||
report := sanitizeJSONForStorageWithReport(input)
|
||||
if report.BinaryCount != 0 {
|
||||
t.Fatalf("short generic Base64-like text should not be sanitized: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeJSONForStorageRecognizesExistingPlaceholder(t *testing.T) {
|
||||
value := storageBinaryPlaceholder([]byte("payload"), "image/png", "raw")
|
||||
report := sanitizeJSONForStorageWithReport(map[string]any{"b64_json": value})
|
||||
if report.BinaryCount != 0 {
|
||||
t.Fatalf("existing placeholder should pass the final guard: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageBinaryPlaceholderIsAlwaysBounded(t *testing.T) {
|
||||
value := storageBinaryPlaceholder(
|
||||
[]byte("payload"),
|
||||
"application/vnd.a-very-long-provider-specific-generated-binary-result+json",
|
||||
"data-uri",
|
||||
)
|
||||
if len(value) > 200 {
|
||||
t.Fatalf("placeholder exceeds 200 bytes: %d %q", len(value), value)
|
||||
}
|
||||
if !strings.Contains(value, ";mime=application/octet-stream;") {
|
||||
t.Fatalf("oversized MIME should use the bounded fallback: %q", value)
|
||||
}
|
||||
}
|
||||
@@ -1993,7 +1993,15 @@ func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *aut
|
||||
}
|
||||
|
||||
func (s *Store) CreateTaskIdempotent(ctx context.Context, input CreateTaskInput, user *auth.User) (CreateTaskResult, error) {
|
||||
requestBody, _ := json.Marshal(input.Request)
|
||||
requestReport := sanitizeJSONForStorageWithReport(input.Request)
|
||||
if requestReport.BinaryCount > 0 {
|
||||
return CreateTaskResult{}, &taskPayloadBinaryError{
|
||||
target: ErrTaskRequestBinaryNotMaterialized,
|
||||
code: "request_binary_not_materialized",
|
||||
count: requestReport.BinaryCount,
|
||||
}
|
||||
}
|
||||
requestBody, _ := json.Marshal(requestReport.Value)
|
||||
runMode := normalizeRunMode(input.RunMode, input.Request)
|
||||
status := "queued"
|
||||
resultBody, _ := json.Marshal(map[string]any(nil))
|
||||
@@ -2048,7 +2056,7 @@ WHERE user_id = $1 AND idempotency_key_hash = $2`, user.ID, strings.TrimSpace(in
|
||||
}
|
||||
events := taskEventsForCreate(task.ID, runMode, status, nil)
|
||||
for _, event := range events {
|
||||
payload, _ := json.Marshal(event.Payload)
|
||||
payload, _ := json.Marshal(sanitizeJSONForStorage(event.Payload))
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated)
|
||||
VALUES ($1::uuid, $2, $3::text, NULLIF($4::text, ''), NULLIF($5::text, ''), $6, NULLIF($7::text, ''), $8::jsonb, $9)`,
|
||||
|
||||
@@ -45,15 +45,15 @@ type CreateResponseChainInput struct {
|
||||
}
|
||||
|
||||
func (s *Store) CreateResponseChain(ctx context.Context, input CreateResponseChainInput) error {
|
||||
requestJSON, err := json.Marshal(input.RequestSnapshot)
|
||||
requestJSON, err := json.Marshal(sanitizeJSONForStorage(input.RequestSnapshot))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
responseJSON, err := json.Marshal(input.ResponseSnapshot)
|
||||
responseJSON, err := json.Marshal(sanitizeJSONForStorage(input.ResponseSnapshot))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
internalJSON, err := json.Marshal(input.InternalSnapshot)
|
||||
internalJSON, err := json.Marshal(sanitizeJSONForStorage(input.InternalSnapshot))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -503,7 +503,7 @@ WHERE id = $1::uuid`, taskID, riverJobID)
|
||||
}
|
||||
|
||||
func (s *Store) SetTaskRemoteTask(ctx context.Context, taskID string, executionToken string, attemptID string, remoteTaskID string, payload map[string]any) error {
|
||||
payloadJSON, _ := json.Marshal(emptyObjectIfNil(payload))
|
||||
payloadJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(payload)))
|
||||
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
@@ -732,7 +732,7 @@ func (s *Store) CreateTaskParamPreprocessingLog(ctx context.Context, input Creat
|
||||
if !input.Changed {
|
||||
return "", nil
|
||||
}
|
||||
changesJSON, _ := json.Marshal(input.Changes)
|
||||
changesJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Changes))
|
||||
if input.Changes == nil {
|
||||
changesJSON = []byte("[]")
|
||||
}
|
||||
@@ -1029,12 +1029,20 @@ WHERE id = $1::uuid`,
|
||||
}
|
||||
|
||||
func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessInput) (GatewayTask, error) {
|
||||
resultJSON, _ := json.Marshal(minimalTaskResult(input.Result))
|
||||
billingsJSON, _ := json.Marshal(input.Billings)
|
||||
usageJSON, _ := json.Marshal(emptyObjectIfNil(input.Usage))
|
||||
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
|
||||
billingSummaryJSON, _ := json.Marshal(emptyObjectIfNil(input.BillingSummary))
|
||||
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
|
||||
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(input.Result))
|
||||
if resultReport.BinaryCount > 0 {
|
||||
return GatewayTask{}, &taskPayloadBinaryError{
|
||||
target: ErrTaskResultBinaryNotMaterialized,
|
||||
code: "result_binary_not_materialized",
|
||||
count: resultReport.BinaryCount,
|
||||
}
|
||||
}
|
||||
resultJSON, _ := json.Marshal(resultReport.Value)
|
||||
billingsJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Billings))
|
||||
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Usage)))
|
||||
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
|
||||
billingSummaryJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.BillingSummary)))
|
||||
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
|
||||
finalChargeAmount := strings.TrimSpace(input.FinalChargeAmountText)
|
||||
if finalChargeAmount == "" {
|
||||
finalChargeAmount = strconv.FormatFloat(input.FinalChargeAmount, 'f', 9, 64)
|
||||
@@ -1158,8 +1166,16 @@ func (s *Store) FinishTaskManualReview(ctx context.Context, input FinishTaskManu
|
||||
if status == "failed" {
|
||||
result = nil
|
||||
}
|
||||
resultJSON, _ := json.Marshal(minimalTaskResult(result))
|
||||
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
|
||||
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(result))
|
||||
if resultReport.BinaryCount > 0 {
|
||||
return GatewayTask{}, &taskPayloadBinaryError{
|
||||
target: ErrTaskResultBinaryNotMaterialized,
|
||||
code: "result_binary_not_materialized",
|
||||
count: resultReport.BinaryCount,
|
||||
}
|
||||
}
|
||||
resultJSON, _ := json.Marshal(resultReport.Value)
|
||||
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if strings.TrimSpace(input.AttemptID) != "" {
|
||||
attemptStatus := "failed"
|
||||
@@ -1261,7 +1277,7 @@ func (s *Store) SettleTaskBilling(ctx context.Context, task GatewayTask) error {
|
||||
"billings": task.Billings,
|
||||
"billingSummary": task.BillingSummary,
|
||||
}
|
||||
metadata, _ := json.Marshal(metadataMap)
|
||||
metadata, _ := json.Marshal(sanitizeJSONForStorage(metadataMap))
|
||||
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_wallet_accounts (
|
||||
@@ -1353,7 +1369,7 @@ ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO N
|
||||
"frozenBefore": roundMoney(frozenBefore),
|
||||
"frozenAfter": frozenAfter,
|
||||
})
|
||||
metadata, _ = json.Marshal(billingMetadata)
|
||||
metadata, _ = json.Marshal(sanitizeJSONForStorage(billingMetadata))
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_wallet_transactions (
|
||||
account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type,
|
||||
@@ -1389,7 +1405,7 @@ func taskBillingString(value any) string {
|
||||
}
|
||||
|
||||
func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureInput) (GatewayTask, error) {
|
||||
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
|
||||
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
|
||||
resultJSON, _ := json.Marshal(minimalTaskResult(nil))
|
||||
message := truncateUTF8Bytes(input.Message, 2048)
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
|
||||
@@ -161,7 +161,7 @@ func (s *Store) ReserveTaskBilling(ctx context.Context, task GatewayTask, user *
|
||||
}
|
||||
|
||||
reservations := make([]WalletBillingReservation, 0, len(amounts))
|
||||
pricingSnapshotJSON, _ := json.Marshal(pricingSnapshot)
|
||||
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(pricingSnapshot))
|
||||
requestFingerprint := walletString(pricingSnapshot["requestFingerprint"])
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
for currency, rawAmount := range amounts {
|
||||
@@ -306,7 +306,7 @@ func (s *Store) reserveTaskBillingExact(ctx context.Context, task GatewayTask, g
|
||||
if currency != "resource" {
|
||||
return nil, fmt.Errorf("unsupported billing currency %q", currency)
|
||||
}
|
||||
pricingSnapshotJSON, _ := json.Marshal(pricingSnapshot)
|
||||
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(pricingSnapshot))
|
||||
requestFingerprint := walletString(pricingSnapshot["requestFingerprint"])
|
||||
var reservations []WalletBillingReservation
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
|
||||
Reference in New Issue
Block a user