perf(storage): 拦截任务二进制并本地暂存结果

原因:任务标准结果中的 Base64、Data URI 和 Buffer 会进入 PostgreSQL JSON,导致 TOAST 与备份体积快速增长。

影响:新增统一 JSON 持久化关口;upload_none 将二进制原子写入本地结果目录,数据库仅保存带 SHA-256 的有界占位符;任务详情、同步响应、异步查询和兼容协议按需校验恢复。补充 24 小时清理、容量上限、历史小批量治理命令及管理端说明。

风险:本地结果超过 TTL、丢失或损坏时分别返回明确的 410/500;空间不足时返回 503 且不重试上游。未自动执行历史治理。

验证:三种真实图片模型同步/异步与幂等重放通过;Go vet/全量测试、前端 111 测试、lint/typecheck/build、OpenAPI、迁移安全、govulncheck、依赖审计、手工发布测试及 Linux amd64 构建通过。
This commit is contained in:
2026-07-24 21:13:09 +08:00
parent 2457de6a56
commit 4f163ea6d7
31 changed files with 2067 additions and 30 deletions
+4
View File
@@ -80,6 +80,10 @@ AI_GATEWAY_TASK_RETENTION_DAYS=30
AI_GATEWAY_TASK_ANALYSIS_RETENTION_DAYS=7
AI_GATEWAY_TASK_CLEANUP_INTERVAL_SECONDS=300
AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE=1000
AI_GATEWAY_LOCAL_RESULT_TTL_HOURS=24
AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES=10737418240
AI_GATEWAY_LOCAL_RESULT_MAX_BYTES=268435456
AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES=536870912
CORS_ALLOWED_ORIGIN=http://localhost:5178,http://127.0.0.1:5178
VITE_GATEWAY_API_BASE_URL=http://localhost:8088
+3 -1
View File
@@ -32,7 +32,8 @@ RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
cd apps/api && \
CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway ./cmd/gateway && \
CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway-migrate ./cmd/migrate
CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway-migrate ./cmd/migrate && \
CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags="-s -w" -o /out/easyai-ai-gateway-backfill-binary-results ./cmd/backfill-binary-results
FROM ${API_RUNTIME_IMAGE} AS api
@@ -42,6 +43,7 @@ WORKDIR /app
COPY --from=api-builder /out/easyai-ai-gateway /app/easyai-ai-gateway
COPY --from=api-builder /out/easyai-ai-gateway-migrate /app/easyai-ai-gateway-migrate
COPY --from=api-builder /out/easyai-ai-gateway-backfill-binary-results /app/easyai-ai-gateway-backfill-binary-results
COPY apps/api/migrations /app/migrations
RUN mkdir -p /app/data/static/generated /app/data/static/uploaded && \
@@ -0,0 +1,113 @@
package main
import (
"context"
"flag"
"log/slog"
"os"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func main() {
apply := flag.Bool("apply", false, "persist compacted results; default is dry-run")
batchSize := flag.Int("batch-size", 100, "rows per batch, maximum 100")
maxBatches := flag.Int("max-batches", 10, "maximum batches for one invocation")
afterID := flag.String("after-id", "", "resume after this task UUID")
flag.Parse()
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
cfg := config.Load()
if err := cfg.Validate(); err != nil {
logger.Error("invalid gateway configuration", "error", err)
os.Exit(1)
}
if *batchSize < 1 || *batchSize > 100 || *maxBatches < 1 {
logger.Error("invalid backfill bounds", "batchSize", *batchSize, "maxBatches", *maxBatches)
os.Exit(1)
}
ctx := context.Background()
db, err := store.Connect(ctx, cfg.DatabaseURL)
if err != nil {
logger.Error("connect postgres failed", "error", err)
os.Exit(1)
}
defer db.Close()
service := runner.New(cfg, db, logger)
cursor := *afterID
scanned := 0
matched := 0
updated := 0
expired := 0
for batch := 0; batch < *maxBatches; batch++ {
items, err := db.ListTaskBinaryResultBackfillBatch(ctx, cursor, *batchSize)
if err != nil {
logger.Error("list binary result backfill batch failed", "afterId", cursor, "error", err)
os.Exit(1)
}
if len(items) == 0 {
break
}
for _, item := range items {
cursor = item.ID
scanned++
if !runner.TaskResultHasInlineBinary(item.Result) {
continue
}
matched++
if !*apply {
continue
}
isExpired := item.FinishedAt.Before(time.Now().Add(-time.Duration(localResultTTLHours(cfg)) * time.Hour))
var persistent map[string]any
var changed bool
if isExpired {
persistent, changed, err = service.CompactExpiredTaskResultForStorage(ctx, item.ID, item.Result)
} else {
persistent, changed, err = service.MaterializeTaskResultForStorage(ctx, item.ID, item.Result)
}
if err != nil {
logger.Error("materialize historical binary result failed", "taskId", item.ID, "error", err)
os.Exit(1)
}
if !changed {
continue
}
ok, err := db.UpdateTaskBinaryResultBackfill(ctx, item.ID, persistent)
if err != nil {
logger.Error("update historical binary result failed", "taskId", item.ID, "error", err)
os.Exit(1)
}
if !ok {
continue
}
updated++
if isExpired {
expired++
}
}
if len(items) < *batchSize {
break
}
}
logger.Info("binary result backfill completed",
"apply", *apply,
"scanned", scanned,
"matched", matched,
"updated", updated,
"expired", expired,
"resumeAfterId", cursor,
)
}
func localResultTTLHours(cfg config.Config) int {
if cfg.LocalResultTTLHours <= 0 {
return 24
}
return cfg.LocalResultTTLHours
}
+44
View File
@@ -4738,6 +4738,12 @@
"schema": {
"$ref": "#/definitions/httpapi.EasyAIGeneratedResponse"
}
},
"410": {
"description": "Gone",
"schema": {
"$ref": "#/definitions/httpapi.EasyAIGeneratedResponse"
}
}
}
}
@@ -5645,6 +5651,12 @@
"schema": {
"$ref": "#/definitions/httpapi.VolcesErrorEnvelope"
}
},
"410": {
"description": "Gone",
"schema": {
"$ref": "#/definitions/httpapi.VolcesErrorEnvelope"
}
}
}
},
@@ -6304,6 +6316,13 @@
"type": "object",
"additionalProperties": true
}
},
"410": {
"description": "Gone",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
}
@@ -6404,6 +6423,13 @@
"type": "object",
"additionalProperties": true
}
},
"410": {
"description": "Gone",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
},
@@ -7883,6 +7909,12 @@
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"410": {
"description": "Gone",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
@@ -8305,6 +8337,12 @@
"$ref": "#/definitions/httpapi.KelingCompatibleEnvelope"
}
},
"410": {
"description": "Gone",
"schema": {
"$ref": "#/definitions/httpapi.KelingCompatibleEnvelope"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
@@ -8766,6 +8804,12 @@
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"410": {
"description": "Gone",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
+30
View File
@@ -6821,6 +6821,10 @@ paths:
description: Not Found
schema:
$ref: '#/definitions/httpapi.EasyAIGeneratedResponse'
"410":
description: Gone
schema:
$ref: '#/definitions/httpapi.EasyAIGeneratedResponse'
security:
- BearerAuth: []
summary: 查询 server-main EasyAIClient 兼容任务结果
@@ -7429,6 +7433,10 @@ paths:
description: Not Found
schema:
$ref: '#/definitions/httpapi.VolcesErrorEnvelope'
"410":
description: Gone
schema:
$ref: '#/definitions/httpapi.VolcesErrorEnvelope'
security:
- BearerAuth: []
summary: 查询火山内容生成任务
@@ -7829,6 +7837,11 @@ paths:
schema:
additionalProperties: true
type: object
"410":
description: Gone
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: 查询可灵 V1 Omni 视频任务
@@ -7893,6 +7906,11 @@ paths:
schema:
additionalProperties: true
type: object
"410":
description: Gone
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: 按 ID 查询可灵 API 2.0 任务
@@ -8852,6 +8870,10 @@ paths:
description: Not Found
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"410":
description: Gone
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"500":
description: Internal Server Error
schema:
@@ -9123,6 +9145,10 @@ paths:
description: Not Found
schema:
$ref: '#/definitions/httpapi.KelingCompatibleEnvelope'
"410":
description: Gone
schema:
$ref: '#/definitions/httpapi.KelingCompatibleEnvelope'
"500":
description: Internal Server Error
schema:
@@ -9421,6 +9447,10 @@ paths:
description: Not Found
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"410":
description: Gone
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"500":
description: Internal Server Error
schema:
+32
View File
@@ -39,6 +39,10 @@ type Config struct {
LocalGeneratedStorageDir string
LocalUploadedStorageDir string
LocalTempAssetTTLHours int
LocalResultTTLHours int
LocalResultMinFreeBytes int64
LocalResultMaxBytes int64
LocalResultMaxTaskBytes int64
TaskProgressCallbackEnabled bool
TaskProgressCallbackURL string
TaskProgressCallbackTimeoutMS int
@@ -88,6 +92,10 @@ func Load() Config {
LocalGeneratedStorageDir: env("AI_GATEWAY_GENERATED_STORAGE_DIR", env("LOCAL_GENERATED_STORAGE_DIR", env("AI_GATEWAY_STATIC_STORAGE_DIR", DefaultLocalGeneratedStorageDir))),
LocalUploadedStorageDir: env("AI_GATEWAY_UPLOADED_STORAGE_DIR", env("LOCAL_UPLOADED_STORAGE_DIR", DefaultLocalUploadedStorageDir)),
LocalTempAssetTTLHours: envInt("AI_GATEWAY_LOCAL_TEMP_ASSET_TTL_HOURS", 24),
LocalResultTTLHours: envIntValidated("AI_GATEWAY_LOCAL_RESULT_TTL_HOURS", 24),
LocalResultMinFreeBytes: envInt64Validated("AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES", 10*1024*1024*1024),
LocalResultMaxBytes: envInt64Validated("AI_GATEWAY_LOCAL_RESULT_MAX_BYTES", 256*1024*1024),
LocalResultMaxTaskBytes: envInt64Validated("AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES", 512*1024*1024),
TaskProgressCallbackEnabled: env("TASK_PROGRESS_CALLBACK_ENABLED", "true") == "true",
TaskProgressCallbackURL: env("TASK_PROGRESS_CALLBACK_URL",
strings.TrimRight(env("SERVER_MAIN_BASE_URL", "http://localhost:3000"), "/")+"/internal/platform/task-progress-callbacks",
@@ -139,6 +147,18 @@ func (c Config) Validate() error {
if c.TaskCleanupBatchSize != 0 && (c.TaskCleanupBatchSize < 100 || c.TaskCleanupBatchSize > 5000) {
return errors.New("AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE must be between 100 and 5000")
}
if c.LocalResultTTLHours != 0 && (c.LocalResultTTLHours < 1 || c.LocalResultTTLHours > 24*30) {
return errors.New("AI_GATEWAY_LOCAL_RESULT_TTL_HOURS must be between 1 and 720")
}
if c.LocalResultMinFreeBytes < 0 {
return errors.New("AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES must not be negative")
}
if c.LocalResultMaxBytes != 0 && c.LocalResultMaxBytes < 1024 {
return errors.New("AI_GATEWAY_LOCAL_RESULT_MAX_BYTES must be at least 1024")
}
if c.LocalResultMaxTaskBytes != 0 && c.LocalResultMaxTaskBytes < c.LocalResultMaxBytes {
return errors.New("AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES must be at least AI_GATEWAY_LOCAL_RESULT_MAX_BYTES")
}
switch strings.ToLower(strings.TrimSpace(c.IdentitySecretStore)) {
case "":
case "file":
@@ -260,6 +280,18 @@ func envIntValidated(key string, fallback int) int {
return parsed
}
func envInt64Validated(key string, fallback int64) int64 {
value := envValue(key)
if value == "" {
return fallback
}
parsed, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return 0
}
return parsed
}
func logLevel(value string) slog.Level {
switch strings.ToLower(value) {
case "debug":
@@ -0,0 +1,30 @@
package httpapi
import (
"context"
"net/http"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func (s *Server) hydrateTaskResult(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) {
if task.Status != "succeeded" || len(task.Result) == 0 {
return task, nil
}
result, err := s.runner.HydrateTaskResult(ctx, task.ID, task.Result)
if err != nil {
return store.GatewayTask{}, err
}
task.Result = result
return task, nil
}
func writeStoredBinaryResultError(w http.ResponseWriter, err error) {
status := statusFromRunError(err)
code := clients.ErrorCode(err)
if code == "" {
code = "binary_result_corrupted"
}
writeError(w, status, err.Error(), code)
}
@@ -170,6 +170,10 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
writeGeminiTaskError(http.StatusConflict, "Idempotency-Key was reused for a different request", nil, "idempotency_key_reused")
return
}
if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) {
writeGeminiTaskError(http.StatusBadRequest, err.Error(), nil, clients.ErrorCode(err))
return
}
s.logger.Error("create gemini task failed", "kind", mapping.Kind, "error_category", "task_create_failed")
writeGeminiTaskError(http.StatusInternalServerError, "create task failed", nil, "task_create_failed")
return
@@ -184,6 +188,11 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
return
}
if task.Status == "succeeded" {
task, err = s.hydrateTaskResult(r.Context(), task)
if err != nil {
writeGeminiTaskError(statusFromRunError(err), err.Error(), nil, clients.ErrorCode(err))
return
}
if _, native := task.Result["candidates"]; native {
writeJSON(w, http.StatusOK, task.Result)
return
+27
View File
@@ -1134,6 +1134,10 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
writeTaskError(http.StatusConflict, "Idempotency-Key was reused for a different request", nil, "idempotency_key_reused")
return
}
if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) {
writeTaskError(http.StatusBadRequest, err.Error(), nil, clients.ErrorCode(err))
return
}
s.logger.Error("create task failed", "kind", kind, "error_category", "task_create_failed")
writeTaskError(http.StatusInternalServerError, "create task failed", nil, "task_create_failed")
return
@@ -1150,6 +1154,17 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
writeTaskError(http.StatusConflict, "streaming idempotent replay is not supported", nil, "idempotency_stream_replay_unsupported")
return
}
if task.Status == "succeeded" && !responsePlan.asyncMode && (targetProtocol != "" || responsePlan.compatibleMode) {
task, err = s.hydrateTaskResult(r.Context(), task)
if err != nil {
if targetProtocol != "" {
writeProtocolError(w, targetProtocol, statusFromRunError(err), err.Error(), nil, clients.ErrorCode(err))
} else {
writeStoredBinaryResultError(w, err)
}
return
}
}
if targetProtocol != "" && !responsePlan.asyncMode {
if task.Status == "succeeded" {
writeJSON(w, http.StatusOK, task.Result)
@@ -1597,6 +1612,12 @@ func scopeForTaskKind(kind string) string {
func statusFromRunError(err error) int {
switch {
case clients.ErrorCode(err) == "binary_result_expired":
return http.StatusGone
case clients.ErrorCode(err) == "binary_result_corrupted" || clients.ErrorCode(err) == "result_binary_not_materialized":
return http.StatusInternalServerError
case clients.ErrorCode(err) == "local_result_storage_unavailable":
return http.StatusServiceUnavailable
case clients.ErrorCode(err) == "billing_hold":
return http.StatusServiceUnavailable
case runner.IsPricingUnavailable(err):
@@ -1901,6 +1922,7 @@ func boolValue(body map[string]any, key string) bool {
// @Success 200 {object} store.GatewayTask
// @Failure 401 {object} ErrorEnvelope
// @Failure 404 {object} ErrorEnvelope
// @Failure 410 {object} ErrorEnvelope
// @Failure 500 {object} ErrorEnvelope
// @Router /api/workspace/tasks/{taskID} [get]
// @Router /api/v1/tasks/{taskID} [get]
@@ -1920,6 +1942,11 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
task.Cancellable = &cancelState.Cancellable
task.Submitted = &cancelState.Submitted
task.Message = cancelState.Message
task, err = s.hydrateTaskResult(r.Context(), task)
if err != nil {
writeStoredBinaryResultError(w, err)
return
}
writeJSON(w, http.StatusOK, task)
return
}
@@ -177,6 +177,10 @@ func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
writeKelingCompatError(w, requestID, kelingCompatGatewayError(staged.Err))
return
}
if errors.Is(createErr, store.ErrTaskRequestBinaryNotMaterialized) {
writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusBadRequest, 1201, clients.ErrorCode(createErr)))
return
}
s.logger.Error("create Kling-compatible task failed", "error", createErr)
writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusInternalServerError, 5000, "create task failed"))
return
@@ -221,6 +225,7 @@ func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
// @Failure 401 {object} KelingCompatibleEnvelope
// @Failure 403 {object} KelingCompatibleEnvelope
// @Failure 404 {object} KelingCompatibleEnvelope
// @Failure 410 {object} KelingCompatibleEnvelope
// @Failure 500 {object} KelingCompatibleEnvelope
// @Router /api/v1/videos/omni-video/{taskID} [get]
func (s *Server) getKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
@@ -247,6 +252,11 @@ func (s *Server) getKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusNotFound, 1203, "task not found"))
return
}
task, err = s.hydrateTaskResult(r.Context(), task)
if err != nil {
writeKelingCompatError(w, requestID, newKelingCompatError(statusFromRunError(err), 5000, clients.ErrorCode(err)))
return
}
writeJSON(w, http.StatusOK, KelingCompatibleEnvelope{
Code: 0,
Message: "SUCCEED",
+14
View File
@@ -144,6 +144,8 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
writeKlingCompatError(w, http.StatusConflict, err.Error(), "idempotency_key_reused")
case errors.Is(err, store.ErrExternalTaskIDReused):
writeKlingCompatError(w, http.StatusConflict, err.Error(), "external_task_id_reused")
case errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized):
writeKlingCompatError(w, http.StatusBadRequest, err.Error(), clients.ErrorCode(err))
default:
s.logger.Error("create Kling compatibility task failed", "version", version, "model", model, "error", err)
writeKlingCompatError(w, http.StatusInternalServerError, "create task failed", "task_create_failed")
@@ -464,6 +466,7 @@ func validateKlingCompatBody(model string, body map[string]any) error {
// @Param taskID path string true "任务 ID"
// @Success 200 {object} map[string]interface{}
// @Failure 404 {object} map[string]interface{}
// @Failure 410 {object} map[string]interface{}
// @Router /api/v1/kling/v1/videos/omni-video/{taskID} [get]
func (s *Server) klingV1GetOmniVideo(w http.ResponseWriter, r *http.Request) {
user, _ := auth.UserFromContext(r.Context())
@@ -476,6 +479,11 @@ func (s *Server) klingV1GetOmniVideo(w http.ResponseWriter, r *http.Request) {
writeKlingCompatError(w, http.StatusInternalServerError, "get task failed", "task_query_failed")
return
}
task, err = s.hydrateTaskResult(r.Context(), task)
if err != nil {
writeKlingCompatError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
return
}
writeJSON(w, http.StatusOK, klingV1Envelope(task))
}
@@ -520,6 +528,7 @@ func (s *Server) klingV1ListOmniVideos(w http.ResponseWriter, r *http.Request) {
// @Param task_ids query string false "逗号分隔的任务 ID"
// @Param external_task_ids query string false "逗号分隔的外部任务 ID"
// @Success 200 {object} map[string]interface{}
// @Failure 410 {object} map[string]interface{}
// @Router /api/v1/kling/v2/tasks [get]
func (s *Server) klingV2GetTasks(w http.ResponseWriter, r *http.Request) {
user, _ := auth.UserFromContext(r.Context())
@@ -543,6 +552,11 @@ func (s *Server) klingV2GetTasks(w http.ResponseWriter, r *http.Request) {
writeKlingCompatError(w, http.StatusInternalServerError, "get task failed", "task_query_failed")
return
}
task, err = s.hydrateTaskResult(r.Context(), task)
if err != nil {
writeKlingCompatError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
return
}
data = append(data, klingV2TaskData(task))
}
writeJSON(w, http.StatusOK, map[string]any{"code": 0, "message": "success", "request_id": klingRequestIDFromAny(data), "data": data})
@@ -45,6 +45,58 @@ func (s *Server) cleanupExpiredLocalTempAssets(ctx context.Context, now time.Tim
for _, target := range targets {
deleted += s.cleanupExpiredLocalTempAssetsInDir(ctx, now, target)
}
deleted += s.cleanupExpiredLocalBinaryResults(now)
return deleted
}
func (s *Server) cleanupExpiredLocalBinaryResults(now time.Time) int {
storageDir := strings.TrimSpace(s.cfg.LocalGeneratedStorageDir)
if storageDir == "" {
storageDir = config.DefaultLocalGeneratedStorageDir
}
root := filepath.Join(storageDir, "results")
ttlHours := s.cfg.LocalResultTTLHours
if ttlHours <= 0 {
ttlHours = 24
}
expiredBefore := now.Add(-time.Duration(ttlHours) * time.Hour)
deleted := 0
var directories []string
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
if !errors.Is(walkErr, os.ErrNotExist) && s.logger != nil {
s.logger.Warn("walk local binary result failed", "path", path, "error", walkErr)
}
return nil
}
if entry.IsDir() {
if path != root {
directories = append(directories, path)
}
return nil
}
if entry.Type()&os.ModeSymlink != 0 {
return nil
}
info, infoErr := entry.Info()
if infoErr != nil || info.ModTime().After(expiredBefore) {
return nil
}
if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
if s.logger != nil {
s.logger.Warn("remove local binary result failed", "path", path, "error", removeErr)
}
return nil
}
deleted++
return nil
})
if err != nil && !errors.Is(err, os.ErrNotExist) && s.logger != nil {
s.logger.Warn("scan local binary result root failed", "dir", root, "error", err)
}
for index := len(directories) - 1; index >= 0; index-- {
_ = os.Remove(directories[index])
}
return deleted
}
@@ -11,6 +11,7 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -235,6 +236,47 @@ func TestCleanupExpiredLocalTempAssetsDeletesExpiredStaticFiles(t *testing.T) {
}
}
func TestCleanupExpiredLocalTempAssetsDeletesNestedBinaryResultsWithIndependentTTL(t *testing.T) {
generatedDir := t.TempDir()
taskDir := filepath.Join(generatedDir, "results", "task-123")
if err := os.MkdirAll(taskDir, 0o750); err != nil {
t.Fatalf("create local result fixture dir: %v", err)
}
oldResult := filepath.Join(taskDir, strings.Repeat("a", 64)+".bin")
freshResult := filepath.Join(taskDir, strings.Repeat("b", 64)+".bin")
for _, path := range []string{oldResult, freshResult} {
if err := os.WriteFile(path, []byte("asset"), 0o640); err != nil {
t.Fatalf("write fixture %s: %v", path, err)
}
}
now := time.Now()
if err := os.Chtimes(oldResult, now.Add(-25*time.Hour), now.Add(-25*time.Hour)); err != nil {
t.Fatalf("age old binary result: %v", err)
}
if err := os.Chtimes(freshResult, now.Add(-23*time.Hour), now.Add(-23*time.Hour)); err != nil {
t.Fatalf("age fresh binary result: %v", err)
}
server := &Server{
cfg: config.Config{
LocalGeneratedStorageDir: generatedDir,
LocalResultTTLHours: 24,
},
logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
}
deleted := server.cleanupExpiredLocalTempAssets(context.Background(), now)
if deleted != 1 {
t.Fatalf("expected one expired binary result delete, got %d", deleted)
}
if _, err := os.Stat(oldResult); !os.IsNotExist(err) {
t.Fatalf("old binary result should be deleted, stat err=%v", err)
}
if _, err := os.Stat(freshResult); err != nil {
t.Fatalf("fresh binary result should remain: %v", err)
}
}
func TestRequestConversationKeyPriority(t *testing.T) {
request := httptest.NewRequest(http.MethodPost, "/api/v1/chat/completions", nil)
request.Header.Set("X-EasyAI-Conversation-ID", "from-header")
@@ -56,12 +56,19 @@ func (s *Server) createVolcesContentsGenerationTask(w http.ResponseWriter, r *ht
// @Param taskID path string true "任务 ID"
// @Success 200 {object} VolcesContentsGenerationTaskResponse
// @Failure 404 {object} VolcesErrorEnvelope
// @Failure 410 {object} VolcesErrorEnvelope
// @Router /api/v1/contents/generations/tasks/{taskID} [get]
func (s *Server) getVolcesContentsGenerationTask(w http.ResponseWriter, r *http.Request) {
task, ok := s.volcesCompatibleTaskForUser(w, r)
if !ok {
return
}
var err error
task, err = s.hydrateTaskResult(r.Context(), task)
if err != nil {
writeVolcesError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
return
}
writeJSON(w, http.StatusOK, volcesCompatibleTask(task))
}
@@ -178,6 +185,7 @@ func (s *Server) createLegacyVolcesVideoGeneration(w http.ResponseWriter, r *htt
// @Param taskID path string true "任务 ID"
// @Success 200 {object} EasyAIGeneratedResponse
// @Failure 404 {object} EasyAIGeneratedResponse
// @Failure 410 {object} EasyAIGeneratedResponse
// @Router /api/v1/ai/result/{taskID} [get]
func (s *Server) getEasyAITaskResult(w http.ResponseWriter, r *http.Request) {
user, ok := auth.UserFromContext(r.Context())
@@ -199,6 +207,11 @@ func (s *Server) getEasyAITaskResult(w http.ResponseWriter, r *http.Request) {
writeEasyAIAsyncError(w, http.StatusNotFound, "task not found", nil, "not_found")
return
}
task, err = s.hydrateTaskResult(r.Context(), task)
if err != nil {
writeEasyAIAsyncError(w, statusFromRunError(err), err.Error(), nil, clients.ErrorCode(err))
return
}
writeJSON(w, http.StatusOK, easyAITaskResultResponse(task))
}
@@ -408,6 +421,8 @@ func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) {
status = clientErr.StatusCode
} else if errors.As(err, &clientErr) {
status = http.StatusBadRequest
} else if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) {
status = http.StatusBadRequest
}
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, clients.ProtocolVolcesContents) {
writeWireResponse(w, wire)
+721
View File
@@ -0,0 +1,721 @@
package runner
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
)
const (
localBinaryResultDirName = "results"
localBinaryPlaceholderPrefix = "[GatewayBinary:v1;"
localBinaryGenericBase64MinLength = 4096
localBinaryMaxDepth = 64
defaultLocalResultTTLHours = 24
defaultLocalResultMinFreeBytes = int64(10 * 1024 * 1024 * 1024)
defaultLocalResultMaxBytes = int64(256 * 1024 * 1024)
defaultLocalResultMaxTaskBytes = int64(512 * 1024 * 1024)
)
type localBinaryDescriptor struct {
Prefix string
SHA256 string
Size int64
ContentType string
Encoding string
}
type localBinaryMaterializer struct {
service *Service
taskDir string
writeFiles bool
enforceLimits bool
totalBytes int64
seen map[string]struct{}
createdFiles []string
}
// materializeLocalBinaryResult replaces every inline binary value with a
// bounded placeholder after atomically writing and verifying the bytes locally.
func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
if _, _, err := s.transformLocalBinaryResult(ctx, taskID, result, false, true); err != nil {
return nil, false, err
}
return s.transformLocalBinaryResult(ctx, taskID, result, true, true)
}
func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string, result map[string]any, writeFiles bool, enforceLimits bool) (map[string]any, bool, error) {
root := s.localBinaryResultRoot()
taskDir := filepath.Join(root, safeLocalBinaryTaskDir(taskID))
materializer := &localBinaryMaterializer{
service: s,
taskDir: taskDir,
writeFiles: writeFiles,
enforceLimits: enforceLimits,
seen: map[string]struct{}{},
}
next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0)
if err != nil {
materializer.rollbackCreatedFiles()
return nil, false, err
}
mapped, ok := next.(map[string]any)
if !ok {
materializer.rollbackCreatedFiles()
return nil, false, &clients.ClientError{
Code: "result_binary_not_materialized",
Message: "generated result is not a JSON object",
StatusCode: 500,
Retryable: false,
}
}
if changed && enforceLimits && !writeFiles {
if err := os.MkdirAll(root, 0o750); err != nil {
return nil, false, localBinaryStorageError(err)
}
if err := ensureLocalBinaryDiskHeadroom(root, materializer.totalBytes, s.localResultMinFreeBytes()); err != nil {
return nil, false, err
}
}
return mapped, changed, nil
}
// MaterializeTaskResultForStorage exposes the same verified materialization
// path to the explicit historical maintenance command.
func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
return s.materializeLocalBinaryResult(ctx, taskID, result)
}
// CompactExpiredTaskResultForStorage computes the same deterministic
// placeholders without creating files that are already outside the recovery
// window.
func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
return s.transformLocalBinaryResult(ctx, taskID, result, false, false)
}
func TaskResultHasInlineBinary(result map[string]any) bool {
return localBinaryValueHasPayload(result, "", nil, 0)
}
func localBinaryValueHasPayload(value any, key string, siblings map[string]any, depth int) bool {
if depth >= localBinaryMaxDepth {
return false
}
switch typed := value.(type) {
case map[string]any:
if _, _, ok := localBufferObjectBytes(typed); ok {
return true
}
for childKey, child := range typed {
if localBinaryValueHasPayload(child, childKey, typed, depth+1) {
return true
}
}
case []any:
if localBinaryKey(key) {
if _, ok := bytesFromNumberArray(typed); ok {
return true
}
}
for _, child := range typed {
if localBinaryValueHasPayload(child, key, siblings, depth+1) {
return true
}
}
case []byte:
return len(typed) > 0
case string:
_, _, _, ok := localBinaryStringBytes(key, typed, siblings)
return ok
}
return false
}
func (m *localBinaryMaterializer) materializeValue(ctx context.Context, value any, key string, siblings map[string]any, depth int) (any, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
if depth >= localBinaryMaxDepth {
return nil, false, &clients.ClientError{
Code: "result_binary_not_materialized",
Message: "generated result exceeds the maximum JSON depth",
StatusCode: 500,
Retryable: false,
}
}
switch typed := value.(type) {
case map[string]any:
if payload, contentType, ok := localBufferObjectBytes(typed); ok {
return m.persistBinary(ctx, payload, contentType, "buffer")
}
next := make(map[string]any, len(typed))
changed := false
for childKey, childValue := range typed {
child, childChanged, err := m.materializeValue(ctx, childValue, childKey, typed, depth+1)
if err != nil {
return nil, false, err
}
next[childKey] = child
changed = changed || childChanged
}
if !changed {
return value, false, nil
}
return next, true, nil
case []any:
if localBinaryKey(key) {
if payload, ok := bytesFromNumberArray(typed); ok {
return m.persistBinary(ctx, payload, mediaContentTypeFromItem(siblings), "buffer")
}
}
next := make([]any, len(typed))
changed := false
for index, item := range typed {
child, childChanged, err := m.materializeValue(ctx, item, key, siblings, depth+1)
if err != nil {
return nil, false, err
}
next[index] = child
changed = changed || childChanged
}
if !changed {
return value, false, nil
}
return next, true, nil
case []byte:
if len(typed) == 0 {
return value, false, nil
}
return m.persistBinary(ctx, append([]byte(nil), typed...), mediaContentTypeFromItem(siblings), "buffer")
case string:
payload, contentType, encoding, ok := localBinaryStringBytes(key, typed, siblings)
if !ok {
return value, false, nil
}
return m.persistBinary(ctx, payload, contentType, encoding)
default:
return value, false, nil
}
}
func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []byte, contentType string, encoding string) (any, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
if len(payload) == 0 {
return nil, false, nil
}
size := int64(len(payload))
if m.enforceLimits && size > m.service.localResultMaxBytes() {
return nil, false, &clients.ClientError{
Code: "binary_result_too_large",
Message: "one generated binary result exceeds the local storage limit",
StatusCode: 502,
Retryable: false,
}
}
digest := sha256.Sum256(payload)
digestHex := hex.EncodeToString(digest[:])
if _, exists := m.seen[digestHex]; !exists {
if m.enforceLimits && m.totalBytes+size > m.service.localResultMaxTaskBytes() {
return nil, false, &clients.ClientError{
Code: "binary_result_too_large",
Message: "generated binary results exceed the per-task local storage limit",
StatusCode: 502,
Retryable: false,
}
}
if m.writeFiles {
created, err := m.service.writeLocalBinaryResult(m.taskDir, digestHex, payload)
if err != nil {
return nil, false, err
}
if created {
m.createdFiles = append(m.createdFiles, filepath.Join(m.taskDir, digestHex+".bin"))
}
}
m.seen[digestHex] = struct{}{}
m.totalBytes += size
}
descriptor := localBinaryDescriptor{
Prefix: localBinaryPrefix(payload),
SHA256: digestHex,
Size: size,
ContentType: normalizedLocalBinaryContentType(contentType),
Encoding: normalizedLocalBinaryEncoding(encoding),
}
return localBinaryPlaceholder(descriptor), true, nil
}
func (m *localBinaryMaterializer) rollbackCreatedFiles() {
for _, path := range m.createdFiles {
_ = os.Remove(path)
}
_ = os.Remove(m.taskDir)
m.createdFiles = nil
}
func (s *Service) writeLocalBinaryResult(taskDir string, digestHex string, payload []byte) (bool, error) {
if err := os.MkdirAll(taskDir, 0o750); err != nil {
return false, localBinaryStorageError(err)
}
targetPath := filepath.Join(taskDir, digestHex+".bin")
if info, err := os.Stat(targetPath); err == nil {
if !info.IsDir() && info.Size() == int64(len(payload)) {
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err == nil {
now := time.Now()
if err := os.Chtimes(targetPath, now, now); err != nil {
return false, localBinaryStorageError(err)
}
return false, nil
}
}
return false, &clients.ClientError{
Code: "binary_result_corrupted",
Message: "existing local result file does not match its content hash",
StatusCode: 500,
Retryable: false,
}
} else if !errors.Is(err, os.ErrNotExist) {
return false, localBinaryStorageError(err)
}
if err := ensureLocalBinaryDiskHeadroom(taskDir, int64(len(payload)), s.localResultMinFreeBytes()); err != nil {
return false, err
}
tempFile, err := os.CreateTemp(taskDir, ".gateway-result-*")
if err != nil {
return false, localBinaryStorageError(err)
}
tempPath := tempFile.Name()
cleanup := func() {
_ = tempFile.Close()
_ = os.Remove(tempPath)
}
if err := tempFile.Chmod(0o640); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if _, err := tempFile.Write(payload); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if err := tempFile.Sync(); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if err := tempFile.Close(); err != nil {
_ = os.Remove(tempPath)
return false, localBinaryStorageError(err)
}
if err := os.Rename(tempPath, targetPath); err != nil {
_ = os.Remove(tempPath)
return false, localBinaryStorageError(err)
}
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err != nil {
_ = os.Remove(targetPath)
return false, err
}
return true, nil
}
func ensureLocalBinaryDiskHeadroom(path string, incomingBytes int64, minFreeBytes int64) error {
var stat syscall.Statfs_t
if err := syscall.Statfs(path, &stat); err != nil {
return localBinaryStorageError(err)
}
freeBytes := int64(stat.Bavail) * int64(stat.Bsize)
if freeBytes-incomingBytes < minFreeBytes {
return &clients.ClientError{
Code: "local_result_storage_unavailable",
Message: "local result storage does not have enough free space",
StatusCode: 503,
Retryable: false,
}
}
return nil
}
func localBinaryStorageError(err error) error {
return &clients.ClientError{
Code: "local_result_storage_unavailable",
Message: "local result storage failed: " + err.Error(),
StatusCode: 503,
Retryable: false,
}
}
func verifyLocalBinaryFile(path string, expectedHash string, expectedSize int64) error {
file, err := os.Open(path)
if err != nil {
return localBinaryStorageError(err)
}
defer file.Close()
hasher := sha256.New()
size, err := io.Copy(hasher, file)
if err != nil {
return localBinaryStorageError(err)
}
if size != expectedSize || hex.EncodeToString(hasher.Sum(nil)) != expectedHash {
return &clients.ClientError{
Code: "binary_result_corrupted",
Message: "local result file failed size or hash verification",
StatusCode: 500,
Retryable: false,
}
}
return nil
}
// HydrateTaskResult restores placeholders from verified local files. It is only
// used by result/detail/replay endpoints, never by task lists or callbacks.
func (s *Service) HydrateTaskResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, error) {
next, changed, err := s.hydrateLocalBinaryValue(ctx, taskID, result, 0)
if err != nil {
return nil, err
}
if !changed {
return result, nil
}
mapped, ok := next.(map[string]any)
if !ok {
return nil, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result is not a JSON object", StatusCode: 500}
}
return mapped, nil
}
func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, value any, depth int) (any, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
if depth >= localBinaryMaxDepth {
return nil, false, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result exceeds the maximum JSON depth", StatusCode: 500}
}
switch typed := value.(type) {
case map[string]any:
next := make(map[string]any, len(typed))
changed := false
for key, childValue := range typed {
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
if err != nil {
return nil, false, err
}
next[key] = child
changed = changed || childChanged
}
if !changed {
return value, false, nil
}
return next, true, nil
case []any:
next := make([]any, len(typed))
changed := false
for index, childValue := range typed {
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
if err != nil {
return nil, false, err
}
next[index] = child
changed = changed || childChanged
}
if !changed {
return value, false, nil
}
return next, true, nil
case string:
descriptor, ok := parseLocalBinaryPlaceholder(typed)
if !ok {
return value, false, nil
}
payload, err := s.readLocalBinaryResult(taskID, descriptor)
if err != nil {
return nil, false, err
}
encoded := base64.StdEncoding.EncodeToString(payload)
if descriptor.Encoding == "data-uri" {
return "data:" + descriptor.ContentType + ";base64," + encoded, true, nil
}
return encoded, true, nil
default:
return value, false, nil
}
}
func (s *Service) readLocalBinaryResult(taskID string, descriptor localBinaryDescriptor) ([]byte, error) {
path := filepath.Join(s.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID), descriptor.SHA256+".bin")
info, err := os.Stat(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, &clients.ClientError{Code: "binary_result_expired", Message: "local binary result has expired", StatusCode: 410, Retryable: false}
}
return nil, localBinaryStorageError(err)
}
if info.IsDir() || info.Size() != descriptor.Size {
return nil, &clients.ClientError{Code: "binary_result_corrupted", Message: "local binary result has an invalid size", StatusCode: 500, Retryable: false}
}
if info.ModTime().Before(time.Now().Add(-time.Duration(s.localResultTTLHours()) * time.Hour)) {
return nil, &clients.ClientError{Code: "binary_result_expired", Message: "local binary result has expired", StatusCode: 410, Retryable: false}
}
payload, err := os.ReadFile(path)
if err != nil {
return nil, localBinaryStorageError(err)
}
digest := sha256.Sum256(payload)
if int64(len(payload)) != descriptor.Size || hex.EncodeToString(digest[:]) != descriptor.SHA256 {
return nil, &clients.ClientError{Code: "binary_result_corrupted", Message: "local binary result failed size or hash verification", StatusCode: 500, Retryable: false}
}
return payload, nil
}
func localBinaryStringBytes(key string, value string, siblings map[string]any) ([]byte, string, string, bool) {
raw := strings.TrimSpace(value)
if raw == "" || strings.HasPrefix(raw, localBinaryPlaceholderPrefix) {
return nil, "", "", false
}
if strings.HasPrefix(strings.ToLower(raw), "data:") {
contentType, encoded, ok, err := parseBase64DataURL(raw)
if err == nil && ok {
payload, decodeErr := decodeBase64Payload(encoded)
if decodeErr == nil && len(payload) > 0 {
return payload, contentType, "data-uri", true
}
}
return nil, "", "", false
}
strict := localBinaryKey(key)
if !strict && len(raw) < localBinaryGenericBase64MinLength {
return nil, "", "", false
}
payload, err := decodeBase64Payload(raw)
if err != nil || len(payload) == 0 {
return nil, "", "", false
}
return payload, firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key)), "raw", true
}
func localBufferObjectBytes(value map[string]any) ([]byte, string, bool) {
if normalizeLocalBinaryKey(stringFromAny(value["type"])) != "buffer" {
return nil, "", false
}
contentType := firstNonEmptyString(
stringFromAny(value["mime_type"]),
stringFromAny(value["mimeType"]),
stringFromAny(value["contentType"]),
)
switch data := value["data"].(type) {
case []byte:
if len(data) == 0 {
return nil, "", false
}
return append([]byte(nil), data...), contentType, true
case []any:
payload, ok := bytesFromNumberArray(data)
return payload, contentType, ok
default:
return nil, "", false
}
}
func localBinaryKey(key string) bool {
normalized := normalizeLocalBinaryKey(key)
return normalized == "b64" ||
normalized == "b64json" ||
normalized == "base64" ||
normalized == "buffer" ||
normalized == "bytes" ||
strings.Contains(normalized, "base64") ||
strings.Contains(normalized, "buffer") ||
strings.Contains(normalized, "binary") ||
strings.HasSuffix(normalized, "b64") ||
strings.HasSuffix(normalized, "bytes")
}
func normalizeLocalBinaryKey(value string) string {
return strings.Map(func(char rune) rune {
switch {
case char >= 'a' && char <= 'z':
return char
case char >= 'A' && char <= 'Z':
return char + ('a' - 'A')
case char >= '0' && char <= '9':
return char
default:
return -1
}
}, value)
}
func localBinaryPrefix(payload []byte) string {
prefix := base64.StdEncoding.EncodeToString(payload)
if len(prefix) > 16 {
prefix = prefix[:16]
}
return prefix
}
func normalizedLocalBinaryContentType(value string) string {
value = normalizeGeneratedContentType(value)
if value == "" || len(value) > 32 || !localBinaryContentTypeSafe(value) {
return "application/octet-stream"
}
return value
}
func localBinaryContentTypeSafe(value string) bool {
for _, char := range value {
switch {
case char >= 'a' && char <= 'z':
case char >= 'A' && char <= 'Z':
case char >= '0' && char <= '9':
case char == '/', char == '.', char == '+', char == '-':
default:
return false
}
}
return true
}
func normalizedLocalBinaryEncoding(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "data-uri":
return "data-uri"
case "buffer":
return "buffer"
default:
return "raw"
}
}
func localBinaryPlaceholder(descriptor localBinaryDescriptor) string {
return fmt.Sprintf(
"[GatewayBinary:v1;prefix=%s;sha256=%s;bytes=%d;mime=%s;encoding=%s]",
descriptor.Prefix,
descriptor.SHA256,
descriptor.Size,
descriptor.ContentType,
descriptor.Encoding,
)
}
func parseLocalBinaryPlaceholder(value string) (localBinaryDescriptor, bool) {
if !strings.HasPrefix(value, localBinaryPlaceholderPrefix) || !strings.HasSuffix(value, "]") {
return localBinaryDescriptor{}, false
}
content := strings.TrimSuffix(strings.TrimPrefix(value, localBinaryPlaceholderPrefix), "]")
fields := map[string]string{}
for _, item := range strings.Split(content, ";") {
key, fieldValue, ok := strings.Cut(item, "=")
if !ok {
continue
}
fields[key] = fieldValue
}
size, err := strconv.ParseInt(fields["bytes"], 10, 64)
if err != nil || size <= 0 || len(fields["sha256"]) != sha256.Size*2 {
return localBinaryDescriptor{}, false
}
if _, err := hex.DecodeString(fields["sha256"]); err != nil {
return localBinaryDescriptor{}, false
}
descriptor := localBinaryDescriptor{
Prefix: fields["prefix"],
SHA256: strings.ToLower(fields["sha256"]),
Size: size,
ContentType: normalizedLocalBinaryContentType(fields["mime"]),
Encoding: normalizedLocalBinaryEncoding(fields["encoding"]),
}
if len(descriptor.Prefix) > 16 {
return localBinaryDescriptor{}, false
}
return descriptor, true
}
func localBinaryResultHasPlaceholders(value any) bool {
switch typed := value.(type) {
case map[string]any:
for _, child := range typed {
if localBinaryResultHasPlaceholders(child) {
return true
}
}
case []any:
for _, child := range typed {
if localBinaryResultHasPlaceholders(child) {
return true
}
}
case string:
return strings.HasPrefix(typed, localBinaryPlaceholderPrefix)
}
return false
}
func safeLocalBinaryTaskDir(taskID string) string {
taskID = strings.TrimSpace(taskID)
if taskID != "" {
safe := true
for _, char := range taskID {
if (char >= 'a' && char <= 'z') ||
(char >= 'A' && char <= 'Z') ||
(char >= '0' && char <= '9') ||
char == '-' || char == '_' {
continue
}
safe = false
break
}
if safe && taskID != "." && taskID != ".." && len(taskID) <= 128 {
return taskID
}
}
digest := sha256.Sum256([]byte(taskID))
return "task-" + hex.EncodeToString(digest[:16])
}
func (s *Service) localBinaryResultRoot() string {
root := strings.TrimSpace(s.cfg.LocalGeneratedStorageDir)
if root == "" {
root = config.DefaultLocalGeneratedStorageDir
}
return filepath.Join(root, localBinaryResultDirName)
}
func (s *Service) localResultTTLHours() int {
if s.cfg.LocalResultTTLHours <= 0 {
return defaultLocalResultTTLHours
}
return s.cfg.LocalResultTTLHours
}
func (s *Service) localResultMinFreeBytes() int64 {
if s.cfg.LocalResultMinFreeBytes <= 0 {
return defaultLocalResultMinFreeBytes
}
return s.cfg.LocalResultMinFreeBytes
}
func (s *Service) localResultMaxBytes() int64 {
if s.cfg.LocalResultMaxBytes <= 0 {
return defaultLocalResultMaxBytes
}
return s.cfg.LocalResultMaxBytes
}
func (s *Service) localResultMaxTaskBytes() int64 {
if s.cfg.LocalResultMaxTaskBytes <= 0 {
return defaultLocalResultMaxTaskBytes
}
return s.cfg.LocalResultMaxTaskBytes
}
@@ -0,0 +1,233 @@
package runner
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func newLocalBinaryTestService(t *testing.T) *Service {
t.Helper()
return &Service{cfg: config.Config{
LocalGeneratedStorageDir: t.TempDir(),
LocalResultTTLHours: 24,
LocalResultMinFreeBytes: 1,
LocalResultMaxBytes: 1024 * 1024,
LocalResultMaxTaskBytes: 2 * 1024 * 1024,
}}
}
func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
service := newLocalBinaryTestService(t)
payload := []byte("one binary result shared across representations")
encoded := base64.StdEncoding.EncodeToString(payload)
input := map[string]any{
"data": []any{
map[string]any{
"b64_json": encoded,
"data_uri": "data:image/png;base64," + encoded,
"buffer": map[string]any{
"type": "Buffer",
"data": bytesToAny(payload),
"mimeType": "image/png",
},
},
},
}
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-123", input)
if err != nil {
t.Fatalf("materialize local binary result: %v", err)
}
if !changed {
t.Fatal("expected binary result to be materialized")
}
item := persistent["data"].([]any)[0].(map[string]any)
for _, key := range []string{"b64_json", "data_uri", "buffer"} {
placeholder, ok := item[key].(string)
if !ok || !strings.HasPrefix(placeholder, localBinaryPlaceholderPrefix) {
t.Fatalf("%s was not replaced with a placeholder: %+v", key, item[key])
}
if len(placeholder) > 200 {
t.Fatalf("%s placeholder exceeds 200 bytes: %d", key, len(placeholder))
}
}
if input["data"].([]any)[0].(map[string]any)["b64_json"] != encoded {
t.Fatal("materializer mutated the provider result")
}
persistentJSON, err := json.Marshal(persistent)
if err != nil {
t.Fatalf("marshal persistent result: %v", err)
}
if len(persistentJSON) > 32*1024 {
t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
}
taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
entries, err := os.ReadDir(taskDir)
if err != nil {
t.Fatalf("read local result dir: %v", err)
}
if len(entries) != 1 {
t.Fatalf("same payload should reuse one local file, got %d", len(entries))
}
taskInfo, err := os.Stat(taskDir)
if err != nil {
t.Fatalf("stat task directory: %v", err)
}
if taskInfo.Mode().Perm() != 0o750 {
t.Fatalf("task directory mode = %v, want 0750", taskInfo.Mode().Perm())
}
fileInfo, err := entries[0].Info()
if err != nil {
t.Fatalf("stat result file: %v", err)
}
if fileInfo.Mode().Perm() != 0o640 {
t.Fatalf("result file mode = %v, want 0640", fileInfo.Mode().Perm())
}
wire, err := service.HydrateTaskResult(context.Background(), "task-123", persistent)
if err != nil {
t.Fatalf("hydrate local binary result: %v", err)
}
wireItem := wire["data"].([]any)[0].(map[string]any)
if wireItem["b64_json"] != encoded {
t.Fatalf("raw Base64 mismatch: got %v", wireItem["b64_json"])
}
if wireItem["data_uri"] != "data:image/png;base64,"+encoded {
t.Fatalf("data URI mismatch: got %v", wireItem["data_uri"])
}
if wireItem["buffer"] != encoded {
t.Fatalf("Buffer should hydrate as Base64: got %v", wireItem["buffer"])
}
}
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
service := newLocalBinaryTestService(t)
service.cfg.LocalResultTTLHours = 1
encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
persistent, _, err := service.materializeLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded})
if err != nil {
t.Fatalf("materialize fixture: %v", err)
}
entries, err := os.ReadDir(filepath.Join(service.localBinaryResultRoot(), "task-expired"))
if err != nil || len(entries) != 1 {
t.Fatalf("read fixture result: entries=%v err=%v", entries, err)
}
path := filepath.Join(service.localBinaryResultRoot(), "task-expired", entries[0].Name())
old := time.Now().Add(-2 * time.Hour)
if err := os.Chtimes(path, old, old); err != nil {
t.Fatalf("age fixture: %v", err)
}
_, err = service.HydrateTaskResult(context.Background(), "task-expired", persistent)
assertClientErrorCode(t, err, "binary_result_expired")
now := time.Now()
if err := os.Chtimes(path, now, now); err != nil {
t.Fatalf("refresh fixture: %v", err)
}
if err := os.WriteFile(path, []byte("tampered"), 0o640); err != nil {
t.Fatalf("tamper fixture: %v", err)
}
_, err = service.HydrateTaskResult(context.Background(), "task-expired", persistent)
assertClientErrorCode(t, err, "binary_result_corrupted")
}
func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
service := newLocalBinaryTestService(t)
service.cfg.LocalResultMaxBytes = 4
_, _, err := service.materializeLocalBinaryResult(context.Background(), "task-large", map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
})
assertClientErrorCode(t, err, "binary_result_too_large")
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-text", map[string]any{
"message": "YWJjZA==",
})
if err != nil || changed || persistent["message"] != "YWJjZA==" {
t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
}
service = newLocalBinaryTestService(t)
service.cfg.LocalResultMaxBytes = 1024
service.cfg.LocalResultMaxTaskBytes = 8
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
"first_base64": base64.StdEncoding.EncodeToString([]byte("12345")),
"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
})
assertClientErrorCode(t, err, "binary_result_too_large")
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("preflight task limit must not create partial files: %v", statErr)
}
}
func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
service := newLocalBinaryTestService(t)
service.cfg.LocalResultMaxBytes = 1
encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))
persistent, changed, err := service.CompactExpiredTaskResultForStorage(context.Background(), "task-old", map[string]any{
"b64_json": encoded,
})
if err != nil {
t.Fatalf("compact expired result: %v", err)
}
if !changed || !localBinaryResultHasPlaceholders(persistent) {
t.Fatalf("expired result was not compacted: %+v", persistent)
}
if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("expired compaction must not create a task directory: %v", err)
}
_, err = service.HydrateTaskResult(context.Background(), "task-old", persistent)
assertClientErrorCode(t, err, "binary_result_expired")
}
func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
err := localBinaryStorageError(errors.New("write failed"))
assertClientErrorCode(t, err, "local_result_storage_unavailable")
if clients.IsRetryable(err) {
t.Fatal("local result storage failure must not call the provider again")
}
if retryDecisionForCandidate(store.RuntimeModelCandidate{}, err).Retry {
t.Fatal("local result storage failure must not retry the same provider client")
}
if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
t.Fatal("local result storage failure must not fail over to another provider")
}
service := newLocalBinaryTestService(t)
service.cfg.LocalResultMinFreeBytes = 1 << 62
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-low-disk", map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("disk preflight")),
})
assertClientErrorCode(t, err, "local_result_storage_unavailable")
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-low-disk")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("disk preflight must not create partial task files: %v", statErr)
}
}
func bytesToAny(payload []byte) []any {
result := make([]any, len(payload))
for index, value := range payload {
result[index] = float64(value)
}
return result
}
func assertClientErrorCode(t *testing.T, err error, code string) {
t.Helper()
var clientErr *clients.ClientError
if !errors.As(err, &clientErr) || clientErr.Code != code {
t.Fatalf("error = %v, want client error code %s", err, code)
}
}
@@ -57,6 +57,9 @@ func shouldRetrySameClient(candidate store.RuntimeModelCandidate, err error) boo
func retryDecisionForCandidate(candidate store.RuntimeModelCandidate, err error) retryDecision {
policy := effectiveRetryPolicy(candidate)
info := failureInfoFromError(err)
if isResultPersistenceFailure(err) {
return retryDecision{Retry: false, Reason: "result_persistence_failed", Match: policyRuleMatch{Source: "gateway_result_storage", Policy: "persistence", Rule: "providerRetry", Value: "disabled"}, Info: info}
}
if errors.Is(err, store.ErrRateLimited) {
return retryDecision{Retry: false, Reason: "local_rate_limit_wait_queue", Match: policyRuleMatch{Source: "gateway_rate_limits", Policy: "rateLimitPolicy", Rule: "localCapacity", Value: "exceeded"}, Info: info}
}
@@ -77,6 +80,9 @@ func retryDecisionForCandidate(candidate store.RuntimeModelCandidate, err error)
func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate store.RuntimeModelCandidate, err error) failoverDecision {
info := failureInfoFromError(err)
if isResultPersistenceFailure(err) {
return failoverDecision{Retry: false, Action: "stop", Reason: "result_persistence_failed", Match: policyRuleMatch{Source: "gateway_result_storage", Policy: "persistence", Rule: "providerFailover", Value: "disabled"}, Info: info}
}
if strings.TrimSpace(runnerPolicy.Status) != "" && runnerPolicy.Status != "active" {
return failoverDecision{Retry: false, Action: "stop", Reason: "runner_policy_disabled", Match: policyRuleMatch{Source: "gateway_runner_policies", Policy: "runnerPolicy", Rule: "status", Value: runnerPolicy.Status}, Info: info}
}
@@ -126,6 +132,9 @@ func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) b
func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err error) priorityDemoteDecision {
info := failureInfoFromError(err)
if isResultPersistenceFailure(err) {
return priorityDemoteDecision{Demote: false, Reason: "result_persistence_failed", Info: info}
}
if strings.TrimSpace(runnerPolicy.Status) != "" && runnerPolicy.Status != "active" {
return priorityDemoteDecision{Demote: false, Reason: "runner_policy_disabled", Info: info}
}
@@ -145,6 +154,19 @@ func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err err
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_no_match", Info: info}
}
func isResultPersistenceFailure(err error) bool {
switch strings.ToLower(strings.TrimSpace(clients.ErrorCode(err))) {
case "local_result_storage_unavailable",
"binary_result_too_large",
"binary_result_corrupted",
"binary_result_expired",
"result_binary_not_materialized":
return true
default:
return false
}
}
func effectiveFailoverPolicy(base map[string]any, override map[string]any) map[string]any {
policy := base
if nested := failoverOverridePolicy(override); len(nested) > 0 {
+49 -3
View File
@@ -630,7 +630,11 @@ candidatesLoop:
}
walletReservationFinalized = true
s.logger.Warn("task succeeded but billing requires manual review", "taskID", task.ID, "error_category", "billing_calculation_failed")
return Result{Task: review, Output: response.Result, Wire: response.Wire}, nil
output, hydrateErr := s.HydrateTaskResult(ctx, task.ID, response.Result)
if hydrateErr != nil {
return Result{Task: review}, hydrateErr
}
return Result{Task: review, Output: output, Wire: response.Wire}, nil
}
finalAmountText = finalAmount.String()
} else {
@@ -661,6 +665,38 @@ candidatesLoop:
ResponseDurationMS: record.ResponseDurationMS,
})
if finishErr != nil {
if errors.Is(finishErr, store.ErrTaskResultBinaryNotMaterialized) {
s.logger.Error("task result rejected by binary persistence guard",
"taskID", task.ID,
"error_category", "result_binary_not_materialized",
)
failureCtx := context.WithoutCancel(ctx)
_ = s.store.FinishTaskAttempt(failureCtx, store.FinishTaskAttemptInput{
AttemptID: response.AttemptID,
Status: "failed",
Retryable: false,
RequestID: response.RequestID,
ResponseStartedAt: response.ResponseStartedAt,
ResponseFinishedAt: response.ResponseFinishedAt,
ResponseDurationMS: response.ResponseDurationMS,
ErrorCode: clients.ErrorCode(finishErr),
ErrorMessage: finishErr.Error(),
})
failed, failErr := s.failTask(
failureCtx,
task.ID,
task.ExecutionToken,
clients.ErrorCode(finishErr),
finishErr.Error(),
isSimulation(task, candidate),
finishErr,
)
if failErr != nil {
return Result{}, failErr
}
walletReservationFinalized = true
return Result{Task: failed, Output: failed.Result}, finishErr
}
if errors.Is(finishErr, store.ErrTaskExecutionLeaseLost) {
latest, latestErr := s.store.GetTask(ctx, task.ID)
if latestErr == nil && latest.Status == "cancelled" {
@@ -688,7 +724,11 @@ candidatesLoop:
}, isSimulation(task, candidate)); err != nil {
return Result{}, err
}
return Result{Task: finished, Output: response.Result, Wire: response.Wire}, nil
output, hydrateErr := s.HydrateTaskResult(ctx, task.ID, response.Result)
if hydrateErr != nil {
return Result{Task: finished}, hydrateErr
}
return Result{Task: finished, Output: output, Wire: response.Wire}, nil
}
var submissionUnknown *upstreamSubmissionUnknownError
if errors.As(err, &submissionUnknown) {
@@ -811,7 +851,7 @@ candidatesLoop:
break
}
decision := failoverDecisionForCandidate(runnerPolicy, candidate, candidateErr)
if !decision.Retry && hasLoadAvoidanceFallback(candidates, index, maxPlatforms) {
if !decision.Retry && !isResultPersistenceFailure(candidateErr) && hasLoadAvoidanceFallback(candidates, index, maxPlatforms) {
decision = loadAvoidanceFallbackDecision(candidateErr)
}
s.recordAttemptTrace(ctx, task.ID, attemptNo, failoverTraceEntry(decision, candidate))
@@ -1216,6 +1256,12 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
return clients.Response{}, err
}
response.Result = uploadedResult
if localBinaryResultHasPlaceholders(response.Result) {
// A provider wire response can still contain its original Base64 body.
// Force protocol responses to be rebuilt from the verified standard
// result instead of bypassing local result materialization.
response.Wire = nil
}
if task.Kind == "responses" {
response.UpstreamProtocol = candidate.ResponseProtocol
response.ParentResponseID = responseExecution.PublicPreviousResponseID
+16 -2
View File
@@ -47,6 +47,7 @@ type generatedAssetUploadPolicy struct {
UploadInlineMedia bool
UploadURLMedia bool
PreserveInlineMedia bool
LocalizeInlineMedia bool
}
type generatedAssetDecision struct {
@@ -79,7 +80,8 @@ func defaultGeneratedAssetUploadPolicy() generatedAssetUploadPolicy {
func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, taskKind string, result map[string]any) (map[string]any, error) {
data, _ := result["data"].([]any)
rawNeedsUpload := generatedRawValueHasInlineMedia(result["raw"], "", nil)
if len(data) == 0 && !rawNeedsUpload {
hasInlineBinary := TaskResultHasInlineBinary(result)
if len(data) == 0 && !rawNeedsUpload && !hasInlineBinary {
redactGeneratedResultRawData(result)
return result, nil
}
@@ -87,6 +89,18 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
if err != nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
}
if policy.LocalizeInlineMedia {
next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result)
if err != nil {
return nil, err
}
redactGeneratedResultRawData(next)
return next, nil
}
if len(data) == 0 && !rawNeedsUpload {
redactGeneratedResultRawData(result)
return result, nil
}
// Topaz download URLs are short-lived. Persist them before the task can be
// marked succeeded even when the global policy normally keeps URL media.
if taskKind == "videos.upscales" {
@@ -542,7 +556,7 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP
case store.FileStorageResultUploadPolicyUploadAll:
return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
case store.FileStorageResultUploadPolicyUploadNone:
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true}
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}
default:
return defaultGeneratedAssetUploadPolicy()
}
+1 -1
View File
@@ -222,7 +222,7 @@ func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
{
name: "upload none",
policyName: store.FileStorageResultUploadPolicyUploadNone,
want: generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true},
want: generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true},
},
}
@@ -0,0 +1,87 @@
package store
import (
"context"
"encoding/json"
"time"
"github.com/jackc/pgx/v5"
)
type TaskBinaryResultBackfillItem struct {
ID string
Result map[string]any
FinishedAt time.Time
}
func (s *Store) ListTaskBinaryResultBackfillBatch(ctx context.Context, afterID string, batchSize int) ([]TaskBinaryResultBackfillItem, error) {
if batchSize < 1 || batchSize > 100 {
batchSize = 100
}
var items []TaskBinaryResultBackfillItem
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
return err
}
rows, err := tx.Query(ctx, `
SELECT id::text, result, COALESCE(finished_at, updated_at)
FROM gateway_tasks
WHERE status = 'succeeded'
AND result <> '{}'::jsonb
AND (NULLIF($1::text, '') IS NULL OR id > NULLIF($1::text, '')::uuid)
ORDER BY id
LIMIT $2`, afterID, batchSize)
if err != nil {
return err
}
defer rows.Close()
items = make([]TaskBinaryResultBackfillItem, 0, batchSize)
for rows.Next() {
var item TaskBinaryResultBackfillItem
var resultJSON []byte
if err := rows.Scan(&item.ID, &resultJSON, &item.FinishedAt); err != nil {
return err
}
item.Result = decodeObject(resultJSON)
items = append(items, item)
}
return rows.Err()
})
return items, err
}
func (s *Store) UpdateTaskBinaryResultBackfill(ctx context.Context, taskID string, result map[string]any) (bool, error) {
report := sanitizeJSONForStorageWithReport(minimalTaskResult(result))
if report.BinaryCount > 0 {
return false, &taskPayloadBinaryError{
target: ErrTaskResultBinaryNotMaterialized,
code: "result_binary_not_materialized",
count: report.BinaryCount,
}
}
resultJSON, err := json.Marshal(report.Value)
if err != nil {
return false, err
}
updated := false
err = pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
return err
}
tag, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET result = $2::jsonb,
updated_at = now()
WHERE id = $1::uuid
AND status = 'succeeded'`,
taskID,
string(resultJSON),
)
if err != nil {
return err
}
updated = tag.RowsAffected() == 1
return nil
})
return updated, err
}
+10 -2
View File
@@ -41,7 +41,7 @@ func (s *Store) EnsureConversation(ctx context.Context, user *auth.User, convers
if userID == "" {
userID = "anonymous"
}
metadataJSON, _ := json.Marshal(emptyObjectIfNil(metadata))
metadataJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(metadata)))
var conversationID string
err := s.pool.QueryRow(ctx, `
INSERT INTO gateway_conversations (user_id, gateway_user_id, conversation_key, metadata)
@@ -67,7 +67,15 @@ func (s *Store) UpsertConversationMessages(ctx context.Context, conversationID s
refs := make([]TaskMessageRefInput, 0, len(messages))
newCount := 0
for index, message := range messages {
snapshotJSON, _ := json.Marshal(emptyObjectIfNil(message.Snapshot))
snapshotReport := sanitizeJSONForStorageWithReport(emptyObjectIfNil(message.Snapshot))
if snapshotReport.BinaryCount > 0 {
return nil, 0, &taskPayloadBinaryError{
target: ErrTaskRequestBinaryNotMaterialized,
code: "request_binary_not_materialized",
count: snapshotReport.BinaryCount,
}
}
snapshotJSON, _ := json.Marshal(snapshotReport.Value)
var messageID string
var inserted bool
if err := tx.QueryRow(ctx, `
@@ -0,0 +1,344 @@
package store
import (
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
)
const (
storageBinaryPrefixChars = 16
storageGenericBase64MinLength = 4096
storageInvalidBinaryMinLength = 512
storageJSONSanitizerMaxDepth = 64
storagePlaceholderPrefix = "[GatewayBinary:v1;"
storageBufferObjectType = "buffer"
storageDataURLPrefix = "data:"
storageDataURLBase64Marker = ";base64"
storageDataURLMaxContentType = 64
)
var (
ErrTaskRequestBinaryNotMaterialized = errors.New("task request binary payload was not materialized")
ErrTaskResultBinaryNotMaterialized = errors.New("task result binary payload was not materialized")
)
type taskPayloadBinaryError struct {
target error
code string
count int
}
func (e *taskPayloadBinaryError) Error() string {
return fmt.Sprintf("%s: detected %d inline binary value(s)", e.code, e.count)
}
func (e *taskPayloadBinaryError) ErrorCode() string {
return e.code
}
func (e *taskPayloadBinaryError) Is(target error) bool {
return target == e.target
}
type storageSanitizeReport struct {
Value any
BinaryCount int
}
// sanitizeJSONForStorage is the final task-domain persistence guard. It always
// returns a detached JSON-compatible value and replaces inline binary payloads
// with a bounded, deterministic placeholder.
func sanitizeJSONForStorage(value any) any {
return sanitizeJSONForStorageWithReport(value).Value
}
func sanitizeJSONForStorageWithReport(value any) storageSanitizeReport {
next, count := sanitizeJSONStorageValue(value, nil, 0)
return storageSanitizeReport{Value: next, BinaryCount: count}
}
func sanitizeJSONStorageValue(value any, path []string, depth int) (any, int) {
if depth >= storageJSONSanitizerMaxDepth {
return "[JSON,max-depth]", 0
}
switch typed := value.(type) {
case map[string]any:
if payload, contentType, ok := storageBufferObjectBytes(typed); ok {
return storageBinaryPlaceholder(payload, contentType, "buffer"), 1
}
next := make(map[string]any, len(typed))
count := 0
for key, child := range typed {
sanitized, childCount := sanitizeJSONStorageValue(child, appendStoragePath(path, key), depth+1)
next[key] = sanitized
count += childCount
}
return next, count
case []any:
if storagePathIsBinary(path) {
if payload, ok := storageNumberArrayBytes(typed); ok {
return storageBinaryPlaceholder(payload, "", "buffer"), 1
}
}
next := make([]any, len(typed))
count := 0
for index, child := range typed {
sanitized, childCount := sanitizeJSONStorageValue(child, path, depth+1)
next[index] = sanitized
count += childCount
}
return next, count
case []byte:
return storageBinaryPlaceholder(typed, "", "buffer"), 1
case string:
if placeholder, ok := storageStringPlaceholder(typed, path); ok {
return placeholder, 1
}
return typed, 0
default:
return value, 0
}
}
func appendStoragePath(path []string, key string) []string {
next := make([]string, len(path)+1)
copy(next, path)
next[len(path)] = key
return next
}
func storageStringPlaceholder(value string, path []string) (string, bool) {
raw := strings.TrimSpace(value)
if raw == "" || strings.HasPrefix(raw, storagePlaceholderPrefix) {
return "", false
}
encoded, contentType, dataURL := storageBase64StringParts(raw)
strict := dataURL || storagePathIsBinary(path)
if !strict && len(encoded) < storageGenericBase64MinLength {
return "", false
}
payload, ok := storageDecodeBase64(encoded)
if ok {
encoding := "raw"
if dataURL {
encoding = "data-uri"
}
return storageBinaryPlaceholder(payload, contentType, encoding), true
}
if strict && len(raw) >= storageInvalidBinaryMinLength {
return storageBinaryPlaceholder([]byte(raw), contentType, "raw"), true
}
return "", false
}
func storageBase64StringParts(value string) (encoded string, contentType string, dataURL bool) {
if !strings.HasPrefix(strings.ToLower(value), storageDataURLPrefix) {
return value, "", false
}
prefix, payload, ok := strings.Cut(value, ",")
if !ok || !strings.Contains(strings.ToLower(prefix), storageDataURLBase64Marker) {
return value, "", false
}
mediaType := strings.TrimSpace(prefix[len(storageDataURLPrefix):])
if before, _, found := strings.Cut(mediaType, ";"); found {
mediaType = before
}
if len(mediaType) > storageDataURLMaxContentType || !storageSafeContentType(mediaType) {
mediaType = ""
}
return payload, strings.ToLower(mediaType), true
}
func storageSafeContentType(value string) bool {
if value == "" {
return true
}
for _, char := range value {
switch {
case char >= 'a' && char <= 'z':
case char >= 'A' && char <= 'Z':
case char >= '0' && char <= '9':
case char == '/', char == '.', char == '+', char == '-':
default:
return false
}
}
return true
}
func storageDecodeBase64(value string) ([]byte, bool) {
normalized := removeStorageASCIIWhitespace(value)
if normalized == "" {
return nil, false
}
for _, encoding := range []*base64.Encoding{
base64.StdEncoding,
base64.RawStdEncoding,
base64.URLEncoding,
base64.RawURLEncoding,
} {
payload, err := encoding.DecodeString(normalized)
if err != nil || len(payload) == 0 {
continue
}
canonical := encoding.EncodeToString(payload)
if strings.TrimRight(normalized, "=") == strings.TrimRight(canonical, "=") {
return payload, true
}
}
return nil, false
}
func removeStorageASCIIWhitespace(value string) string {
return strings.Map(func(char rune) rune {
switch char {
case ' ', '\n', '\r', '\t':
return -1
default:
return char
}
}, value)
}
func storagePathIsBinary(path []string) bool {
if len(path) == 0 {
return false
}
key := normalizeStorageBinaryKey(path[len(path)-1])
if storageBinaryKey(key) {
return true
}
if len(path) < 2 {
return false
}
parent := normalizeStorageBinaryKey(path[len(path)-2])
return (parent == "inlinedata" || parent == "binary" || parent == "media") &&
(key == "data" || key == "content")
}
func normalizeStorageBinaryKey(value string) string {
return strings.Map(func(char rune) rune {
switch {
case char >= 'a' && char <= 'z':
return char
case char >= 'A' && char <= 'Z':
return char + ('a' - 'A')
case char >= '0' && char <= '9':
return char
default:
return -1
}
}, value)
}
func storageBinaryKey(key string) bool {
return key == "b64" ||
key == "b64json" ||
key == "base64" ||
key == "buffer" ||
key == "bytes" ||
strings.Contains(key, "base64") ||
strings.Contains(key, "buffer") ||
strings.Contains(key, "binary") ||
strings.HasSuffix(key, "b64") ||
strings.HasSuffix(key, "bytes")
}
func storageBufferObjectBytes(value map[string]any) ([]byte, string, bool) {
kind, _ := value["type"].(string)
if normalizeStorageBinaryKey(kind) != storageBufferObjectType {
return nil, "", false
}
contentType := firstNonEmpty(
stringFromAny(value["mime_type"]),
stringFromAny(value["mimeType"]),
stringFromAny(value["contentType"]),
)
switch data := value["data"].(type) {
case []byte:
if len(data) == 0 {
return nil, "", false
}
return append([]byte(nil), data...), contentType, true
case []any:
payload, ok := storageNumberArrayBytes(data)
return payload, contentType, ok
default:
return nil, "", false
}
}
func storageNumberArrayBytes(values []any) ([]byte, bool) {
if len(values) == 0 {
return nil, false
}
payload := make([]byte, len(values))
for index, value := range values {
next, ok := storageByteFromAny(value)
if !ok {
return nil, false
}
payload[index] = next
}
return payload, true
}
func storageByteFromAny(value any) (byte, bool) {
switch typed := value.(type) {
case byte:
return typed, true
case int:
if typed >= 0 && typed <= 255 {
return byte(typed), true
}
case int32:
if typed >= 0 && typed <= 255 {
return byte(typed), true
}
case int64:
if typed >= 0 && typed <= 255 {
return byte(typed), true
}
case float64:
asInt := int(typed)
if typed == float64(asInt) && asInt >= 0 && asInt <= 255 {
return byte(asInt), true
}
case json.Number:
asInt, err := strconv.ParseInt(string(typed), 10, 16)
if err == nil && asInt >= 0 && asInt <= 255 {
return byte(asInt), true
}
}
return 0, false
}
func storageBinaryPlaceholder(payload []byte, contentType string, encoding string) string {
digest := sha256.Sum256(payload)
prefix := base64.StdEncoding.EncodeToString(payload)
if len(prefix) > storageBinaryPrefixChars {
prefix = prefix[:storageBinaryPrefixChars]
}
contentType = strings.ToLower(strings.TrimSpace(contentType))
if contentType == "" || len(contentType) > 32 || !storageSafeContentType(contentType) {
contentType = "application/octet-stream"
}
switch encoding {
case "data-uri", "buffer":
default:
encoding = "raw"
}
return fmt.Sprintf(
"[GatewayBinary:v1;prefix=%s;sha256=%x;bytes=%d;mime=%s;encoding=%s]",
prefix,
digest,
len(payload),
contentType,
encoding,
)
}
@@ -0,0 +1,103 @@
package store
import (
"encoding/base64"
"encoding/json"
"strings"
"testing"
)
func TestSanitizeJSONForStorageReplacesBinaryWithoutMutatingInput(t *testing.T) {
payload := []byte("shared binary payload")
encoded := base64.StdEncoding.EncodeToString(payload)
input := map[string]any{
"data": []any{
map[string]any{"b64_json": encoded},
},
"buffer": map[string]any{
"type": "Buffer",
"data": []any{float64(1), float64(2), float64(3)},
},
"text": "ordinary text",
}
report := sanitizeJSONForStorageWithReport(input)
if report.BinaryCount != 2 {
t.Fatalf("binary count = %d, want 2", report.BinaryCount)
}
next := report.Value.(map[string]any)
data := next["data"].([]any)
placeholder := data[0].(map[string]any)["b64_json"].(string)
if !strings.HasPrefix(placeholder, storagePlaceholderPrefix) {
t.Fatalf("unexpected placeholder: %q", placeholder)
}
if len(placeholder) > 200 {
t.Fatalf("placeholder exceeds 200 bytes: %d", len(placeholder))
}
if !strings.Contains(placeholder, ";prefix="+encoded[:16]+";") {
t.Fatalf("placeholder should retain the bounded Base64 prefix: %q", placeholder)
}
if got := input["data"].([]any)[0].(map[string]any)["b64_json"]; got != encoded {
t.Fatalf("sanitizer mutated input: %v", got)
}
if next["text"] != "ordinary text" {
t.Fatalf("ordinary text changed: %+v", next)
}
}
func TestSanitizeJSONForStorageUsesDecodedBytesForEquivalentRepresentations(t *testing.T) {
payload := []byte("equivalent payload")
encoded := base64.StdEncoding.EncodeToString(payload)
report := sanitizeJSONForStorageWithReport(map[string]any{
"rawBase64": encoded,
"dataURL": "data:image/png;base64," + encoded,
"bytes": []any{float64('e'), float64('q'), float64('u'), float64('i'), float64('v'), float64('a'), float64('l'), float64('e'), float64('n'), float64('t'), float64(' '), float64('p'), float64('a'), float64('y'), float64('l'), float64('o'), float64('a'), float64('d')},
})
if report.BinaryCount != 3 {
t.Fatalf("binary count = %d, want 3", report.BinaryCount)
}
next := report.Value.(map[string]any)
hashes := map[string]struct{}{}
for _, key := range []string{"rawBase64", "dataURL", "bytes"} {
value := next[key].(string)
hashStart := strings.Index(value, ";sha256=")
hashEnd := strings.Index(value[hashStart+1:], ";bytes=")
if hashStart < 0 || hashEnd < 0 {
t.Fatalf("missing hash in %s placeholder: %q", key, value)
}
hashes[value[hashStart+8:hashStart+1+hashEnd]] = struct{}{}
}
if len(hashes) != 1 {
t.Fatalf("equivalent binary values produced different hashes: %+v", next)
}
}
func TestSanitizeJSONForStorageAvoidsShortGenericTextFalsePositive(t *testing.T) {
input := map[string]any{"message": "YWJjZA==", "count": json.Number("12")}
report := sanitizeJSONForStorageWithReport(input)
if report.BinaryCount != 0 {
t.Fatalf("short generic Base64-like text should not be sanitized: %+v", report)
}
}
func TestSanitizeJSONForStorageRecognizesExistingPlaceholder(t *testing.T) {
value := storageBinaryPlaceholder([]byte("payload"), "image/png", "raw")
report := sanitizeJSONForStorageWithReport(map[string]any{"b64_json": value})
if report.BinaryCount != 0 {
t.Fatalf("existing placeholder should pass the final guard: %+v", report)
}
}
func TestStorageBinaryPlaceholderIsAlwaysBounded(t *testing.T) {
value := storageBinaryPlaceholder(
[]byte("payload"),
"application/vnd.a-very-long-provider-specific-generated-binary-result+json",
"data-uri",
)
if len(value) > 200 {
t.Fatalf("placeholder exceeds 200 bytes: %d %q", len(value), value)
}
if !strings.Contains(value, ";mime=application/octet-stream;") {
t.Fatalf("oversized MIME should use the bounded fallback: %q", value)
}
}
+10 -2
View File
@@ -1993,7 +1993,15 @@ func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *aut
}
func (s *Store) CreateTaskIdempotent(ctx context.Context, input CreateTaskInput, user *auth.User) (CreateTaskResult, error) {
requestBody, _ := json.Marshal(input.Request)
requestReport := sanitizeJSONForStorageWithReport(input.Request)
if requestReport.BinaryCount > 0 {
return CreateTaskResult{}, &taskPayloadBinaryError{
target: ErrTaskRequestBinaryNotMaterialized,
code: "request_binary_not_materialized",
count: requestReport.BinaryCount,
}
}
requestBody, _ := json.Marshal(requestReport.Value)
runMode := normalizeRunMode(input.RunMode, input.Request)
status := "queued"
resultBody, _ := json.Marshal(map[string]any(nil))
@@ -2048,7 +2056,7 @@ WHERE user_id = $1 AND idempotency_key_hash = $2`, user.ID, strings.TrimSpace(in
}
events := taskEventsForCreate(task.ID, runMode, status, nil)
for _, event := range events {
payload, _ := json.Marshal(event.Payload)
payload, _ := json.Marshal(sanitizeJSONForStorage(event.Payload))
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated)
VALUES ($1::uuid, $2, $3::text, NULLIF($4::text, ''), NULLIF($5::text, ''), $6, NULLIF($7::text, ''), $8::jsonb, $9)`,
+3 -3
View File
@@ -45,15 +45,15 @@ type CreateResponseChainInput struct {
}
func (s *Store) CreateResponseChain(ctx context.Context, input CreateResponseChainInput) error {
requestJSON, err := json.Marshal(input.RequestSnapshot)
requestJSON, err := json.Marshal(sanitizeJSONForStorage(input.RequestSnapshot))
if err != nil {
return err
}
responseJSON, err := json.Marshal(input.ResponseSnapshot)
responseJSON, err := json.Marshal(sanitizeJSONForStorage(input.ResponseSnapshot))
if err != nil {
return err
}
internalJSON, err := json.Marshal(input.InternalSnapshot)
internalJSON, err := json.Marshal(sanitizeJSONForStorage(input.InternalSnapshot))
if err != nil {
return err
}
+29 -13
View File
@@ -503,7 +503,7 @@ WHERE id = $1::uuid`, taskID, riverJobID)
}
func (s *Store) SetTaskRemoteTask(ctx context.Context, taskID string, executionToken string, attemptID string, remoteTaskID string, payload map[string]any) error {
payloadJSON, _ := json.Marshal(emptyObjectIfNil(payload))
payloadJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(payload)))
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
tag, err := tx.Exec(ctx, `
UPDATE gateway_tasks
@@ -732,7 +732,7 @@ func (s *Store) CreateTaskParamPreprocessingLog(ctx context.Context, input Creat
if !input.Changed {
return "", nil
}
changesJSON, _ := json.Marshal(input.Changes)
changesJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Changes))
if input.Changes == nil {
changesJSON = []byte("[]")
}
@@ -1029,12 +1029,20 @@ WHERE id = $1::uuid`,
}
func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessInput) (GatewayTask, error) {
resultJSON, _ := json.Marshal(minimalTaskResult(input.Result))
billingsJSON, _ := json.Marshal(input.Billings)
usageJSON, _ := json.Marshal(emptyObjectIfNil(input.Usage))
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
billingSummaryJSON, _ := json.Marshal(emptyObjectIfNil(input.BillingSummary))
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(input.Result))
if resultReport.BinaryCount > 0 {
return GatewayTask{}, &taskPayloadBinaryError{
target: ErrTaskResultBinaryNotMaterialized,
code: "result_binary_not_materialized",
count: resultReport.BinaryCount,
}
}
resultJSON, _ := json.Marshal(resultReport.Value)
billingsJSON, _ := json.Marshal(sanitizeJSONForStorage(input.Billings))
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Usage)))
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
billingSummaryJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.BillingSummary)))
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
finalChargeAmount := strings.TrimSpace(input.FinalChargeAmountText)
if finalChargeAmount == "" {
finalChargeAmount = strconv.FormatFloat(input.FinalChargeAmount, 'f', 9, 64)
@@ -1158,8 +1166,16 @@ func (s *Store) FinishTaskManualReview(ctx context.Context, input FinishTaskManu
if status == "failed" {
result = nil
}
resultJSON, _ := json.Marshal(minimalTaskResult(result))
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
resultReport := sanitizeJSONForStorageWithReport(minimalTaskResult(result))
if resultReport.BinaryCount > 0 {
return GatewayTask{}, &taskPayloadBinaryError{
target: ErrTaskResultBinaryNotMaterialized,
code: "result_binary_not_materialized",
count: resultReport.BinaryCount,
}
}
resultJSON, _ := json.Marshal(resultReport.Value)
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.PricingSnapshot)))
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if strings.TrimSpace(input.AttemptID) != "" {
attemptStatus := "failed"
@@ -1261,7 +1277,7 @@ func (s *Store) SettleTaskBilling(ctx context.Context, task GatewayTask) error {
"billings": task.Billings,
"billingSummary": task.BillingSummary,
}
metadata, _ := json.Marshal(metadataMap)
metadata, _ := json.Marshal(sanitizeJSONForStorage(metadataMap))
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_wallet_accounts (
@@ -1353,7 +1369,7 @@ ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO N
"frozenBefore": roundMoney(frozenBefore),
"frozenAfter": frozenAfter,
})
metadata, _ = json.Marshal(billingMetadata)
metadata, _ = json.Marshal(sanitizeJSONForStorage(billingMetadata))
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_wallet_transactions (
account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type,
@@ -1389,7 +1405,7 @@ func taskBillingString(value any) string {
}
func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureInput) (GatewayTask, error) {
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
resultJSON, _ := json.Marshal(minimalTaskResult(nil))
message := truncateUTF8Bytes(input.Message, 2048)
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
+2 -2
View File
@@ -161,7 +161,7 @@ func (s *Store) ReserveTaskBilling(ctx context.Context, task GatewayTask, user *
}
reservations := make([]WalletBillingReservation, 0, len(amounts))
pricingSnapshotJSON, _ := json.Marshal(pricingSnapshot)
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(pricingSnapshot))
requestFingerprint := walletString(pricingSnapshot["requestFingerprint"])
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
for currency, rawAmount := range amounts {
@@ -306,7 +306,7 @@ func (s *Store) reserveTaskBillingExact(ctx context.Context, task GatewayTask, g
if currency != "resource" {
return nil, fmt.Errorf("unsupported billing currency %q", currency)
}
pricingSnapshotJSON, _ := json.Marshal(pricingSnapshot)
pricingSnapshotJSON, _ := json.Marshal(sanitizeJSONForStorage(pricingSnapshot))
requestFingerprint := walletString(pricingSnapshot["requestFingerprint"])
var reservations []WalletBillingReservation
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
@@ -51,7 +51,7 @@ const sceneOptions = [
const resultUploadPolicyOptions = [
{ value: 'default', label: '默认:仅非链接资源转存', description: 'URL 结果直接保存;base64 / buffer 等结果转存后保存 URL' },
{ value: 'upload_all', label: '全部转存', description: 'URL、base64、buffer 等生成媒体结果都会转存到当前文件渠道' },
{ value: 'upload_none', label: '全部不转存', description: '链接结果直接保存;base64 / buffer 结果保留在对应响应字段中' },
{ value: 'upload_none', label: '不做外部转存', description: 'base64 / buffer 临时写入本地静态文件;数据库仅保存占位符,默认 24 小时内按需恢复' },
];
export function SystemSettingsPanel(props: {
+4
View File
@@ -29,6 +29,10 @@ x-api-environment: &api-environment
AI_GATEWAY_WEB_BASE_URL: ${AI_GATEWAY_COMPOSE_WEB_BASE_URL:-http://localhost:5178}
AI_GATEWAY_GENERATED_STORAGE_DIR: /app/data/static/generated
AI_GATEWAY_UPLOADED_STORAGE_DIR: /app/data/static/uploaded
AI_GATEWAY_LOCAL_RESULT_TTL_HOURS: ${AI_GATEWAY_LOCAL_RESULT_TTL_HOURS:-24}
AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES: ${AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES:-10737418240}
AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: ${AI_GATEWAY_LOCAL_RESULT_MAX_BYTES:-268435456}
AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: ${AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES:-536870912}
services:
postgres:
+7
View File
@@ -1823,11 +1823,18 @@ AI_GATEWAY_TASK_RETENTION_DAYS=30
AI_GATEWAY_TASK_ANALYSIS_RETENTION_DAYS=7
AI_GATEWAY_TASK_CLEANUP_INTERVAL_SECONDS=300
AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE=1000
AI_GATEWAY_LOCAL_RESULT_TTL_HOURS=24
AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES=10737418240
AI_GATEWAY_LOCAL_RESULT_MAX_BYTES=268435456
AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES=536870912
```
运行时要求:
- 首次上线先保持清理关闭;兼容协议和异步恢复验收通过后,显式改为 `AI_GATEWAY_TASK_CLEANUP_ENABLED=true`,再启动历史压缩和 7/30 天删除。
- 结果策略为 `upload_none` 时,Base64、Data URI 和 Buffer 只写入 `data/static/generated/results/<task-id>/<sha256>.bin`;任务 JSON 保存固定占位符,查询结果时校验大小与 SHA-256 后恢复 Base64。
- 本地二进制结果默认保留 24 小时;文件过期返回 `410 binary_result_expired`,文件损坏返回 `500 binary_result_corrupted`。写入前保留至少 10GiB 空间,单文件和单任务上限分别为 256MiB、512MiB。
- 历史二进制治理使用镜像内 `/app/easyai-ai-gateway-backfill-binary-results` 先 dry-run;确认范围后带 `--apply`,每批最多 100 行,使用输出的 `resumeAfterId` 分段继续。该命令不会随服务启动自动执行。
- runtime 仅在任务真实状态变化时写 `gateway_task_events`;事件 payload 固定为空,轮询不会生成事件。
- callback worker 以 `0083_task_history_minimal_storage` 的应用时间为边界,只投递新 outbox;旧 pending/processing callback 在启用治理后分批标记为 `failed`,不会向业务端补发形成历史回调风暴。
- callback worker 使用 `SERVER_MAIN_INTERNAL_TOKEN` 调用 `TASK_PROGRESS_CALLBACK_URL`