fix(media): 统一图片结果 URL 化并限制同步 Base64

将上游 URL 直接持久化,内联媒体经对象存储后仅保留 URL 与内部定位元数据;异步轮询、任务详情和幂等重放统一使用零对象读取的 URL 投影,并增加 64KiB 响应门禁。

OpenAI 图片接口接受 url 与 b64_json,同步 Base64 限制为 20MiB 和每 Pod 2 并发;新增历史结果迁移清零门禁、结果指标和 API GOMEMLIMIT。

验证:API go test ./...、go vet、聚焦 race、pnpm openapi、pnpm lint/test/build、迁移安全检查与 docker compose config 均通过。
This commit is contained in:
2026-08-05 18:15:06 +08:00
parent f9b945e4aa
commit b13392ef50
22 changed files with 1367 additions and 176 deletions
+496 -2
View File
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sort"
@@ -21,6 +22,9 @@ import (
)
const (
// MaxSynchronousInlineResponseBytes bounds the raw media payload restored for
// an explicit synchronous OpenAI-compatible b64_json response.
MaxSynchronousInlineResponseBytes = int64(20 << 20)
localBinaryResultDirName = "results"
localBinaryPlaceholderPrefix = "[GatewayBinary:v1;"
localBinaryGenericBase64MinLength = 4096
@@ -96,6 +100,225 @@ func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID st
return next, hadInline && !TaskResultHasInlineBinary(next), nil
}
// MigrateTaskResultToURLs materializes historical inline payloads and rewrites
// legacy assetRef wrappers into the canonical URL-plus-upload storage shape.
func (s *Service) MigrateTaskResultToURLs(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
next := result
changed := false
if TaskResultHasLocalPlaceholder(next) {
restored, restoredChanged, err := s.restoreLocalPlaceholderValue(ctx, taskID, next, 0)
if err != nil {
return nil, false, err
}
mapped, ok := restored.(map[string]any)
if !ok {
return nil, false, &clients.ClientError{Code: "result_binary_not_materialized", Message: "generated result is not a JSON object", StatusCode: http.StatusInternalServerError}
}
next = mapped
changed = restoredChanged
}
if TaskResultHasInlineBinary(next) {
materialized, materializedChanged, err := s.MaterializeTaskResultForStorage(ctx, taskID, next)
if err != nil {
return nil, false, err
}
next = materialized
changed = materializedChanged
}
rewritten, rewrittenChanged := migrateStoredResultURLValue(next, 0)
mapped, ok := rewritten.(map[string]any)
if !ok {
return nil, false, &clients.ClientError{Code: "result_binary_not_materialized", Message: "generated result is not a JSON object", StatusCode: http.StatusInternalServerError}
}
return mapped, changed || rewrittenChanged, nil
}
// TaskResultNeedsURLMigration reports whether a stored result contains inline
// binary, a local placeholder, or the legacy assetRef representation.
func TaskResultNeedsURLMigration(result map[string]any) bool {
if TaskResultHasInlineBinary(result) {
return true
}
return storedResultURLValueNeedsMigration(result, 0)
}
// TaskResultHasLocalPlaceholder reports whether a historical result still
// references a node-local GatewayBinary file.
func TaskResultHasLocalPlaceholder(result map[string]any) bool {
return localBinaryResultHasPlaceholders(result)
}
func (s *Service) restoreLocalPlaceholderValue(ctx context.Context, taskID string, value any, depth int) (any, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
if depth >= localBinaryMaxDepth {
return nil, false, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result exceeds the maximum JSON depth", StatusCode: http.StatusInternalServerError}
}
switch typed := value.(type) {
case map[string]any:
next := make(map[string]any, len(typed))
changed := false
for key, childValue := range typed {
child, childChanged, err := s.restoreLocalPlaceholderValue(ctx, taskID, childValue, depth+1)
if err != nil {
return nil, false, err
}
next[key] = child
changed = changed || childChanged
}
if !changed {
return value, false, nil
}
return next, true, nil
case []any:
next := make([]any, len(typed))
changed := false
for index, childValue := range typed {
child, childChanged, err := s.restoreLocalPlaceholderValue(ctx, taskID, childValue, depth+1)
if err != nil {
return nil, false, err
}
next[index] = child
changed = changed || childChanged
}
if !changed {
return value, false, nil
}
return next, true, nil
case string:
descriptor, ok := parseLocalBinaryPlaceholder(typed)
if !ok {
return value, false, nil
}
payload, err := s.readLocalBinaryResult(taskID, descriptor)
if err != nil {
return nil, false, err
}
encoded := base64.StdEncoding.EncodeToString(payload)
if descriptor.Encoding == "data-uri" {
return "data:" + descriptor.ContentType + ";base64," + encoded, true, nil
}
return encoded, true, nil
default:
return value, false, nil
}
}
func storedResultURLValueNeedsMigration(value any, depth int) bool {
if depth >= localBinaryMaxDepth {
return true
}
switch typed := value.(type) {
case map[string]any:
if _, ok := typed["assetRef"].(map[string]any); ok {
return true
}
for _, child := range typed {
if storedResultURLValueNeedsMigration(child, depth+1) {
return true
}
}
case []any:
for _, child := range typed {
if storedResultURLValueNeedsMigration(child, depth+1) {
return true
}
}
case string:
_, ok := parseLocalBinaryPlaceholder(typed)
return ok
}
return false
}
func migrateStoredResultURLValue(value any, depth int) (any, bool) {
if depth >= localBinaryMaxDepth {
return value, false
}
switch typed := value.(type) {
case map[string]any:
if asset, ok := generatedResultAssetReference(typed); ok {
upload, _ := typed["upload"].(map[string]any)
if upload == nil {
upload = uploadMetadataFromLegacyAsset(asset)
}
accessURL := firstNonEmptyString(stringFromAny(upload["url"]), asset.URL)
next := map[string]any{
"url": accessURL,
"upload": upload,
"assetStorage": map[string]any{
"scene": store.FileStorageSceneImageResult,
"source": stringFromAny(typed["assetStorage"].(map[string]any)["source"]),
"strategy": "migrate_asset_ref",
"contentType": asset.ContentType,
},
}
for _, key := range []string{"type", "mime_type", "width", "height", "duration", "format", "seed", "revised_prompt"} {
if item, exists := typed[key]; exists {
next[key] = item
}
}
if stringFromAny(next["mime_type"]) == "" && asset.ContentType != "" {
next["mime_type"] = asset.ContentType
}
return next, true
}
next := make(map[string]any, len(typed))
changed := false
for key, childValue := range typed {
child, childChanged := migrateStoredResultURLValue(childValue, depth+1)
if resultCanonicalInlineField(key) {
if item, ok := child.(map[string]any); ok && stringFromAny(item["url"]) != "" {
mergeProjectedResultURL(next, item)
changed = true
continue
}
}
next[key] = child
changed = changed || childChanged
}
if !changed {
return value, false
}
return next, true
case []any:
next := make([]any, len(typed))
changed := false
for index, childValue := range typed {
child, childChanged := migrateStoredResultURLValue(childValue, depth+1)
next[index] = child
changed = changed || childChanged
}
if !changed {
return value, false
}
return next, true
default:
return value, false
}
}
func uploadMetadataFromLegacyAsset(asset store.RequestAsset) map[string]any {
upload := map[string]any{
"url": asset.URL,
"objectKey": asset.ObjectKey,
"contentType": asset.ContentType,
"size": asset.ByteSize,
"sha256": asset.SHA256,
"accessScope": asset.AccessScope,
"storageChannel": map[string]any{
"id": asset.StorageChannelID,
"channelKey": asset.StorageChannelKey,
"provider": asset.StorageProvider,
},
}
if asset.ExpiresAt != nil {
upload["objectExpiresAt"] = asset.ExpiresAt.Format(time.RFC3339)
}
return upload
}
// CompactExpiredTaskResultForStorage keeps the historical maintenance API but
// now uses the same object-storage path as live results. New GatewayBinary
// placeholders are never created; their parser remains read-only compatibility.
@@ -325,8 +548,9 @@ func verifyLocalBinaryFile(path string, expectedHash string, expectedSize int64)
return nil
}
// HydrateTaskResult restores placeholders from verified local files. It is only
// used by result/detail/replay endpoints, never by task lists or callbacks.
// HydrateTaskResult restores verified stored payloads only for bounded,
// explicit synchronous inline responses. Asynchronous readers use
// ProjectTaskResultURLs and never call this path.
func (s *Service) HydrateTaskResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, error) {
next, changed, err := s.hydrateLocalBinaryValue(ctx, taskID, result, 0)
if err != nil {
@@ -342,6 +566,244 @@ func (s *Service) HydrateTaskResult(ctx context.Context, taskID string, result m
return mapped, nil
}
// SynchronousInlineResultBytes returns the stored raw byte count that the
// synchronous hydration path would read. It deliberately inspects metadata
// only, so callers can reject oversized responses before any object GET or
// local file read occurs.
func SynchronousInlineResultBytes(result map[string]any) int64 {
return synchronousInlineResultValueBytes(result, 0)
}
func synchronousInlineResultValueBytes(value any, depth int) int64 {
if depth >= localBinaryMaxDepth {
return 0
}
switch typed := value.(type) {
case map[string]any:
if asset, _, ok := generatedResultUploadReference(typed); ok {
storage, _ := typed["assetStorage"].(map[string]any)
if resultCanonicalInlineField(stringFromAny(storage["source"])) {
return asset.ByteSize
}
}
if asset, ok := generatedResultAssetReference(typed); ok {
return asset.ByteSize
}
var total int64
for _, child := range typed {
total += synchronousInlineResultValueBytes(child, depth+1)
if total > MaxSynchronousInlineResponseBytes {
return total
}
}
return total
case []any:
var total int64
for _, child := range typed {
total += synchronousInlineResultValueBytes(child, depth+1)
if total > MaxSynchronousInlineResponseBytes {
return total
}
}
return total
case string:
if descriptor, ok := parseLocalBinaryPlaceholder(typed); ok {
return descriptor.Size
}
}
return 0
}
// ProjectTaskResultURLs returns the public, URL-only representation of a
// stored task result. It may refresh a signed object-storage URL, but it never
// reads object bytes or restores historical inline binary payloads.
func (s *Service) ProjectTaskResultURLs(ctx context.Context, taskID string, result map[string]any) (map[string]any, error) {
next, changed, err := s.projectTaskResultURLValue(ctx, taskID, result, 0)
if err != nil {
return nil, err
}
if !changed {
return result, nil
}
mapped, ok := next.(map[string]any)
if !ok {
return nil, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result is not a JSON object", StatusCode: http.StatusInternalServerError}
}
return mapped, nil
}
func (s *Service) projectTaskResultURLValue(ctx context.Context, taskID string, value any, depth int) (any, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
if depth >= localBinaryMaxDepth {
return nil, false, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result exceeds the maximum JSON depth", StatusCode: http.StatusInternalServerError}
}
switch typed := value.(type) {
case map[string]any:
if asset, _, ok := generatedResultUploadReference(typed); ok {
accessURL, err := s.resultAssetAccessURL(ctx, asset)
if err != nil {
return nil, false, err
}
next := publicResultURLItem(typed, accessURL, asset.ContentType)
return next, true, nil
}
if asset, ok := generatedResultAssetReference(typed); ok {
accessURL, err := s.resultAssetAccessURL(ctx, asset)
if err != nil {
return nil, false, err
}
next := publicResultURLItem(typed, accessURL, asset.ContentType)
return next, true, nil
}
if _, _, ok := localBufferObjectBytes(typed); ok {
return nil, false, resultMaterializationRequired(taskID)
}
next := make(map[string]any, len(typed))
changed := false
for key, childValue := range typed {
if resultInternalField(key) {
changed = true
continue
}
child, childChanged, err := s.projectTaskResultURLValue(ctx, taskID, childValue, depth+1)
if err != nil {
return nil, false, err
}
if resultCanonicalInlineField(key) {
if item, ok := child.(map[string]any); ok && stringFromAny(item["url"]) != "" {
mergeProjectedResultURL(next, item)
changed = true
continue
}
if raw, ok := child.(string); ok {
if requestAssetStringIsHTTPURL(raw) {
next["url"] = strings.TrimSpace(raw)
changed = true
continue
}
if strings.TrimSpace(raw) != "" {
return nil, false, resultMaterializationRequired(taskID)
}
}
}
if localBinaryKey(key) {
switch inline := child.(type) {
case map[string]any:
if stringFromAny(inline["url"]) != "" {
mergeProjectedResultURL(next, inline)
changed = true
continue
}
case string:
if strings.TrimSpace(inline) != "" {
return nil, false, resultMaterializationRequired(taskID)
}
case []any:
if len(inline) > 0 {
return nil, false, resultMaterializationRequired(taskID)
}
}
}
next[key] = child
changed = changed || childChanged
}
if !changed {
return value, false, nil
}
return next, true, nil
case []any:
next := make([]any, len(typed))
changed := false
for index, childValue := range typed {
child, childChanged, err := s.projectTaskResultURLValue(ctx, taskID, childValue, depth+1)
if err != nil {
return nil, false, err
}
next[index] = child
changed = changed || childChanged
}
if !changed {
return value, false, nil
}
return next, true, nil
case []byte:
if len(typed) > 0 {
return nil, false, resultMaterializationRequired(taskID)
}
return value, false, nil
case string:
if _, ok := parseLocalBinaryPlaceholder(typed); ok {
return nil, false, resultMaterializationRequired(taskID)
}
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(typed)), "data:") {
return nil, false, resultMaterializationRequired(taskID)
}
return value, false, nil
default:
return value, false, nil
}
}
func (s *Service) resultAssetAccessURL(ctx context.Context, asset store.RequestAsset) (string, error) {
accessURL, err := s.requestAssetAccessURL(ctx, asset)
if err != nil || strings.TrimSpace(accessURL) == "" {
message := "stored result URL is unavailable"
if err != nil {
message = err.Error()
}
return "", &clients.ClientError{Code: "result_url_unavailable", Message: message, StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
return accessURL, nil
}
func resultMaterializationRequired(taskID string) error {
return &clients.ClientError{
Code: "result_materialization_required",
Message: "stored result must be migrated to object storage before it can be returned",
Details: map[string]any{"task_id": strings.TrimSpace(taskID)},
StatusCode: http.StatusServiceUnavailable,
Retryable: true,
}
}
func publicResultURLItem(source map[string]any, accessURL string, contentType string) map[string]any {
next := map[string]any{"url": accessURL}
for _, key := range []string{"type", "mime_type", "mimeType", "content_type", "contentType", "width", "height", "duration", "format", "seed", "revised_prompt"} {
if value, ok := source[key]; ok && value != nil {
next[key] = value
}
}
if stringFromAny(next["mime_type"]) == "" && strings.TrimSpace(contentType) != "" {
next["mime_type"] = strings.TrimSpace(contentType)
}
return next
}
func mergeProjectedResultURL(target map[string]any, item map[string]any) {
for key, value := range item {
if _, exists := target[key]; !exists || key == "url" {
target[key] = value
}
}
}
func resultInternalField(key string) bool {
switch normalizeLocalBinaryKey(key) {
case "assetref", "assetstorage", "upload", "raw", "rawdata", "rawresponse", "providerpayload", "providerresponse", "providerraw", "upstreamresponse", "thinkingbytes", "thoughtsignature", "signaturebuffer":
return true
default:
return false
}
}
func resultCanonicalInlineField(key string) bool {
normalized := normalizeLocalBinaryKey(key)
return normalized == "b64json" || normalized == "base64" || normalized == "b64" ||
normalized == "datauri" || strings.Contains(normalized, "base64") || strings.HasSuffix(normalized, "b64")
}
func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, value any, depth int) (any, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
@@ -374,6 +836,31 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
}
typed = next
refreshedAccessURL = true
storage, _ := typed["assetStorage"].(map[string]any)
sourceKey := strings.TrimSpace(stringFromAny(storage["source"]))
if resultCanonicalInlineField(sourceKey) && asset.SHA256 != "" && asset.ByteSize > 0 {
payload, contentType, err := s.readGeneratedResultAsset(ctx, asset)
if err != nil {
return nil, false, err
}
encoded := base64.StdEncoding.EncodeToString(payload)
inline := make(map[string]any, len(typed))
for key, item := range typed {
if resultInternalField(key) || key == "url" || key == "image_url" || key == "video_url" || key == "audio_url" {
continue
}
inline[key] = item
}
if generatedResultAssetUsesDataURI(typed) {
inline[sourceKey] = "data:" + contentType + ";base64," + encoded
} else {
inline[sourceKey] = encoded
}
if stringFromAny(inline["mime_type"]) == "" {
inline["mime_type"] = contentType
}
return inline, true, nil
}
}
if ref, ok := generatedResultAssetReference(typed); ok {
payload, contentType, err := s.readGeneratedResultAsset(ctx, ref)
@@ -389,6 +876,10 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
next := make(map[string]any, len(typed))
changed := refreshedAccessURL
for key, childValue := range typed {
if resultInternalField(key) {
changed = true
continue
}
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
if err != nil {
return nil, false, err
@@ -446,6 +937,9 @@ func generatedResultUploadReference(value map[string]any) (store.RequestAsset, m
return store.RequestAsset{}, nil, false
}
return store.RequestAsset{
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(upload["sha256"]))),
ContentType: stringFromAny(upload["contentType"]),
ByteSize: int64(floatFromAny(upload["size"])),
URL: stringFromAny(upload["url"]),
StorageProvider: stringFromAny(channel["provider"]),
StorageChannelID: stringFromAny(channel["id"]),
@@ -8,6 +8,7 @@ import (
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
@@ -209,6 +210,181 @@ func TestHydrateGeneratedResultRefreshesPrivateObjectURL(t *testing.T) {
}
}
func TestHydrateCanonicalUploadedResultForExplicitSynchronousBase64(t *testing.T) {
payload := []byte("canonical uploaded image")
digest := sha256.Sum256(payload)
getCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
getCount++
_, _ = w.Write(payload)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
cfg := config.Config{
MediaOSSDirectEnabled: true, MediaOSSEndpoint: server.URL, MediaOSSBucket: "media-bucket",
MediaOSSAccessKeyID: "access-id", MediaOSSAccessKeySecret: "access-secret", MediaOSSObjectPrefix: "media",
}
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
result := map[string]any{"thinking_bytes": strings.Repeat("opaque", 1024), "thought_signature": "signature", "data": []any{map[string]any{
"type": "image", "url": "https://expired.example/result.png", "mime_type": "image/png",
"assetStorage": map[string]any{"scene": store.FileStorageSceneImageResult, "source": "b64_json"},
"upload": map[string]any{
"url": "https://expired.example/result.png", "objectKey": "media/image_result/hash.png", "accessScope": "private",
"sha256": hex.EncodeToString(digest[:]), "size": len(payload), "contentType": "image/png",
"storageChannel": map[string]any{"channelKey": "environment-direct-oss", "provider": "aliyun_oss"},
},
}}}
projected, err := service.ProjectTaskResultURLs(t.Context(), "task-sync-b64", result)
if err != nil {
t.Fatal(err)
}
projectedItem := projected["data"].([]any)[0].(map[string]any)
if stringFromAny(projectedItem["url"]) == "" || getCount != 0 {
t.Fatalf("URL projection read object: item=%#v getCount=%d", projectedItem, getCount)
}
if projected["thinking_bytes"] != nil || projected["thought_signature"] != nil {
t.Fatalf("provider metadata leaked: %#v", projected)
}
hydrated, err := service.HydrateTaskResult(t.Context(), "task-sync-b64", result)
if err != nil {
t.Fatal(err)
}
item := hydrated["data"].([]any)[0].(map[string]any)
if got := stringFromAny(item["b64_json"]); got != base64.StdEncoding.EncodeToString(payload) {
t.Fatalf("Base64=%q", got)
}
if item["url"] != nil || item["upload"] != nil || getCount != 1 {
t.Fatalf("unexpected hydrated item=%#v getCount=%d", item, getCount)
}
if hydrated["thinking_bytes"] != nil || hydrated["thought_signature"] != nil {
t.Fatalf("provider metadata leaked into synchronous response: %#v", hydrated)
}
}
func TestSynchronousInlineResultBytesUsesMetadataBeforeObjectRead(t *testing.T) {
result := map[string]any{"data": []any{
map[string]any{
"assetStorage": map[string]any{"scene": store.FileStorageSceneImageResult, "source": "b64_json"},
"upload": map[string]any{
"objectKey": "media/image_result/large.png", "sha256": strings.Repeat("a", 64),
"size": MaxSynchronousInlineResponseBytes + 1, "contentType": "image/png",
"storageChannel": map[string]any{"channelKey": "environment-direct-oss"},
},
},
}}
if got := SynchronousInlineResultBytes(result); got != MaxSynchronousInlineResponseBytes+1 {
t.Fatalf("stored bytes=%d", got)
}
}
func TestProjectTaskResultURLsConvertsLegacyAssetWithoutReadingObject(t *testing.T) {
payload := []byte("must never be downloaded")
digest := sha256.Sum256(payload)
getCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
getCount++
}
http.Error(w, "object read is forbidden", http.StatusInternalServerError)
}))
defer server.Close()
service := &Service{}
result := map[string]any{"data": []any{map[string]any{
"b64_json": map[string]any{
"assetRef": map[string]any{
"sha256": hex.EncodeToString(digest[:]), "contentType": "image/png", "size": len(payload), "url": server.URL + "/result.png",
},
"assetStorage": map[string]any{"scene": store.FileStorageSceneImageResult, "source": "b64_json"},
},
}}}
projected, err := service.ProjectTaskResultURLs(t.Context(), "task-url-only", result)
if err != nil {
t.Fatal(err)
}
item := projected["data"].([]any)[0].(map[string]any)
if got := stringFromAny(item["url"]); got != server.URL+"/result.png" {
t.Fatalf("projected URL=%q", got)
}
if item["b64_json"] != nil || item["assetRef"] != nil || item["assetStorage"] != nil || item["upload"] != nil {
t.Fatalf("internal or inline fields leaked: %#v", item)
}
if getCount != 0 {
t.Fatalf("projector downloaded object %d time(s)", getCount)
}
}
func TestProjectTaskResultURLsRejectsHistoricalInlinePayload(t *testing.T) {
service := &Service{}
_, err := service.ProjectTaskResultURLs(t.Context(), "task-inline", map[string]any{
"data": []any{map[string]any{"b64_json": base64.StdEncoding.EncodeToString([]byte("inline"))}},
})
assertClientErrorCode(t, err, "result_materialization_required")
}
func TestMigrateTaskResultToURLsRewritesLegacyAssetReference(t *testing.T) {
payload := []byte("legacy")
digest := sha256.Sum256(payload)
service := &Service{}
result := map[string]any{"data": []any{map[string]any{
"b64_json": map[string]any{
"assetRef": map[string]any{
"sha256": hex.EncodeToString(digest[:]), "contentType": "image/png", "size": len(payload), "url": "https://cdn.example/result.png",
},
"assetStorage": map[string]any{"scene": store.FileStorageSceneImageResult, "source": "b64_json"},
},
}}}
migrated, changed, err := service.MigrateTaskResultToURLs(t.Context(), "task-legacy", result)
if err != nil {
t.Fatal(err)
}
if !changed || TaskResultNeedsURLMigration(migrated) {
t.Fatalf("legacy result was not fully migrated: %#v", migrated)
}
item := migrated["data"].([]any)[0].(map[string]any)
if stringFromAny(item["url"]) != "https://cdn.example/result.png" || item["upload"] == nil || item["b64_json"] != nil {
t.Fatalf("unexpected migrated result: %#v", item)
}
}
func TestMigrateTaskResultToURLsUploadsActiveLocalPlaceholder(t *testing.T) {
payload := []byte("historical local image")
digest := sha256.Sum256(payload)
putCount := 0
storageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPut {
putCount++
w.WriteHeader(http.StatusOK)
return
}
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
}))
defer storageServer.Close()
service := newLocalBinaryTestService(t)
service.directOSS = &directOSSUploader{
endpoint: storageServer.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media",
}
writeHistoricalLocalBinaryFixture(t, service, "task-local-migrate", payload)
placeholder := fmt.Sprintf("%ssha256=%s;bytes=%d;mime=image/png;encoding=base64]", localBinaryPlaceholderPrefix, hex.EncodeToString(digest[:]), len(payload))
result := map[string]any{"data": []any{map[string]any{"b64_json": placeholder}}}
migrated, changed, err := service.MigrateTaskResultToURLs(t.Context(), "task-local-migrate", result)
if err != nil {
t.Fatal(err)
}
if !changed || TaskResultNeedsURLMigration(migrated) || putCount != 1 {
t.Fatalf("placeholder migration changed=%t putCount=%d result=%#v", changed, putCount, migrated)
}
}
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
service := newLocalBinaryTestService(t)
service.cfg.LocalResultTTLHours = 1
+37 -2
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"regexp"
"strconv"
@@ -243,6 +244,13 @@ func (s *Service) observeObjectStorage(event string, provider string, bytes int,
}
}
func (s *Service) observeResultStorage(source string) {
observer, ok := s.billingMetrics.(interface{ ObserveResultStorage(string) })
if ok {
observer.ObserveResultStorage(source)
}
}
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
return s.execute(ctx, task, user, nil)
}
@@ -982,7 +990,7 @@ candidatesLoop:
}
walletReservationFinalized = true
s.logger.Warn("task succeeded but billing requires manual review", "taskID", task.ID, "error_category", "billing_calculation_failed")
output, hydrateErr := s.HydrateTaskResult(ctx, task.ID, response.Result)
output, hydrateErr := s.taskResultForSynchronousResponse(ctx, task, response.Result)
if hydrateErr != nil {
return Result{Task: review}, hydrateErr
}
@@ -1075,7 +1083,7 @@ candidatesLoop:
// time after the task is already durably complete.
return Result{Task: finished, Output: response.Result}, nil
}
output, hydrateErr := s.HydrateTaskResult(ctx, task.ID, response.Result)
output, hydrateErr := s.taskResultForSynchronousResponse(ctx, task, response.Result)
if hydrateErr != nil {
return Result{Task: finished}, hydrateErr
}
@@ -1272,6 +1280,33 @@ candidatesLoop:
return Result{Task: failed, Output: failed.Result}, lastErr
}
func (s *Service) taskResultForSynchronousResponse(ctx context.Context, task store.GatewayTask, result map[string]any) (map[string]any, error) {
if taskRequestsInlineMediaResult(task.Request) {
if rawBytes := SynchronousInlineResultBytes(result); rawBytes > MaxSynchronousInlineResponseBytes {
return nil, &clients.ClientError{
Code: "response_format_too_large",
Message: "synchronous Base64 response exceeds the 20 MiB limit",
StatusCode: http.StatusRequestEntityTooLarge,
Retryable: false,
Details: map[string]any{
"task_id": task.ID,
"query_url": "/api/v1/ai/result/" + task.ID,
"limit_bytes": MaxSynchronousInlineResponseBytes,
"actual_bytes": rawBytes,
"response_format": "url",
},
}
}
return s.HydrateTaskResult(ctx, task.ID, result)
}
return s.ProjectTaskResultURLs(ctx, task.ID, result)
}
func taskRequestsInlineMediaResult(request map[string]any) bool {
value, _ := request["response_format"].(string)
return strings.EqualFold(strings.TrimSpace(value), "b64_json")
}
func pricingCandidateKey(candidate store.RuntimeModelCandidate) string {
return firstNonEmptyString(candidate.PlatformModelID, candidate.PlatformID+":"+candidate.ModelName)
}
+69 -49
View File
@@ -50,6 +50,13 @@ type generatedAssetDecision struct {
StripKeys []string
}
type generatedAssetUploadResult struct {
upload map[string]any
contentType string
kind string
strategy string
}
type generatedInlineAsset struct {
Bytes []byte
ContentType string
@@ -150,6 +157,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
return nil, err
}
decisions[index] = decision
if _, mediaURL := mediaURLSourceFromItem(item); mediaURL != "" && decision.URL == nil {
s.observeResultStorage("upstream_url")
}
if decision.Inline != nil || decision.URL != nil {
needsUpload = true
}
@@ -175,6 +185,7 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
next[key] = value
}
nextData := make([]any, 0, len(data))
uploadCache := make(map[string]generatedAssetUploadResult)
for index, rawItem := range data {
item, _ := rawItem.(map[string]any)
if item == nil {
@@ -197,7 +208,15 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
var contentType string
var err error
if decision.Inline != nil {
upload, contentType, kind, strategy, err = s.uploadGeneratedAsset(ctx, taskID, decision.Inline, index, channels)
cacheKey := generatedInlineAssetCacheKey(decision.Inline)
if cached, ok := uploadCache[cacheKey]; ok {
upload, contentType, kind, strategy = cached.upload, cached.contentType, cached.kind, cached.strategy
} else {
upload, contentType, kind, strategy, err = s.uploadGeneratedAsset(ctx, taskID, decision.Inline, index, channels)
if err == nil {
uploadCache[cacheKey] = generatedAssetUploadResult{upload: upload, contentType: contentType, kind: kind, strategy: strategy}
}
}
sourceKey = decision.Inline.SourceKey
} else {
upload, contentType, kind, strategy, err = s.uploadGeneratedURLAsset(ctx, taskID, decision.URL, index, channels, acceptanceEmulatorBaseURL)
@@ -233,9 +252,6 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
if contentType != "" && stringFromAny(merged["mime_type"]) == "" {
merged["mime_type"] = contentType
}
if decision.Inline != nil && strings.TrimSpace(sourceKey) != "" {
merged[sourceKey] = generatedRawMediaReference(decision.Inline, upload, contentType, kind, strategy)
}
}
nextData = append(nextData, merged)
}
@@ -255,6 +271,15 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
return s.finalizeGeneratedAssets(ctx, taskID, taskKind, next, policy, channels, channelsLoaded, len(nextData))
}
func generatedInlineAssetCacheKey(asset *generatedInlineAsset) string {
if asset == nil {
return ""
}
digest := sha256.Sum256(asset.Bytes)
contentType := resolvedGeneratedAssetContentType(asset.ContentType, asset.Kind, asset.Bytes)
return hex.EncodeToString(digest[:]) + ":" + contentType
}
func generatedAssetUploadPolicyForAcceptanceRun(policy generatedAssetUploadPolicy, acceptanceRunID string) generatedAssetUploadPolicy {
if strings.TrimSpace(acceptanceRunID) != "" {
policy.UploadURLMedia = true
@@ -274,7 +299,7 @@ func (s *Service) finalizeGeneratedAssets(
) (map[string]any, error) {
redactGeneratedResultRawData(result)
if !TaskResultHasInlineBinary(result) {
return result, nil
return canonicalStoredResultURLs(result), nil
}
next := result
if policy.UploadInlineMedia {
@@ -304,7 +329,7 @@ func (s *Service) finalizeGeneratedAssets(
}
}
if !TaskResultHasInlineBinary(next) {
return next, nil
return canonicalStoredResultURLs(next), nil
}
diagnostics := taskResultInlineBinaryDiagnostics(next)
if s.logger != nil {
@@ -322,6 +347,18 @@ func (s *Service) finalizeGeneratedAssets(
}
}
func canonicalStoredResultURLs(result map[string]any) map[string]any {
next, changed := migrateStoredResultURLValue(result, 0)
if !changed {
return result
}
mapped, ok := next.(map[string]any)
if !ok {
return result
}
return mapped
}
func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool {
switch typed := value.(type) {
case map[string]any:
@@ -448,42 +485,9 @@ func generatedRawInlineMediaAsset(key string, value string, siblings map[string]
}
func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]any, contentType string, kind string, strategy string) map[string]any {
digest := sha256.Sum256(asset.Bytes)
urlValue := stringFromAny(upload["url"])
ref := map[string]any{
"sha256": hex.EncodeToString(digest[:]),
"contentType": contentType,
"size": len(asset.Bytes),
}
if urlValue != "" {
ref["url"] = urlValue
}
if fileName := stringFromAny(upload["fileName"]); fileName != "" {
ref["fileName"] = fileName
}
if expiresAt := stringFromAny(upload["expiresAt"]); expiresAt != "" {
ref["expiresAt"] = expiresAt
}
if channel, ok := upload["storageChannel"].(map[string]any); ok {
if provider := stringFromAny(channel["provider"]); provider != "" {
ref["storageProvider"] = provider
}
if id := stringFromAny(channel["id"]); id != "" {
ref["storageChannelId"] = id
}
if key := stringFromAny(channel["channelKey"]); key != "" {
ref["storageChannelKey"] = key
}
}
if objectKey := stringFromAny(upload["objectKey"]); objectKey != "" {
ref["objectKey"] = objectKey
}
if accessScope := stringFromAny(upload["accessScope"]); accessScope != "" {
ref["accessScope"] = accessScope
}
out := map[string]any{
"assetRef": ref,
"upload": upload,
"upload": upload,
"assetStorage": map[string]any{
"scene": store.FileStorageSceneImageResult,
"source": asset.SourceKey,
@@ -497,6 +501,9 @@ func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]a
if kind != "" {
out["type"] = kind
}
if contentType != "" {
out["mime_type"] = contentType
}
return out
}
@@ -713,6 +720,9 @@ func (s *Service) uploadGeneratedAsset(ctx context.Context, taskID string, asset
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
upload, err := s.uploadFileWithFailover(ctx, payload, channels)
if err == nil {
s.observeResultStorage("uploaded")
}
return upload, contentType, kind, "upload_inline_media", err
}
@@ -734,6 +744,9 @@ func (s *Service) uploadGeneratedURLAsset(ctx context.Context, taskID string, as
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
upload, err := s.uploadFileWithFailover(ctx, uploadPayload, channels)
if err == nil {
s.observeResultStorage("uploaded")
}
return upload, contentType, kind, "upload_url_media", err
}
@@ -984,7 +997,7 @@ func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUpload
func storageFailureAllowsFailover(channel store.FileStorageChannel, err error) bool {
code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err)))
switch code {
case "upload_source_too_large", "upload_decode_failed", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio":
case "upload_source_too_large", "upload_decode_failed", "invalid_upstream_result", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio":
return false
}
var clientErr *clients.ClientError
@@ -1120,6 +1133,12 @@ func stripDataURLPrefix(value string) string {
func generatedAssetDecisionForItem(taskKind string, item map[string]any, policy generatedAssetUploadPolicy) (generatedAssetDecision, error) {
decision := generatedAssetDecision{}
for _, key := range mediaURLCandidateKeys() {
value := strings.TrimSpace(stringFromAny(item[key]))
if value != "" && strings.Contains(value, "://") && !mediaURLString(value) {
return decision, &clients.ClientError{Code: "invalid_upstream_result", Message: "generated media URL must use http or https", StatusCode: http.StatusBadGateway, Retryable: false}
}
}
urlKey, mediaURL := mediaURLSourceFromItem(item)
if mediaURL != "" {
if !policy.UploadURLMedia {
@@ -1225,7 +1244,7 @@ func inlineMediaPayloadFromString(value string, strictBase64 bool) ([]byte, stri
}
payload, err := decodeBase64Payload(encoded)
if err != nil {
return nil, "", false, &clients.ClientError{Code: "upload_decode_failed", Message: err.Error(), Retryable: false}
return nil, "", false, &clients.ClientError{Code: "invalid_upstream_result", Message: err.Error(), StatusCode: http.StatusBadGateway, Retryable: false}
}
return payload, contentType, true, nil
}
@@ -1235,7 +1254,7 @@ func inlineMediaPayloadFromString(value string, strictBase64 bool) ([]byte, stri
payload, err := decodeBase64Payload(raw)
if err != nil {
if strictBase64 {
return nil, "", false, &clients.ClientError{Code: "upload_decode_failed", Message: err.Error(), Retryable: false}
return nil, "", false, &clients.ClientError{Code: "invalid_upstream_result", Message: err.Error(), StatusCode: http.StatusBadGateway, Retryable: false}
}
return nil, "", false, nil
}
@@ -1245,7 +1264,7 @@ func inlineMediaPayloadFromString(value string, strictBase64 bool) ([]byte, stri
func parseBase64DataURL(value string) (string, string, bool, error) {
prefix, payload, ok := strings.Cut(value, ",")
if !ok {
return "", "", false, &clients.ClientError{Code: "upload_decode_failed", Message: "invalid data URL media payload", Retryable: false}
return "", "", false, &clients.ClientError{Code: "invalid_upstream_result", Message: "invalid data URL media payload", StatusCode: http.StatusBadGateway, Retryable: false}
}
meta := strings.TrimPrefix(prefix, "data:")
meta = strings.TrimPrefix(meta, "DATA:")
@@ -1259,7 +1278,7 @@ func parseBase64DataURL(value string) (string, string, bool, error) {
}
}
if !isBase64 {
return "", "", false, &clients.ClientError{Code: "upload_decode_failed", Message: "data URL media payload is not base64 encoded", Retryable: false}
return "", "", false, &clients.ClientError{Code: "invalid_upstream_result", Message: "data URL media payload is not base64 encoded", StatusCode: http.StatusBadGateway, Retryable: false}
}
return contentType, payload, true, nil
}
@@ -1456,10 +1475,11 @@ func mediaURLString(value string) bool {
if strings.HasPrefix(lower, "data:") {
return false
}
return strings.HasPrefix(lower, "http://") ||
strings.HasPrefix(lower, "https://") ||
strings.HasPrefix(lower, "/") ||
strings.Contains(lower, "://")
if strings.HasPrefix(lower, "/") {
return true
}
parsed, err := url.Parse(raw)
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
}
func mediaContentTypeFromItem(item map[string]any) string {
+64 -14
View File
@@ -55,6 +55,43 @@ func TestGeneratedAssetDecisionUploadsInlineImageBase64(t *testing.T) {
}
}
func TestUploadGeneratedAssetsReusesDuplicateSHAWithinTask(t *testing.T) {
putCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
return
}
putCount++
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
service := &Service{}
service.directOSS = &directOSSUploader{
endpoint: server.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media",
}
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 32)...)
encoded := base64.StdEncoding.EncodeToString(payload)
result := map[string]any{"data": []any{
map[string]any{"b64_json": encoded, "mime_type": "image/png"},
map[string]any{"b64_json": encoded, "mime_type": "image/png"},
}}
stored, err := service.uploadGeneratedAssets(t.Context(), "task-duplicate", "images.generations", "", result)
if err != nil {
t.Fatal(err)
}
items := stored["data"].([]any)
first := items[0].(map[string]any)
second := items[1].(map[string]any)
if putCount != 1 || stringFromAny(first["url"]) == "" || first["url"] != second["url"] {
t.Fatalf("duplicate SHA was not reused: puts=%d first=%#v second=%#v", putCount, first, second)
}
if TaskResultHasInlineBinary(stored) || first["b64_json"] != nil || second["b64_json"] != nil {
t.Fatalf("inline payload remained after deduplicated upload: %#v", stored)
}
}
func TestMediaResultMaterializationConcurrencyIsBounded(t *testing.T) {
service := New(config.Config{MediaMaterializationConcurrency: 1}, nil, nil)
result := map[string]any{
@@ -227,6 +264,15 @@ func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) {
}
}
func TestGeneratedAssetDecisionRejectsUnsupportedURLScheme(t *testing.T) {
_, err := generatedAssetDecisionForItem("images.generations", map[string]any{
"url": "ftp://files.example/result.png", "type": "image",
}, defaultGeneratedAssetUploadPolicy())
if clients.ErrorCode(err) != "invalid_upstream_result" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
tests := []struct {
name string
@@ -329,13 +375,15 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
t.Fatal("finalized result still contains inline binary")
}
nested := finalized["provider_result"].(map[string]any)
reference, ok := nested["binary_data_base64"].(map[string]any)
if !ok {
t.Fatalf("nested binary should be replaced by an asset reference: %+v", nested)
if _, exists := nested["binary_data_base64"]; exists {
t.Fatalf("nested Base64 field was retained: %+v", nested)
}
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") {
if urlValue := stringFromAny(nested["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") {
t.Fatalf("unexpected object storage URL: %s", urlValue)
}
if nested["upload"] == nil {
t.Fatalf("nested upload metadata was not retained: %+v", nested)
}
}
func TestFinalizeGeneratedAssetsMaterializesDetectedMediaAndKeepsOpaqueMetadata(t *testing.T) {
@@ -368,7 +416,7 @@ func TestFinalizeGeneratedAssetsMaterializesDetectedMediaAndKeepsOpaqueMetadata(
t.Fatalf("finalized result still contains generated media: %#v", finalized)
}
reference, ok := finalized["provider_payload"].(map[string]any)
if !ok || reference["assetRef"] == nil || reference["upload"] == nil {
if !ok || reference["url"] == nil || reference["upload"] == nil || reference["assetRef"] != nil {
t.Fatalf("detected media was not objectified: %#v", finalized["provider_payload"])
}
if finalized["thought_signature"] != opaque {
@@ -407,9 +455,11 @@ func TestFinalizeGeneratedAssetsMaterializesGeminiImageData(t *testing.T) {
}
data := finalized["data"].([]any)
item := data[0].(map[string]any)
reference, ok := item["b64_json"].(map[string]any)
if !ok || reference["assetRef"] == nil || reference["upload"] == nil {
t.Fatalf("Gemini b64_json was not replaced by an object reference: %#v", item)
if _, exists := item["b64_json"]; exists {
t.Fatalf("Gemini b64_json was retained: %#v", item)
}
if stringFromAny(item["url"]) == "" || item["upload"] == nil || item["assetRef"] != nil {
t.Fatalf("Gemini b64_json was not replaced by a URL result: %#v", item)
}
}
@@ -474,7 +524,7 @@ func TestUploadGeneratedAudioFailsWithoutObjectStorage(t *testing.T) {
}
}
func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *testing.T) {
func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithURL(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
@@ -516,9 +566,9 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
if !ok {
t.Fatalf("inlineData.data should be an asset reference, got %+v", inlineData["data"])
}
ref, _ := data["assetRef"].(map[string]any)
if ref["sha256"] == "" || ref["contentType"] != "image/png" || ref["size"] != len(payload) {
t.Fatalf("unexpected asset ref: %+v", ref)
upload, _ := data["upload"].(map[string]any)
if upload["sha256"] == "" || upload["contentType"] != "image/png" || upload["size"] != len(payload) || data["assetRef"] != nil {
t.Fatalf("unexpected URL storage metadata: %+v", data)
}
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") || !strings.HasSuffix(urlValue, ".png") {
t.Fatalf("unexpected raw media URL: %s", urlValue)
@@ -528,7 +578,7 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
}
}
func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithAssetRefs(t *testing.T) {
func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithURLs(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
@@ -551,7 +601,7 @@ func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithAssetRefs(t *test
next := uploaded.(map[string]any)
for _, key := range []string{"buffer", "audio_bytes", "direct"} {
item, ok := next[key].(map[string]any)
if !ok || item["assetRef"] == nil || item["upload"] == nil {
if !ok || item["url"] == nil || item["upload"] == nil || item["assetRef"] != nil {
t.Fatalf("%s was not objectified: %#v", key, next[key])
}
}