wangbo
b13392ef50
将上游 URL 直接持久化，内联媒体经对象存储后仅保留 URL 与内部定位元数据；异步轮询、任务详情和幂等重放统一使用零对象读取的 URL 投影，并增加 64KiB 响应门禁。 OpenAI 图片接口接受 url 与 b64_json，同步 Base64 限制为 20MiB 和每 Pod 2 并发；新增历史结果迁移清零门禁、结果指标和 API GOMEMLIMIT。 验证：API go test ./...、go vet、聚焦 race、pnpm openapi、pnpm lint/test/build、迁移安全检查与 docker compose config 均通过。
1283 lines
40 KiB
Go
1283 lines
40 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
const (
|
|
// MaxSynchronousInlineResponseBytes bounds the raw media payload restored for
|
|
// an explicit synchronous OpenAI-compatible b64_json response.
|
|
MaxSynchronousInlineResponseBytes = int64(20 << 20)
|
|
localBinaryResultDirName = "results"
|
|
localBinaryPlaceholderPrefix = "[GatewayBinary:v1;"
|
|
localBinaryGenericBase64MinLength = 4096
|
|
localBinaryMaxDepth = 64
|
|
defaultLocalResultTTLHours = 24
|
|
defaultLocalResultMaxBytes = int64(256 * 1024 * 1024)
|
|
defaultLocalResultMaxTaskBytes = int64(512 * 1024 * 1024)
|
|
)
|
|
|
|
type localBinaryDescriptor struct {
|
|
Prefix string
|
|
SHA256 string
|
|
Size int64
|
|
ContentType string
|
|
Encoding string
|
|
}
|
|
|
|
type localBinaryMaterializer struct {
|
|
service *Service
|
|
enforceLimits bool
|
|
totalBytes int64
|
|
seen map[string]struct{}
|
|
}
|
|
|
|
// materializeLocalBinaryResult is retained as an internal compatibility seam,
|
|
// but new materialization always targets object storage. Historical local
|
|
// placeholders remain readable through HydrateTaskResult.
|
|
func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
|
hadInline := TaskResultHasInlineBinary(result)
|
|
if !hadInline {
|
|
return result, false, nil
|
|
}
|
|
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
return next, !TaskResultHasInlineBinary(next), nil
|
|
}
|
|
|
|
func (s *Service) transformLocalBinaryResult(ctx context.Context, _ string, result map[string]any, _ bool, enforceLimits bool) (map[string]any, bool, error) {
|
|
materializer := &localBinaryMaterializer{
|
|
service: s,
|
|
enforceLimits: enforceLimits,
|
|
seen: map[string]struct{}{},
|
|
}
|
|
next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
mapped, ok := next.(map[string]any)
|
|
if !ok {
|
|
return nil, false, &clients.ClientError{
|
|
Code: "result_binary_not_materialized",
|
|
Message: "generated result is not a JSON object",
|
|
StatusCode: 500,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
return mapped, changed, nil
|
|
}
|
|
|
|
// MaterializeTaskResultForStorage exposes the same verified materialization
|
|
// path to the explicit historical maintenance command.
|
|
func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
|
hadInline := TaskResultHasInlineBinary(result)
|
|
if !hadInline {
|
|
return result, false, nil
|
|
}
|
|
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
return next, hadInline && !TaskResultHasInlineBinary(next), nil
|
|
}
|
|
|
|
// MigrateTaskResultToURLs materializes historical inline payloads and rewrites
|
|
// legacy assetRef wrappers into the canonical URL-plus-upload storage shape.
|
|
func (s *Service) MigrateTaskResultToURLs(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
|
next := result
|
|
changed := false
|
|
if TaskResultHasLocalPlaceholder(next) {
|
|
restored, restoredChanged, err := s.restoreLocalPlaceholderValue(ctx, taskID, next, 0)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
mapped, ok := restored.(map[string]any)
|
|
if !ok {
|
|
return nil, false, &clients.ClientError{Code: "result_binary_not_materialized", Message: "generated result is not a JSON object", StatusCode: http.StatusInternalServerError}
|
|
}
|
|
next = mapped
|
|
changed = restoredChanged
|
|
}
|
|
if TaskResultHasInlineBinary(next) {
|
|
materialized, materializedChanged, err := s.MaterializeTaskResultForStorage(ctx, taskID, next)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next = materialized
|
|
changed = materializedChanged
|
|
}
|
|
rewritten, rewrittenChanged := migrateStoredResultURLValue(next, 0)
|
|
mapped, ok := rewritten.(map[string]any)
|
|
if !ok {
|
|
return nil, false, &clients.ClientError{Code: "result_binary_not_materialized", Message: "generated result is not a JSON object", StatusCode: http.StatusInternalServerError}
|
|
}
|
|
return mapped, changed || rewrittenChanged, nil
|
|
}
|
|
|
|
// TaskResultNeedsURLMigration reports whether a stored result contains inline
|
|
// binary, a local placeholder, or the legacy assetRef representation.
|
|
func TaskResultNeedsURLMigration(result map[string]any) bool {
|
|
if TaskResultHasInlineBinary(result) {
|
|
return true
|
|
}
|
|
return storedResultURLValueNeedsMigration(result, 0)
|
|
}
|
|
|
|
// TaskResultHasLocalPlaceholder reports whether a historical result still
|
|
// references a node-local GatewayBinary file.
|
|
func TaskResultHasLocalPlaceholder(result map[string]any) bool {
|
|
return localBinaryResultHasPlaceholders(result)
|
|
}
|
|
|
|
func (s *Service) restoreLocalPlaceholderValue(ctx context.Context, taskID string, value any, depth int) (any, bool, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, false, err
|
|
}
|
|
if depth >= localBinaryMaxDepth {
|
|
return nil, false, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result exceeds the maximum JSON depth", StatusCode: http.StatusInternalServerError}
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
next := make(map[string]any, len(typed))
|
|
changed := false
|
|
for key, childValue := range typed {
|
|
child, childChanged, err := s.restoreLocalPlaceholderValue(ctx, taskID, childValue, depth+1)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next[key] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false, nil
|
|
}
|
|
return next, true, nil
|
|
case []any:
|
|
next := make([]any, len(typed))
|
|
changed := false
|
|
for index, childValue := range typed {
|
|
child, childChanged, err := s.restoreLocalPlaceholderValue(ctx, taskID, childValue, depth+1)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next[index] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false, nil
|
|
}
|
|
return next, true, nil
|
|
case string:
|
|
descriptor, ok := parseLocalBinaryPlaceholder(typed)
|
|
if !ok {
|
|
return value, false, nil
|
|
}
|
|
payload, err := s.readLocalBinaryResult(taskID, descriptor)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
encoded := base64.StdEncoding.EncodeToString(payload)
|
|
if descriptor.Encoding == "data-uri" {
|
|
return "data:" + descriptor.ContentType + ";base64," + encoded, true, nil
|
|
}
|
|
return encoded, true, nil
|
|
default:
|
|
return value, false, nil
|
|
}
|
|
}
|
|
|
|
func storedResultURLValueNeedsMigration(value any, depth int) bool {
|
|
if depth >= localBinaryMaxDepth {
|
|
return true
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
if _, ok := typed["assetRef"].(map[string]any); ok {
|
|
return true
|
|
}
|
|
for _, child := range typed {
|
|
if storedResultURLValueNeedsMigration(child, depth+1) {
|
|
return true
|
|
}
|
|
}
|
|
case []any:
|
|
for _, child := range typed {
|
|
if storedResultURLValueNeedsMigration(child, depth+1) {
|
|
return true
|
|
}
|
|
}
|
|
case string:
|
|
_, ok := parseLocalBinaryPlaceholder(typed)
|
|
return ok
|
|
}
|
|
return false
|
|
}
|
|
|
|
func migrateStoredResultURLValue(value any, depth int) (any, bool) {
|
|
if depth >= localBinaryMaxDepth {
|
|
return value, false
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
if asset, ok := generatedResultAssetReference(typed); ok {
|
|
upload, _ := typed["upload"].(map[string]any)
|
|
if upload == nil {
|
|
upload = uploadMetadataFromLegacyAsset(asset)
|
|
}
|
|
accessURL := firstNonEmptyString(stringFromAny(upload["url"]), asset.URL)
|
|
next := map[string]any{
|
|
"url": accessURL,
|
|
"upload": upload,
|
|
"assetStorage": map[string]any{
|
|
"scene": store.FileStorageSceneImageResult,
|
|
"source": stringFromAny(typed["assetStorage"].(map[string]any)["source"]),
|
|
"strategy": "migrate_asset_ref",
|
|
"contentType": asset.ContentType,
|
|
},
|
|
}
|
|
for _, key := range []string{"type", "mime_type", "width", "height", "duration", "format", "seed", "revised_prompt"} {
|
|
if item, exists := typed[key]; exists {
|
|
next[key] = item
|
|
}
|
|
}
|
|
if stringFromAny(next["mime_type"]) == "" && asset.ContentType != "" {
|
|
next["mime_type"] = asset.ContentType
|
|
}
|
|
return next, true
|
|
}
|
|
next := make(map[string]any, len(typed))
|
|
changed := false
|
|
for key, childValue := range typed {
|
|
child, childChanged := migrateStoredResultURLValue(childValue, depth+1)
|
|
if resultCanonicalInlineField(key) {
|
|
if item, ok := child.(map[string]any); ok && stringFromAny(item["url"]) != "" {
|
|
mergeProjectedResultURL(next, item)
|
|
changed = true
|
|
continue
|
|
}
|
|
}
|
|
next[key] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false
|
|
}
|
|
return next, true
|
|
case []any:
|
|
next := make([]any, len(typed))
|
|
changed := false
|
|
for index, childValue := range typed {
|
|
child, childChanged := migrateStoredResultURLValue(childValue, depth+1)
|
|
next[index] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false
|
|
}
|
|
return next, true
|
|
default:
|
|
return value, false
|
|
}
|
|
}
|
|
|
|
func uploadMetadataFromLegacyAsset(asset store.RequestAsset) map[string]any {
|
|
upload := map[string]any{
|
|
"url": asset.URL,
|
|
"objectKey": asset.ObjectKey,
|
|
"contentType": asset.ContentType,
|
|
"size": asset.ByteSize,
|
|
"sha256": asset.SHA256,
|
|
"accessScope": asset.AccessScope,
|
|
"storageChannel": map[string]any{
|
|
"id": asset.StorageChannelID,
|
|
"channelKey": asset.StorageChannelKey,
|
|
"provider": asset.StorageProvider,
|
|
},
|
|
}
|
|
if asset.ExpiresAt != nil {
|
|
upload["objectExpiresAt"] = asset.ExpiresAt.Format(time.RFC3339)
|
|
}
|
|
return upload
|
|
}
|
|
|
|
// CompactExpiredTaskResultForStorage keeps the historical maintenance API but
|
|
// now uses the same object-storage path as live results. New GatewayBinary
|
|
// placeholders are never created; their parser remains read-only compatibility.
|
|
func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
|
return s.MaterializeTaskResultForStorage(ctx, taskID, result)
|
|
}
|
|
|
|
func TaskResultHasInlineBinary(result map[string]any) bool {
|
|
return localBinaryValueHasPayload(result, "", nil, 0)
|
|
}
|
|
|
|
func taskResultInlineBinaryDiagnostics(result map[string]any) []string {
|
|
diagnostics := make([]string, 0, 4)
|
|
appendInlineBinaryDiagnostics(result, "", nil, "$", 0, &diagnostics)
|
|
return diagnostics
|
|
}
|
|
|
|
func appendInlineBinaryDiagnostics(value any, key string, siblings map[string]any, path string, depth int, diagnostics *[]string) {
|
|
if depth >= localBinaryMaxDepth || len(*diagnostics) >= 8 {
|
|
return
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
if payload, contentType, ok := localBufferObjectBytes(typed); ok {
|
|
*diagnostics = append(*diagnostics, fmt.Sprintf("%s buffer bytes=%d contentType=%s", path, len(payload), contentType))
|
|
return
|
|
}
|
|
keys := make([]string, 0, len(typed))
|
|
for childKey := range typed {
|
|
keys = append(keys, childKey)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, childKey := range keys {
|
|
appendInlineBinaryDiagnostics(typed[childKey], childKey, typed, path+"."+childKey, depth+1, diagnostics)
|
|
}
|
|
case []any:
|
|
if localBinaryKey(key) {
|
|
if payload, ok := bytesFromNumberArray(typed); ok {
|
|
*diagnostics = append(*diagnostics, fmt.Sprintf("%s number-array bytes=%d", path, len(payload)))
|
|
return
|
|
}
|
|
}
|
|
for index, child := range typed {
|
|
appendInlineBinaryDiagnostics(child, key, siblings, fmt.Sprintf("%s[%d]", path, index), depth+1, diagnostics)
|
|
}
|
|
case []byte:
|
|
if len(typed) > 0 {
|
|
*diagnostics = append(*diagnostics, fmt.Sprintf("%s bytes=%d", path, len(typed)))
|
|
}
|
|
case string:
|
|
payload, contentType, encoding, ok := localBinaryStringBytes(key, typed, siblings)
|
|
if ok {
|
|
*diagnostics = append(*diagnostics, fmt.Sprintf("%s string bytes=%d contentType=%s encoding=%s", path, len(payload), contentType, encoding))
|
|
}
|
|
}
|
|
}
|
|
|
|
func localBinaryValueHasPayload(value any, key string, siblings map[string]any, depth int) bool {
|
|
if depth >= localBinaryMaxDepth {
|
|
return false
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
if _, _, ok := localBufferObjectBytes(typed); ok {
|
|
return true
|
|
}
|
|
for childKey, child := range typed {
|
|
if localBinaryValueHasPayload(child, childKey, typed, depth+1) {
|
|
return true
|
|
}
|
|
}
|
|
case []any:
|
|
if localBinaryKey(key) {
|
|
if _, ok := bytesFromNumberArray(typed); ok {
|
|
return true
|
|
}
|
|
}
|
|
for _, child := range typed {
|
|
if localBinaryValueHasPayload(child, key, siblings, depth+1) {
|
|
return true
|
|
}
|
|
}
|
|
case []byte:
|
|
return len(typed) > 0
|
|
case string:
|
|
_, _, _, ok := localBinaryStringBytes(key, typed, siblings)
|
|
return ok
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (m *localBinaryMaterializer) materializeValue(ctx context.Context, value any, key string, siblings map[string]any, depth int) (any, bool, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, false, err
|
|
}
|
|
if depth >= localBinaryMaxDepth {
|
|
return nil, false, &clients.ClientError{
|
|
Code: "result_binary_not_materialized",
|
|
Message: "generated result exceeds the maximum JSON depth",
|
|
StatusCode: 500,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
if payload, contentType, ok := localBufferObjectBytes(typed); ok {
|
|
return m.persistBinary(ctx, payload, contentType, "buffer")
|
|
}
|
|
next := make(map[string]any, len(typed))
|
|
changed := false
|
|
for childKey, childValue := range typed {
|
|
child, childChanged, err := m.materializeValue(ctx, childValue, childKey, typed, depth+1)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next[childKey] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false, nil
|
|
}
|
|
return next, true, nil
|
|
case []any:
|
|
if localBinaryKey(key) {
|
|
if payload, ok := bytesFromNumberArray(typed); ok {
|
|
return m.persistBinary(ctx, payload, mediaContentTypeFromItem(siblings), "buffer")
|
|
}
|
|
}
|
|
next := make([]any, len(typed))
|
|
changed := false
|
|
for index, item := range typed {
|
|
child, childChanged, err := m.materializeValue(ctx, item, key, siblings, depth+1)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next[index] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false, nil
|
|
}
|
|
return next, true, nil
|
|
case []byte:
|
|
if len(typed) == 0 {
|
|
return value, false, nil
|
|
}
|
|
return m.persistBinary(ctx, append([]byte(nil), typed...), mediaContentTypeFromItem(siblings), "buffer")
|
|
case string:
|
|
payload, contentType, encoding, ok := localBinaryStringBytes(key, typed, siblings)
|
|
if !ok {
|
|
return value, false, nil
|
|
}
|
|
return m.persistBinary(ctx, payload, contentType, encoding)
|
|
default:
|
|
return value, false, nil
|
|
}
|
|
}
|
|
|
|
func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []byte, contentType string, encoding string) (any, bool, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, false, err
|
|
}
|
|
if len(payload) == 0 {
|
|
return nil, false, nil
|
|
}
|
|
size := int64(len(payload))
|
|
if m.enforceLimits && size > m.service.localResultMaxBytes() {
|
|
return nil, false, &clients.ClientError{
|
|
Code: "binary_result_too_large",
|
|
Message: "one generated binary result exceeds the local storage limit",
|
|
StatusCode: 502,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
digest := sha256.Sum256(payload)
|
|
digestHex := hex.EncodeToString(digest[:])
|
|
if _, exists := m.seen[digestHex]; !exists {
|
|
if m.enforceLimits && m.totalBytes+size > m.service.localResultMaxTaskBytes() {
|
|
return nil, false, &clients.ClientError{
|
|
Code: "binary_result_too_large",
|
|
Message: "generated binary results exceed the per-task local storage limit",
|
|
StatusCode: 502,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
m.seen[digestHex] = struct{}{}
|
|
m.totalBytes += size
|
|
}
|
|
descriptor := localBinaryDescriptor{
|
|
Prefix: localBinaryPrefix(payload),
|
|
SHA256: digestHex,
|
|
Size: size,
|
|
ContentType: normalizedLocalBinaryContentType(contentType),
|
|
Encoding: normalizedLocalBinaryEncoding(encoding),
|
|
}
|
|
return localBinaryPlaceholder(descriptor), true, nil
|
|
}
|
|
|
|
func localBinaryStorageError(err error) error {
|
|
return &clients.ClientError{
|
|
Code: "local_result_storage_unavailable",
|
|
Message: "local result storage failed: " + err.Error(),
|
|
StatusCode: 503,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
|
|
func verifyLocalBinaryFile(path string, expectedHash string, expectedSize int64) error {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return localBinaryStorageError(err)
|
|
}
|
|
defer file.Close()
|
|
hasher := sha256.New()
|
|
size, err := io.Copy(hasher, file)
|
|
if err != nil {
|
|
return localBinaryStorageError(err)
|
|
}
|
|
if size != expectedSize || hex.EncodeToString(hasher.Sum(nil)) != expectedHash {
|
|
return &clients.ClientError{
|
|
Code: "binary_result_corrupted",
|
|
Message: "local result file failed size or hash verification",
|
|
StatusCode: 500,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// HydrateTaskResult restores verified stored payloads only for bounded,
|
|
// explicit synchronous inline responses. Asynchronous readers use
|
|
// ProjectTaskResultURLs and never call this path.
|
|
func (s *Service) HydrateTaskResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, error) {
|
|
next, changed, err := s.hydrateLocalBinaryValue(ctx, taskID, result, 0)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !changed {
|
|
return result, nil
|
|
}
|
|
mapped, ok := next.(map[string]any)
|
|
if !ok {
|
|
return nil, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result is not a JSON object", StatusCode: 500}
|
|
}
|
|
return mapped, nil
|
|
}
|
|
|
|
// SynchronousInlineResultBytes returns the stored raw byte count that the
|
|
// synchronous hydration path would read. It deliberately inspects metadata
|
|
// only, so callers can reject oversized responses before any object GET or
|
|
// local file read occurs.
|
|
func SynchronousInlineResultBytes(result map[string]any) int64 {
|
|
return synchronousInlineResultValueBytes(result, 0)
|
|
}
|
|
|
|
func synchronousInlineResultValueBytes(value any, depth int) int64 {
|
|
if depth >= localBinaryMaxDepth {
|
|
return 0
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
if asset, _, ok := generatedResultUploadReference(typed); ok {
|
|
storage, _ := typed["assetStorage"].(map[string]any)
|
|
if resultCanonicalInlineField(stringFromAny(storage["source"])) {
|
|
return asset.ByteSize
|
|
}
|
|
}
|
|
if asset, ok := generatedResultAssetReference(typed); ok {
|
|
return asset.ByteSize
|
|
}
|
|
var total int64
|
|
for _, child := range typed {
|
|
total += synchronousInlineResultValueBytes(child, depth+1)
|
|
if total > MaxSynchronousInlineResponseBytes {
|
|
return total
|
|
}
|
|
}
|
|
return total
|
|
case []any:
|
|
var total int64
|
|
for _, child := range typed {
|
|
total += synchronousInlineResultValueBytes(child, depth+1)
|
|
if total > MaxSynchronousInlineResponseBytes {
|
|
return total
|
|
}
|
|
}
|
|
return total
|
|
case string:
|
|
if descriptor, ok := parseLocalBinaryPlaceholder(typed); ok {
|
|
return descriptor.Size
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// ProjectTaskResultURLs returns the public, URL-only representation of a
|
|
// stored task result. It may refresh a signed object-storage URL, but it never
|
|
// reads object bytes or restores historical inline binary payloads.
|
|
func (s *Service) ProjectTaskResultURLs(ctx context.Context, taskID string, result map[string]any) (map[string]any, error) {
|
|
next, changed, err := s.projectTaskResultURLValue(ctx, taskID, result, 0)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !changed {
|
|
return result, nil
|
|
}
|
|
mapped, ok := next.(map[string]any)
|
|
if !ok {
|
|
return nil, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result is not a JSON object", StatusCode: http.StatusInternalServerError}
|
|
}
|
|
return mapped, nil
|
|
}
|
|
|
|
func (s *Service) projectTaskResultURLValue(ctx context.Context, taskID string, value any, depth int) (any, bool, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, false, err
|
|
}
|
|
if depth >= localBinaryMaxDepth {
|
|
return nil, false, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result exceeds the maximum JSON depth", StatusCode: http.StatusInternalServerError}
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
if asset, _, ok := generatedResultUploadReference(typed); ok {
|
|
accessURL, err := s.resultAssetAccessURL(ctx, asset)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next := publicResultURLItem(typed, accessURL, asset.ContentType)
|
|
return next, true, nil
|
|
}
|
|
if asset, ok := generatedResultAssetReference(typed); ok {
|
|
accessURL, err := s.resultAssetAccessURL(ctx, asset)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next := publicResultURLItem(typed, accessURL, asset.ContentType)
|
|
return next, true, nil
|
|
}
|
|
if _, _, ok := localBufferObjectBytes(typed); ok {
|
|
return nil, false, resultMaterializationRequired(taskID)
|
|
}
|
|
next := make(map[string]any, len(typed))
|
|
changed := false
|
|
for key, childValue := range typed {
|
|
if resultInternalField(key) {
|
|
changed = true
|
|
continue
|
|
}
|
|
child, childChanged, err := s.projectTaskResultURLValue(ctx, taskID, childValue, depth+1)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
if resultCanonicalInlineField(key) {
|
|
if item, ok := child.(map[string]any); ok && stringFromAny(item["url"]) != "" {
|
|
mergeProjectedResultURL(next, item)
|
|
changed = true
|
|
continue
|
|
}
|
|
if raw, ok := child.(string); ok {
|
|
if requestAssetStringIsHTTPURL(raw) {
|
|
next["url"] = strings.TrimSpace(raw)
|
|
changed = true
|
|
continue
|
|
}
|
|
if strings.TrimSpace(raw) != "" {
|
|
return nil, false, resultMaterializationRequired(taskID)
|
|
}
|
|
}
|
|
}
|
|
if localBinaryKey(key) {
|
|
switch inline := child.(type) {
|
|
case map[string]any:
|
|
if stringFromAny(inline["url"]) != "" {
|
|
mergeProjectedResultURL(next, inline)
|
|
changed = true
|
|
continue
|
|
}
|
|
case string:
|
|
if strings.TrimSpace(inline) != "" {
|
|
return nil, false, resultMaterializationRequired(taskID)
|
|
}
|
|
case []any:
|
|
if len(inline) > 0 {
|
|
return nil, false, resultMaterializationRequired(taskID)
|
|
}
|
|
}
|
|
}
|
|
next[key] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false, nil
|
|
}
|
|
return next, true, nil
|
|
case []any:
|
|
next := make([]any, len(typed))
|
|
changed := false
|
|
for index, childValue := range typed {
|
|
child, childChanged, err := s.projectTaskResultURLValue(ctx, taskID, childValue, depth+1)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next[index] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false, nil
|
|
}
|
|
return next, true, nil
|
|
case []byte:
|
|
if len(typed) > 0 {
|
|
return nil, false, resultMaterializationRequired(taskID)
|
|
}
|
|
return value, false, nil
|
|
case string:
|
|
if _, ok := parseLocalBinaryPlaceholder(typed); ok {
|
|
return nil, false, resultMaterializationRequired(taskID)
|
|
}
|
|
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(typed)), "data:") {
|
|
return nil, false, resultMaterializationRequired(taskID)
|
|
}
|
|
return value, false, nil
|
|
default:
|
|
return value, false, nil
|
|
}
|
|
}
|
|
|
|
func (s *Service) resultAssetAccessURL(ctx context.Context, asset store.RequestAsset) (string, error) {
|
|
accessURL, err := s.requestAssetAccessURL(ctx, asset)
|
|
if err != nil || strings.TrimSpace(accessURL) == "" {
|
|
message := "stored result URL is unavailable"
|
|
if err != nil {
|
|
message = err.Error()
|
|
}
|
|
return "", &clients.ClientError{Code: "result_url_unavailable", Message: message, StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
|
}
|
|
return accessURL, nil
|
|
}
|
|
|
|
func resultMaterializationRequired(taskID string) error {
|
|
return &clients.ClientError{
|
|
Code: "result_materialization_required",
|
|
Message: "stored result must be migrated to object storage before it can be returned",
|
|
Details: map[string]any{"task_id": strings.TrimSpace(taskID)},
|
|
StatusCode: http.StatusServiceUnavailable,
|
|
Retryable: true,
|
|
}
|
|
}
|
|
|
|
func publicResultURLItem(source map[string]any, accessURL string, contentType string) map[string]any {
|
|
next := map[string]any{"url": accessURL}
|
|
for _, key := range []string{"type", "mime_type", "mimeType", "content_type", "contentType", "width", "height", "duration", "format", "seed", "revised_prompt"} {
|
|
if value, ok := source[key]; ok && value != nil {
|
|
next[key] = value
|
|
}
|
|
}
|
|
if stringFromAny(next["mime_type"]) == "" && strings.TrimSpace(contentType) != "" {
|
|
next["mime_type"] = strings.TrimSpace(contentType)
|
|
}
|
|
return next
|
|
}
|
|
|
|
func mergeProjectedResultURL(target map[string]any, item map[string]any) {
|
|
for key, value := range item {
|
|
if _, exists := target[key]; !exists || key == "url" {
|
|
target[key] = value
|
|
}
|
|
}
|
|
}
|
|
|
|
func resultInternalField(key string) bool {
|
|
switch normalizeLocalBinaryKey(key) {
|
|
case "assetref", "assetstorage", "upload", "raw", "rawdata", "rawresponse", "providerpayload", "providerresponse", "providerraw", "upstreamresponse", "thinkingbytes", "thoughtsignature", "signaturebuffer":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func resultCanonicalInlineField(key string) bool {
|
|
normalized := normalizeLocalBinaryKey(key)
|
|
return normalized == "b64json" || normalized == "base64" || normalized == "b64" ||
|
|
normalized == "datauri" || strings.Contains(normalized, "base64") || strings.HasSuffix(normalized, "b64")
|
|
}
|
|
|
|
func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, value any, depth int) (any, bool, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, false, err
|
|
}
|
|
if depth >= localBinaryMaxDepth {
|
|
return nil, false, &clients.ClientError{Code: "binary_result_corrupted", Message: "stored result exceeds the maximum JSON depth", StatusCode: 500}
|
|
}
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
refreshedAccessURL := false
|
|
if asset, upload, ok := generatedResultUploadReference(typed); ok {
|
|
accessURL, err := s.requestAssetAccessURL(ctx, asset)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next := make(map[string]any, len(typed))
|
|
for key, item := range typed {
|
|
next[key] = item
|
|
}
|
|
nextUpload := make(map[string]any, len(upload))
|
|
for key, item := range upload {
|
|
nextUpload[key] = item
|
|
}
|
|
nextUpload["url"] = accessURL
|
|
next["upload"] = nextUpload
|
|
for _, key := range []string{"url", "image_url", "video_url", "audio_url"} {
|
|
if _, exists := next[key]; exists {
|
|
next[key] = accessURL
|
|
}
|
|
}
|
|
typed = next
|
|
refreshedAccessURL = true
|
|
storage, _ := typed["assetStorage"].(map[string]any)
|
|
sourceKey := strings.TrimSpace(stringFromAny(storage["source"]))
|
|
if resultCanonicalInlineField(sourceKey) && asset.SHA256 != "" && asset.ByteSize > 0 {
|
|
payload, contentType, err := s.readGeneratedResultAsset(ctx, asset)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
encoded := base64.StdEncoding.EncodeToString(payload)
|
|
inline := make(map[string]any, len(typed))
|
|
for key, item := range typed {
|
|
if resultInternalField(key) || key == "url" || key == "image_url" || key == "video_url" || key == "audio_url" {
|
|
continue
|
|
}
|
|
inline[key] = item
|
|
}
|
|
if generatedResultAssetUsesDataURI(typed) {
|
|
inline[sourceKey] = "data:" + contentType + ";base64," + encoded
|
|
} else {
|
|
inline[sourceKey] = encoded
|
|
}
|
|
if stringFromAny(inline["mime_type"]) == "" {
|
|
inline["mime_type"] = contentType
|
|
}
|
|
return inline, true, nil
|
|
}
|
|
}
|
|
if ref, ok := generatedResultAssetReference(typed); ok {
|
|
payload, contentType, err := s.readGeneratedResultAsset(ctx, ref)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
encoded := base64.StdEncoding.EncodeToString(payload)
|
|
if generatedResultAssetUsesDataURI(typed) {
|
|
return "data:" + contentType + ";base64," + encoded, true, nil
|
|
}
|
|
return encoded, true, nil
|
|
}
|
|
next := make(map[string]any, len(typed))
|
|
changed := refreshedAccessURL
|
|
for key, childValue := range typed {
|
|
if resultInternalField(key) {
|
|
changed = true
|
|
continue
|
|
}
|
|
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next[key] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false, nil
|
|
}
|
|
return next, true, nil
|
|
case []any:
|
|
next := make([]any, len(typed))
|
|
changed := false
|
|
for index, childValue := range typed {
|
|
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
next[index] = child
|
|
changed = changed || childChanged
|
|
}
|
|
if !changed {
|
|
return value, false, nil
|
|
}
|
|
return next, true, nil
|
|
case string:
|
|
descriptor, ok := parseLocalBinaryPlaceholder(typed)
|
|
if !ok {
|
|
return value, false, nil
|
|
}
|
|
payload, err := s.readLocalBinaryResult(taskID, descriptor)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
encoded := base64.StdEncoding.EncodeToString(payload)
|
|
if descriptor.Encoding == "data-uri" {
|
|
return "data:" + descriptor.ContentType + ";base64," + encoded, true, nil
|
|
}
|
|
return encoded, true, nil
|
|
default:
|
|
return value, false, nil
|
|
}
|
|
}
|
|
|
|
func generatedResultUploadReference(value map[string]any) (store.RequestAsset, map[string]any, bool) {
|
|
upload, ok := value["upload"].(map[string]any)
|
|
if !ok {
|
|
return store.RequestAsset{}, nil, false
|
|
}
|
|
objectKey := strings.TrimSpace(stringFromAny(upload["objectKey"]))
|
|
channel, _ := upload["storageChannel"].(map[string]any)
|
|
channelKey := strings.TrimSpace(stringFromAny(channel["channelKey"]))
|
|
if objectKey == "" || channelKey == "" {
|
|
return store.RequestAsset{}, nil, false
|
|
}
|
|
return store.RequestAsset{
|
|
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(upload["sha256"]))),
|
|
ContentType: stringFromAny(upload["contentType"]),
|
|
ByteSize: int64(floatFromAny(upload["size"])),
|
|
URL: stringFromAny(upload["url"]),
|
|
StorageProvider: stringFromAny(channel["provider"]),
|
|
StorageChannelID: stringFromAny(channel["id"]),
|
|
StorageChannelKey: channelKey,
|
|
ObjectKey: objectKey,
|
|
AccessScope: stringFromAny(upload["accessScope"]),
|
|
}, upload, true
|
|
}
|
|
|
|
func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bool) {
|
|
ref, ok := value["assetRef"].(map[string]any)
|
|
if !ok {
|
|
return store.RequestAsset{}, false
|
|
}
|
|
storage, _ := value["assetStorage"].(map[string]any)
|
|
if stringFromAny(storage["scene"]) != store.FileStorageSceneImageResult {
|
|
return store.RequestAsset{}, false
|
|
}
|
|
asset := store.RequestAsset{
|
|
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
|
|
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
|
|
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
|
|
StorageProvider: stringFromAny(ref["storageProvider"]),
|
|
StorageChannelID: stringFromAny(ref["storageChannelId"]),
|
|
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
|
|
ObjectKey: stringFromAny(ref["objectKey"]),
|
|
AccessScope: stringFromAny(ref["accessScope"]),
|
|
}
|
|
if size := floatFromAny(ref["size"]); size > 0 {
|
|
asset.ByteSize = int64(size)
|
|
}
|
|
if expiresAt := stringFromAny(ref["expiresAt"]); expiresAt != "" {
|
|
if parsed, err := time.Parse(time.RFC3339, expiresAt); err == nil {
|
|
asset.ExpiresAt = &parsed
|
|
}
|
|
}
|
|
if asset.URL == "" || asset.SHA256 == "" || asset.ByteSize <= 0 {
|
|
return store.RequestAsset{}, false
|
|
}
|
|
return asset, true
|
|
}
|
|
|
|
func generatedResultAssetUsesDataURI(value map[string]any) bool {
|
|
storage, _ := value["assetStorage"].(map[string]any)
|
|
source := normalizeLocalBinaryKey(stringFromAny(storage["source"]))
|
|
return source == "datauri"
|
|
}
|
|
|
|
func (s *Service) readGeneratedResultAsset(ctx context.Context, asset store.RequestAsset) ([]byte, string, error) {
|
|
payload, err := s.readRequestAssetBytes(ctx, asset)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
digest := sha256.Sum256(payload)
|
|
if int64(len(payload)) != asset.ByteSize || hex.EncodeToString(digest[:]) != asset.SHA256 {
|
|
return nil, "", &clients.ClientError{
|
|
Code: "binary_result_corrupted",
|
|
Message: "stored result asset failed size or hash verification",
|
|
StatusCode: 500,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
contentType := strings.TrimSpace(asset.ContentType)
|
|
if contentType == "" {
|
|
contentType = "application/octet-stream"
|
|
}
|
|
return payload, contentType, nil
|
|
}
|
|
|
|
func (s *Service) readLocalBinaryResult(taskID string, descriptor localBinaryDescriptor) ([]byte, error) {
|
|
path := filepath.Join(s.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID), descriptor.SHA256+".bin")
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil, &clients.ClientError{Code: "binary_result_expired", Message: "local binary result has expired", StatusCode: 410, Retryable: false}
|
|
}
|
|
return nil, localBinaryStorageError(err)
|
|
}
|
|
if info.IsDir() || info.Size() != descriptor.Size {
|
|
return nil, &clients.ClientError{Code: "binary_result_corrupted", Message: "local binary result has an invalid size", StatusCode: 500, Retryable: false}
|
|
}
|
|
if info.ModTime().Before(time.Now().Add(-time.Duration(s.localResultTTLHours()) * time.Hour)) {
|
|
return nil, &clients.ClientError{Code: "binary_result_expired", Message: "local binary result has expired", StatusCode: 410, Retryable: false}
|
|
}
|
|
payload, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, localBinaryStorageError(err)
|
|
}
|
|
digest := sha256.Sum256(payload)
|
|
if int64(len(payload)) != descriptor.Size || hex.EncodeToString(digest[:]) != descriptor.SHA256 {
|
|
return nil, &clients.ClientError{Code: "binary_result_corrupted", Message: "local binary result failed size or hash verification", StatusCode: 500, Retryable: false}
|
|
}
|
|
return payload, nil
|
|
}
|
|
|
|
func localBinaryStringBytes(key string, value string, siblings map[string]any) ([]byte, string, string, bool) {
|
|
raw := strings.TrimSpace(value)
|
|
if raw == "" || strings.HasPrefix(raw, localBinaryPlaceholderPrefix) {
|
|
return nil, "", "", false
|
|
}
|
|
if strings.HasPrefix(strings.ToLower(raw), "data:") {
|
|
contentType, encoded, ok, err := parseBase64DataURL(raw)
|
|
if err == nil && ok {
|
|
payload, decodeErr := decodeBase64Payload(encoded)
|
|
if decodeErr == nil && len(payload) > 0 {
|
|
return payload, contentType, "data-uri", true
|
|
}
|
|
}
|
|
return nil, "", "", false
|
|
}
|
|
strict := localBinaryKey(key)
|
|
if !strict && len(raw) < localBinaryGenericBase64MinLength {
|
|
return nil, "", "", false
|
|
}
|
|
payload, err := decodeBase64Payload(raw)
|
|
if err != nil || len(payload) == 0 {
|
|
return nil, "", "", false
|
|
}
|
|
contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
|
|
// Keys such as thinking_bytes and signature_buffer can carry opaque provider
|
|
// metadata. Treat them as generated media only when a sibling content type or
|
|
// the payload signature proves that they are media/document bytes. Explicit
|
|
// Base64 media keys (b64_json, image_data, binary_data_base64, ...) retain the
|
|
// strict behavior expected by compatible image protocols.
|
|
if contentType == "" && (!strict || !generatedRawDataMediaPayloadKey(key)) {
|
|
contentType = detectGeneratedAssetContentType(payload)
|
|
if !generatedContentTypeIsMedia(contentType) && !generatedContentTypeIsDocument(contentType) {
|
|
return nil, "", "", false
|
|
}
|
|
}
|
|
return payload, contentType, "raw", true
|
|
}
|
|
|
|
func localBufferObjectBytes(value map[string]any) ([]byte, string, bool) {
|
|
if normalizeLocalBinaryKey(stringFromAny(value["type"])) != "buffer" {
|
|
return nil, "", false
|
|
}
|
|
contentType := firstNonEmptyString(
|
|
stringFromAny(value["mime_type"]),
|
|
stringFromAny(value["mimeType"]),
|
|
stringFromAny(value["contentType"]),
|
|
)
|
|
switch data := value["data"].(type) {
|
|
case []byte:
|
|
if len(data) == 0 {
|
|
return nil, "", false
|
|
}
|
|
return append([]byte(nil), data...), contentType, true
|
|
case []any:
|
|
payload, ok := bytesFromNumberArray(data)
|
|
return payload, contentType, ok
|
|
default:
|
|
return nil, "", false
|
|
}
|
|
}
|
|
|
|
func localBinaryKey(key string) bool {
|
|
normalized := normalizeLocalBinaryKey(key)
|
|
return normalized == "b64" ||
|
|
normalized == "b64json" ||
|
|
normalized == "base64" ||
|
|
normalized == "buffer" ||
|
|
normalized == "bytes" ||
|
|
strings.Contains(normalized, "base64") ||
|
|
strings.Contains(normalized, "buffer") ||
|
|
strings.Contains(normalized, "binary") ||
|
|
strings.HasSuffix(normalized, "b64") ||
|
|
strings.HasSuffix(normalized, "bytes")
|
|
}
|
|
|
|
func normalizeLocalBinaryKey(value string) string {
|
|
return strings.Map(func(char rune) rune {
|
|
switch {
|
|
case char >= 'a' && char <= 'z':
|
|
return char
|
|
case char >= 'A' && char <= 'Z':
|
|
return char + ('a' - 'A')
|
|
case char >= '0' && char <= '9':
|
|
return char
|
|
default:
|
|
return -1
|
|
}
|
|
}, value)
|
|
}
|
|
|
|
func localBinaryPrefix(payload []byte) string {
|
|
prefix := base64.StdEncoding.EncodeToString(payload)
|
|
if len(prefix) > 16 {
|
|
prefix = prefix[:16]
|
|
}
|
|
return prefix
|
|
}
|
|
|
|
func normalizedLocalBinaryContentType(value string) string {
|
|
value = normalizeGeneratedContentType(value)
|
|
if value == "" || len(value) > 32 || !localBinaryContentTypeSafe(value) {
|
|
return "application/octet-stream"
|
|
}
|
|
return value
|
|
}
|
|
|
|
func localBinaryContentTypeSafe(value string) bool {
|
|
for _, char := range value {
|
|
switch {
|
|
case char >= 'a' && char <= 'z':
|
|
case char >= 'A' && char <= 'Z':
|
|
case char >= '0' && char <= '9':
|
|
case char == '/', char == '.', char == '+', char == '-':
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func normalizedLocalBinaryEncoding(value string) string {
|
|
switch strings.ToLower(strings.TrimSpace(value)) {
|
|
case "data-uri":
|
|
return "data-uri"
|
|
case "buffer":
|
|
return "buffer"
|
|
default:
|
|
return "raw"
|
|
}
|
|
}
|
|
|
|
func localBinaryPlaceholder(descriptor localBinaryDescriptor) string {
|
|
return fmt.Sprintf(
|
|
"[GatewayBinary:v1;prefix=%s;sha256=%s;bytes=%d;mime=%s;encoding=%s]",
|
|
descriptor.Prefix,
|
|
descriptor.SHA256,
|
|
descriptor.Size,
|
|
descriptor.ContentType,
|
|
descriptor.Encoding,
|
|
)
|
|
}
|
|
|
|
func parseLocalBinaryPlaceholder(value string) (localBinaryDescriptor, bool) {
|
|
if !strings.HasPrefix(value, localBinaryPlaceholderPrefix) || !strings.HasSuffix(value, "]") {
|
|
return localBinaryDescriptor{}, false
|
|
}
|
|
content := strings.TrimSuffix(strings.TrimPrefix(value, localBinaryPlaceholderPrefix), "]")
|
|
fields := map[string]string{}
|
|
for _, item := range strings.Split(content, ";") {
|
|
key, fieldValue, ok := strings.Cut(item, "=")
|
|
if !ok {
|
|
continue
|
|
}
|
|
fields[key] = fieldValue
|
|
}
|
|
size, err := strconv.ParseInt(fields["bytes"], 10, 64)
|
|
if err != nil || size <= 0 || len(fields["sha256"]) != sha256.Size*2 {
|
|
return localBinaryDescriptor{}, false
|
|
}
|
|
if _, err := hex.DecodeString(fields["sha256"]); err != nil {
|
|
return localBinaryDescriptor{}, false
|
|
}
|
|
descriptor := localBinaryDescriptor{
|
|
Prefix: fields["prefix"],
|
|
SHA256: strings.ToLower(fields["sha256"]),
|
|
Size: size,
|
|
ContentType: normalizedLocalBinaryContentType(fields["mime"]),
|
|
Encoding: normalizedLocalBinaryEncoding(fields["encoding"]),
|
|
}
|
|
if len(descriptor.Prefix) > 16 {
|
|
return localBinaryDescriptor{}, false
|
|
}
|
|
return descriptor, true
|
|
}
|
|
|
|
func localBinaryResultHasPlaceholders(value any) bool {
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
for _, child := range typed {
|
|
if localBinaryResultHasPlaceholders(child) {
|
|
return true
|
|
}
|
|
}
|
|
case []any:
|
|
for _, child := range typed {
|
|
if localBinaryResultHasPlaceholders(child) {
|
|
return true
|
|
}
|
|
}
|
|
case string:
|
|
return strings.HasPrefix(typed, localBinaryPlaceholderPrefix)
|
|
}
|
|
return false
|
|
}
|
|
|
|
func safeLocalBinaryTaskDir(taskID string) string {
|
|
taskID = strings.TrimSpace(taskID)
|
|
if taskID != "" {
|
|
safe := true
|
|
for _, char := range taskID {
|
|
if (char >= 'a' && char <= 'z') ||
|
|
(char >= 'A' && char <= 'Z') ||
|
|
(char >= '0' && char <= '9') ||
|
|
char == '-' || char == '_' {
|
|
continue
|
|
}
|
|
safe = false
|
|
break
|
|
}
|
|
if safe && taskID != "." && taskID != ".." && len(taskID) <= 128 {
|
|
return taskID
|
|
}
|
|
}
|
|
digest := sha256.Sum256([]byte(taskID))
|
|
return "task-" + hex.EncodeToString(digest[:16])
|
|
}
|
|
|
|
func (s *Service) localBinaryResultRoot() string {
|
|
root := strings.TrimSpace(s.cfg.LocalGeneratedStorageDir)
|
|
if root == "" {
|
|
root = config.DefaultLocalGeneratedStorageDir
|
|
}
|
|
return filepath.Join(root, localBinaryResultDirName)
|
|
}
|
|
|
|
func (s *Service) localResultTTLHours() int {
|
|
if s.cfg.LocalResultTTLHours <= 0 {
|
|
return defaultLocalResultTTLHours
|
|
}
|
|
return s.cfg.LocalResultTTLHours
|
|
}
|
|
|
|
func (s *Service) localResultMaxBytes() int64 {
|
|
if s.cfg.LocalResultMaxBytes <= 0 {
|
|
return defaultLocalResultMaxBytes
|
|
}
|
|
return s.cfg.LocalResultMaxBytes
|
|
}
|
|
|
|
func (s *Service) localResultMaxTaskBytes() int64 {
|
|
if s.cfg.LocalResultMaxTaskBytes <= 0 {
|
|
return defaultLocalResultMaxTaskBytes
|
|
}
|
|
return s.cfg.LocalResultMaxTaskBytes
|
|
}
|