Handle Gemini inline data in asset and size processing

apps/api/internal/runner/upload.go

@@ -76,7 +76,8 @@ func defaultGeneratedAssetUploadPolicy() generatedAssetUploadPolicy {
 
 func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, taskKind string, result map[string]any) (map[string]any, error) {
 	data, _ := result["data"].([]any)
-	if len(data) == 0 {
+	rawNeedsUpload := generatedRawValueHasInlineMedia(result["raw"], "", nil)
+	if len(data) == 0 && !rawNeedsUpload {
 		redactGeneratedResultRawData(result)
 		return result, nil
 	}
@@ -104,12 +105,13 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
 			changed = true
 		}
 	}
-	if !needsUpload && !changed {
+	rawNeedsUpload = rawNeedsUpload && policy.UploadInlineMedia
+	if !needsUpload && !changed && !rawNeedsUpload {
 		redactGeneratedResultRawData(result)
 		return result, nil
 	}
 	var channels []store.FileStorageChannel
-	if needsUpload && generatedAssetNeedsChannelLookup(policy, decisions) {
+	if (needsUpload && generatedAssetNeedsChannelLookup(policy, decisions)) || (rawNeedsUpload && !policy.StoreInlineMediaLocally) {
 		channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
 		if err != nil {
 			return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
@@ -181,11 +183,166 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
 		}
 		nextData = append(nextData, merged)
 	}
-	next["data"] = nextData
+	if len(data) > 0 {
+		next["data"] = nextData
+	}
+	if rawNeedsUpload {
+		rawIndex := len(nextData)
+		raw, rawChanged, err := s.uploadGeneratedRawMediaValue(ctx, taskID, taskKind, next["raw"], "", nil, policy, channels, &rawIndex)
+		if err != nil {
+			return nil, err
+		}
+		if rawChanged {
+			next["raw"] = raw
+		}
+	}
 	redactGeneratedResultRawData(next)
 	return next, nil
 }
 
+func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool {
+	switch typed := value.(type) {
+	case map[string]any:
+		for childKey, childValue := range typed {
+			if generatedRawValueHasInlineMedia(childValue, childKey, typed) {
+				return true
+			}
+		}
+	case []any:
+		for _, item := range typed {
+			if generatedRawValueHasInlineMedia(item, key, siblings) {
+				return true
+			}
+		}
+	case string:
+		_, ok := generatedRawInlineMediaAsset(key, typed, siblings, "")
+		return ok
+	}
+	return false
+}
+
+func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID string, taskKind string, value any, key string, siblings map[string]any, policy generatedAssetUploadPolicy, channels []store.FileStorageChannel, index *int) (any, bool, error) {
+	switch typed := value.(type) {
+	case map[string]any:
+		next := make(map[string]any, len(typed))
+		changed := false
+		for childKey, childValue := range typed {
+			uploaded, childChanged, err := s.uploadGeneratedRawMediaValue(ctx, taskID, taskKind, childValue, childKey, typed, policy, channels, index)
+			if err != nil {
+				return nil, false, err
+			}
+			next[childKey] = uploaded
+			if childChanged {
+				changed = true
+			}
+		}
+		if changed {
+			return next, true, nil
+		}
+		return value, false, nil
+	case []any:
+		next := make([]any, len(typed))
+		changed := false
+		for itemIndex, item := range typed {
+			uploaded, itemChanged, err := s.uploadGeneratedRawMediaValue(ctx, taskID, taskKind, item, key, siblings, policy, channels, index)
+			if err != nil {
+				return nil, false, err
+			}
+			next[itemIndex] = uploaded
+			if itemChanged {
+				changed = true
+			}
+		}
+		if changed {
+			return next, true, nil
+		}
+		return value, false, nil
+	case string:
+		asset, ok := generatedRawInlineMediaAsset(key, typed, siblings, taskKind)
+		if !ok {
+			return value, false, nil
+		}
+		upload, contentType, kind, strategy, err := s.uploadGeneratedAsset(ctx, taskID, asset, *index, channels, policy.StoreInlineMediaLocally)
+		if err != nil {
+			return nil, false, err
+		}
+		*index = *index + 1
+		return generatedRawMediaReference(asset, upload, contentType, kind, strategy), true, nil
+	default:
+		return value, false, nil
+	}
+}
+
+func generatedRawInlineMediaAsset(key string, value string, siblings map[string]any, taskKind string) (*generatedInlineAsset, bool) {
+	raw := strings.TrimSpace(value)
+	if raw == "" {
+		return nil, false
+	}
+	contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
+	if !generatedRawDataMediaPayloadKey(key) && !generatedContentTypeIsMedia(contentType) {
+		return nil, false
+	}
+	if !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
+		return nil, false
+	}
+	payload, payloadContentType, ok, err := inlineMediaPayload(raw, generatedRawDataMediaPayloadKey(key))
+	if err != nil || !ok || len(payload) == 0 {
+		return nil, false
+	}
+	contentType = firstNonEmptyString(payloadContentType, contentType)
+	if !generatedContentTypeIsMedia(contentType) {
+		return nil, false
+	}
+	kind := mediaKindForAsset(taskKind, siblings, key, contentType)
+	return &generatedInlineAsset{
+		Bytes:       payload,
+		ContentType: contentType,
+		Kind:        kind,
+		SourceKey:   key,
+	}, true
+}
+
+func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]any, contentType string, kind string, strategy string) map[string]any {
+	digest := sha256.Sum256(asset.Bytes)
+	urlValue := stringFromAny(upload["url"])
+	ref := map[string]any{
+		"sha256":      hex.EncodeToString(digest[:]),
+		"contentType": contentType,
+		"size":        len(asset.Bytes),
+	}
+	if urlValue != "" {
+		ref["url"] = urlValue
+	}
+	if fileName := stringFromAny(upload["fileName"]); fileName != "" {
+		ref["fileName"] = fileName
+	}
+	if expiresAt := stringFromAny(upload["expiresAt"]); expiresAt != "" {
+		ref["expiresAt"] = expiresAt
+	}
+	if channel, ok := upload["storageChannel"].(map[string]any); ok {
+		if provider := stringFromAny(channel["provider"]); provider != "" {
+			ref["storageProvider"] = provider
+		}
+	}
+	out := map[string]any{
+		"assetRef": ref,
+		"upload":   upload,
+		"assetStorage": map[string]any{
+			"scene":       store.FileStorageSceneImageResult,
+			"source":      asset.SourceKey,
+			"strategy":    strategy,
+			"contentType": contentType,
+		},
+	}
+	if urlValue != "" {
+		out["url"] = urlValue
+	}
+	if kind != "" {
+		out["type"] = kind
+	}
+	return out
+}
+
 func redactGeneratedResultRawData(result map[string]any) bool {
 	if result == nil {
 		return false