Expire local static assets after 24 hours

2026-06-07 19:29:04 +08:00
parent f47132a653
commit 4d1a01ec71
5 changed files with 395 additions and 30 deletions
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
@@ -76,6 +77,7 @@ func defaultGeneratedAssetUploadPolicy() generatedAssetUploadPolicy {
 func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, taskKind string, result map[string]any) (map[string]any, error) {
 	data, _ := result["data"].([]any)
 	if len(data) == 0 {
+		redactGeneratedResultRawData(result)
 		return result, nil
 	}
 	policy, err := s.generatedAssetUploadPolicy(ctx)
@@ -103,6 +105,7 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
 		}
 	}
 	if !needsUpload && !changed {
+		redactGeneratedResultRawData(result)
 		return result, nil
 	}
 	var channels []store.FileStorageChannel
@@ -165,6 +168,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
 				if kind == "image" {
 					merged["image_url"] = urlValue
 				}
+				if kind == "audio" {
+					merged["audio_url"] = urlValue
+				}
 			}
 			if kind != "" && stringFromAny(merged["type"]) == "" {
 				merged["type"] = kind
@@ -176,9 +182,185 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
 		nextData = append(nextData, merged)
 	}
 	next["data"] = nextData
+	redactGeneratedResultRawData(next)
 	return next, nil
 }

+func redactGeneratedResultRawData(result map[string]any) bool {
+	if result == nil {
+		return false
+	}
+	changed := false
+	for _, key := range []string{"raw_data", "rawData"} {
+		value, ok := result[key]
+		if !ok || value == nil {
+			continue
+		}
+		next, nextChanged := redactGeneratedRawDataValue(value, key, nil)
+		if nextChanged {
+			result[key] = next
+			changed = true
+		}
+	}
+	return changed
+}
+
+func redactGeneratedRawDataValue(value any, key string, siblings map[string]any) (any, bool) {
+	switch typed := value.(type) {
+	case map[string]any:
+		next := make(map[string]any, len(typed))
+		changed := false
+		for childKey, childValue := range typed {
+			redacted, childChanged := redactGeneratedRawDataValue(childValue, childKey, typed)
+			next[childKey] = redacted
+			if childChanged {
+				changed = true
+			}
+		}
+		if changed {
+			return next, true
+		}
+		return value, false
+	case []any:
+		next := make([]any, len(typed))
+		changed := false
+		for index, item := range typed {
+			redacted, itemChanged := redactGeneratedRawDataValue(item, key, siblings)
+			next[index] = redacted
+			if itemChanged {
+				changed = true
+			}
+		}
+		if changed {
+			return next, true
+		}
+		return value, false
+	case string:
+		redacted, ok := generatedRawDataMediaRedaction(key, typed, siblings)
+		if ok {
+			return redacted, true
+		}
+		return value, false
+	default:
+		return value, false
+	}
+}
+
+func generatedRawDataMediaRedaction(key string, value string, siblings map[string]any) (map[string]any, bool) {
+	raw := strings.TrimSpace(value)
+	if raw == "" {
+		return nil, false
+	}
+	contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
+	if strings.HasPrefix(strings.ToLower(raw), "data:") {
+		declared, encoded, ok, err := parseBase64DataURL(raw)
+		if err != nil || !ok || !generatedContentTypeIsMedia(declared) {
+			return nil, false
+		}
+		payload, err := decodeBase64Payload(encoded)
+		if err != nil {
+			return nil, false
+		}
+		return generatedRawDataRedaction("data_url", declared, payload, raw), true
+	}
+	if len(raw) < 128 {
+		return nil, false
+	}
+	keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key)
+	if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) {
+		return nil, false
+	}
+	if keyLooksLikeMediaPayload && looksHexMediaPayload(raw) {
+		normalized := removeASCIIWhitespace(raw)
+		payload, err := hex.DecodeString(normalized)
+		if err == nil && len(payload) > 0 {
+			return generatedRawDataRedaction("hex", contentType, payload, raw), true
+		}
+	}
+	if payload, err := decodeBase64Payload(raw); err == nil && len(payload) > 0 {
+		return generatedRawDataRedaction("base64", contentType, payload, raw), true
+	}
+	if keyLooksLikeMediaPayload && len(raw) > 2048 {
+		return generatedRawDataRedaction("unknown", contentType, nil, raw), true
+	}
+	return nil, false
+}
+
+func generatedRawDataMediaPayloadKey(key string) bool {
+	normalized := strings.ToLower(strings.TrimSpace(key))
+	normalized = strings.ReplaceAll(normalized, "-", "_")
+	return normalized == "audio" ||
+		normalized == "image" ||
+		normalized == "video" ||
+		normalized == "b64_json" ||
+		normalized == "base64" ||
+		normalized == "b64" ||
+		normalized == "binary_data_base64" ||
+		strings.Contains(normalized, "base64") ||
+		strings.Contains(normalized, "_b64") ||
+		strings.Contains(normalized, "audio_data") ||
+		strings.Contains(normalized, "image_data") ||
+		strings.Contains(normalized, "video_data")
+}
+
+func defaultContentTypeForRawMediaKey(key string) string {
+	normalized := strings.ToLower(strings.TrimSpace(key))
+	if strings.Contains(normalized, "audio") {
+		return "audio/mpeg"
+	}
+	if strings.Contains(normalized, "video") {
+		return "video/mp4"
+	}
+	if strings.Contains(normalized, "image") || strings.Contains(normalized, "b64_json") {
+		return "image/png"
+	}
+	return ""
+}
+
+func generatedRawDataRedaction(encoding string, contentType string, payload []byte, raw string) map[string]any {
+	out := map[string]any{
+		"redacted": true,
+		"reason":   "inline_media_payload",
+		"encoding": encoding,
+	}
+	if contentType = normalizeGeneratedContentType(contentType); contentType != "" {
+		out["contentType"] = contentType
+	}
+	if len(payload) > 0 {
+		digest := sha256.Sum256(payload)
+		out["size"] = len(payload)
+		out["sha256"] = hex.EncodeToString(digest[:])
+	} else {
+		out["size"] = len(raw)
+	}
+	return out
+}
+
+func looksHexMediaPayload(value string) bool {
+	normalized := removeASCIIWhitespace(value)
+	if len(normalized) < 128 || len(normalized)%2 != 0 {
+		return false
+	}
+	for _, item := range normalized {
+		if (item >= '0' && item <= '9') || (item >= 'a' && item <= 'f') || (item >= 'A' && item <= 'F') {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+func removeASCIIWhitespace(value string) string {
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case '\n', '\r', '\t', ' ':
+			return -1
+		default:
+			return r
+		}
+	}, value)
+}
+
 func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) {
 	settings, err := s.store.GetFileStorageSettings(ctx)
 	if err != nil {
@@ -282,11 +464,13 @@ func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string,
 		_ = os.Remove(targetPath)
 		return nil, &clients.ClientError{Code: "local_static_store_failed", Message: closeErr.Error(), Retryable: true}
 	}
+	expiresAt := time.Now().Add(time.Duration(s.localStaticAssetTTLHours()) * time.Hour).UTC().Format(time.RFC3339)
 	return map[string]any{
 		"url":         s.localStaticFileURL(fileName, pathPrefix),
 		"fileName":    fileName,
 		"contentType": payload.ContentType,
 		"size":        len(payload.Bytes),
+		"expiresAt":   expiresAt,
 		"storageChannel": map[string]any{
 			"id":         "local-static",
 			"channelKey": "local-static",
@@ -296,6 +480,13 @@ func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string,
 	}, nil
 }

+func (s *Service) localStaticAssetTTLHours() int {
+	if s.cfg.LocalTempAssetTTLHours <= 0 {
+		return 24
+	}
+	return s.cfg.LocalTempAssetTTLHours
+}
+
 func (s *Service) localStaticFileURL(fileName string, pathPrefix string) string {
 	if strings.TrimSpace(pathPrefix) == "" {
 		pathPrefix = localStaticUploadedPathPrefix
@@ -643,6 +834,9 @@ func inlineAssetFromItem(taskKind string, item map[string]any) (*generatedInline
 		if !ok || value == nil {
 			continue
 		}
+		if !inlineMediaKeyAllowedForItem(item, key, value) {
+			continue
+		}
 		strictBase64 := inlineMediaKeyIsStrictBase64(key)
 		payload, contentType, ok, err := inlineMediaPayload(value, strictBase64)
 		if err != nil {
@@ -818,6 +1012,9 @@ func inlineMediaKeys(item map[string]any) []string {
 		if !ok || value == nil {
 			continue
 		}
+		if !inlineMediaKeyAllowedForItem(item, key, value) {
+			continue
+		}
 		strictBase64 := inlineMediaKeyIsStrictBase64(key)
 		if strictBase64 && stringFromAny(value) != "" {
 			keys = append(keys, key)
@@ -835,6 +1032,8 @@ func inlineMediaCandidateKeys() []string {
 		"b64_json",
 		"image_base64",
 		"image_b64",
+		"audio_base64",
+		"audio_b64",
 		"video_base64",
 		"video_b64",
 		"base64",
@@ -842,24 +1041,56 @@ func inlineMediaCandidateKeys() []string {
 		"url",
 		"image_url",
 		"imageUrl",
+		"audio_url",
+		"audioUrl",
 		"video_url",
 		"videoUrl",
 		"output_url",
 		"outputUrl",
+		"output_audio_url",
+		"outputAudioUrl",
 		"output_video_url",
 		"outputVideoUrl",
 		"image",
+		"audio",
 		"video",
 		"image_buffer",
 		"image_bytes",
+		"audio_buffer",
+		"audio_bytes",
 		"video_buffer",
 		"video_bytes",
 		"buffer",
 		"bytes",
 		"data",
+		"content",
 	}
 }

+func inlineMediaKeyAllowedForItem(item map[string]any, key string, value any) bool {
+	if !strings.EqualFold(strings.TrimSpace(key), "content") {
+		return true
+	}
+	raw, ok := value.(string)
+	if !ok {
+		return generatedContentTypeIsMedia(mediaContentTypeFromItem(item)) || generatedResultItemIsMedia(item)
+	}
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return false
+	}
+	if strings.HasPrefix(strings.ToLower(raw), "data:") {
+		contentType, _, ok, err := parseBase64DataURL(raw)
+		return err == nil && ok && generatedContentTypeIsMedia(contentType)
+	}
+	return generatedContentTypeIsMedia(mediaContentTypeFromItem(item)) || generatedResultItemIsMedia(item)
+}
+
+func generatedResultItemIsMedia(item map[string]any) bool {
+	itemType := strings.ToLower(strings.TrimSpace(stringFromAny(item["type"])))
+	return strings.Contains(itemType, "image") || strings.Contains(itemType, "video") || strings.Contains(itemType, "audio")
+}
+
 func inlineMediaKeyIsStrictBase64(key string) bool {
 	lower := strings.ToLower(key)
 	return lower == "b64_json" || lower == "base64" || lower == "b64" || strings.Contains(lower, "base64") || strings.Contains(lower, "_b64")
@@ -887,7 +1118,7 @@ func mediaURLKeys(item map[string]any) []string {
 }

 func mediaURLCandidateKeys() []string {
-	return []string{"url", "image_url", "imageUrl", "video_url", "videoUrl", "output_url", "outputUrl", "output_video_url", "outputVideoUrl", "download_url", "downloadUrl", "file_url", "fileUrl"}
+	return []string{"url", "image_url", "imageUrl", "audio_url", "audioUrl", "video_url", "videoUrl", "output_url", "outputUrl", "output_audio_url", "outputAudioUrl", "output_video_url", "outputVideoUrl", "download_url", "downloadUrl", "file_url", "fileUrl"}
 }

 func mediaURLString(value string) bool {