Expire local static assets after 24 hours
This commit is contained in:
parent
f47132a653
commit
4d1a01ec71
@ -29,9 +29,35 @@ func (s *Server) startLocalTempAssetCleanup(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (s *Server) cleanupExpiredLocalTempAssets(ctx context.Context, now time.Time) int {
|
||||
storageDir := strings.TrimSpace(s.cfg.LocalUploadedStorageDir)
|
||||
targets := []localTempAssetCleanupTarget{
|
||||
{
|
||||
StorageDir: s.cfg.LocalGeneratedStorageDir,
|
||||
FallbackDir: config.DefaultLocalGeneratedStorageDir,
|
||||
MarkRequestAsset: false,
|
||||
},
|
||||
{
|
||||
StorageDir: s.cfg.LocalUploadedStorageDir,
|
||||
FallbackDir: config.DefaultLocalUploadedStorageDir,
|
||||
MarkRequestAsset: true,
|
||||
},
|
||||
}
|
||||
deleted := 0
|
||||
for _, target := range targets {
|
||||
deleted += s.cleanupExpiredLocalTempAssetsInDir(ctx, now, target)
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
|
||||
type localTempAssetCleanupTarget struct {
|
||||
StorageDir string
|
||||
FallbackDir string
|
||||
MarkRequestAsset bool
|
||||
}
|
||||
|
||||
func (s *Server) cleanupExpiredLocalTempAssetsInDir(ctx context.Context, now time.Time, target localTempAssetCleanupTarget) int {
|
||||
storageDir := strings.TrimSpace(target.StorageDir)
|
||||
if storageDir == "" {
|
||||
storageDir = config.DefaultLocalUploadedStorageDir
|
||||
storageDir = target.FallbackDir
|
||||
}
|
||||
entries, err := os.ReadDir(storageDir)
|
||||
if err != nil {
|
||||
@ -44,7 +70,7 @@ func (s *Server) cleanupExpiredLocalTempAssets(ctx context.Context, now time.Tim
|
||||
expiredBefore := now.Add(-ttl)
|
||||
deleted := 0
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasPrefix(entry.Name(), requestAssetFilePrefix) {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
@ -60,7 +86,7 @@ func (s *Server) cleanupExpiredLocalTempAssets(ctx context.Context, now time.Tim
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
if s.store != nil {
|
||||
if target.MarkRequestAsset && strings.HasPrefix(entry.Name(), requestAssetFilePrefix) && s.store != nil {
|
||||
if err := s.store.MarkRequestAssetExpiredByLocalPath(ctx, localPath, now); err != nil && !store.IsUndefinedDatabaseObject(err) {
|
||||
s.logger.Warn("mark local temp asset expired failed", "path", localPath, "error", err)
|
||||
}
|
||||
|
||||
@ -75,42 +75,51 @@ func TestCanonicalConversationMessageHashUsesTextAndAssetRefs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupExpiredLocalTempAssetsOnlyDeletesExpiredPrefixedFiles(t *testing.T) {
|
||||
storageDir := t.TempDir()
|
||||
oldTemp := filepath.Join(storageDir, requestAssetFilePrefix+"old.png")
|
||||
freshTemp := filepath.Join(storageDir, requestAssetFilePrefix+"fresh.png")
|
||||
oldGenerated := filepath.Join(storageDir, "gateway-result-old.png")
|
||||
for _, path := range []string{oldTemp, freshTemp, oldGenerated} {
|
||||
func TestCleanupExpiredLocalTempAssetsDeletesExpiredStaticFiles(t *testing.T) {
|
||||
uploadedDir := t.TempDir()
|
||||
generatedDir := t.TempDir()
|
||||
oldUploaded := filepath.Join(uploadedDir, requestAssetFilePrefix+"old.png")
|
||||
freshUploaded := filepath.Join(uploadedDir, requestAssetFilePrefix+"fresh.png")
|
||||
oldGenerated := filepath.Join(generatedDir, "gateway-result-old.png")
|
||||
freshGenerated := filepath.Join(generatedDir, "gateway-result-fresh.png")
|
||||
for _, path := range []string{oldUploaded, freshUploaded, oldGenerated, freshGenerated} {
|
||||
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
|
||||
t.Fatalf("write fixture %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
now := time.Now()
|
||||
if err := os.Chtimes(oldTemp, now.Add(-25*time.Hour), now.Add(-25*time.Hour)); err != nil {
|
||||
t.Fatalf("touch old temp: %v", err)
|
||||
for _, path := range []string{oldUploaded, oldGenerated} {
|
||||
if err := os.Chtimes(path, now.Add(-25*time.Hour), now.Add(-25*time.Hour)); err != nil {
|
||||
t.Fatalf("touch old static asset %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
if err := os.Chtimes(freshTemp, now.Add(-23*time.Hour), now.Add(-23*time.Hour)); err != nil {
|
||||
t.Fatalf("touch fresh temp: %v", err)
|
||||
}
|
||||
if err := os.Chtimes(oldGenerated, now.Add(-25*time.Hour), now.Add(-25*time.Hour)); err != nil {
|
||||
t.Fatalf("touch old generated: %v", err)
|
||||
for _, path := range []string{freshUploaded, freshGenerated} {
|
||||
if err := os.Chtimes(path, now.Add(-23*time.Hour), now.Add(-23*time.Hour)); err != nil {
|
||||
t.Fatalf("touch fresh static asset %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
server := &Server{
|
||||
cfg: config.Config{LocalUploadedStorageDir: storageDir, LocalTempAssetTTLHours: 24},
|
||||
cfg: config.Config{
|
||||
LocalGeneratedStorageDir: generatedDir,
|
||||
LocalUploadedStorageDir: uploadedDir,
|
||||
LocalTempAssetTTLHours: 24,
|
||||
},
|
||||
logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
}
|
||||
|
||||
deleted := server.cleanupExpiredLocalTempAssets(context.Background(), now)
|
||||
|
||||
if deleted != 1 {
|
||||
t.Fatalf("expected one expired temp asset delete, got %d", deleted)
|
||||
if deleted != 2 {
|
||||
t.Fatalf("expected two expired static asset deletes, got %d", deleted)
|
||||
}
|
||||
if _, err := os.Stat(oldTemp); !os.IsNotExist(err) {
|
||||
t.Fatalf("old prefixed temp asset should be deleted, stat err=%v", err)
|
||||
for _, path := range []string{oldUploaded, oldGenerated} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("old static asset should be deleted %s, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{freshTemp, oldGenerated} {
|
||||
for _, path := range []string{freshUploaded, freshGenerated} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("non-expired or non-prefixed file should remain %s: %v", path, err)
|
||||
t.Fatalf("fresh static asset should remain %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
@ -76,6 +77,7 @@ func defaultGeneratedAssetUploadPolicy() generatedAssetUploadPolicy {
|
||||
func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, taskKind string, result map[string]any) (map[string]any, error) {
|
||||
data, _ := result["data"].([]any)
|
||||
if len(data) == 0 {
|
||||
redactGeneratedResultRawData(result)
|
||||
return result, nil
|
||||
}
|
||||
policy, err := s.generatedAssetUploadPolicy(ctx)
|
||||
@ -103,6 +105,7 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
||||
}
|
||||
}
|
||||
if !needsUpload && !changed {
|
||||
redactGeneratedResultRawData(result)
|
||||
return result, nil
|
||||
}
|
||||
var channels []store.FileStorageChannel
|
||||
@ -165,6 +168,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
||||
if kind == "image" {
|
||||
merged["image_url"] = urlValue
|
||||
}
|
||||
if kind == "audio" {
|
||||
merged["audio_url"] = urlValue
|
||||
}
|
||||
}
|
||||
if kind != "" && stringFromAny(merged["type"]) == "" {
|
||||
merged["type"] = kind
|
||||
@ -176,9 +182,185 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
||||
nextData = append(nextData, merged)
|
||||
}
|
||||
next["data"] = nextData
|
||||
redactGeneratedResultRawData(next)
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func redactGeneratedResultRawData(result map[string]any) bool {
|
||||
if result == nil {
|
||||
return false
|
||||
}
|
||||
changed := false
|
||||
for _, key := range []string{"raw_data", "rawData"} {
|
||||
value, ok := result[key]
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
next, nextChanged := redactGeneratedRawDataValue(value, key, nil)
|
||||
if nextChanged {
|
||||
result[key] = next
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
func redactGeneratedRawDataValue(value any, key string, siblings map[string]any) (any, bool) {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
next := make(map[string]any, len(typed))
|
||||
changed := false
|
||||
for childKey, childValue := range typed {
|
||||
redacted, childChanged := redactGeneratedRawDataValue(childValue, childKey, typed)
|
||||
next[childKey] = redacted
|
||||
if childChanged {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
return next, true
|
||||
}
|
||||
return value, false
|
||||
case []any:
|
||||
next := make([]any, len(typed))
|
||||
changed := false
|
||||
for index, item := range typed {
|
||||
redacted, itemChanged := redactGeneratedRawDataValue(item, key, siblings)
|
||||
next[index] = redacted
|
||||
if itemChanged {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
return next, true
|
||||
}
|
||||
return value, false
|
||||
case string:
|
||||
redacted, ok := generatedRawDataMediaRedaction(key, typed, siblings)
|
||||
if ok {
|
||||
return redacted, true
|
||||
}
|
||||
return value, false
|
||||
default:
|
||||
return value, false
|
||||
}
|
||||
}
|
||||
|
||||
func generatedRawDataMediaRedaction(key string, value string, siblings map[string]any) (map[string]any, bool) {
|
||||
raw := strings.TrimSpace(value)
|
||||
if raw == "" {
|
||||
return nil, false
|
||||
}
|
||||
contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
|
||||
if strings.HasPrefix(strings.ToLower(raw), "data:") {
|
||||
declared, encoded, ok, err := parseBase64DataURL(raw)
|
||||
if err != nil || !ok || !generatedContentTypeIsMedia(declared) {
|
||||
return nil, false
|
||||
}
|
||||
payload, err := decodeBase64Payload(encoded)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return generatedRawDataRedaction("data_url", declared, payload, raw), true
|
||||
}
|
||||
if len(raw) < 128 {
|
||||
return nil, false
|
||||
}
|
||||
keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key)
|
||||
if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) {
|
||||
return nil, false
|
||||
}
|
||||
if keyLooksLikeMediaPayload && looksHexMediaPayload(raw) {
|
||||
normalized := removeASCIIWhitespace(raw)
|
||||
payload, err := hex.DecodeString(normalized)
|
||||
if err == nil && len(payload) > 0 {
|
||||
return generatedRawDataRedaction("hex", contentType, payload, raw), true
|
||||
}
|
||||
}
|
||||
if payload, err := decodeBase64Payload(raw); err == nil && len(payload) > 0 {
|
||||
return generatedRawDataRedaction("base64", contentType, payload, raw), true
|
||||
}
|
||||
if keyLooksLikeMediaPayload && len(raw) > 2048 {
|
||||
return generatedRawDataRedaction("unknown", contentType, nil, raw), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func generatedRawDataMediaPayloadKey(key string) bool {
|
||||
normalized := strings.ToLower(strings.TrimSpace(key))
|
||||
normalized = strings.ReplaceAll(normalized, "-", "_")
|
||||
return normalized == "audio" ||
|
||||
normalized == "image" ||
|
||||
normalized == "video" ||
|
||||
normalized == "b64_json" ||
|
||||
normalized == "base64" ||
|
||||
normalized == "b64" ||
|
||||
normalized == "binary_data_base64" ||
|
||||
strings.Contains(normalized, "base64") ||
|
||||
strings.Contains(normalized, "_b64") ||
|
||||
strings.Contains(normalized, "audio_data") ||
|
||||
strings.Contains(normalized, "image_data") ||
|
||||
strings.Contains(normalized, "video_data")
|
||||
}
|
||||
|
||||
func defaultContentTypeForRawMediaKey(key string) string {
|
||||
normalized := strings.ToLower(strings.TrimSpace(key))
|
||||
if strings.Contains(normalized, "audio") {
|
||||
return "audio/mpeg"
|
||||
}
|
||||
if strings.Contains(normalized, "video") {
|
||||
return "video/mp4"
|
||||
}
|
||||
if strings.Contains(normalized, "image") || strings.Contains(normalized, "b64_json") {
|
||||
return "image/png"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func generatedRawDataRedaction(encoding string, contentType string, payload []byte, raw string) map[string]any {
|
||||
out := map[string]any{
|
||||
"redacted": true,
|
||||
"reason": "inline_media_payload",
|
||||
"encoding": encoding,
|
||||
}
|
||||
if contentType = normalizeGeneratedContentType(contentType); contentType != "" {
|
||||
out["contentType"] = contentType
|
||||
}
|
||||
if len(payload) > 0 {
|
||||
digest := sha256.Sum256(payload)
|
||||
out["size"] = len(payload)
|
||||
out["sha256"] = hex.EncodeToString(digest[:])
|
||||
} else {
|
||||
out["size"] = len(raw)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func looksHexMediaPayload(value string) bool {
|
||||
normalized := removeASCIIWhitespace(value)
|
||||
if len(normalized) < 128 || len(normalized)%2 != 0 {
|
||||
return false
|
||||
}
|
||||
for _, item := range normalized {
|
||||
if (item >= '0' && item <= '9') || (item >= 'a' && item <= 'f') || (item >= 'A' && item <= 'F') {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func removeASCIIWhitespace(value string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
switch r {
|
||||
case '\n', '\r', '\t', ' ':
|
||||
return -1
|
||||
default:
|
||||
return r
|
||||
}
|
||||
}, value)
|
||||
}
|
||||
|
||||
func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) {
|
||||
settings, err := s.store.GetFileStorageSettings(ctx)
|
||||
if err != nil {
|
||||
@ -282,11 +464,13 @@ func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string,
|
||||
_ = os.Remove(targetPath)
|
||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: closeErr.Error(), Retryable: true}
|
||||
}
|
||||
expiresAt := time.Now().Add(time.Duration(s.localStaticAssetTTLHours()) * time.Hour).UTC().Format(time.RFC3339)
|
||||
return map[string]any{
|
||||
"url": s.localStaticFileURL(fileName, pathPrefix),
|
||||
"fileName": fileName,
|
||||
"contentType": payload.ContentType,
|
||||
"size": len(payload.Bytes),
|
||||
"expiresAt": expiresAt,
|
||||
"storageChannel": map[string]any{
|
||||
"id": "local-static",
|
||||
"channelKey": "local-static",
|
||||
@ -296,6 +480,13 @@ func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) localStaticAssetTTLHours() int {
|
||||
if s.cfg.LocalTempAssetTTLHours <= 0 {
|
||||
return 24
|
||||
}
|
||||
return s.cfg.LocalTempAssetTTLHours
|
||||
}
|
||||
|
||||
func (s *Service) localStaticFileURL(fileName string, pathPrefix string) string {
|
||||
if strings.TrimSpace(pathPrefix) == "" {
|
||||
pathPrefix = localStaticUploadedPathPrefix
|
||||
@ -643,6 +834,9 @@ func inlineAssetFromItem(taskKind string, item map[string]any) (*generatedInline
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
if !inlineMediaKeyAllowedForItem(item, key, value) {
|
||||
continue
|
||||
}
|
||||
strictBase64 := inlineMediaKeyIsStrictBase64(key)
|
||||
payload, contentType, ok, err := inlineMediaPayload(value, strictBase64)
|
||||
if err != nil {
|
||||
@ -818,6 +1012,9 @@ func inlineMediaKeys(item map[string]any) []string {
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
if !inlineMediaKeyAllowedForItem(item, key, value) {
|
||||
continue
|
||||
}
|
||||
strictBase64 := inlineMediaKeyIsStrictBase64(key)
|
||||
if strictBase64 && stringFromAny(value) != "" {
|
||||
keys = append(keys, key)
|
||||
@ -835,6 +1032,8 @@ func inlineMediaCandidateKeys() []string {
|
||||
"b64_json",
|
||||
"image_base64",
|
||||
"image_b64",
|
||||
"audio_base64",
|
||||
"audio_b64",
|
||||
"video_base64",
|
||||
"video_b64",
|
||||
"base64",
|
||||
@ -842,24 +1041,56 @@ func inlineMediaCandidateKeys() []string {
|
||||
"url",
|
||||
"image_url",
|
||||
"imageUrl",
|
||||
"audio_url",
|
||||
"audioUrl",
|
||||
"video_url",
|
||||
"videoUrl",
|
||||
"output_url",
|
||||
"outputUrl",
|
||||
"output_audio_url",
|
||||
"outputAudioUrl",
|
||||
"output_video_url",
|
||||
"outputVideoUrl",
|
||||
"image",
|
||||
"audio",
|
||||
"video",
|
||||
"image_buffer",
|
||||
"image_bytes",
|
||||
"audio_buffer",
|
||||
"audio_bytes",
|
||||
"video_buffer",
|
||||
"video_bytes",
|
||||
"buffer",
|
||||
"bytes",
|
||||
"data",
|
||||
"content",
|
||||
}
|
||||
}
|
||||
|
||||
func inlineMediaKeyAllowedForItem(item map[string]any, key string, value any) bool {
|
||||
if !strings.EqualFold(strings.TrimSpace(key), "content") {
|
||||
return true
|
||||
}
|
||||
raw, ok := value.(string)
|
||||
if !ok {
|
||||
return generatedContentTypeIsMedia(mediaContentTypeFromItem(item)) || generatedResultItemIsMedia(item)
|
||||
}
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(strings.ToLower(raw), "data:") {
|
||||
contentType, _, ok, err := parseBase64DataURL(raw)
|
||||
return err == nil && ok && generatedContentTypeIsMedia(contentType)
|
||||
}
|
||||
return generatedContentTypeIsMedia(mediaContentTypeFromItem(item)) || generatedResultItemIsMedia(item)
|
||||
}
|
||||
|
||||
func generatedResultItemIsMedia(item map[string]any) bool {
|
||||
itemType := strings.ToLower(strings.TrimSpace(stringFromAny(item["type"])))
|
||||
return strings.Contains(itemType, "image") || strings.Contains(itemType, "video") || strings.Contains(itemType, "audio")
|
||||
}
|
||||
|
||||
func inlineMediaKeyIsStrictBase64(key string) bool {
|
||||
lower := strings.ToLower(key)
|
||||
return lower == "b64_json" || lower == "base64" || lower == "b64" || strings.Contains(lower, "base64") || strings.Contains(lower, "_b64")
|
||||
@ -887,7 +1118,7 @@ func mediaURLKeys(item map[string]any) []string {
|
||||
}
|
||||
|
||||
func mediaURLCandidateKeys() []string {
|
||||
return []string{"url", "image_url", "imageUrl", "video_url", "videoUrl", "output_url", "outputUrl", "output_video_url", "outputVideoUrl", "download_url", "downloadUrl", "file_url", "fileUrl"}
|
||||
return []string{"url", "image_url", "imageUrl", "audio_url", "audioUrl", "video_url", "videoUrl", "output_url", "outputUrl", "output_audio_url", "outputAudioUrl", "output_video_url", "outputVideoUrl", "download_url", "downloadUrl", "file_url", "fileUrl"}
|
||||
}
|
||||
|
||||
func mediaURLString(value string) bool {
|
||||
|
||||
@ -93,6 +93,43 @@ func TestGeneratedAssetDecisionUploadsDataURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedAssetDecisionUploadsAudioContentDataURL(t *testing.T) {
|
||||
item := map[string]any{
|
||||
"type": "audio",
|
||||
"content": "data:audio/mpeg;base64," + base64.StdEncoding.EncodeToString([]byte("inline audio")),
|
||||
"mime_type": "audio/mpeg",
|
||||
}
|
||||
|
||||
decision, err := generatedAssetDecisionForItem("speech.generations", item, defaultGeneratedAssetUploadPolicy())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if decision.Inline == nil {
|
||||
t.Fatalf("expected inline audio content to be uploaded")
|
||||
}
|
||||
if decision.Inline.Kind != "audio" || decision.Inline.ContentType != "audio/mpeg" || decision.Inline.SourceKey != "content" {
|
||||
t.Fatalf("unexpected inline audio metadata: %+v", decision.Inline)
|
||||
}
|
||||
if !containsString(decision.StripKeys, "content") {
|
||||
t.Fatalf("uploaded audio content should be stripped: %+v", decision.StripKeys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedAssetDecisionDoesNotUploadPlainTextContent(t *testing.T) {
|
||||
item := map[string]any{
|
||||
"type": "text",
|
||||
"content": base64.StdEncoding.EncodeToString([]byte(strings.Repeat("plain text ", 20))),
|
||||
}
|
||||
|
||||
decision, err := generatedAssetDecisionForItem("speech.generations", item, defaultGeneratedAssetUploadPolicy())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if decision.Inline != nil || len(decision.StripKeys) > 0 {
|
||||
t.Fatalf("plain text content should not be treated as generated media: %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) {
|
||||
item := map[string]any{
|
||||
"type": "video",
|
||||
@ -216,6 +253,9 @@ func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
|
||||
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-123-01-") || !strings.HasSuffix(urlValue, ".png") {
|
||||
t.Fatalf("unexpected local static URL: %s", urlValue)
|
||||
}
|
||||
if stringFromAny(upload["expiresAt"]) == "" {
|
||||
t.Fatalf("local static upload should expose expiresAt: %+v", upload)
|
||||
}
|
||||
entries, err := os.ReadDir(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read local static dir: %v", err)
|
||||
@ -232,6 +272,39 @@ func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
|
||||
storageDir := t.TempDir()
|
||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
||||
asset := &generatedInlineAsset{
|
||||
Bytes: []byte("inline audio payload"),
|
||||
ContentType: "audio/mpeg",
|
||||
Kind: "audio",
|
||||
SourceKey: "content",
|
||||
}
|
||||
|
||||
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if contentType != "audio/mpeg" || kind != "audio" || strategy != "local_static_inline_media" {
|
||||
t.Fatalf("unexpected local audio metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
|
||||
}
|
||||
urlValue := stringFromAny(upload["url"])
|
||||
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-tts-01-") || !strings.HasSuffix(urlValue, ".mp3") {
|
||||
t.Fatalf("unexpected local audio URL: %s", urlValue)
|
||||
}
|
||||
if stringFromAny(upload["expiresAt"]) == "" {
|
||||
t.Fatalf("local audio static upload should expose expiresAt: %+v", upload)
|
||||
}
|
||||
entries, err := os.ReadDir(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read local static dir: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".mp3") {
|
||||
t.Fatalf("expected one MP3 file in local static dir, got %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
|
||||
storageDir := t.TempDir()
|
||||
service := &Service{cfg: config.Config{
|
||||
@ -254,6 +327,9 @@ func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
|
||||
if !strings.HasPrefix(urlValue, "/static/uploaded/") || !strings.HasSuffix(urlValue, ".pdf") {
|
||||
t.Fatalf("unexpected uploaded local static URL: %s", urlValue)
|
||||
}
|
||||
if stringFromAny(upload["expiresAt"]) == "" {
|
||||
t.Fatalf("local uploaded static file should expose expiresAt: %+v", upload)
|
||||
}
|
||||
storageChannel, _ := upload["storageChannel"].(map[string]any)
|
||||
if stringFromAny(storageChannel["provider"]) != "local_static" {
|
||||
t.Fatalf("expected local static provider metadata, got %+v", upload["storageChannel"])
|
||||
@ -277,3 +353,26 @@ func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
|
||||
t.Fatalf("stored uploaded payload does not match source payload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactGeneratedResultRawDataAudioPayload(t *testing.T) {
|
||||
audioHex := strings.Repeat("49443304", 40)
|
||||
result := map[string]any{
|
||||
"raw_data": map[string]any{
|
||||
"base_resp": map[string]any{"status_code": float64(0)},
|
||||
"data": map[string]any{"audio": audioHex},
|
||||
},
|
||||
}
|
||||
|
||||
if !redactGeneratedResultRawData(result) {
|
||||
t.Fatalf("expected raw audio payload to be redacted")
|
||||
}
|
||||
rawData, _ := result["raw_data"].(map[string]any)
|
||||
data, _ := rawData["data"].(map[string]any)
|
||||
audio, _ := data["audio"].(map[string]any)
|
||||
if audio["redacted"] != true || audio["encoding"] != "hex" || audio["contentType"] != "audio/mpeg" {
|
||||
t.Fatalf("unexpected redacted audio payload: %+v", audio)
|
||||
}
|
||||
if _, ok := audio["sha256"].(string); !ok {
|
||||
t.Fatalf("expected redacted audio payload to include sha256: %+v", audio)
|
||||
}
|
||||
}
|
||||
|
||||
@ -37,13 +37,13 @@ const providerOptions = [
|
||||
const defaultScenes = ['upload', 'image_result'];
|
||||
const sceneOptions = [
|
||||
{ value: 'upload', label: '上传', description: 'OpenAPI / 管理端主动上传文件' },
|
||||
{ value: 'image_result', label: '返图', description: '模型返回 base64 / buffer 图片或视频后的转存' },
|
||||
{ value: 'image_result', label: '生成媒体结果', description: '模型返回 base64 / buffer 图片、音频或视频后的转存' },
|
||||
];
|
||||
|
||||
const resultUploadPolicyOptions = [
|
||||
{ value: 'default', label: '默认：仅非链接资源转存', description: 'URL 结果直接保存；base64 / buffer 等结果转存后保存 URL' },
|
||||
{ value: 'upload_all', label: '全部转存', description: 'URL、base64、buffer 等返图结果都会转存到当前文件渠道' },
|
||||
{ value: 'upload_none', label: '全部不转存', description: '链接结果直接保存；base64 / buffer 结果写入网关本地静态托管后保存 URL' },
|
||||
{ value: 'upload_all', label: '全部转存', description: 'URL、base64、buffer 等生成媒体结果都会转存到当前文件渠道' },
|
||||
{ value: 'upload_none', label: '全部不转存', description: '链接结果直接保存；base64 / buffer 结果写入 24 小时本地静态托管后保存 URL' },
|
||||
];
|
||||
|
||||
export function SystemSettingsPanel(props: {
|
||||
@ -146,8 +146,8 @@ export function SystemSettingsPanel(props: {
|
||||
<section className="fileStoragePanel">
|
||||
<div className="fileStorageSettingsCard">
|
||||
<div>
|
||||
<strong>全局返图转存策略</strong>
|
||||
<span>对所有返图场景统一生效；渠道只负责上传目标、凭证、重试和轮转。</span>
|
||||
<strong>全局生成媒体转存策略</strong>
|
||||
<span>对图片、音频和视频生成结果统一生效；网关本地静态资源仅保留 24 小时。</span>
|
||||
</div>
|
||||
<Label>
|
||||
策略
|
||||
|
||||
Loading…
Reference in New Issue
Block a user