feat: add file storage settings and uploads

This commit is contained in:
2026-05-13 20:23:45 +08:00
parent 0d0d0b9115
commit fc5dfd6bc5
21 changed files with 3401 additions and 72 deletions
@@ -0,0 +1,58 @@
package httpapi
import (
"io"
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
)
const maxGatewayUploadBytes = 256 << 20
func (s *Server) uploadFile(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxGatewayUploadBytes)
if err := r.ParseMultipartForm(32 << 20); err != nil {
writeError(w, http.StatusBadRequest, "invalid multipart upload")
return
}
file, header, err := r.FormFile("file")
if err != nil {
writeError(w, http.StatusBadRequest, "file is required")
return
}
defer file.Close()
payload, err := io.ReadAll(file)
if err != nil {
writeError(w, http.StatusBadRequest, "read upload file failed")
return
}
contentType := strings.TrimSpace(header.Header.Get("Content-Type"))
if contentType == "" && len(payload) > 0 {
contentType = http.DetectContentType(payload)
}
upload, err := s.runner.UploadFile(r.Context(), runner.FileUploadPayload{
Bytes: payload,
ContentType: contentType,
FileName: header.Filename,
Source: firstNonEmptyFormValue(r, "source", "ai-gateway-openapi"),
})
if err != nil {
s.logger.Error("upload file failed", "error", err)
status := http.StatusBadGateway
if clients.ErrorCode(err) == "upload_no_channel" {
status = http.StatusServiceUnavailable
}
writeError(w, status, err.Error())
return
}
writeJSON(w, http.StatusOK, upload)
}
func firstNonEmptyFormValue(r *http.Request, key string, fallback string) string {
if value := strings.TrimSpace(r.FormValue(key)); value != "" {
return value
}
return fallback
}
+8
View File
@@ -102,6 +102,12 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("GET /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getRunnerPolicy)))
mux.Handle("PATCH /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateRunnerPolicy)))
mux.Handle("GET /api/admin/config/network-proxy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getNetworkProxyConfig)))
mux.Handle("GET /api/admin/system/file-storage/settings", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getFileStorageSettings)))
mux.Handle("PATCH /api/admin/system/file-storage/settings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageSettings)))
mux.Handle("GET /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listFileStorageChannels)))
mux.Handle("POST /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createFileStorageChannel)))
mux.Handle("PATCH /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageChannel)))
mux.Handle("DELETE /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteFileStorageChannel)))
mux.Handle("GET /api/admin/platforms", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listPlatforms)))
mux.Handle("POST /api/admin/platforms", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createPlatform)))
mux.Handle("PATCH /api/admin/platforms/{platformID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updatePlatform)))
@@ -123,6 +129,7 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("POST /api/v1/images/generations", server.auth.Require(auth.PermissionBasic, server.createTask("images.generations", false)))
mux.Handle("POST /api/v1/images/edits", server.auth.Require(auth.PermissionBasic, server.createTask("images.edits", false)))
mux.Handle("POST /api/v1/videos/generations", server.auth.Require(auth.PermissionBasic, server.createTask("videos.generations", false)))
mux.Handle("POST /api/v1/files/upload", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.uploadFile)))
mux.Handle("GET /api/v1/tasks", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.listTasks)))
mux.Handle("GET /api/v1/tasks/{taskID}", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.getTask)))
mux.Handle("GET /api/v1/tasks/{taskID}/param-preprocessing", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.taskParamPreprocessing)))
@@ -135,6 +142,7 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("POST /v1/images/generations", server.auth.Require(auth.PermissionBasic, server.createTask("images.generations", true)))
mux.Handle("POST /images/edits", server.auth.Require(auth.PermissionBasic, server.createTask("images.edits", true)))
mux.Handle("POST /v1/images/edits", server.auth.Require(auth.PermissionBasic, server.createTask("images.edits", true)))
mux.Handle("POST /v1/files/upload", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.uploadFile)))
return server.recover(server.cors(mux))
}
@@ -0,0 +1,150 @@
package httpapi
import (
"encoding/json"
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func (s *Server) listFileStorageChannels(w http.ResponseWriter, r *http.Request) {
items, err := s.store.ListFileStorageChannels(r.Context())
if err != nil {
s.logger.Error("list file storage channels failed", "error", err)
writeError(w, http.StatusInternalServerError, "list file storage channels failed")
return
}
writeJSON(w, http.StatusOK, map[string]any{"items": items})
}
func (s *Server) getFileStorageSettings(w http.ResponseWriter, r *http.Request) {
settings, err := s.store.GetFileStorageSettings(r.Context())
if err != nil {
if store.IsUndefinedDatabaseObject(err) {
writeJSON(w, http.StatusOK, store.DefaultFileStorageSettings())
return
}
s.logger.Error("get file storage settings failed", "error", err)
writeError(w, http.StatusInternalServerError, "get file storage settings failed")
return
}
writeJSON(w, http.StatusOK, settings)
}
func (s *Server) updateFileStorageSettings(w http.ResponseWriter, r *http.Request) {
var input store.FileStorageSettingsInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
settings, err := s.store.UpdateFileStorageSettings(r.Context(), input)
if err != nil {
s.logger.Error("update file storage settings failed", "error", err)
writeError(w, http.StatusInternalServerError, "update file storage settings failed")
return
}
writeJSON(w, http.StatusOK, settings)
}
func (s *Server) createFileStorageChannel(w http.ResponseWriter, r *http.Request) {
var input store.FileStorageChannelInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
if message := validateFileStorageChannelInput(input, nil); message != "" {
writeError(w, http.StatusBadRequest, message)
return
}
item, err := s.store.CreateFileStorageChannel(r.Context(), input)
if err != nil {
if store.IsUniqueViolation(err) {
writeError(w, http.StatusConflict, "file storage channel key already exists")
return
}
s.logger.Error("create file storage channel failed", "error", err)
writeError(w, http.StatusInternalServerError, "create file storage channel failed")
return
}
writeJSON(w, http.StatusCreated, item)
}
func (s *Server) updateFileStorageChannel(w http.ResponseWriter, r *http.Request) {
var input store.FileStorageChannelInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
existing, err := s.store.GetFileStorageChannel(r.Context(), r.PathValue("channelID"))
if err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusNotFound, "file storage channel not found")
return
}
s.logger.Error("get file storage channel failed", "error", err)
writeError(w, http.StatusInternalServerError, "get file storage channel failed")
return
}
if message := validateFileStorageChannelInput(input, &existing); message != "" {
writeError(w, http.StatusBadRequest, message)
return
}
item, err := s.store.UpdateFileStorageChannel(r.Context(), r.PathValue("channelID"), input)
if err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusNotFound, "file storage channel not found")
return
}
if store.IsUniqueViolation(err) {
writeError(w, http.StatusConflict, "file storage channel key already exists")
return
}
s.logger.Error("update file storage channel failed", "error", err)
writeError(w, http.StatusInternalServerError, "update file storage channel failed")
return
}
writeJSON(w, http.StatusOK, item)
}
func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request) {
if err := s.store.DeleteFileStorageChannel(r.Context(), r.PathValue("channelID")); err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusNotFound, "file storage channel not found")
return
}
s.logger.Error("delete file storage channel failed", "error", err)
writeError(w, http.StatusInternalServerError, "delete file storage channel failed")
return
}
w.WriteHeader(http.StatusNoContent)
}
func validateFileStorageChannelInput(input store.FileStorageChannelInput, existing *store.FileStorageChannel) string {
provider := strings.ToLower(strings.TrimSpace(input.Provider))
if provider == "" {
provider = "server_main_openapi"
}
status := strings.ToLower(strings.TrimSpace(input.Status))
if status == "" {
status = "disabled"
}
if strings.TrimSpace(input.ChannelKey) == "" || strings.TrimSpace(input.Name) == "" {
return "channelKey and name are required"
}
if status != "enabled" && status != "disabled" {
return "status must be enabled or disabled"
}
if provider == "server_main_openapi" {
hasAPIKey := false
if input.APIKey != nil {
hasAPIKey = strings.TrimSpace(*input.APIKey) != ""
} else if existing != nil {
hasAPIKey = strings.TrimSpace(existing.APIKey) != ""
}
if status == "enabled" && !hasAPIKey {
return "server-main OpenAPI channel requires API key before enabling"
}
}
return ""
}
+1 -1
View File
@@ -481,7 +481,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated)
return clients.Response{}, err
}
uploadedResult, err := s.uploadGeneratedAssets(ctx, response.Result)
uploadedResult, err := s.uploadGeneratedAssets(ctx, task.ID, task.Kind, response.Result)
if err != nil {
metrics := mergeMetrics(taskMetrics(task, user, body, candidate, response, simulated), parameterPreprocessingMetrics(preprocessing), map[string]any{
"error": err.Error(),
File diff suppressed because it is too large Load Diff
+187
View File
@@ -0,0 +1,187 @@
package runner
import (
"encoding/base64"
"strings"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestGeneratedAssetDecisionSkipsURLResultAndStripsInlinePayload(t *testing.T) {
item := map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
"url": "https://cdn.example.com/generated.png",
}
decision, err := generatedAssetDecisionForItem("images.generations", item, defaultGeneratedAssetUploadPolicy())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline != nil {
t.Fatalf("URL media should not be uploaded by the default policy")
}
if !containsString(decision.StripKeys, "b64_json") {
t.Fatalf("inline payload should be stripped when URL is already available: %+v", decision.StripKeys)
}
}
func TestGeneratedAssetDecisionUploadsInlineImageBase64(t *testing.T) {
item := map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
"mime_type": "image/jpeg",
}
decision, err := generatedAssetDecisionForItem("images.generations", item, defaultGeneratedAssetUploadPolicy())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline == nil {
t.Fatalf("expected inline image to be uploaded")
}
if decision.Inline.Kind != "image" || decision.Inline.ContentType != "image/jpeg" {
t.Fatalf("unexpected inline image metadata: %+v", decision.Inline)
}
if !containsString(decision.StripKeys, "b64_json") {
t.Fatalf("uploaded inline payload should be stripped: %+v", decision.StripKeys)
}
}
func TestGeneratedAssetDecisionUploadsInlineVideoBuffer(t *testing.T) {
item := map[string]any{
"type": "video",
"video_buffer": []any{float64(0), float64(1), float64(2), float64(3)},
}
decision, err := generatedAssetDecisionForItem("videos.generations", item, defaultGeneratedAssetUploadPolicy())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline == nil {
t.Fatalf("expected inline video buffer to be uploaded")
}
if decision.Inline.Kind != "video" || decision.Inline.ContentType != "video/mp4" {
t.Fatalf("unexpected inline video metadata: %+v", decision.Inline)
}
if !containsString(decision.StripKeys, "video_buffer") {
t.Fatalf("uploaded video buffer should be stripped: %+v", decision.StripKeys)
}
}
func TestGeneratedAssetDecisionUploadsDataURL(t *testing.T) {
item := map[string]any{
"url": "data:image/webp;base64," + base64.StdEncoding.EncodeToString([]byte("inline webp")),
}
decision, err := generatedAssetDecisionForItem("images.generations", item, defaultGeneratedAssetUploadPolicy())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline == nil {
t.Fatalf("expected data URL to be uploaded")
}
if decision.Inline.SourceKey != "url" || decision.Inline.ContentType != "image/webp" {
t.Fatalf("unexpected data URL metadata: %+v", decision.Inline)
}
if !containsString(decision.StripKeys, "url") {
t.Fatalf("uploaded data URL field should be stripped: %+v", decision.StripKeys)
}
}
func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) {
item := map[string]any{
"type": "video",
"video_url": "https://cdn.example.com/generated.mp4",
}
decision, err := generatedAssetDecisionForItem("videos.generations", item, generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.URL == nil {
t.Fatalf("expected URL media to be uploaded")
}
if decision.URL.Kind != "video" || decision.URL.SourceKey != "video_url" {
t.Fatalf("unexpected URL media metadata: %+v", decision.URL)
}
if !containsString(decision.StripKeys, "video_url") {
t.Fatalf("uploaded URL field should be stripped: %+v", decision.StripKeys)
}
}
func TestGeneratedAssetDecisionSkipsAllWhenPolicyUploadNone(t *testing.T) {
item := map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
}
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline != nil || decision.URL != nil || len(decision.StripKeys) != 0 {
t.Fatalf("upload_none should keep the result unchanged: %+v", decision)
}
}
func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
tests := []struct {
name string
policyName string
want generatedAssetUploadPolicy
}{
{
name: "default",
policyName: store.FileStorageResultUploadPolicyDefault,
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: false},
},
{
name: "upload all",
policyName: store.FileStorageResultUploadPolicyUploadAll,
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true},
},
{
name: "upload none",
policyName: store.FileStorageResultUploadPolicyUploadNone,
want: generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := generatedAssetUploadPolicyFromName(tt.policyName)
if got != tt.want {
t.Fatalf("unexpected policy: got %+v, want %+v", got, tt.want)
}
})
}
}
func TestResolvedGeneratedAssetContentTypePrefersDetectedMedia(t *testing.T) {
pngPayload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}
contentType := resolvedGeneratedAssetContentType("image/jpeg", "image", pngPayload)
if contentType != "image/png" {
t.Fatalf("expected detected PNG content type, got %s", contentType)
}
if extension := fileExtensionForContentType(contentType, "image"); extension != ".png" {
t.Fatalf("expected PNG extension, got %s", extension)
}
}
func TestResolvedGeneratedAssetContentTypeKeepsDeclaredMediaWhenDetectionIsGeneric(t *testing.T) {
contentType := resolvedGeneratedAssetContentType("image/webp", "image", []byte("not enough media bytes"))
if contentType != "image/webp" {
t.Fatalf("expected declared webp content type, got %s", contentType)
}
}
func TestGeneratedAssetFileNameIsUniqueAndTyped(t *testing.T) {
first := generatedAssetFileName("663e19cd4fa9d8078385c7c9", 0, "image/png", "image")
second := generatedAssetFileName("663e19cd4fa9d8078385c7c9", 0, "image/png", "image")
if first == second {
t.Fatalf("expected generated file names to be unique, both were %s", first)
}
if !strings.HasPrefix(first, "gateway-result-663e19cd4fa9d8078385c7c9-01-") || !strings.HasSuffix(first, ".png") {
t.Fatalf("unexpected generated file name: %s", first)
}
}
@@ -0,0 +1,499 @@
package store
import (
"context"
"encoding/json"
"strings"
"time"
"github.com/jackc/pgx/v5"
)
const defaultServerMainUploadURL = "http://127.0.0.1:3001/v1/files/upload"
const (
FileStorageSceneUpload = "upload"
FileStorageSceneImageResult = "image_result"
)
const (
FileStorageResultUploadPolicyDefault = "default"
FileStorageResultUploadPolicyUploadAll = "upload_all"
FileStorageResultUploadPolicyUploadNone = "upload_none"
)
const SystemSettingFileStorage = "file_storage"
const fileStorageChannelColumns = `
id::text, channel_key, name, provider, COALESCE(upload_url, ''), credentials,
config, retry_policy, priority, status, COALESCE(last_error, ''),
COALESCE(last_failed_at::text, ''), COALESCE(last_succeeded_at::text, ''),
created_at, updated_at`
type FileStorageChannel struct {
ID string `json:"id"`
ChannelKey string `json:"channelKey"`
Name string `json:"name"`
Provider string `json:"provider"`
UploadURL string `json:"uploadUrl,omitempty"`
APIKey string `json:"-"`
CredentialsPreview map[string]any `json:"credentialsPreview,omitempty"`
Scenes []string `json:"scenes,omitempty"`
Config map[string]any `json:"config,omitempty"`
RetryPolicy map[string]any `json:"retryPolicy,omitempty"`
Priority int `json:"priority"`
Status string `json:"status"`
LastError string `json:"lastError,omitempty"`
LastFailedAt string `json:"lastFailedAt,omitempty"`
LastSucceededAt string `json:"lastSucceededAt,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type FileStorageChannelInput struct {
ChannelKey string `json:"channelKey"`
Name string `json:"name"`
Provider string `json:"provider"`
UploadURL string `json:"uploadUrl"`
APIKey *string `json:"apiKey"`
Scenes []string `json:"scenes"`
Config map[string]any `json:"config"`
RetryPolicy map[string]any `json:"retryPolicy"`
Priority int `json:"priority"`
Status string `json:"status"`
}
type FileStorageSettings struct {
ResultUploadPolicy string `json:"resultUploadPolicy"`
}
type FileStorageSettingsInput struct {
ResultUploadPolicy string `json:"resultUploadPolicy"`
}
type fileStorageChannelScanner interface {
Scan(dest ...any) error
}
func (s *Store) ListFileStorageChannels(ctx context.Context) ([]FileStorageChannel, error) {
rows, err := s.pool.Query(ctx, `
SELECT `+fileStorageChannelColumns+`
FROM file_storage_channels
WHERE deleted_at IS NULL
ORDER BY priority ASC, created_at ASC`)
if err != nil {
return nil, err
}
defer rows.Close()
items := make([]FileStorageChannel, 0)
for rows.Next() {
item, err := scanFileStorageChannel(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Store) ListEnabledFileStorageChannels(ctx context.Context) ([]FileStorageChannel, error) {
return s.listEnabledFileStorageChannels(ctx, "")
}
func (s *Store) ListEnabledFileStorageChannelsForScene(ctx context.Context, scene string) ([]FileStorageChannel, error) {
return s.listEnabledFileStorageChannels(ctx, normalizeFileStorageScene(scene))
}
func (s *Store) listEnabledFileStorageChannels(ctx context.Context, scene string) ([]FileStorageChannel, error) {
rows, err := s.pool.Query(ctx, `
SELECT `+fileStorageChannelColumns+`
FROM file_storage_channels
WHERE deleted_at IS NULL
AND status = 'enabled'
AND (
$1 = ''
OR NOT (config ? 'scenes')
OR jsonb_typeof(config->'scenes') <> 'array'
OR (config->'scenes') ? $1
)
ORDER BY priority ASC, created_at ASC`, scene)
if err != nil {
return nil, err
}
defer rows.Close()
items := make([]FileStorageChannel, 0)
for rows.Next() {
item, err := scanFileStorageChannel(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Store) GetFileStorageChannel(ctx context.Context, id string) (FileStorageChannel, error) {
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
SELECT `+fileStorageChannelColumns+`
FROM file_storage_channels
WHERE id = $1::uuid
AND deleted_at IS NULL`, id))
}
func (s *Store) CreateFileStorageChannel(ctx context.Context, input FileStorageChannelInput) (FileStorageChannel, error) {
input = normalizeFileStorageChannelInput(input)
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
config, _ := json.Marshal(configFromFileStorageInput(input))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy))
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
INSERT INTO file_storage_channels (
channel_key, name, provider, upload_url, credentials, config, retry_policy, priority, status
)
VALUES ($1, $2, $3, NULLIF($4, ''), $5, $6, $7, $8, $9)
RETURNING `+fileStorageChannelColumns,
input.ChannelKey,
input.Name,
input.Provider,
input.UploadURL,
credentials,
config,
retryPolicy,
input.Priority,
input.Status,
))
}
func (s *Store) UpdateFileStorageChannel(ctx context.Context, id string, input FileStorageChannelInput) (FileStorageChannel, error) {
input = normalizeFileStorageChannelInput(input)
replaceCredentials := input.APIKey != nil
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
config, _ := json.Marshal(configFromFileStorageInput(input))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy))
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
UPDATE file_storage_channels
SET channel_key = $2,
name = $3,
provider = $4,
upload_url = NULLIF($5, ''),
credentials = CASE WHEN $6::boolean THEN $7 ELSE credentials END,
config = $8,
retry_policy = $9,
priority = $10,
status = $11,
updated_at = now()
WHERE id = $1::uuid
AND deleted_at IS NULL
RETURNING `+fileStorageChannelColumns,
id,
input.ChannelKey,
input.Name,
input.Provider,
input.UploadURL,
replaceCredentials,
credentials,
config,
retryPolicy,
input.Priority,
input.Status,
))
}
func (s *Store) DeleteFileStorageChannel(ctx context.Context, id string) error {
result, err := s.pool.Exec(ctx, `
UPDATE file_storage_channels
SET deleted_at = now(),
status = 'disabled',
updated_at = now()
WHERE id = $1::uuid
AND deleted_at IS NULL`, id)
if err != nil {
return err
}
if result.RowsAffected() == 0 {
return pgx.ErrNoRows
}
return nil
}
func (s *Store) MarkFileStorageChannelFailure(ctx context.Context, id string, message string) error {
if strings.TrimSpace(id) == "" {
return nil
}
_, err := s.pool.Exec(ctx, `
UPDATE file_storage_channels
SET last_error = NULLIF($2, ''),
last_failed_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND deleted_at IS NULL`, id, strings.TrimSpace(message))
return err
}
func (s *Store) MarkFileStorageChannelSuccess(ctx context.Context, id string) error {
if strings.TrimSpace(id) == "" {
return nil
}
_, err := s.pool.Exec(ctx, `
UPDATE file_storage_channels
SET last_error = NULL,
last_succeeded_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND deleted_at IS NULL`, id)
return err
}
func scanFileStorageChannel(scanner fileStorageChannelScanner) (FileStorageChannel, error) {
var item FileStorageChannel
var credentials []byte
var config []byte
var retryPolicy []byte
if err := scanner.Scan(
&item.ID,
&item.ChannelKey,
&item.Name,
&item.Provider,
&item.UploadURL,
&credentials,
&config,
&retryPolicy,
&item.Priority,
&item.Status,
&item.LastError,
&item.LastFailedAt,
&item.LastSucceededAt,
&item.CreatedAt,
&item.UpdatedAt,
); err != nil {
return FileStorageChannel{}, err
}
credentialObject := decodeObject(credentials)
item.APIKey = stringFromObject(credentialObject, "apiKey")
item.CredentialsPreview = maskCredentialsPreview(credentials)
configObject := decodeObject(config)
item.Scenes = fileStorageScenesFromConfig(configObject)
item.Config = fileStorageConfigWithoutManagedFields(configObject)
item.RetryPolicy = decodeObject(retryPolicy)
return item, nil
}
func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorageChannelInput {
input.ChannelKey = strings.TrimSpace(input.ChannelKey)
input.Name = strings.TrimSpace(input.Name)
input.Provider = strings.ToLower(strings.TrimSpace(input.Provider))
input.UploadURL = strings.TrimSpace(input.UploadURL)
if input.APIKey != nil {
apiKey := strings.TrimSpace(*input.APIKey)
input.APIKey = &apiKey
}
input.Scenes = normalizeFileStorageScenes(input.Scenes)
input.Status = strings.ToLower(strings.TrimSpace(input.Status))
if input.Provider == "" {
input.Provider = "server_main_openapi"
}
if input.Provider == "server_main_openapi" && input.UploadURL == "" {
input.UploadURL = defaultServerMainUploadURL
}
if input.Status == "" {
input.Status = "disabled"
}
if input.Priority <= 0 {
input.Priority = 100
}
return input
}
func credentialsFromFileStorageInput(input FileStorageChannelInput) map[string]any {
apiKey := fileStorageInputAPIKey(input)
if apiKey == "" {
return map[string]any{}
}
return map[string]any{"apiKey": apiKey}
}
func fileStorageInputAPIKey(input FileStorageChannelInput) string {
if input.APIKey == nil {
return ""
}
return strings.TrimSpace(*input.APIKey)
}
func configFromFileStorageInput(input FileStorageChannelInput) map[string]any {
config := map[string]any{}
for key, value := range emptyObjectIfNil(input.Config) {
config[key] = value
}
config["scenes"] = normalizeFileStorageScenes(input.Scenes)
return config
}
func fileStorageConfigWithoutManagedFields(config map[string]any) map[string]any {
out := map[string]any{}
for key, value := range config {
if key == "scenes" || key == "resultUploadPolicy" {
continue
}
out[key] = value
}
if len(out) == 0 {
return nil
}
return out
}
func DefaultFileStorageSettings() FileStorageSettings {
return FileStorageSettings{ResultUploadPolicy: FileStorageResultUploadPolicyDefault}
}
func (s *Store) GetFileStorageSettings(ctx context.Context) (FileStorageSettings, error) {
var value []byte
err := s.pool.QueryRow(ctx, `
SELECT value
FROM system_settings
WHERE setting_key = $1`, SystemSettingFileStorage).Scan(&value)
if err != nil {
if IsNotFound(err) {
return DefaultFileStorageSettings(), nil
}
return FileStorageSettings{}, err
}
return fileStorageSettingsFromValue(decodeObject(value)), nil
}
func (s *Store) UpdateFileStorageSettings(ctx context.Context, input FileStorageSettingsInput) (FileStorageSettings, error) {
settings := FileStorageSettings{ResultUploadPolicy: NormalizeFileStorageResultUploadPolicy(input.ResultUploadPolicy)}
value, _ := json.Marshal(settings)
var saved []byte
err := s.upsertFileStorageSettings(ctx, value, &saved)
if err != nil && IsUndefinedDatabaseObject(err) {
if ensureErr := s.ensureSystemSettingsTable(ctx); ensureErr != nil {
return FileStorageSettings{}, ensureErr
}
err = s.upsertFileStorageSettings(ctx, value, &saved)
}
if err != nil {
return FileStorageSettings{}, err
}
return fileStorageSettingsFromValue(decodeObject(saved)), nil
}
func (s *Store) upsertFileStorageSettings(ctx context.Context, value []byte, saved *[]byte) error {
return s.pool.QueryRow(ctx, `
INSERT INTO system_settings (setting_key, value)
VALUES ($1, $2)
ON CONFLICT (setting_key)
DO UPDATE SET value = EXCLUDED.value, updated_at = now()
RETURNING value`, SystemSettingFileStorage, value).Scan(saved)
}
func (s *Store) ensureSystemSettingsTable(ctx context.Context) error {
_, err := s.pool.Exec(ctx, `
CREATE TABLE IF NOT EXISTS system_settings (
setting_key text PRIMARY KEY,
value jsonb NOT NULL DEFAULT '{}'::jsonb,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
)`)
return err
}
func fileStorageSettingsFromValue(value map[string]any) FileStorageSettings {
settings := DefaultFileStorageSettings()
if value == nil {
return settings
}
settings.ResultUploadPolicy = NormalizeFileStorageResultUploadPolicy(stringFromAny(value["resultUploadPolicy"]))
return settings
}
func NormalizeFileStorageResultUploadPolicy(policy string) string {
normalized := strings.ToLower(strings.TrimSpace(policy))
normalized = strings.ReplaceAll(normalized, "-", "_")
switch normalized {
case "", "default", "non_link_only", "inline_only", "nonlink_only", "non_link":
return FileStorageResultUploadPolicyDefault
case "upload_all", "all", "always", "all_upload":
return FileStorageResultUploadPolicyUploadAll
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
return FileStorageResultUploadPolicyUploadNone
default:
return FileStorageResultUploadPolicyDefault
}
}
func fileStorageScenesFromConfig(config map[string]any) []string {
if config == nil {
return defaultFileStorageScenes()
}
raw, ok := config["scenes"]
if !ok {
return defaultFileStorageScenes()
}
items, ok := raw.([]any)
if !ok {
return defaultFileStorageScenes()
}
scenes := make([]string, 0, len(items))
for _, item := range items {
if value, ok := item.(string); ok {
scenes = append(scenes, value)
}
}
return normalizeFileStorageScenes(scenes)
}
func normalizeFileStorageScenes(scenes []string) []string {
seen := map[string]bool{}
out := make([]string, 0, len(scenes))
for _, item := range scenes {
scene := normalizeFileStorageScene(item)
if scene == "" || seen[scene] {
continue
}
seen[scene] = true
out = append(out, scene)
}
if len(out) == 0 {
return defaultFileStorageScenes()
}
return out
}
func normalizeFileStorageScene(scene string) string {
return strings.ToLower(strings.TrimSpace(scene))
}
func defaultFileStorageScenes() []string {
return []string{FileStorageSceneUpload, FileStorageSceneImageResult}
}
func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any) map[string]any {
if len(policy) > 0 {
return policy
}
return map[string]any{
"enabled": true,
"maxRetries": 3,
"backoffSeconds": []any{60, 120, 180},
"strategy": "exponential",
}
}
func stringFromObject(value map[string]any, key string) string {
if value == nil {
return ""
}
raw, _ := value[key].(string)
return strings.TrimSpace(raw)
}
func stringFromAny(value any) string {
switch typed := value.(type) {
case string:
return strings.TrimSpace(typed)
default:
return ""
}
}