BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(storage): 完善对象存储配置与过期策略

Browse Source 
补齐 OSS/S3 的 Endpoint、Region、Bucket、CDN、对象前缀和签名有效期配置，并为生成结果与请求素材自动维护分级生命周期规则。普通上传继续保持永久，私有资源按配置生成限时签名 URL，管理端连接测试覆盖生命周期、上传、读取和删除。\n\n新增可重复的真实 OSS 验收脚本，凭据仅从本地环境读取，接口响应继续保持脱敏。\n\n验证：Go 全量测试、迁移安全检查、pnpm lint、pnpm test、pnpm build、本地阿里云 OSS 真实上传下载删除验收。
This commit is contained in:
wangbo
2026-08-04 12:44:24 +08:00
parent fe56aa46b9
commit 6a64d3936a
16 changed files with 1274 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/httpapi/openapi_models.go
+7 -5
View File
@@ -238,11 +238,13 @@ type FileStorageChannelListResponse struct {
}
type FileStorageChannelTestResponse struct {
Provider string `json:"provider"`
PutSucceeded bool `json:"putSucceeded"`
HeadSucceeded bool `json:"headSucceeded"`
DeleteSucceeded bool `json:"deleteSucceeded"`
DurationMS int64 `json:"durationMs"`
Provider string `json:"provider"`
PutSucceeded bool `json:"putSucceeded"`
HeadSucceeded bool `json:"headSucceeded"`
DeleteSucceeded bool `json:"deleteSucceeded"`
LifecycleReady bool `json:"lifecycleReady"`
ExpirationPolicy string `json:"expirationPolicy"`
DurationMS int64 `json:"durationMs"`
}
type FileUploadResponse struct {
apps/api/internal/httpapi/request_preparation.go
+22 -1
View File
@@ -417,6 +417,8 @@ func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestA
expiry := now.Add(time.Duration(s.localTempAssetTTLHours()) * time.Hour)
expiresAt = &expiry
localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"]))
} else {
expiresAt = requestAssetUploadExpiresAt(upload)
}
asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{
SHA256: sha,
@@ -576,7 +578,7 @@ func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
if asset.ExpiredAt != nil {
return false
}
if asset.ExpiresAt != nil && !asset.ExpiresAt.After(now) {
if asset.ExpiresAt != nil && !asset.ExpiresAt.After(now.Add(30*time.Second)) {
return false
}
if strings.EqualFold(strings.TrimSpace(asset.StorageProvider), "local_static") {
@@ -592,6 +594,25 @@ func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
return strings.TrimSpace(asset.URL) != ""
}
func requestAssetUploadExpiresAt(upload map[string]any) *time.Time {
var earliest *time.Time
for _, key := range []string{"urlExpiresAt", "objectExpiresAt"} {
value := strings.TrimSpace(stringFromRequestAny(upload[key]))
if value == "" {
continue
}
parsed, err := time.Parse(time.RFC3339, value)
if err != nil {
continue
}
if earliest == nil || parsed.Before(*earliest) {
candidate := parsed
earliest = &candidate
}
}
return earliest
}
func requestAssetStorageProvider(upload map[string]any) string {
if channel, ok := upload["storageChannel"].(map[string]any); ok {
if provider := stringFromRequestAny(channel["provider"]); provider != "" {
apps/api/internal/httpapi/request_preparation_test.go
+17
View File
@@ -176,6 +176,23 @@ func TestRequestAssetStillUsableRequiresExistingLocalFile(t *testing.T) {
}
}
func TestRequestAssetUsesEarliestObjectStorageExpiry(t *testing.T) {
expiresAt := requestAssetUploadExpiresAt(map[string]any{
"urlExpiresAt": "2026-08-04T01:00:00Z",
"objectExpiresAt": "2026-09-03T00:00:00Z",
})
if expiresAt == nil || expiresAt.Format(time.RFC3339) != "2026-08-04T01:00:00Z" {
t.Fatalf("unexpected request asset expiry: %v", expiresAt)
}
asset := store.RequestAsset{URL: "https://signed.example/object", StorageProvider: "s3", ExpiresAt: expiresAt}
if !requestAssetStillUsable(asset, time.Date(2026, time.August, 4, 0, 58, 0, 0, time.UTC)) {
t.Fatal("request asset was refreshed before the safety window")
}
if requestAssetStillUsable(asset, time.Date(2026, time.August, 4, 0, 59, 45, 0, time.UTC)) {
t.Fatal("request asset inside the signed URL safety window was reused")
}
}
func TestCanonicalConversationMessageHashUsesTextAndAssetRefs(t *testing.T) {
message := map[string]any{
"role": "user",
apps/api/internal/httpapi/system_settings_handlers.go
+47 -2
View File
@@ -280,7 +280,7 @@ func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request
// testFileStorageChannel godoc
// @Summary 测试对象存储通道
// @Description 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。
// @Description 对指定 OSS 或 S3 通道检查并初始化临时文件生命周期规则，然后执行隔离的 Put、Head、Delete 探针；不返回凭据或对象键。
// @Tags system
// @Produce json
// @Security BearerAuth
@@ -358,9 +358,26 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
if !validFileStorageBaseURL(endpoint) {
return "object storage config.endpoint must be an http or https URL without embedded credentials"
}
if publicBaseURL := firstNonEmpty(fileStorageConfigString(input.Config, "publicBaseUrl"), fileStorageConfigString(input.Config, "publicBaseURL")); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
if publicBaseURL := firstNonEmpty(
fileStorageConfigString(input.Config, "publicBaseUrl"),
fileStorageConfigString(input.Config, "publicBaseURL"),
fileStorageConfigString(input.Config, "cdnDomain"),
fileStorageConfigString(input.Config, "publicDomain"),
); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials"
}
if policy := firstNonEmpty(
fileStorageConfigString(input.Config, "temporaryFileExpirePolicy"),
fileStorageConfigString(input.Config, "apiReturnExpirePolicy"),
); policy != "" && !validFileStorageExpirationPolicy(policy) {
return "object storage config.temporaryFileExpirePolicy must be never, 1d, 1m, 3m or 6m"
}
if value, exists := input.Config["signedUrlExpiresSeconds"]; exists {
seconds, ok := fileStorageConfigInteger(value)
if !ok || seconds < 60 || seconds > 7*24*60*60 {
return "object storage config.signedUrlExpiresSeconds must be an integer between 60 and 604800"
}
}
accessKeyID := input.AccessKeyID
if accessKeyID == nil {
accessKeyID = input.AccessKey
@@ -378,6 +395,34 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
return ""
}
func validFileStorageExpirationPolicy(value string) bool {
switch strings.ToLower(strings.TrimSpace(value)) {
case "never", "1d", "1m", "3m", "6m":
return true
default:
return false
}
}
func fileStorageConfigInteger(value any) (int64, bool) {
switch typed := value.(type) {
case int:
return int64(typed), true
case int32:
return int64(typed), true
case int64:
return typed, true
case float64:
converted := int64(typed)
return converted, float64(converted) == typed
case float32:
converted := int64(typed)
return converted, float32(converted) == typed
default:
return 0, false
}
}
func validFileStorageBaseURL(value string) bool {
parsed, err := url.Parse(strings.TrimSpace(value))
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
apps/api/internal/httpapi/system_settings_handlers_test.go
+31
View File
@@ -53,3 +53,34 @@ func TestValidateObjectStorageChannelRejectsCredentialedEndpoint(t *testing.T) {
t.Fatalf("credentialed endpoint was accepted: %q", message)
}
}
func TestValidateObjectStorageChannelAcceptsExpirationAndCDNAliases(t *testing.T) {
input := store.FileStorageChannelInput{
ChannelKey: "oss-expiring", Name: "OSS expiring", Provider: "aliyun_oss", Status: "disabled",
Config: map[string]any{
"endpoint": "https://oss-cn-shanghai.aliyuncs.com", "region": "cn-shanghai", "bucket": "media",
"cdnDomain": "https://cdn.example.com", "temporaryFileExpirePolicy": "3m", "signedUrlExpiresSeconds": float64(3600),
},
}
if message := validateFileStorageChannelInput(input, nil); message != "" {
t.Fatalf("valid object storage expiration config rejected: %s", message)
}
}
func TestValidateObjectStorageChannelRejectsInvalidExpiration(t *testing.T) {
base := store.FileStorageChannelInput{
ChannelKey: "oss-expiring", Name: "OSS expiring", Provider: "aliyun_oss", Status: "disabled",
Config: map[string]any{
"endpoint": "https://oss-cn-shanghai.aliyuncs.com", "region": "cn-shanghai", "bucket": "media",
"temporaryFileExpirePolicy": "7d", "signedUrlExpiresSeconds": float64(3600),
},
}
if message := validateFileStorageChannelInput(base, nil); !strings.Contains(message, "temporaryFileExpirePolicy") {
t.Fatalf("invalid expiration policy was accepted: %q", message)
}
base.Config["temporaryFileExpirePolicy"] = "1d"
base.Config["signedUrlExpiresSeconds"] = float64(30)
if message := validateFileStorageChannelInput(base, nil); !strings.Contains(message, "signedUrlExpiresSeconds") {
t.Fatalf("invalid signed URL TTL was accepted: %q", message)
}
}