feat(storage): 完善对象存储配置与过期策略
补齐 OSS/S3 的 Endpoint、Region、Bucket、CDN、对象前缀和签名有效期配置,并为生成结果与请求素材自动维护分级生命周期规则。普通上传继续保持永久,私有资源按配置生成限时签名 URL,管理端连接测试覆盖生命周期、上传、读取和删除。\n\n新增可重复的真实 OSS 验收脚本,凭据仅从本地环境读取,接口响应继续保持脱敏。\n\n验证:Go 全量测试、迁移安全检查、pnpm lint、pnpm test、pnpm build、本地阿里云 OSS 真实上传下载删除验收。
This commit is contained in:
@@ -3361,7 +3361,7 @@
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针,不返回凭据或对象键。",
|
||||
"description": "对指定 OSS 或 S3 通道检查并初始化临时文件生命周期规则,然后执行隔离的 Put、Head、Delete 探针;不返回凭据或对象键。",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
@@ -11340,9 +11340,15 @@
|
||||
"durationMs": {
|
||||
"type": "integer"
|
||||
},
|
||||
"expirationPolicy": {
|
||||
"type": "string"
|
||||
},
|
||||
"headSucceeded": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"lifecycleReady": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"provider": {
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
@@ -530,8 +530,12 @@ definitions:
|
||||
type: boolean
|
||||
durationMs:
|
||||
type: integer
|
||||
expirationPolicy:
|
||||
type: string
|
||||
headSucceeded:
|
||||
type: boolean
|
||||
lifecycleReady:
|
||||
type: boolean
|
||||
provider:
|
||||
type: string
|
||||
putSucceeded:
|
||||
@@ -6558,7 +6562,7 @@ paths:
|
||||
- system
|
||||
/api/admin/system/file-storage/channels/{channelID}/test:
|
||||
post:
|
||||
description: 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针,不返回凭据或对象键。
|
||||
description: 对指定 OSS 或 S3 通道检查并初始化临时文件生命周期规则,然后执行隔离的 Put、Head、Delete 探针;不返回凭据或对象键。
|
||||
parameters:
|
||||
- description: 文件存储通道 ID
|
||||
in: path
|
||||
|
||||
@@ -238,11 +238,13 @@ type FileStorageChannelListResponse struct {
|
||||
}
|
||||
|
||||
type FileStorageChannelTestResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
PutSucceeded bool `json:"putSucceeded"`
|
||||
HeadSucceeded bool `json:"headSucceeded"`
|
||||
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||
DurationMS int64 `json:"durationMs"`
|
||||
Provider string `json:"provider"`
|
||||
PutSucceeded bool `json:"putSucceeded"`
|
||||
HeadSucceeded bool `json:"headSucceeded"`
|
||||
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||
LifecycleReady bool `json:"lifecycleReady"`
|
||||
ExpirationPolicy string `json:"expirationPolicy"`
|
||||
DurationMS int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
type FileUploadResponse struct {
|
||||
|
||||
@@ -417,6 +417,8 @@ func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestA
|
||||
expiry := now.Add(time.Duration(s.localTempAssetTTLHours()) * time.Hour)
|
||||
expiresAt = &expiry
|
||||
localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"]))
|
||||
} else {
|
||||
expiresAt = requestAssetUploadExpiresAt(upload)
|
||||
}
|
||||
asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{
|
||||
SHA256: sha,
|
||||
@@ -576,7 +578,7 @@ func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
|
||||
if asset.ExpiredAt != nil {
|
||||
return false
|
||||
}
|
||||
if asset.ExpiresAt != nil && !asset.ExpiresAt.After(now) {
|
||||
if asset.ExpiresAt != nil && !asset.ExpiresAt.After(now.Add(30*time.Second)) {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(asset.StorageProvider), "local_static") {
|
||||
@@ -592,6 +594,25 @@ func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
|
||||
return strings.TrimSpace(asset.URL) != ""
|
||||
}
|
||||
|
||||
func requestAssetUploadExpiresAt(upload map[string]any) *time.Time {
|
||||
var earliest *time.Time
|
||||
for _, key := range []string{"urlExpiresAt", "objectExpiresAt"} {
|
||||
value := strings.TrimSpace(stringFromRequestAny(upload[key]))
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if earliest == nil || parsed.Before(*earliest) {
|
||||
candidate := parsed
|
||||
earliest = &candidate
|
||||
}
|
||||
}
|
||||
return earliest
|
||||
}
|
||||
|
||||
func requestAssetStorageProvider(upload map[string]any) string {
|
||||
if channel, ok := upload["storageChannel"].(map[string]any); ok {
|
||||
if provider := stringFromRequestAny(channel["provider"]); provider != "" {
|
||||
|
||||
@@ -176,6 +176,23 @@ func TestRequestAssetStillUsableRequiresExistingLocalFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestAssetUsesEarliestObjectStorageExpiry(t *testing.T) {
|
||||
expiresAt := requestAssetUploadExpiresAt(map[string]any{
|
||||
"urlExpiresAt": "2026-08-04T01:00:00Z",
|
||||
"objectExpiresAt": "2026-09-03T00:00:00Z",
|
||||
})
|
||||
if expiresAt == nil || expiresAt.Format(time.RFC3339) != "2026-08-04T01:00:00Z" {
|
||||
t.Fatalf("unexpected request asset expiry: %v", expiresAt)
|
||||
}
|
||||
asset := store.RequestAsset{URL: "https://signed.example/object", StorageProvider: "s3", ExpiresAt: expiresAt}
|
||||
if !requestAssetStillUsable(asset, time.Date(2026, time.August, 4, 0, 58, 0, 0, time.UTC)) {
|
||||
t.Fatal("request asset was refreshed before the safety window")
|
||||
}
|
||||
if requestAssetStillUsable(asset, time.Date(2026, time.August, 4, 0, 59, 45, 0, time.UTC)) {
|
||||
t.Fatal("request asset inside the signed URL safety window was reused")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalConversationMessageHashUsesTextAndAssetRefs(t *testing.T) {
|
||||
message := map[string]any{
|
||||
"role": "user",
|
||||
|
||||
@@ -280,7 +280,7 @@ func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request
|
||||
|
||||
// testFileStorageChannel godoc
|
||||
// @Summary 测试对象存储通道
|
||||
// @Description 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针,不返回凭据或对象键。
|
||||
// @Description 对指定 OSS 或 S3 通道检查并初始化临时文件生命周期规则,然后执行隔离的 Put、Head、Delete 探针;不返回凭据或对象键。
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
@@ -358,9 +358,26 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
|
||||
if !validFileStorageBaseURL(endpoint) {
|
||||
return "object storage config.endpoint must be an http or https URL without embedded credentials"
|
||||
}
|
||||
if publicBaseURL := firstNonEmpty(fileStorageConfigString(input.Config, "publicBaseUrl"), fileStorageConfigString(input.Config, "publicBaseURL")); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
|
||||
if publicBaseURL := firstNonEmpty(
|
||||
fileStorageConfigString(input.Config, "publicBaseUrl"),
|
||||
fileStorageConfigString(input.Config, "publicBaseURL"),
|
||||
fileStorageConfigString(input.Config, "cdnDomain"),
|
||||
fileStorageConfigString(input.Config, "publicDomain"),
|
||||
); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
|
||||
return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials"
|
||||
}
|
||||
if policy := firstNonEmpty(
|
||||
fileStorageConfigString(input.Config, "temporaryFileExpirePolicy"),
|
||||
fileStorageConfigString(input.Config, "apiReturnExpirePolicy"),
|
||||
); policy != "" && !validFileStorageExpirationPolicy(policy) {
|
||||
return "object storage config.temporaryFileExpirePolicy must be never, 1d, 1m, 3m or 6m"
|
||||
}
|
||||
if value, exists := input.Config["signedUrlExpiresSeconds"]; exists {
|
||||
seconds, ok := fileStorageConfigInteger(value)
|
||||
if !ok || seconds < 60 || seconds > 7*24*60*60 {
|
||||
return "object storage config.signedUrlExpiresSeconds must be an integer between 60 and 604800"
|
||||
}
|
||||
}
|
||||
accessKeyID := input.AccessKeyID
|
||||
if accessKeyID == nil {
|
||||
accessKeyID = input.AccessKey
|
||||
@@ -378,6 +395,34 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
|
||||
return ""
|
||||
}
|
||||
|
||||
func validFileStorageExpirationPolicy(value string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "never", "1d", "1m", "3m", "6m":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func fileStorageConfigInteger(value any) (int64, bool) {
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return int64(typed), true
|
||||
case int32:
|
||||
return int64(typed), true
|
||||
case int64:
|
||||
return typed, true
|
||||
case float64:
|
||||
converted := int64(typed)
|
||||
return converted, float64(converted) == typed
|
||||
case float32:
|
||||
converted := int64(typed)
|
||||
return converted, float32(converted) == typed
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func validFileStorageBaseURL(value string) bool {
|
||||
parsed, err := url.Parse(strings.TrimSpace(value))
|
||||
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
|
||||
|
||||
@@ -53,3 +53,34 @@ func TestValidateObjectStorageChannelRejectsCredentialedEndpoint(t *testing.T) {
|
||||
t.Fatalf("credentialed endpoint was accepted: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateObjectStorageChannelAcceptsExpirationAndCDNAliases(t *testing.T) {
|
||||
input := store.FileStorageChannelInput{
|
||||
ChannelKey: "oss-expiring", Name: "OSS expiring", Provider: "aliyun_oss", Status: "disabled",
|
||||
Config: map[string]any{
|
||||
"endpoint": "https://oss-cn-shanghai.aliyuncs.com", "region": "cn-shanghai", "bucket": "media",
|
||||
"cdnDomain": "https://cdn.example.com", "temporaryFileExpirePolicy": "3m", "signedUrlExpiresSeconds": float64(3600),
|
||||
},
|
||||
}
|
||||
if message := validateFileStorageChannelInput(input, nil); message != "" {
|
||||
t.Fatalf("valid object storage expiration config rejected: %s", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateObjectStorageChannelRejectsInvalidExpiration(t *testing.T) {
|
||||
base := store.FileStorageChannelInput{
|
||||
ChannelKey: "oss-expiring", Name: "OSS expiring", Provider: "aliyun_oss", Status: "disabled",
|
||||
Config: map[string]any{
|
||||
"endpoint": "https://oss-cn-shanghai.aliyuncs.com", "region": "cn-shanghai", "bucket": "media",
|
||||
"temporaryFileExpirePolicy": "7d", "signedUrlExpiresSeconds": float64(3600),
|
||||
},
|
||||
}
|
||||
if message := validateFileStorageChannelInput(base, nil); !strings.Contains(message, "temporaryFileExpirePolicy") {
|
||||
t.Fatalf("invalid expiration policy was accepted: %q", message)
|
||||
}
|
||||
base.Config["temporaryFileExpirePolicy"] = "1d"
|
||||
base.Config["signedUrlExpiresSeconds"] = float64(30)
|
||||
if message := validateFileStorageChannelInput(base, nil); !strings.Contains(message, "signedUrlExpiresSeconds") {
|
||||
t.Fatalf("invalid signed URL TTL was accepted: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,11 +47,13 @@ type objectStorageAdapter struct {
|
||||
}
|
||||
|
||||
type FileStorageChannelTestResult struct {
|
||||
Provider string `json:"provider"`
|
||||
PutSucceeded bool `json:"putSucceeded"`
|
||||
HeadSucceeded bool `json:"headSucceeded"`
|
||||
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||
DurationMS int64 `json:"durationMs"`
|
||||
Provider string `json:"provider"`
|
||||
PutSucceeded bool `json:"putSucceeded"`
|
||||
HeadSucceeded bool `json:"headSucceeded"`
|
||||
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||
LifecycleReady bool `json:"lifecycleReady"`
|
||||
ExpirationPolicy string `json:"expirationPolicy"`
|
||||
DurationMS int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
// TestFileStorageChannel performs an isolated write, metadata read and cleanup
|
||||
@@ -59,16 +61,29 @@ type FileStorageChannelTestResult struct {
|
||||
// content-addressed business object and no object key or credential is exposed.
|
||||
func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) {
|
||||
startedAt := time.Now()
|
||||
result := FileStorageChannelTestResult{Provider: strings.ToLower(strings.TrimSpace(channel.Provider))}
|
||||
result := FileStorageChannelTestResult{
|
||||
Provider: strings.ToLower(strings.TrimSpace(channel.Provider)),
|
||||
ExpirationPolicy: objectStorageExpirationPolicy(channel.Config),
|
||||
}
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if result.ExpirationPolicy != objectStorageExpirationNever {
|
||||
if err := adapter.ensureLifecycle(ctx); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.LifecycleReady = true
|
||||
}
|
||||
probeScene := store.FileStorageSceneUpload
|
||||
if result.LifecycleReady {
|
||||
probeScene = store.FileStorageSceneImageResult
|
||||
}
|
||||
payload := FileUploadPayload{
|
||||
Bytes: []byte("easyai-storage-probe:" + uuid.NewString()),
|
||||
ContentType: "application/octet-stream",
|
||||
FileName: "probe.bin",
|
||||
Scene: store.FileStorageSceneUpload,
|
||||
Scene: probeScene,
|
||||
Source: "admin-connection-test",
|
||||
}
|
||||
upload, err := adapter.put(ctx, payload)
|
||||
@@ -112,6 +127,12 @@ func newObjectStorageAdapter(channel store.FileStorageChannel) (*objectStorageAd
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
|
||||
retention := objectStorageRetentionFor(a.channel.Config, payload.Scene, a.now().UTC())
|
||||
if retention.Enabled {
|
||||
if err := a.ensureLifecycle(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
objectKey := a.objectKey(payload)
|
||||
requestURL, err := a.objectURL(objectKey)
|
||||
if err != nil {
|
||||
@@ -126,6 +147,14 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
|
||||
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
if retention.Enabled {
|
||||
tagging := retention.taggingHeader()
|
||||
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||
req.Header.Set("x-amz-tagging", tagging)
|
||||
} else {
|
||||
req.Header.Set("x-oss-tagging", tagging)
|
||||
}
|
||||
}
|
||||
if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -148,11 +177,12 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
|
||||
}
|
||||
urlExpiresAt := ""
|
||||
if publicURL == "" {
|
||||
publicURL, err = a.presignGet(objectKey, objectStorageSignedURLTTL)
|
||||
signedURLTTL := objectStorageSignedURLTTLForConfig(a.channel.Config)
|
||||
publicURL, err = a.presignGet(objectKey, signedURLTTL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
urlExpiresAt = a.now().UTC().Add(objectStorageSignedURLTTL).Format(time.RFC3339)
|
||||
urlExpiresAt = a.now().UTC().Add(signedURLTTL).Format(time.RFC3339)
|
||||
}
|
||||
digest := sha256.Sum256(payload.Bytes)
|
||||
result := map[string]any{
|
||||
@@ -173,6 +203,10 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
|
||||
if urlExpiresAt != "" {
|
||||
result["urlExpiresAt"] = urlExpiresAt
|
||||
}
|
||||
if retention.Enabled {
|
||||
result["objectExpiresAt"] = retention.ExpiresAt.Format(time.RFC3339)
|
||||
result["expirationPolicy"] = retention.Policy
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -301,10 +335,7 @@ func (a *objectStorageAdapter) objectURL(objectKey string) (string, error) {
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) publicURL(objectKey string) string {
|
||||
baseURL := strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseUrl"), "/")
|
||||
if baseURL == "" {
|
||||
baseURL = strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseURL"), "/")
|
||||
}
|
||||
baseURL := strings.TrimRight(objectStoragePublicBaseURL(a.channel.Config), "/")
|
||||
if baseURL == "" {
|
||||
return ""
|
||||
}
|
||||
@@ -322,19 +353,45 @@ func (a *objectStorageAdapter) sign(req *http.Request, payloadHash string, now t
|
||||
|
||||
func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) {
|
||||
date := now.UTC().Format(http.TimeFormat)
|
||||
contentMD5 := req.Header.Get("Content-MD5")
|
||||
contentType := req.Header.Get("Content-Type")
|
||||
canonicalHeaders := ""
|
||||
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||
req.Header.Set("x-oss-security-token", token)
|
||||
canonicalHeaders = "x-oss-security-token:" + token + "\n"
|
||||
}
|
||||
canonicalHeaders := canonicalOSSHeaders(req)
|
||||
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath()
|
||||
stringToSign := req.Method + "\n\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
|
||||
if _, ok := req.URL.Query()["lifecycle"]; ok {
|
||||
canonicalResource += "?lifecycle"
|
||||
}
|
||||
stringToSign := req.Method + "\n" + contentMD5 + "\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
|
||||
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign)
|
||||
req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature)
|
||||
req.Header.Set("Date", date)
|
||||
}
|
||||
|
||||
func canonicalOSSHeaders(req *http.Request) string {
|
||||
headers := map[string]string{}
|
||||
for key, values := range req.Header {
|
||||
lower := strings.ToLower(strings.TrimSpace(key))
|
||||
if strings.HasPrefix(lower, "x-oss-") {
|
||||
headers[lower] = strings.Join(values, ",")
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(headers))
|
||||
for key := range headers {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var canonical strings.Builder
|
||||
for _, key := range keys {
|
||||
canonical.WriteString(key)
|
||||
canonical.WriteByte(':')
|
||||
canonical.WriteString(strings.Join(strings.Fields(headers[key]), " "))
|
||||
canonical.WriteByte('\n')
|
||||
}
|
||||
return canonical.String()
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) {
|
||||
parsed, err := url.Parse(requestURL)
|
||||
if err != nil {
|
||||
@@ -487,18 +544,27 @@ func objectStorageConfigBool(config map[string]any, key string) bool {
|
||||
}
|
||||
|
||||
func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string {
|
||||
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
|
||||
return scope
|
||||
}
|
||||
if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset {
|
||||
return "private"
|
||||
}
|
||||
if strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseUrl")) != "" || strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseURL")) != "" {
|
||||
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
|
||||
return scope
|
||||
}
|
||||
if objectStoragePublicBaseURL(channel.Config) != "" {
|
||||
return "public"
|
||||
}
|
||||
return "private"
|
||||
}
|
||||
|
||||
func objectStoragePublicBaseURL(config map[string]any) string {
|
||||
return firstNonEmptyString(
|
||||
objectStorageConfigString(config, "publicBaseUrl"),
|
||||
objectStorageConfigString(config, "publicBaseURL"),
|
||||
objectStorageConfigString(config, "cdnDomain"),
|
||||
objectStorageConfigString(config, "publicDomain"),
|
||||
)
|
||||
}
|
||||
|
||||
func escapeObjectKey(objectKey string) string {
|
||||
parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/")
|
||||
for index, part := range parts {
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
objectStorageExpirationNever = "never"
|
||||
objectStorageExpiration1Day = "1d"
|
||||
objectStorageExpiration1Month = "1m"
|
||||
objectStorageExpiration3Months = "3m"
|
||||
objectStorageExpiration6Months = "6m"
|
||||
|
||||
objectStorageSignedURLTTLMin = time.Minute
|
||||
objectStorageSignedURLTTLMax = 7 * 24 * time.Hour
|
||||
)
|
||||
|
||||
type objectStorageLifecycleDefinition struct {
|
||||
ID string
|
||||
TagValue string
|
||||
Days int
|
||||
}
|
||||
|
||||
var objectStorageLifecycleDefinitions = []objectStorageLifecycleDefinition{
|
||||
{ID: "DeleteTempFiles-1d", TagValue: objectStorageExpiration1Day, Days: 1},
|
||||
{ID: "DeleteTempFiles-1m", TagValue: objectStorageExpiration1Month, Days: 30},
|
||||
{ID: "DeleteTempFiles-3m", TagValue: objectStorageExpiration3Months, Days: 90},
|
||||
{ID: "DeleteTempFiles-6m", TagValue: objectStorageExpiration6Months, Days: 180},
|
||||
}
|
||||
|
||||
type objectStorageRetention struct {
|
||||
Enabled bool
|
||||
Policy string
|
||||
TagValue string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
func (r objectStorageRetention) taggingHeader() string {
|
||||
values := url.Values{}
|
||||
values.Set("TempFile", "true")
|
||||
values.Set("ExpiresAt", strconv.FormatInt(r.ExpiresAt.Unix(), 10))
|
||||
values.Set("ExpiresIn", r.TagValue)
|
||||
return values.Encode()
|
||||
}
|
||||
|
||||
func objectStorageExpirationPolicy(config map[string]any) string {
|
||||
policy := strings.ToLower(firstNonEmptyString(
|
||||
objectStorageConfigString(config, "temporaryFileExpirePolicy"),
|
||||
objectStorageConfigString(config, "apiReturnExpirePolicy"),
|
||||
))
|
||||
switch policy {
|
||||
case objectStorageExpiration1Day,
|
||||
objectStorageExpiration1Month,
|
||||
objectStorageExpiration3Months,
|
||||
objectStorageExpiration6Months:
|
||||
return policy
|
||||
default:
|
||||
return objectStorageExpirationNever
|
||||
}
|
||||
}
|
||||
|
||||
func objectStorageRetentionFor(config map[string]any, scene string, now time.Time) objectStorageRetention {
|
||||
if strings.TrimSpace(scene) != "image_result" && strings.TrimSpace(scene) != "request_asset" {
|
||||
return objectStorageRetention{Policy: objectStorageExpirationNever}
|
||||
}
|
||||
policy := objectStorageExpirationPolicy(config)
|
||||
duration := objectStorageExpirationDuration(policy)
|
||||
if duration <= 0 {
|
||||
return objectStorageRetention{Policy: objectStorageExpirationNever}
|
||||
}
|
||||
return objectStorageRetention{
|
||||
Enabled: true,
|
||||
Policy: policy,
|
||||
TagValue: policy,
|
||||
ExpiresAt: now.Add(duration),
|
||||
}
|
||||
}
|
||||
|
||||
func objectStorageExpirationDuration(policy string) time.Duration {
|
||||
switch policy {
|
||||
case objectStorageExpiration1Day:
|
||||
return 24 * time.Hour
|
||||
case objectStorageExpiration1Month:
|
||||
return 30 * 24 * time.Hour
|
||||
case objectStorageExpiration3Months:
|
||||
return 90 * 24 * time.Hour
|
||||
case objectStorageExpiration6Months:
|
||||
return 180 * 24 * time.Hour
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func objectStorageSignedURLTTLForConfig(config map[string]any) time.Duration {
|
||||
seconds := int64FromAny(config["signedUrlExpiresSeconds"])
|
||||
if seconds <= 0 {
|
||||
return objectStorageSignedURLTTL
|
||||
}
|
||||
ttl := time.Duration(seconds) * time.Second
|
||||
if ttl < objectStorageSignedURLTTLMin {
|
||||
return objectStorageSignedURLTTLMin
|
||||
}
|
||||
if ttl > objectStorageSignedURLTTLMax {
|
||||
return objectStorageSignedURLTTLMax
|
||||
}
|
||||
return ttl
|
||||
}
|
||||
|
||||
func int64FromAny(value any) int64 {
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return int64(typed)
|
||||
case int32:
|
||||
return int64(typed)
|
||||
case int64:
|
||||
return typed
|
||||
case float32:
|
||||
return int64(typed)
|
||||
case float64:
|
||||
return int64(typed)
|
||||
case string:
|
||||
parsed, _ := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||
return parsed
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
objectStorageLifecycleReady sync.Map
|
||||
objectStorageLifecycleLocks sync.Map
|
||||
)
|
||||
|
||||
func (a *objectStorageAdapter) ensureLifecycle(ctx context.Context) error {
|
||||
cacheKey := strings.Join([]string{
|
||||
strings.ToLower(strings.TrimSpace(a.channel.Provider)),
|
||||
objectStorageConfigString(a.channel.Config, "endpoint"),
|
||||
objectStorageConfigString(a.channel.Config, "bucket"),
|
||||
}, "\x00")
|
||||
if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
lockValue, _ := objectStorageLifecycleLocks.LoadOrStore(cacheKey, &sync.Mutex{})
|
||||
lifecycleLock := lockValue.(*sync.Mutex)
|
||||
lifecycleLock.Lock()
|
||||
defer lifecycleLock.Unlock()
|
||||
if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
current, err := a.readLifecycle(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next, changed, err := mergeObjectStorageLifecycle(a.channel.Provider, current)
|
||||
if err != nil {
|
||||
return storageClientError("storage_config_invalid", "object storage lifecycle configuration is invalid", 0, false)
|
||||
}
|
||||
if changed {
|
||||
if err := a.writeLifecycle(ctx, next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
objectStorageLifecycleReady.Store(cacheKey, struct{}{})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) readLifecycle(ctx context.Context) ([]byte, error) {
|
||||
requestURL, err := a.bucketLifecycleURL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_write_failed", "object storage lifecycle check failed", 0, true)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
|
||||
if readErr != nil {
|
||||
return nil, storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) writeLifecycle(ctx context.Context, payload []byte) error {
|
||||
requestURL, err := a.bucketLifecycleURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
digest := md5.Sum(payload)
|
||||
req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(digest[:]))
|
||||
req.Header.Set("Content-Type", "application/xml")
|
||||
if err := a.sign(req, sha256Hex(payload), a.now().UTC()); err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return storageClientError("storage_write_failed", "object storage lifecycle configuration failed", 0, true)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
if readErr != nil {
|
||||
return storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) bucketLifecycleURL() (string, error) {
|
||||
endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/"))
|
||||
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil {
|
||||
return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false)
|
||||
}
|
||||
bucket := objectStorageConfigString(a.channel.Config, "bucket")
|
||||
hostname := endpoint.Hostname()
|
||||
forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle")
|
||||
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||
if forcePathStyle {
|
||||
endpoint.Path = path.Join(endpoint.Path, bucket)
|
||||
} else if !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
||||
if port := endpoint.Port(); port != "" {
|
||||
endpoint.Host = bucket + "." + hostname + ":" + port
|
||||
} else {
|
||||
endpoint.Host = bucket + "." + endpoint.Host
|
||||
}
|
||||
}
|
||||
} else if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
||||
if port := endpoint.Port(); port != "" {
|
||||
endpoint.Host = bucket + "." + hostname + ":" + port
|
||||
} else {
|
||||
endpoint.Host = bucket + "." + endpoint.Host
|
||||
}
|
||||
}
|
||||
if endpoint.Path == "" {
|
||||
endpoint.Path = "/"
|
||||
}
|
||||
endpoint.RawQuery = "lifecycle"
|
||||
endpoint.Fragment = ""
|
||||
return endpoint.String(), nil
|
||||
}
|
||||
|
||||
type rawLifecycleConfiguration struct {
|
||||
Rules []rawLifecycleRule `xml:"Rule"`
|
||||
}
|
||||
|
||||
type rawLifecycleRule struct {
|
||||
ID string `xml:"ID"`
|
||||
Status string `xml:"Status"`
|
||||
Tag rawLifecycleTag `xml:"Tag"`
|
||||
Filter rawLifecycleFilter `xml:"Filter"`
|
||||
Expiration rawLifecycleExpiration `xml:"Expiration"`
|
||||
InnerXML string `xml:",innerxml"`
|
||||
}
|
||||
|
||||
type rawLifecycleFilter struct {
|
||||
Tag rawLifecycleTag `xml:"Tag"`
|
||||
}
|
||||
|
||||
type rawLifecycleTag struct {
|
||||
Key string `xml:"Key"`
|
||||
Value string `xml:"Value"`
|
||||
}
|
||||
|
||||
type rawLifecycleExpiration struct {
|
||||
Days int `xml:"Days"`
|
||||
}
|
||||
|
||||
func mergeObjectStorageLifecycle(provider string, current []byte) ([]byte, bool, error) {
|
||||
configuration := rawLifecycleConfiguration{}
|
||||
if len(bytes.TrimSpace(current)) > 0 {
|
||||
if err := xml.Unmarshal(current, &configuration); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
managed := map[string]objectStorageLifecycleDefinition{}
|
||||
for _, definition := range objectStorageLifecycleDefinitions {
|
||||
managed[definition.ID] = definition
|
||||
}
|
||||
allManagedReady := true
|
||||
for _, definition := range objectStorageLifecycleDefinitions {
|
||||
matched := false
|
||||
for _, rule := range configuration.Rules {
|
||||
if rule.ID == definition.ID {
|
||||
matched = lifecycleRuleMatches(rule, definition)
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
allManagedReady = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allManagedReady {
|
||||
return current, false, nil
|
||||
}
|
||||
|
||||
var output strings.Builder
|
||||
output.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
|
||||
if strings.EqualFold(strings.TrimSpace(provider), "s3") {
|
||||
output.WriteString(`<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`)
|
||||
} else {
|
||||
output.WriteString(`<LifecycleConfiguration>`)
|
||||
}
|
||||
for _, rule := range configuration.Rules {
|
||||
if _, isManaged := managed[rule.ID]; isManaged {
|
||||
continue
|
||||
}
|
||||
output.WriteString("<Rule>")
|
||||
output.WriteString(rule.InnerXML)
|
||||
output.WriteString("</Rule>")
|
||||
}
|
||||
for _, definition := range objectStorageLifecycleDefinitions {
|
||||
output.WriteString(lifecycleRuleXML(provider, definition))
|
||||
}
|
||||
output.WriteString(`</LifecycleConfiguration>`)
|
||||
return []byte(output.String()), true, nil
|
||||
}
|
||||
|
||||
func lifecycleRuleMatches(rule rawLifecycleRule, definition objectStorageLifecycleDefinition) bool {
|
||||
tag := rule.Tag
|
||||
if tag.Key == "" {
|
||||
tag = rule.Filter.Tag
|
||||
}
|
||||
return strings.EqualFold(strings.TrimSpace(rule.Status), "Enabled") &&
|
||||
tag.Key == "ExpiresIn" && tag.Value == definition.TagValue &&
|
||||
rule.Expiration.Days == definition.Days
|
||||
}
|
||||
|
||||
func lifecycleRuleXML(provider string, definition objectStorageLifecycleDefinition) string {
|
||||
if strings.EqualFold(strings.TrimSpace(provider), "s3") {
|
||||
return fmt.Sprintf(
|
||||
"<Rule><ID>%s</ID><Filter><Tag><Key>ExpiresIn</Key><Value>%s</Value></Tag></Filter><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule>",
|
||||
definition.ID,
|
||||
definition.TagValue,
|
||||
definition.Days,
|
||||
)
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"<Rule><ID>%s</ID><Prefix></Prefix><Tag><Key>ExpiresIn</Key><Value>%s</Value></Tag><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule>",
|
||||
definition.ID,
|
||||
definition.TagValue,
|
||||
definition.Days,
|
||||
)
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -80,6 +81,7 @@ func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.
|
||||
defer server.Close()
|
||||
channel := testObjectStorageChannel("s3", server.URL, "s3-private")
|
||||
channel.Config["publicBaseUrl"] = "https://cdn.example"
|
||||
channel.Config["accessScope"] = "public"
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -101,6 +103,112 @@ func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrivateSignedURLUsesConfiguredTTL(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
channel := testObjectStorageChannel("s3", server.URL, "s3-private-ttl")
|
||||
channel.Config["signedUrlExpiresSeconds"] = 3600
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
|
||||
|
||||
result, err := adapter.put(t.Context(), FileUploadPayload{
|
||||
Bytes: []byte("private input"), ContentType: "image/png", FileName: "input.png", Scene: store.FileStorageSceneRequestAsset,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parsed, err := url.Parse(stringFromAny(result["url"]))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := parsed.Query().Get("X-Amz-Expires"); got != "3600" {
|
||||
t.Fatalf("X-Amz-Expires=%q, want 3600", got)
|
||||
}
|
||||
if got := stringFromAny(result["urlExpiresAt"]); got != "2026-08-04T01:00:00Z" {
|
||||
t.Fatalf("urlExpiresAt=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemporaryObjectEnsuresS3LifecycleAndAddsTagging(t *testing.T) {
|
||||
var methods []string
|
||||
var lifecyclePayload string
|
||||
var tagging string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, lifecycle := r.URL.Query()["lifecycle"]; lifecycle {
|
||||
methods = append(methods, r.Method+" lifecycle")
|
||||
if r.Method == http.MethodGet {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
lifecyclePayload = string(body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
methods = append(methods, r.Method+" object")
|
||||
tagging = r.Header.Get("x-amz-tagging")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
channel := testObjectStorageChannel("s3", server.URL, "s3-expiring")
|
||||
channel.Config["temporaryFileExpirePolicy"] = "1m"
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
|
||||
result, err := adapter.put(t.Context(), FileUploadPayload{
|
||||
Bytes: []byte("temporary image"), ContentType: "image/png", Scene: store.FileStorageSceneImageResult,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(methods, ","); got != "GET lifecycle,PUT lifecycle,PUT object" {
|
||||
t.Fatalf("methods=%s", got)
|
||||
}
|
||||
for _, fragment := range []string{"DeleteTempFiles-1d", "DeleteTempFiles-1m", "DeleteTempFiles-3m", "DeleteTempFiles-6m", "<Value>1m</Value>"} {
|
||||
if !strings.Contains(lifecyclePayload, fragment) {
|
||||
t.Fatalf("lifecycle payload missing %q: %s", fragment, lifecyclePayload)
|
||||
}
|
||||
}
|
||||
values, err := url.ParseQuery(tagging)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if values.Get("TempFile") != "true" || values.Get("ExpiresIn") != "1m" || values.Get("ExpiresAt") != "1788393600" {
|
||||
t.Fatalf("unexpected tagging: %q", tagging)
|
||||
}
|
||||
if got := stringFromAny(result["objectExpiresAt"]); got != "2026-09-03T00:00:00Z" {
|
||||
t.Fatalf("objectExpiresAt=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeObjectStorageLifecyclePreservesUnrelatedRules(t *testing.T) {
|
||||
current := []byte(`<?xml version="1.0"?><LifecycleConfiguration><Rule><ID>KeepArchive</ID><Prefix>archive/</Prefix><Status>Enabled</Status><Expiration><Days>365</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||
merged, changed, err := mergeObjectStorageLifecycle("aliyun_oss", current)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !changed {
|
||||
t.Fatal("lifecycle merge unexpectedly reported no change")
|
||||
}
|
||||
value := string(merged)
|
||||
if !strings.Contains(value, "<ID>KeepArchive</ID>") || !strings.Contains(value, "<Prefix>archive/</Prefix>") {
|
||||
t.Fatalf("unrelated lifecycle rule was not preserved: %s", value)
|
||||
}
|
||||
for _, definition := range objectStorageLifecycleDefinitions {
|
||||
if !strings.Contains(value, "<ID>"+definition.ID+"</ID>") {
|
||||
t.Fatalf("managed lifecycle rule %s missing", definition.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) {
|
||||
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
||||
channel.Config["forcePathStyle"] = false
|
||||
@@ -138,6 +246,37 @@ func TestAliyunOSSRequestUsesSignatureV1(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAliyunOSSSignatureIncludesLifecycleTaggingHeader(t *testing.T) {
|
||||
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-tagged")
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request, err := http.NewRequest(http.MethodPut, "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/image_result/object.png", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.Header.Set("Content-Type", "image/png")
|
||||
request.Header.Set("x-oss-tagging", "ExpiresAt=1788393600&ExpiresIn=1m&TempFile=true")
|
||||
now := time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC)
|
||||
adapter.signOSSRequest(request, now)
|
||||
stringToSign := "PUT\n\nimage/png\nTue, 04 Aug 2026 01:02:03 GMT\n" +
|
||||
"x-oss-tagging:ExpiresAt=1788393600&ExpiresIn=1m&TempFile=true\n" +
|
||||
"/bucket/media/image_result/object.png"
|
||||
want := "OSS access-id:" + hmacSHA1Base64([]byte("access-secret"), stringToSign)
|
||||
if got := request.Header.Get("Authorization"); got != want {
|
||||
t.Fatalf("authorization=%q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemporaryRetentionDoesNotApplyToPersistentUploadScene(t *testing.T) {
|
||||
config := map[string]any{"temporaryFileExpirePolicy": "1d"}
|
||||
retention := objectStorageRetentionFor(config, store.FileStorageSceneUpload, time.Now())
|
||||
if retention.Enabled || retention.Policy != objectStorageExpirationNever {
|
||||
t.Fatalf("persistent upload unexpectedly expires: %+v", retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) {
|
||||
var primaryCalls atomic.Int64
|
||||
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
@@ -1191,7 +1191,10 @@ export function App() {
|
||||
const refreshed = await listFileStorageChannels(token);
|
||||
setFileStorageChannels(refreshed.items);
|
||||
setCoreState('ready');
|
||||
setCoreMessage(`对象存储连接测试通过,Put / Head / Delete 均成功,耗时 ${result.durationMs}ms。`);
|
||||
const lifecycleMessage = result.expirationPolicy !== 'never'
|
||||
? `生命周期 ${result.expirationPolicy} 已就绪,`
|
||||
: '';
|
||||
setCoreMessage(`对象存储连接测试通过,${lifecycleMessage}Put / Head / Delete 均成功,耗时 ${result.durationMs}ms。`);
|
||||
} catch (err) {
|
||||
setCoreState('error');
|
||||
setCoreMessage(err instanceof Error ? err.message : '对象存储连接测试失败');
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { FileStorageChannel } from '@easyai-ai-gateway/contracts';
|
||||
import { channelToForm, formToPayload } from './SystemSettingsPanel';
|
||||
|
||||
describe('SystemSettingsPanel object storage form', () => {
|
||||
it('promotes legacy CDN and expiration config while preserving unknown extensions', () => {
|
||||
const channel: FileStorageChannel = {
|
||||
id: 'channel-1',
|
||||
channelKey: 'oss-primary',
|
||||
name: 'OSS primary',
|
||||
provider: 'aliyun_oss',
|
||||
credentialsPreview: {},
|
||||
scenes: ['upload', 'image_result', 'request_asset'],
|
||||
config: {
|
||||
endpoint: 'https://oss-cn-shanghai.aliyuncs.com',
|
||||
region: 'cn-shanghai',
|
||||
bucket: 'media',
|
||||
cdnDomain: 'https://cdn.example.com/',
|
||||
apiReturnExpirePolicy: '3m',
|
||||
signedUrlExpiresSeconds: 3600,
|
||||
customHeader: 'retained',
|
||||
},
|
||||
retryPolicy: {},
|
||||
priority: 100,
|
||||
status: 'enabled',
|
||||
createdAt: '2026-08-04T00:00:00Z',
|
||||
updatedAt: '2026-08-04T00:00:00Z',
|
||||
};
|
||||
|
||||
const form = channelToForm(channel);
|
||||
expect(form.publicBaseUrl).toBe('https://cdn.example.com/');
|
||||
expect(form.temporaryFileExpirePolicy).toBe('3m');
|
||||
expect(form.signedUrlExpiresMinutes).toBe('60');
|
||||
expect(JSON.parse(form.configJson)).toEqual({ customHeader: 'retained' });
|
||||
|
||||
const payload = formToPayload({
|
||||
...form,
|
||||
objectPrefix: '/gateway/',
|
||||
temporaryFileExpirePolicy: '1m',
|
||||
});
|
||||
expect(payload.config).toEqual({
|
||||
endpoint: 'https://oss-cn-shanghai.aliyuncs.com',
|
||||
region: 'cn-shanghai',
|
||||
bucket: 'media',
|
||||
publicBaseUrl: 'https://cdn.example.com',
|
||||
objectPrefix: 'gateway',
|
||||
temporaryFileExpirePolicy: '1m',
|
||||
signedUrlExpiresSeconds: 3600,
|
||||
customHeader: 'retained',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -21,16 +21,24 @@ type FileStorageChannelForm = {
|
||||
accessKeySecretPreview: string;
|
||||
apiKey: string;
|
||||
apiKeyPreview: string;
|
||||
bucket: string;
|
||||
channelKey: string;
|
||||
configJson: string;
|
||||
endpoint: string;
|
||||
forcePathStyle: boolean;
|
||||
name: string;
|
||||
objectPrefix: string;
|
||||
priority: string;
|
||||
provider: string;
|
||||
publicBaseUrl: string;
|
||||
region: string;
|
||||
retryPolicyJson: string;
|
||||
sessionToken: string;
|
||||
sessionTokenPreview: string;
|
||||
signedUrlExpiresMinutes: string;
|
||||
scenes: string[];
|
||||
status: string;
|
||||
temporaryFileExpirePolicy: string;
|
||||
uploadUrl: string;
|
||||
};
|
||||
|
||||
@@ -55,6 +63,14 @@ const providerOptions = [
|
||||
{ value: 's3', label: 'S3 / S3 兼容存储' },
|
||||
];
|
||||
|
||||
const expirationPolicyOptions = [
|
||||
{ value: 'never', label: '永久保留' },
|
||||
{ value: '1d', label: '1 天' },
|
||||
{ value: '1m', label: '1 个月(30 天)' },
|
||||
{ value: '3m', label: '3 个月(90 天)' },
|
||||
{ value: '6m', label: '6 个月(180 天)' },
|
||||
];
|
||||
|
||||
const defaultScenes = ['upload', 'image_result', 'request_asset'];
|
||||
const sceneOptions = [
|
||||
{ value: 'upload', label: '上传', description: 'OpenAPI / 管理端主动上传文件' },
|
||||
@@ -126,6 +142,15 @@ export function SystemSettingsPanel(props: {
|
||||
setLocalError('请至少选择一个适用场景。');
|
||||
return;
|
||||
}
|
||||
if (form.provider !== 'server_main_openapi' && (!form.endpoint.trim() || !form.region.trim() || !form.bucket.trim())) {
|
||||
setLocalError('对象存储渠道必须填写 Endpoint、Region 和 Bucket。');
|
||||
return;
|
||||
}
|
||||
const signedUrlExpiresMinutes = Number(form.signedUrlExpiresMinutes);
|
||||
if (form.provider !== 'server_main_openapi' && (!Number.isInteger(signedUrlExpiresMinutes) || signedUrlExpiresMinutes < 1 || signedUrlExpiresMinutes > 10080)) {
|
||||
setLocalError('私有链接有效期必须是 1 至 10080 分钟之间的整数。');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await props.onSaveFileStorageChannel(formToPayload(form), editingChannel?.id);
|
||||
closeDialog();
|
||||
@@ -245,6 +270,10 @@ export function SystemSettingsPanel(props: {
|
||||
<span>场景: {sceneSummary(channel.scenes)}</span>
|
||||
<span>优先级: {channel.priority}</span>
|
||||
<span>重试: {retryPolicySummary(channel.retryPolicy)}</span>
|
||||
{channel.provider !== 'server_main_openapi' && configString(channel.config, 'region') && <span>Region: {configString(channel.config, 'region')}</span>}
|
||||
{channel.provider !== 'server_main_openapi' && configString(channel.config, 'bucket') && <span>Bucket: {configString(channel.config, 'bucket')}</span>}
|
||||
{channel.provider !== 'server_main_openapi' && publicBaseUrlFromConfig(channel.config) && <span>CDN: {publicBaseUrlFromConfig(channel.config)}</span>}
|
||||
{channel.provider !== 'server_main_openapi' && <span>临时文件: {expirationPolicyLabel(expirationPolicyFromConfig(channel.config))}</span>}
|
||||
{channel.uploadUrl && <span>上传路由: {channel.uploadUrl}</span>}
|
||||
{apiKeyPreview(channel) && <span>API Key: {apiKeyPreview(channel)}</span>}
|
||||
{channel.lastError && <span>最近错误: {channel.lastError}</span>}
|
||||
@@ -348,6 +377,7 @@ export function SystemSettingsPanel(props: {
|
||||
const provider = event.target.value;
|
||||
setForm({
|
||||
...form,
|
||||
name: editingChannel ? form.name : providerLabel(provider),
|
||||
provider,
|
||||
retryPolicyJson: editingChannel
|
||||
? form.retryPolicyJson
|
||||
@@ -388,6 +418,35 @@ export function SystemSettingsPanel(props: {
|
||||
<small>保持脱敏值不变表示不修改;填写新值表示覆盖;清空后保存表示清除。</small>
|
||||
</Label>}
|
||||
{form.provider !== 'server_main_openapi' && <>
|
||||
<Label className="spanTwo">
|
||||
上传 Endpoint
|
||||
<Input value={form.endpoint} onChange={(event) => setForm({ ...form, endpoint: event.target.value })} placeholder={form.provider === 'aliyun_oss' ? 'https://oss-cn-shanghai.aliyuncs.com' : 'https://s3.example.com'} />
|
||||
<small>服务端写入对象使用的地址,可填写同地域内网 Endpoint;不要填写 CDN 域名。</small>
|
||||
</Label>
|
||||
<Label>
|
||||
Region
|
||||
<Input value={form.region} onChange={(event) => setForm({ ...form, region: event.target.value })} placeholder={form.provider === 'aliyun_oss' ? 'cn-shanghai' : 'us-east-1'} />
|
||||
</Label>
|
||||
<Label>
|
||||
Bucket
|
||||
<Input value={form.bucket} onChange={(event) => setForm({ ...form, bucket: event.target.value })} placeholder="easyai-media" />
|
||||
</Label>
|
||||
<Label className="spanTwo">
|
||||
CDN / 公网访问域名(可选)
|
||||
<Input value={form.publicBaseUrl} onChange={(event) => setForm({ ...form, publicBaseUrl: event.target.value })} placeholder="https://cdn.example.com" />
|
||||
<small>生成媒体将返回此域名;请求素材仍使用短期私有签名 URL。</small>
|
||||
</Label>
|
||||
<Label>
|
||||
对象路径前缀(可选)
|
||||
<Input value={form.objectPrefix} onChange={(event) => setForm({ ...form, objectPrefix: event.target.value })} placeholder="gateway" />
|
||||
</Label>
|
||||
<Label>
|
||||
临时文件保留时间
|
||||
<Select value={form.temporaryFileExpirePolicy} onChange={(event) => setForm({ ...form, temporaryFileExpirePolicy: event.target.value })}>
|
||||
{expirationPolicyOptions.map((item) => <option value={item.value} key={item.value}>{item.label}</option>)}
|
||||
</Select>
|
||||
<small>仅用于生成媒体和请求素材;云端生命周期按天异步删除。</small>
|
||||
</Label>
|
||||
<Label>
|
||||
Access Key ID
|
||||
<Input value={form.accessKeyId} onChange={(event) => setForm({ ...form, accessKeyId: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeyIdPreview)} />
|
||||
@@ -396,23 +455,43 @@ export function SystemSettingsPanel(props: {
|
||||
Access Key Secret
|
||||
<Input type="password" value={form.accessKeySecret} onChange={(event) => setForm({ ...form, accessKeySecret: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeySecretPreview)} />
|
||||
</Label>
|
||||
<Label className="spanTwo">
|
||||
Session Token(可选)
|
||||
<Input type="password" value={form.sessionToken} onChange={(event) => setForm({ ...form, sessionToken: event.target.value })} placeholder={credentialInputPlaceholder(form.sessionTokenPreview)} />
|
||||
</Label>
|
||||
</>}
|
||||
<Label>
|
||||
优先级
|
||||
<Input type="number" min={1} value={form.priority} onChange={(event) => setForm({ ...form, priority: event.target.value })} />
|
||||
</Label>
|
||||
<Label className="spanTwo">
|
||||
重试策略 JSON
|
||||
<Textarea value={form.retryPolicyJson} onChange={(event) => setForm({ ...form, retryPolicyJson: event.target.value })} />
|
||||
</Label>
|
||||
<Label className="spanTwo">
|
||||
扩展配置 JSON
|
||||
<Textarea value={form.configJson} onChange={(event) => setForm({ ...form, configJson: event.target.value })} />
|
||||
</Label>
|
||||
<details className="fileStorageAdvanced spanTwo">
|
||||
<summary>高级配置</summary>
|
||||
<div className="fileStorageAdvancedGrid">
|
||||
{form.provider !== 'server_main_openapi' && <>
|
||||
<Label className="spanTwo">
|
||||
Session Token(可选)
|
||||
<Input type="password" value={form.sessionToken} onChange={(event) => setForm({ ...form, sessionToken: event.target.value })} placeholder={credentialInputPlaceholder(form.sessionTokenPreview)} />
|
||||
</Label>
|
||||
<Label>
|
||||
私有链接有效期(分钟)
|
||||
<Input type="number" min={1} max={10080} value={form.signedUrlExpiresMinutes} onChange={(event) => setForm({ ...form, signedUrlExpiresMinutes: event.target.value })} />
|
||||
<small>只控制签名 URL,不会删除对象。</small>
|
||||
</Label>
|
||||
{form.provider === 's3' && <Label>
|
||||
Bucket 寻址方式
|
||||
<Select value={form.forcePathStyle ? 'path' : 'virtual'} onChange={(event) => setForm({ ...form, forcePathStyle: event.target.value === 'path' })}>
|
||||
<option value="virtual">Virtual Host(推荐)</option>
|
||||
<option value="path">Path Style</option>
|
||||
</Select>
|
||||
</Label>}
|
||||
</>}
|
||||
<Label className="spanTwo">
|
||||
重试策略 JSON
|
||||
<Textarea value={form.retryPolicyJson} onChange={(event) => setForm({ ...form, retryPolicyJson: event.target.value })} />
|
||||
</Label>
|
||||
<Label className="spanTwo">
|
||||
其他扩展配置 JSON
|
||||
<Textarea value={form.configJson} onChange={(event) => setForm({ ...form, configJson: event.target.value })} />
|
||||
<small>仅保存未被上方结构化字段管理的扩展项。</small>
|
||||
</Label>
|
||||
</div>
|
||||
</details>
|
||||
</FormDialog>
|
||||
|
||||
<ConfirmDialog
|
||||
@@ -464,22 +543,31 @@ function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
|
||||
accessKeySecretPreview: '',
|
||||
apiKey: '',
|
||||
apiKeyPreview: '',
|
||||
bucket: '',
|
||||
channelKey,
|
||||
configJson: '{}',
|
||||
endpoint: '',
|
||||
forcePathStyle: false,
|
||||
name: 'server-main OpenAPI',
|
||||
objectPrefix: '',
|
||||
priority: '100',
|
||||
provider: 'server_main_openapi',
|
||||
publicBaseUrl: '',
|
||||
region: '',
|
||||
retryPolicyJson: stringifyJson(defaultServerMainRetryPolicy),
|
||||
sessionToken: '',
|
||||
sessionTokenPreview: '',
|
||||
signedUrlExpiresMinutes: '15',
|
||||
scenes: defaultScenes,
|
||||
status: 'disabled',
|
||||
temporaryFileExpirePolicy: 'never',
|
||||
uploadUrl: defaultUploadUrl,
|
||||
};
|
||||
}
|
||||
|
||||
function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
||||
export function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
||||
const preview = apiKeyPreview(channel);
|
||||
const config = channel.config ?? {};
|
||||
return {
|
||||
accessKeyId: credentialPreview(channel, 'accessKeyId'),
|
||||
accessKeyIdPreview: credentialPreview(channel, 'accessKeyId'),
|
||||
@@ -487,27 +575,36 @@ function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
||||
accessKeySecretPreview: credentialPreview(channel, 'accessKeySecret'),
|
||||
apiKey: preview,
|
||||
apiKeyPreview: preview,
|
||||
bucket: configString(config, 'bucket'),
|
||||
channelKey: channel.channelKey,
|
||||
configJson: stringifyJson(channel.config ?? {}),
|
||||
configJson: stringifyJson(channel.provider === 'server_main_openapi' ? config : unmanagedObjectStorageConfig(config)),
|
||||
endpoint: configString(config, 'endpoint'),
|
||||
forcePathStyle: configBoolean(config, 'forcePathStyle'),
|
||||
name: channel.name,
|
||||
objectPrefix: configString(config, 'objectPrefix'),
|
||||
priority: String(channel.priority || 100),
|
||||
provider: channel.provider || 'server_main_openapi',
|
||||
publicBaseUrl: publicBaseUrlFromConfig(config),
|
||||
region: configString(config, 'region'),
|
||||
retryPolicyJson: stringifyJson(channel.retryPolicy ?? defaultRetryPolicyForProvider(channel.provider)),
|
||||
sessionToken: credentialPreview(channel, 'sessionToken'),
|
||||
sessionTokenPreview: credentialPreview(channel, 'sessionToken'),
|
||||
signedUrlExpiresMinutes: String(Math.max(1, Math.round(configNumber(config, 'signedUrlExpiresSeconds', 900) / 60))),
|
||||
scenes: normalizeScenes(channel.scenes),
|
||||
status: channel.status || 'disabled',
|
||||
temporaryFileExpirePolicy: expirationPolicyFromConfig(config),
|
||||
uploadUrl: channel.uploadUrl || defaultUploadUrl,
|
||||
};
|
||||
}
|
||||
|
||||
function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRequest {
|
||||
export function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRequest {
|
||||
const config = parseJsonObject(form.configJson, '其他扩展配置 JSON');
|
||||
return {
|
||||
accessKeyId: credentialPayloadValue(form.accessKeyId, form.accessKeyIdPreview),
|
||||
accessKeySecret: credentialPayloadValue(form.accessKeySecret, form.accessKeySecretPreview),
|
||||
apiKey: apiKeyPayloadValue(form),
|
||||
channelKey: form.channelKey.trim(),
|
||||
config: parseJsonObject(form.configJson, '扩展配置 JSON'),
|
||||
config: form.provider === 'server_main_openapi' ? config : objectStorageConfigPayload(form, config),
|
||||
name: form.name.trim(),
|
||||
priority: Number(form.priority) || 100,
|
||||
provider: form.provider,
|
||||
@@ -519,6 +616,73 @@ function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRe
|
||||
};
|
||||
}
|
||||
|
||||
const managedObjectStorageConfigKeys = new Set([
|
||||
'apiReturnExpirePolicy',
|
||||
'bucket',
|
||||
'cdnDomain',
|
||||
'endpoint',
|
||||
'forcePathStyle',
|
||||
'objectPrefix',
|
||||
'publicBaseURL',
|
||||
'publicBaseUrl',
|
||||
'publicDomain',
|
||||
'region',
|
||||
'signedUrlExpiresSeconds',
|
||||
'temporaryFileExpirePolicy',
|
||||
]);
|
||||
|
||||
function unmanagedObjectStorageConfig(config: Record<string, unknown>) {
|
||||
return Object.fromEntries(Object.entries(config).filter(([key]) => !managedObjectStorageConfigKeys.has(key)));
|
||||
}
|
||||
|
||||
function objectStorageConfigPayload(form: FileStorageChannelForm, extras: Record<string, unknown>) {
|
||||
const config: Record<string, unknown> = { ...unmanagedObjectStorageConfig(extras) };
|
||||
config.endpoint = form.endpoint.trim();
|
||||
config.region = form.region.trim();
|
||||
config.bucket = form.bucket.trim();
|
||||
config.temporaryFileExpirePolicy = normalizeExpirationPolicy(form.temporaryFileExpirePolicy);
|
||||
config.signedUrlExpiresSeconds = Math.max(1, Math.round(Number(form.signedUrlExpiresMinutes) || 15)) * 60;
|
||||
if (form.publicBaseUrl.trim()) config.publicBaseUrl = form.publicBaseUrl.trim().replace(/\/+$/, '');
|
||||
if (form.objectPrefix.trim()) config.objectPrefix = form.objectPrefix.trim().replace(/^\/+|\/+$/g, '');
|
||||
if (form.provider === 's3') config.forcePathStyle = form.forcePathStyle;
|
||||
return config;
|
||||
}
|
||||
|
||||
function configString(config: Record<string, unknown> | undefined, key: string) {
|
||||
const value = config?.[key];
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function configBoolean(config: Record<string, unknown> | undefined, key: string) {
|
||||
return config?.[key] === true;
|
||||
}
|
||||
|
||||
function configNumber(config: Record<string, unknown> | undefined, key: string, fallback: number) {
|
||||
const value = Number(config?.[key]);
|
||||
return Number.isFinite(value) && value > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function publicBaseUrlFromConfig(config: Record<string, unknown> | undefined) {
|
||||
return configString(config, 'publicBaseUrl')
|
||||
|| configString(config, 'publicBaseURL')
|
||||
|| configString(config, 'cdnDomain')
|
||||
|| configString(config, 'publicDomain');
|
||||
}
|
||||
|
||||
function expirationPolicyFromConfig(config: Record<string, unknown> | undefined) {
|
||||
return normalizeExpirationPolicy(configString(config, 'temporaryFileExpirePolicy') || configString(config, 'apiReturnExpirePolicy'));
|
||||
}
|
||||
|
||||
function normalizeExpirationPolicy(value: string) {
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return expirationPolicyOptions.some((item) => item.value === normalized) ? normalized : 'never';
|
||||
}
|
||||
|
||||
function expirationPolicyLabel(value: string) {
|
||||
const normalized = normalizeExpirationPolicy(value);
|
||||
return expirationPolicyOptions.find((item) => item.value === normalized)?.label ?? '永久保留';
|
||||
}
|
||||
|
||||
function parseJsonObject(value: string, label: string) {
|
||||
try {
|
||||
const parsed = JSON.parse(value || '{}') as unknown;
|
||||
|
||||
@@ -2441,6 +2441,27 @@
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.fileStorageAdvanced {
|
||||
padding: 0.75rem;
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: 0.625rem;
|
||||
background: var(--surface-subtle);
|
||||
}
|
||||
|
||||
.fileStorageAdvanced summary {
|
||||
cursor: pointer;
|
||||
color: var(--text-strong);
|
||||
font-size: 0.875rem;
|
||||
font-weight: var(--font-weight-semibold);
|
||||
}
|
||||
|
||||
.fileStorageAdvancedGrid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 0.875rem;
|
||||
padding-top: 0.875rem;
|
||||
}
|
||||
|
||||
.fileStorageSceneGrid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
@@ -2822,6 +2843,7 @@
|
||||
.fileStorageGrid,
|
||||
.fileStorageSettingsCard,
|
||||
.fileStorageSceneGrid,
|
||||
.fileStorageAdvancedGrid,
|
||||
.runtimePolicyFormBody,
|
||||
.fileStorageDialogBody,
|
||||
.runtimePolicyRows,
|
||||
|
||||
@@ -1077,6 +1077,8 @@ export interface FileStorageChannelTestResult {
|
||||
putSucceeded: boolean;
|
||||
headSucceeded: boolean;
|
||||
deleteSucceeded: boolean;
|
||||
lifecycleReady: boolean;
|
||||
expirationPolicy: 'never' | '1d' | '1m' | '3m' | '6m' | string;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHmac } from 'node:crypto';
|
||||
|
||||
const requiredEnvironment = [
|
||||
'AI_GATEWAY_TEST_OSS_ENDPOINT',
|
||||
'AI_GATEWAY_TEST_OSS_REGION',
|
||||
'AI_GATEWAY_TEST_OSS_ACCESS_KEY_ID',
|
||||
'AI_GATEWAY_TEST_OSS_ACCESS_KEY_SECRET',
|
||||
'AI_GATEWAY_TEST_OSS_BUCKET',
|
||||
];
|
||||
|
||||
for (const key of requiredEnvironment) {
|
||||
if (!String(process.env[key] ?? '').trim()) {
|
||||
throw new Error(`missing required environment: ${key}`);
|
||||
}
|
||||
}
|
||||
|
||||
const gatewayBaseURL = String(process.env.AI_GATEWAY_TEST_BASE_URL ?? 'http://127.0.0.1:8088').replace(/\/$/, '');
|
||||
const endpoint = new URL(process.env.AI_GATEWAY_TEST_OSS_ENDPOINT);
|
||||
const region = process.env.AI_GATEWAY_TEST_OSS_REGION.trim();
|
||||
const accessKeyID = process.env.AI_GATEWAY_TEST_OSS_ACCESS_KEY_ID.trim();
|
||||
const accessKeySecret = process.env.AI_GATEWAY_TEST_OSS_ACCESS_KEY_SECRET.trim();
|
||||
const bucket = process.env.AI_GATEWAY_TEST_OSS_BUCKET.trim();
|
||||
const expirationPolicy = String(process.env.AI_GATEWAY_TEST_OSS_EXPIRATION_POLICY ?? '1d').trim();
|
||||
const signedURLExpiresSeconds = Number(process.env.AI_GATEWAY_TEST_OSS_SIGNED_URL_EXPIRES_SECONDS ?? 900);
|
||||
const channelKey = String(
|
||||
process.env.AI_GATEWAY_TEST_OSS_CHANNEL_KEY ?? `aliyun-oss-${region}-${bucket}-local`,
|
||||
).trim();
|
||||
const channelName = String(
|
||||
process.env.AI_GATEWAY_TEST_OSS_CHANNEL_NAME ?? `Aliyun OSS ${region} ${bucket} 本地验收`,
|
||||
).trim();
|
||||
const managedLifecycleRuleIDs = [
|
||||
'DeleteTempFiles-1d',
|
||||
'DeleteTempFiles-1m',
|
||||
'DeleteTempFiles-3m',
|
||||
'DeleteTempFiles-6m',
|
||||
];
|
||||
|
||||
function base64URL(value) {
|
||||
return Buffer.from(value).toString('base64url');
|
||||
}
|
||||
|
||||
function managerToken() {
|
||||
if (!String(process.env.CONFIG_JWT_SECRET ?? '').trim()) {
|
||||
throw new Error('missing required environment: CONFIG_JWT_SECRET');
|
||||
}
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const header = base64URL(JSON.stringify({ alg: 'HS256', typ: 'JWT' }));
|
||||
const payload = base64URL(JSON.stringify({
|
||||
sub: 'local-oss-acceptance-manager',
|
||||
username: 'local-oss-acceptance-manager',
|
||||
role: ['manager'],
|
||||
source: 'gateway',
|
||||
gatewayUserId: 'local-oss-acceptance-manager',
|
||||
tokenPurpose: 'local_break_glass_manager',
|
||||
iat: now,
|
||||
exp: now + 15 * 60,
|
||||
}));
|
||||
const signature = createHmac('sha256', process.env.CONFIG_JWT_SECRET)
|
||||
.update(`${header}.${payload}`)
|
||||
.digest('base64url');
|
||||
return `${header}.${payload}.${signature}`;
|
||||
}
|
||||
|
||||
async function gatewayAuthorization() {
|
||||
const account = String(process.env.AI_GATEWAY_ONLINE_ACCOUNT ?? '').trim();
|
||||
const password = String(process.env.AI_GATEWAY_ONLINE_PASSWORD ?? '');
|
||||
const gatewayHostname = new URL(gatewayBaseURL).hostname;
|
||||
const localGateway = gatewayHostname === '127.0.0.1' || gatewayHostname === 'localhost' || gatewayHostname === '::1';
|
||||
if (account && password && !localGateway) {
|
||||
const response = await fetch(`${gatewayBaseURL}/api/v1/auth/login`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ account, password }),
|
||||
});
|
||||
const payload = await response.json().catch(() => ({}));
|
||||
if (!response.ok || !payload.accessToken) {
|
||||
throw new Error(`Gateway online login failed: HTTP ${response.status}`);
|
||||
}
|
||||
return `Bearer ${payload.accessToken}`;
|
||||
}
|
||||
return `Bearer ${managerToken()}`;
|
||||
}
|
||||
|
||||
const authorization = await gatewayAuthorization();
|
||||
|
||||
async function gatewayJSON(path, init = {}) {
|
||||
const response = await fetch(`${gatewayBaseURL}${path}`, {
|
||||
...init,
|
||||
headers: {
|
||||
Authorization: authorization,
|
||||
...(init.body && !(init.body instanceof FormData) ? { 'Content-Type': 'application/json' } : {}),
|
||||
...init.headers,
|
||||
},
|
||||
});
|
||||
const text = await response.text();
|
||||
let payload = {};
|
||||
if (text) {
|
||||
try {
|
||||
payload = JSON.parse(text);
|
||||
} catch {
|
||||
payload = { message: 'non-JSON response' };
|
||||
}
|
||||
}
|
||||
if (!response.ok) {
|
||||
const code = payload.code ?? payload.error?.code ?? 'unknown';
|
||||
const message = payload.message ?? payload.error?.message ?? 'request failed';
|
||||
throw new Error(`gateway ${init.method ?? 'GET'} ${path} failed: HTTP ${response.status} ${code} ${message}`);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
function objectURL(objectKey = '', lifecycle = false) {
|
||||
const url = new URL(endpoint.toString());
|
||||
url.hostname = `${bucket}.${url.hostname}`;
|
||||
url.pathname = objectKey
|
||||
? `/${objectKey.split('/').map((part) => encodeURIComponent(part)).join('/')}`
|
||||
: '/';
|
||||
url.search = lifecycle ? '?lifecycle' : '';
|
||||
return url;
|
||||
}
|
||||
|
||||
async function signedOSSRequest(method, objectKey = '', lifecycle = false) {
|
||||
const date = new Date().toUTCString();
|
||||
const canonicalResource = lifecycle
|
||||
? `/${bucket}/?lifecycle`
|
||||
: `/${bucket}/${objectKey}`;
|
||||
const stringToSign = `${method}\n\n\n${date}\n${canonicalResource}`;
|
||||
const signature = createHmac('sha1', accessKeySecret).update(stringToSign).digest('base64');
|
||||
return fetch(objectURL(objectKey, lifecycle), {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: `OSS ${accessKeyID}:${signature}`,
|
||||
Date: date,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function channelInput() {
|
||||
return {
|
||||
channelKey,
|
||||
name: channelName,
|
||||
provider: 'aliyun_oss',
|
||||
accessKeyId: accessKeyID,
|
||||
accessKeySecret,
|
||||
scenes: ['upload', 'image_result', 'request_asset'],
|
||||
config: {
|
||||
endpoint: endpoint.toString().replace(/\/$/, ''),
|
||||
region,
|
||||
bucket,
|
||||
objectKeyPrefix: 'easyai-ai-gateway/live-acceptance',
|
||||
accessScope: 'private',
|
||||
temporaryFileExpirePolicy: expirationPolicy,
|
||||
signedUrlExpiresSeconds: signedURLExpiresSeconds,
|
||||
},
|
||||
retryPolicy: {
|
||||
enabled: true,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: [0.25],
|
||||
},
|
||||
priority: 10,
|
||||
status: 'enabled',
|
||||
};
|
||||
}
|
||||
|
||||
const health = await gatewayJSON('/api/v1/healthz');
|
||||
const ready = await gatewayJSON('/api/v1/readyz');
|
||||
if (health.ok !== true || ready.ok !== true) {
|
||||
throw new Error('gateway health or readiness check failed');
|
||||
}
|
||||
|
||||
const listed = await gatewayJSON('/api/admin/system/file-storage/channels');
|
||||
const existing = Array.isArray(listed.items)
|
||||
? listed.items.find((item) => item.channelKey === channelKey)
|
||||
: undefined;
|
||||
const channel = existing
|
||||
? await gatewayJSON(`/api/admin/system/file-storage/channels/${encodeURIComponent(existing.id)}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(channelInput()),
|
||||
})
|
||||
: await gatewayJSON('/api/admin/system/file-storage/channels', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(channelInput()),
|
||||
});
|
||||
|
||||
const connectionTest = await gatewayJSON(
|
||||
`/api/admin/system/file-storage/channels/${encodeURIComponent(channel.id)}/test`,
|
||||
{ method: 'POST' },
|
||||
);
|
||||
for (const key of ['putSucceeded', 'headSucceeded', 'deleteSucceeded', 'lifecycleReady']) {
|
||||
if (connectionTest[key] !== true) {
|
||||
throw new Error(`object storage connection test did not prove ${key}`);
|
||||
}
|
||||
}
|
||||
if (connectionTest.expirationPolicy !== expirationPolicy) {
|
||||
throw new Error('object storage connection test returned an unexpected expiration policy');
|
||||
}
|
||||
|
||||
const lifecycleResponse = await signedOSSRequest('GET', '', true);
|
||||
const lifecycleXML = await lifecycleResponse.text();
|
||||
if (!lifecycleResponse.ok) {
|
||||
throw new Error(`OSS lifecycle read failed: HTTP ${lifecycleResponse.status}`);
|
||||
}
|
||||
const lifecycleRuleIDs = [...lifecycleXML.matchAll(/<ID>([^<]+)<\/ID>/g)].map((match) => match[1]);
|
||||
const missingLifecycleRules = managedLifecycleRuleIDs.filter((id) => !lifecycleRuleIDs.includes(id));
|
||||
if (missingLifecycleRules.length > 0) {
|
||||
throw new Error(`OSS lifecycle rules missing: ${missingLifecycleRules.join(', ')}`);
|
||||
}
|
||||
|
||||
const expectedContent = `easyai-oss-live-acceptance:${new Date().toISOString()}`;
|
||||
const form = new FormData();
|
||||
form.append('file', new Blob([expectedContent], { type: 'text/plain' }), 'oss-live-acceptance.txt');
|
||||
form.append('source', 'local-oss-live-acceptance');
|
||||
const upload = await gatewayJSON('/api/v1/files/upload', { method: 'POST', body: form });
|
||||
if (!upload.url || !upload.objectKey || upload.accessScope !== 'private') {
|
||||
throw new Error('Gateway upload response did not include a private OSS object and signed URL');
|
||||
}
|
||||
if (upload.storageChannel?.channelKey !== channelKey) {
|
||||
throw new Error(`Gateway upload did not select the expected OSS channel: ${upload.storageChannel?.channelKey ?? 'none'}`);
|
||||
}
|
||||
if (!upload.urlExpiresAt || upload.objectExpiresAt || upload.expirationPolicy) {
|
||||
throw new Error('normal upload must have a signed URL TTL but remain exempt from temporary-file expiration');
|
||||
}
|
||||
|
||||
const downloadResponse = await fetch(upload.url);
|
||||
const downloadedContent = await downloadResponse.text();
|
||||
if (!downloadResponse.ok || downloadedContent !== expectedContent) {
|
||||
throw new Error(`signed URL download failed: HTTP ${downloadResponse.status}`);
|
||||
}
|
||||
|
||||
const deleteResponse = await signedOSSRequest('DELETE', upload.objectKey);
|
||||
if (!deleteResponse.ok) {
|
||||
throw new Error(`OSS cleanup failed: HTTP ${deleteResponse.status}`);
|
||||
}
|
||||
const afterDeleteResponse = await fetch(upload.url);
|
||||
if (afterDeleteResponse.status !== 404) {
|
||||
throw new Error(`deleted OSS object remained readable: HTTP ${afterDeleteResponse.status}`);
|
||||
}
|
||||
|
||||
const signedURL = new URL(upload.url);
|
||||
console.log(JSON.stringify({
|
||||
gateway: {
|
||||
health: health.ok,
|
||||
ready: ready.ok,
|
||||
},
|
||||
channel: {
|
||||
action: existing ? 'updated' : 'created',
|
||||
id: channel.id,
|
||||
channelKey: channel.channelKey,
|
||||
provider: channel.provider,
|
||||
status: channel.status,
|
||||
region: channel.config?.region,
|
||||
bucket: channel.config?.bucket,
|
||||
expirationPolicy: channel.config?.temporaryFileExpirePolicy,
|
||||
credentialsConfigured: Boolean(channel.credentialsPreview?.accessKeyId)
|
||||
&& Boolean(channel.credentialsPreview?.accessKeySecret),
|
||||
},
|
||||
connectionTest,
|
||||
lifecycle: {
|
||||
status: lifecycleResponse.status,
|
||||
managedRuleIDs: managedLifecycleRuleIDs,
|
||||
},
|
||||
upload: {
|
||||
status: 'success',
|
||||
bytes: Buffer.byteLength(expectedContent),
|
||||
accessScope: upload.accessScope,
|
||||
signedURLHost: signedURL.host,
|
||||
signedURLExpiresAt: upload.urlExpiresAt,
|
||||
normalUploadIsPermanent: !upload.objectExpiresAt,
|
||||
downloadStatus: downloadResponse.status,
|
||||
deleteStatus: deleteResponse.status,
|
||||
afterDeleteStatus: afterDeleteResponse.status,
|
||||
},
|
||||
}, null, 2));
|
||||
Reference in New Issue
Block a user