feat(storage): 完善对象存储配置与过期策略
补齐 OSS/S3 的 Endpoint、Region、Bucket、CDN、对象前缀和签名有效期配置,并为生成结果与请求素材自动维护分级生命周期规则。普通上传继续保持永久,私有资源按配置生成限时签名 URL,管理端连接测试覆盖生命周期、上传、读取和删除。\n\n新增可重复的真实 OSS 验收脚本,凭据仅从本地环境读取,接口响应继续保持脱敏。\n\n验证:Go 全量测试、迁移安全检查、pnpm lint、pnpm test、pnpm build、本地阿里云 OSS 真实上传下载删除验收。
This commit is contained in:
@@ -3361,7 +3361,7 @@
|
|||||||
"BearerAuth": []
|
"BearerAuth": []
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"description": "对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针,不返回凭据或对象键。",
|
"description": "对指定 OSS 或 S3 通道检查并初始化临时文件生命周期规则,然后执行隔离的 Put、Head、Delete 探针;不返回凭据或对象键。",
|
||||||
"produces": [
|
"produces": [
|
||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
@@ -11340,9 +11340,15 @@
|
|||||||
"durationMs": {
|
"durationMs": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
|
"expirationPolicy": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"headSucceeded": {
|
"headSucceeded": {
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
|
"lifecycleReady": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
"provider": {
|
"provider": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -530,8 +530,12 @@ definitions:
|
|||||||
type: boolean
|
type: boolean
|
||||||
durationMs:
|
durationMs:
|
||||||
type: integer
|
type: integer
|
||||||
|
expirationPolicy:
|
||||||
|
type: string
|
||||||
headSucceeded:
|
headSucceeded:
|
||||||
type: boolean
|
type: boolean
|
||||||
|
lifecycleReady:
|
||||||
|
type: boolean
|
||||||
provider:
|
provider:
|
||||||
type: string
|
type: string
|
||||||
putSucceeded:
|
putSucceeded:
|
||||||
@@ -6558,7 +6562,7 @@ paths:
|
|||||||
- system
|
- system
|
||||||
/api/admin/system/file-storage/channels/{channelID}/test:
|
/api/admin/system/file-storage/channels/{channelID}/test:
|
||||||
post:
|
post:
|
||||||
description: 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针,不返回凭据或对象键。
|
description: 对指定 OSS 或 S3 通道检查并初始化临时文件生命周期规则,然后执行隔离的 Put、Head、Delete 探针;不返回凭据或对象键。
|
||||||
parameters:
|
parameters:
|
||||||
- description: 文件存储通道 ID
|
- description: 文件存储通道 ID
|
||||||
in: path
|
in: path
|
||||||
|
|||||||
@@ -238,11 +238,13 @@ type FileStorageChannelListResponse struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type FileStorageChannelTestResponse struct {
|
type FileStorageChannelTestResponse struct {
|
||||||
Provider string `json:"provider"`
|
Provider string `json:"provider"`
|
||||||
PutSucceeded bool `json:"putSucceeded"`
|
PutSucceeded bool `json:"putSucceeded"`
|
||||||
HeadSucceeded bool `json:"headSucceeded"`
|
HeadSucceeded bool `json:"headSucceeded"`
|
||||||
DeleteSucceeded bool `json:"deleteSucceeded"`
|
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||||
DurationMS int64 `json:"durationMs"`
|
LifecycleReady bool `json:"lifecycleReady"`
|
||||||
|
ExpirationPolicy string `json:"expirationPolicy"`
|
||||||
|
DurationMS int64 `json:"durationMs"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type FileUploadResponse struct {
|
type FileUploadResponse struct {
|
||||||
|
|||||||
@@ -417,6 +417,8 @@ func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestA
|
|||||||
expiry := now.Add(time.Duration(s.localTempAssetTTLHours()) * time.Hour)
|
expiry := now.Add(time.Duration(s.localTempAssetTTLHours()) * time.Hour)
|
||||||
expiresAt = &expiry
|
expiresAt = &expiry
|
||||||
localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"]))
|
localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"]))
|
||||||
|
} else {
|
||||||
|
expiresAt = requestAssetUploadExpiresAt(upload)
|
||||||
}
|
}
|
||||||
asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{
|
asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{
|
||||||
SHA256: sha,
|
SHA256: sha,
|
||||||
@@ -576,7 +578,7 @@ func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
|
|||||||
if asset.ExpiredAt != nil {
|
if asset.ExpiredAt != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if asset.ExpiresAt != nil && !asset.ExpiresAt.After(now) {
|
if asset.ExpiresAt != nil && !asset.ExpiresAt.After(now.Add(30*time.Second)) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if strings.EqualFold(strings.TrimSpace(asset.StorageProvider), "local_static") {
|
if strings.EqualFold(strings.TrimSpace(asset.StorageProvider), "local_static") {
|
||||||
@@ -592,6 +594,25 @@ func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
|
|||||||
return strings.TrimSpace(asset.URL) != ""
|
return strings.TrimSpace(asset.URL) != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func requestAssetUploadExpiresAt(upload map[string]any) *time.Time {
|
||||||
|
var earliest *time.Time
|
||||||
|
for _, key := range []string{"urlExpiresAt", "objectExpiresAt"} {
|
||||||
|
value := strings.TrimSpace(stringFromRequestAny(upload[key]))
|
||||||
|
if value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
parsed, err := time.Parse(time.RFC3339, value)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if earliest == nil || parsed.Before(*earliest) {
|
||||||
|
candidate := parsed
|
||||||
|
earliest = &candidate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return earliest
|
||||||
|
}
|
||||||
|
|
||||||
func requestAssetStorageProvider(upload map[string]any) string {
|
func requestAssetStorageProvider(upload map[string]any) string {
|
||||||
if channel, ok := upload["storageChannel"].(map[string]any); ok {
|
if channel, ok := upload["storageChannel"].(map[string]any); ok {
|
||||||
if provider := stringFromRequestAny(channel["provider"]); provider != "" {
|
if provider := stringFromRequestAny(channel["provider"]); provider != "" {
|
||||||
|
|||||||
@@ -176,6 +176,23 @@ func TestRequestAssetStillUsableRequiresExistingLocalFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRequestAssetUsesEarliestObjectStorageExpiry(t *testing.T) {
|
||||||
|
expiresAt := requestAssetUploadExpiresAt(map[string]any{
|
||||||
|
"urlExpiresAt": "2026-08-04T01:00:00Z",
|
||||||
|
"objectExpiresAt": "2026-09-03T00:00:00Z",
|
||||||
|
})
|
||||||
|
if expiresAt == nil || expiresAt.Format(time.RFC3339) != "2026-08-04T01:00:00Z" {
|
||||||
|
t.Fatalf("unexpected request asset expiry: %v", expiresAt)
|
||||||
|
}
|
||||||
|
asset := store.RequestAsset{URL: "https://signed.example/object", StorageProvider: "s3", ExpiresAt: expiresAt}
|
||||||
|
if !requestAssetStillUsable(asset, time.Date(2026, time.August, 4, 0, 58, 0, 0, time.UTC)) {
|
||||||
|
t.Fatal("request asset was refreshed before the safety window")
|
||||||
|
}
|
||||||
|
if requestAssetStillUsable(asset, time.Date(2026, time.August, 4, 0, 59, 45, 0, time.UTC)) {
|
||||||
|
t.Fatal("request asset inside the signed URL safety window was reused")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCanonicalConversationMessageHashUsesTextAndAssetRefs(t *testing.T) {
|
func TestCanonicalConversationMessageHashUsesTextAndAssetRefs(t *testing.T) {
|
||||||
message := map[string]any{
|
message := map[string]any{
|
||||||
"role": "user",
|
"role": "user",
|
||||||
|
|||||||
@@ -280,7 +280,7 @@ func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request
|
|||||||
|
|
||||||
// testFileStorageChannel godoc
|
// testFileStorageChannel godoc
|
||||||
// @Summary 测试对象存储通道
|
// @Summary 测试对象存储通道
|
||||||
// @Description 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针,不返回凭据或对象键。
|
// @Description 对指定 OSS 或 S3 通道检查并初始化临时文件生命周期规则,然后执行隔离的 Put、Head、Delete 探针;不返回凭据或对象键。
|
||||||
// @Tags system
|
// @Tags system
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Security BearerAuth
|
// @Security BearerAuth
|
||||||
@@ -358,9 +358,26 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
|
|||||||
if !validFileStorageBaseURL(endpoint) {
|
if !validFileStorageBaseURL(endpoint) {
|
||||||
return "object storage config.endpoint must be an http or https URL without embedded credentials"
|
return "object storage config.endpoint must be an http or https URL without embedded credentials"
|
||||||
}
|
}
|
||||||
if publicBaseURL := firstNonEmpty(fileStorageConfigString(input.Config, "publicBaseUrl"), fileStorageConfigString(input.Config, "publicBaseURL")); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
|
if publicBaseURL := firstNonEmpty(
|
||||||
|
fileStorageConfigString(input.Config, "publicBaseUrl"),
|
||||||
|
fileStorageConfigString(input.Config, "publicBaseURL"),
|
||||||
|
fileStorageConfigString(input.Config, "cdnDomain"),
|
||||||
|
fileStorageConfigString(input.Config, "publicDomain"),
|
||||||
|
); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
|
||||||
return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials"
|
return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials"
|
||||||
}
|
}
|
||||||
|
if policy := firstNonEmpty(
|
||||||
|
fileStorageConfigString(input.Config, "temporaryFileExpirePolicy"),
|
||||||
|
fileStorageConfigString(input.Config, "apiReturnExpirePolicy"),
|
||||||
|
); policy != "" && !validFileStorageExpirationPolicy(policy) {
|
||||||
|
return "object storage config.temporaryFileExpirePolicy must be never, 1d, 1m, 3m or 6m"
|
||||||
|
}
|
||||||
|
if value, exists := input.Config["signedUrlExpiresSeconds"]; exists {
|
||||||
|
seconds, ok := fileStorageConfigInteger(value)
|
||||||
|
if !ok || seconds < 60 || seconds > 7*24*60*60 {
|
||||||
|
return "object storage config.signedUrlExpiresSeconds must be an integer between 60 and 604800"
|
||||||
|
}
|
||||||
|
}
|
||||||
accessKeyID := input.AccessKeyID
|
accessKeyID := input.AccessKeyID
|
||||||
if accessKeyID == nil {
|
if accessKeyID == nil {
|
||||||
accessKeyID = input.AccessKey
|
accessKeyID = input.AccessKey
|
||||||
@@ -378,6 +395,34 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validFileStorageExpirationPolicy(value string) bool {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||||
|
case "never", "1d", "1m", "3m", "6m":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileStorageConfigInteger(value any) (int64, bool) {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case int:
|
||||||
|
return int64(typed), true
|
||||||
|
case int32:
|
||||||
|
return int64(typed), true
|
||||||
|
case int64:
|
||||||
|
return typed, true
|
||||||
|
case float64:
|
||||||
|
converted := int64(typed)
|
||||||
|
return converted, float64(converted) == typed
|
||||||
|
case float32:
|
||||||
|
converted := int64(typed)
|
||||||
|
return converted, float32(converted) == typed
|
||||||
|
default:
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func validFileStorageBaseURL(value string) bool {
|
func validFileStorageBaseURL(value string) bool {
|
||||||
parsed, err := url.Parse(strings.TrimSpace(value))
|
parsed, err := url.Parse(strings.TrimSpace(value))
|
||||||
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
|
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
|
||||||
|
|||||||
@@ -53,3 +53,34 @@ func TestValidateObjectStorageChannelRejectsCredentialedEndpoint(t *testing.T) {
|
|||||||
t.Fatalf("credentialed endpoint was accepted: %q", message)
|
t.Fatalf("credentialed endpoint was accepted: %q", message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateObjectStorageChannelAcceptsExpirationAndCDNAliases(t *testing.T) {
|
||||||
|
input := store.FileStorageChannelInput{
|
||||||
|
ChannelKey: "oss-expiring", Name: "OSS expiring", Provider: "aliyun_oss", Status: "disabled",
|
||||||
|
Config: map[string]any{
|
||||||
|
"endpoint": "https://oss-cn-shanghai.aliyuncs.com", "region": "cn-shanghai", "bucket": "media",
|
||||||
|
"cdnDomain": "https://cdn.example.com", "temporaryFileExpirePolicy": "3m", "signedUrlExpiresSeconds": float64(3600),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if message := validateFileStorageChannelInput(input, nil); message != "" {
|
||||||
|
t.Fatalf("valid object storage expiration config rejected: %s", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateObjectStorageChannelRejectsInvalidExpiration(t *testing.T) {
|
||||||
|
base := store.FileStorageChannelInput{
|
||||||
|
ChannelKey: "oss-expiring", Name: "OSS expiring", Provider: "aliyun_oss", Status: "disabled",
|
||||||
|
Config: map[string]any{
|
||||||
|
"endpoint": "https://oss-cn-shanghai.aliyuncs.com", "region": "cn-shanghai", "bucket": "media",
|
||||||
|
"temporaryFileExpirePolicy": "7d", "signedUrlExpiresSeconds": float64(3600),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if message := validateFileStorageChannelInput(base, nil); !strings.Contains(message, "temporaryFileExpirePolicy") {
|
||||||
|
t.Fatalf("invalid expiration policy was accepted: %q", message)
|
||||||
|
}
|
||||||
|
base.Config["temporaryFileExpirePolicy"] = "1d"
|
||||||
|
base.Config["signedUrlExpiresSeconds"] = float64(30)
|
||||||
|
if message := validateFileStorageChannelInput(base, nil); !strings.Contains(message, "signedUrlExpiresSeconds") {
|
||||||
|
t.Fatalf("invalid signed URL TTL was accepted: %q", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -47,11 +47,13 @@ type objectStorageAdapter struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type FileStorageChannelTestResult struct {
|
type FileStorageChannelTestResult struct {
|
||||||
Provider string `json:"provider"`
|
Provider string `json:"provider"`
|
||||||
PutSucceeded bool `json:"putSucceeded"`
|
PutSucceeded bool `json:"putSucceeded"`
|
||||||
HeadSucceeded bool `json:"headSucceeded"`
|
HeadSucceeded bool `json:"headSucceeded"`
|
||||||
DeleteSucceeded bool `json:"deleteSucceeded"`
|
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||||
DurationMS int64 `json:"durationMs"`
|
LifecycleReady bool `json:"lifecycleReady"`
|
||||||
|
ExpirationPolicy string `json:"expirationPolicy"`
|
||||||
|
DurationMS int64 `json:"durationMs"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFileStorageChannel performs an isolated write, metadata read and cleanup
|
// TestFileStorageChannel performs an isolated write, metadata read and cleanup
|
||||||
@@ -59,16 +61,29 @@ type FileStorageChannelTestResult struct {
|
|||||||
// content-addressed business object and no object key or credential is exposed.
|
// content-addressed business object and no object key or credential is exposed.
|
||||||
func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) {
|
func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) {
|
||||||
startedAt := time.Now()
|
startedAt := time.Now()
|
||||||
result := FileStorageChannelTestResult{Provider: strings.ToLower(strings.TrimSpace(channel.Provider))}
|
result := FileStorageChannelTestResult{
|
||||||
|
Provider: strings.ToLower(strings.TrimSpace(channel.Provider)),
|
||||||
|
ExpirationPolicy: objectStorageExpirationPolicy(channel.Config),
|
||||||
|
}
|
||||||
adapter, err := newObjectStorageAdapter(channel)
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
if result.ExpirationPolicy != objectStorageExpirationNever {
|
||||||
|
if err := adapter.ensureLifecycle(ctx); err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.LifecycleReady = true
|
||||||
|
}
|
||||||
|
probeScene := store.FileStorageSceneUpload
|
||||||
|
if result.LifecycleReady {
|
||||||
|
probeScene = store.FileStorageSceneImageResult
|
||||||
|
}
|
||||||
payload := FileUploadPayload{
|
payload := FileUploadPayload{
|
||||||
Bytes: []byte("easyai-storage-probe:" + uuid.NewString()),
|
Bytes: []byte("easyai-storage-probe:" + uuid.NewString()),
|
||||||
ContentType: "application/octet-stream",
|
ContentType: "application/octet-stream",
|
||||||
FileName: "probe.bin",
|
FileName: "probe.bin",
|
||||||
Scene: store.FileStorageSceneUpload,
|
Scene: probeScene,
|
||||||
Source: "admin-connection-test",
|
Source: "admin-connection-test",
|
||||||
}
|
}
|
||||||
upload, err := adapter.put(ctx, payload)
|
upload, err := adapter.put(ctx, payload)
|
||||||
@@ -112,6 +127,12 @@ func newObjectStorageAdapter(channel store.FileStorageChannel) (*objectStorageAd
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
|
func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
|
||||||
|
retention := objectStorageRetentionFor(a.channel.Config, payload.Scene, a.now().UTC())
|
||||||
|
if retention.Enabled {
|
||||||
|
if err := a.ensureLifecycle(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
objectKey := a.objectKey(payload)
|
objectKey := a.objectKey(payload)
|
||||||
requestURL, err := a.objectURL(objectKey)
|
requestURL, err := a.objectURL(objectKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -126,6 +147,14 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
|
|||||||
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", contentType)
|
req.Header.Set("Content-Type", contentType)
|
||||||
|
if retention.Enabled {
|
||||||
|
tagging := retention.taggingHeader()
|
||||||
|
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||||
|
req.Header.Set("x-amz-tagging", tagging)
|
||||||
|
} else {
|
||||||
|
req.Header.Set("x-oss-tagging", tagging)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil {
|
if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -148,11 +177,12 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
|
|||||||
}
|
}
|
||||||
urlExpiresAt := ""
|
urlExpiresAt := ""
|
||||||
if publicURL == "" {
|
if publicURL == "" {
|
||||||
publicURL, err = a.presignGet(objectKey, objectStorageSignedURLTTL)
|
signedURLTTL := objectStorageSignedURLTTLForConfig(a.channel.Config)
|
||||||
|
publicURL, err = a.presignGet(objectKey, signedURLTTL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
urlExpiresAt = a.now().UTC().Add(objectStorageSignedURLTTL).Format(time.RFC3339)
|
urlExpiresAt = a.now().UTC().Add(signedURLTTL).Format(time.RFC3339)
|
||||||
}
|
}
|
||||||
digest := sha256.Sum256(payload.Bytes)
|
digest := sha256.Sum256(payload.Bytes)
|
||||||
result := map[string]any{
|
result := map[string]any{
|
||||||
@@ -173,6 +203,10 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
|
|||||||
if urlExpiresAt != "" {
|
if urlExpiresAt != "" {
|
||||||
result["urlExpiresAt"] = urlExpiresAt
|
result["urlExpiresAt"] = urlExpiresAt
|
||||||
}
|
}
|
||||||
|
if retention.Enabled {
|
||||||
|
result["objectExpiresAt"] = retention.ExpiresAt.Format(time.RFC3339)
|
||||||
|
result["expirationPolicy"] = retention.Policy
|
||||||
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,10 +335,7 @@ func (a *objectStorageAdapter) objectURL(objectKey string) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *objectStorageAdapter) publicURL(objectKey string) string {
|
func (a *objectStorageAdapter) publicURL(objectKey string) string {
|
||||||
baseURL := strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseUrl"), "/")
|
baseURL := strings.TrimRight(objectStoragePublicBaseURL(a.channel.Config), "/")
|
||||||
if baseURL == "" {
|
|
||||||
baseURL = strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseURL"), "/")
|
|
||||||
}
|
|
||||||
if baseURL == "" {
|
if baseURL == "" {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
@@ -322,19 +353,45 @@ func (a *objectStorageAdapter) sign(req *http.Request, payloadHash string, now t
|
|||||||
|
|
||||||
func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) {
|
func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) {
|
||||||
date := now.UTC().Format(http.TimeFormat)
|
date := now.UTC().Format(http.TimeFormat)
|
||||||
|
contentMD5 := req.Header.Get("Content-MD5")
|
||||||
contentType := req.Header.Get("Content-Type")
|
contentType := req.Header.Get("Content-Type")
|
||||||
canonicalHeaders := ""
|
|
||||||
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||||
req.Header.Set("x-oss-security-token", token)
|
req.Header.Set("x-oss-security-token", token)
|
||||||
canonicalHeaders = "x-oss-security-token:" + token + "\n"
|
|
||||||
}
|
}
|
||||||
|
canonicalHeaders := canonicalOSSHeaders(req)
|
||||||
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath()
|
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath()
|
||||||
stringToSign := req.Method + "\n\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
|
if _, ok := req.URL.Query()["lifecycle"]; ok {
|
||||||
|
canonicalResource += "?lifecycle"
|
||||||
|
}
|
||||||
|
stringToSign := req.Method + "\n" + contentMD5 + "\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
|
||||||
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign)
|
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign)
|
||||||
req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature)
|
req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature)
|
||||||
req.Header.Set("Date", date)
|
req.Header.Set("Date", date)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func canonicalOSSHeaders(req *http.Request) string {
|
||||||
|
headers := map[string]string{}
|
||||||
|
for key, values := range req.Header {
|
||||||
|
lower := strings.ToLower(strings.TrimSpace(key))
|
||||||
|
if strings.HasPrefix(lower, "x-oss-") {
|
||||||
|
headers[lower] = strings.Join(values, ",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keys := make([]string, 0, len(headers))
|
||||||
|
for key := range headers {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
var canonical strings.Builder
|
||||||
|
for _, key := range keys {
|
||||||
|
canonical.WriteString(key)
|
||||||
|
canonical.WriteByte(':')
|
||||||
|
canonical.WriteString(strings.Join(strings.Fields(headers[key]), " "))
|
||||||
|
canonical.WriteByte('\n')
|
||||||
|
}
|
||||||
|
return canonical.String()
|
||||||
|
}
|
||||||
|
|
||||||
func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) {
|
func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) {
|
||||||
parsed, err := url.Parse(requestURL)
|
parsed, err := url.Parse(requestURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -487,18 +544,27 @@ func objectStorageConfigBool(config map[string]any, key string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string {
|
func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string {
|
||||||
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
|
|
||||||
return scope
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset {
|
if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset {
|
||||||
return "private"
|
return "private"
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseUrl")) != "" || strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseURL")) != "" {
|
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
|
||||||
|
return scope
|
||||||
|
}
|
||||||
|
if objectStoragePublicBaseURL(channel.Config) != "" {
|
||||||
return "public"
|
return "public"
|
||||||
}
|
}
|
||||||
return "private"
|
return "private"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func objectStoragePublicBaseURL(config map[string]any) string {
|
||||||
|
return firstNonEmptyString(
|
||||||
|
objectStorageConfigString(config, "publicBaseUrl"),
|
||||||
|
objectStorageConfigString(config, "publicBaseURL"),
|
||||||
|
objectStorageConfigString(config, "cdnDomain"),
|
||||||
|
objectStorageConfigString(config, "publicDomain"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
func escapeObjectKey(objectKey string) string {
|
func escapeObjectKey(objectKey string) string {
|
||||||
parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/")
|
parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/")
|
||||||
for index, part := range parts {
|
for index, part := range parts {
|
||||||
|
|||||||
@@ -0,0 +1,378 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
objectStorageExpirationNever = "never"
|
||||||
|
objectStorageExpiration1Day = "1d"
|
||||||
|
objectStorageExpiration1Month = "1m"
|
||||||
|
objectStorageExpiration3Months = "3m"
|
||||||
|
objectStorageExpiration6Months = "6m"
|
||||||
|
|
||||||
|
objectStorageSignedURLTTLMin = time.Minute
|
||||||
|
objectStorageSignedURLTTLMax = 7 * 24 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
type objectStorageLifecycleDefinition struct {
|
||||||
|
ID string
|
||||||
|
TagValue string
|
||||||
|
Days int
|
||||||
|
}
|
||||||
|
|
||||||
|
var objectStorageLifecycleDefinitions = []objectStorageLifecycleDefinition{
|
||||||
|
{ID: "DeleteTempFiles-1d", TagValue: objectStorageExpiration1Day, Days: 1},
|
||||||
|
{ID: "DeleteTempFiles-1m", TagValue: objectStorageExpiration1Month, Days: 30},
|
||||||
|
{ID: "DeleteTempFiles-3m", TagValue: objectStorageExpiration3Months, Days: 90},
|
||||||
|
{ID: "DeleteTempFiles-6m", TagValue: objectStorageExpiration6Months, Days: 180},
|
||||||
|
}
|
||||||
|
|
||||||
|
type objectStorageRetention struct {
|
||||||
|
Enabled bool
|
||||||
|
Policy string
|
||||||
|
TagValue string
|
||||||
|
ExpiresAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r objectStorageRetention) taggingHeader() string {
|
||||||
|
values := url.Values{}
|
||||||
|
values.Set("TempFile", "true")
|
||||||
|
values.Set("ExpiresAt", strconv.FormatInt(r.ExpiresAt.Unix(), 10))
|
||||||
|
values.Set("ExpiresIn", r.TagValue)
|
||||||
|
return values.Encode()
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStorageExpirationPolicy(config map[string]any) string {
|
||||||
|
policy := strings.ToLower(firstNonEmptyString(
|
||||||
|
objectStorageConfigString(config, "temporaryFileExpirePolicy"),
|
||||||
|
objectStorageConfigString(config, "apiReturnExpirePolicy"),
|
||||||
|
))
|
||||||
|
switch policy {
|
||||||
|
case objectStorageExpiration1Day,
|
||||||
|
objectStorageExpiration1Month,
|
||||||
|
objectStorageExpiration3Months,
|
||||||
|
objectStorageExpiration6Months:
|
||||||
|
return policy
|
||||||
|
default:
|
||||||
|
return objectStorageExpirationNever
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStorageRetentionFor(config map[string]any, scene string, now time.Time) objectStorageRetention {
|
||||||
|
if strings.TrimSpace(scene) != "image_result" && strings.TrimSpace(scene) != "request_asset" {
|
||||||
|
return objectStorageRetention{Policy: objectStorageExpirationNever}
|
||||||
|
}
|
||||||
|
policy := objectStorageExpirationPolicy(config)
|
||||||
|
duration := objectStorageExpirationDuration(policy)
|
||||||
|
if duration <= 0 {
|
||||||
|
return objectStorageRetention{Policy: objectStorageExpirationNever}
|
||||||
|
}
|
||||||
|
return objectStorageRetention{
|
||||||
|
Enabled: true,
|
||||||
|
Policy: policy,
|
||||||
|
TagValue: policy,
|
||||||
|
ExpiresAt: now.Add(duration),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStorageExpirationDuration(policy string) time.Duration {
|
||||||
|
switch policy {
|
||||||
|
case objectStorageExpiration1Day:
|
||||||
|
return 24 * time.Hour
|
||||||
|
case objectStorageExpiration1Month:
|
||||||
|
return 30 * 24 * time.Hour
|
||||||
|
case objectStorageExpiration3Months:
|
||||||
|
return 90 * 24 * time.Hour
|
||||||
|
case objectStorageExpiration6Months:
|
||||||
|
return 180 * 24 * time.Hour
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStorageSignedURLTTLForConfig(config map[string]any) time.Duration {
|
||||||
|
seconds := int64FromAny(config["signedUrlExpiresSeconds"])
|
||||||
|
if seconds <= 0 {
|
||||||
|
return objectStorageSignedURLTTL
|
||||||
|
}
|
||||||
|
ttl := time.Duration(seconds) * time.Second
|
||||||
|
if ttl < objectStorageSignedURLTTLMin {
|
||||||
|
return objectStorageSignedURLTTLMin
|
||||||
|
}
|
||||||
|
if ttl > objectStorageSignedURLTTLMax {
|
||||||
|
return objectStorageSignedURLTTLMax
|
||||||
|
}
|
||||||
|
return ttl
|
||||||
|
}
|
||||||
|
|
||||||
|
func int64FromAny(value any) int64 {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case int:
|
||||||
|
return int64(typed)
|
||||||
|
case int32:
|
||||||
|
return int64(typed)
|
||||||
|
case int64:
|
||||||
|
return typed
|
||||||
|
case float32:
|
||||||
|
return int64(typed)
|
||||||
|
case float64:
|
||||||
|
return int64(typed)
|
||||||
|
case string:
|
||||||
|
parsed, _ := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||||
|
return parsed
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
objectStorageLifecycleReady sync.Map
|
||||||
|
objectStorageLifecycleLocks sync.Map
|
||||||
|
)
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) ensureLifecycle(ctx context.Context) error {
|
||||||
|
cacheKey := strings.Join([]string{
|
||||||
|
strings.ToLower(strings.TrimSpace(a.channel.Provider)),
|
||||||
|
objectStorageConfigString(a.channel.Config, "endpoint"),
|
||||||
|
objectStorageConfigString(a.channel.Config, "bucket"),
|
||||||
|
}, "\x00")
|
||||||
|
if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
lockValue, _ := objectStorageLifecycleLocks.LoadOrStore(cacheKey, &sync.Mutex{})
|
||||||
|
lifecycleLock := lockValue.(*sync.Mutex)
|
||||||
|
lifecycleLock.Lock()
|
||||||
|
defer lifecycleLock.Unlock()
|
||||||
|
if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
current, err := a.readLifecycle(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
next, changed, err := mergeObjectStorageLifecycle(a.channel.Provider, current)
|
||||||
|
if err != nil {
|
||||||
|
return storageClientError("storage_config_invalid", "object storage lifecycle configuration is invalid", 0, false)
|
||||||
|
}
|
||||||
|
if changed {
|
||||||
|
if err := a.writeLifecycle(ctx, next); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
objectStorageLifecycleReady.Store(cacheKey, struct{}{})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) readLifecycle(ctx context.Context) ([]byte, error) {
|
||||||
|
requestURL, err := a.bucketLifecycleURL()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||||
|
}
|
||||||
|
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := a.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, storageClientError("storage_write_failed", "object storage lifecycle check failed", 0, true)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
|
||||||
|
if readErr != nil {
|
||||||
|
return nil, storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
||||||
|
}
|
||||||
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return nil, objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) writeLifecycle(ctx context.Context, payload []byte) error {
|
||||||
|
requestURL, err := a.bucketLifecycleURL()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload))
|
||||||
|
if err != nil {
|
||||||
|
return storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||||
|
}
|
||||||
|
digest := md5.Sum(payload)
|
||||||
|
req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(digest[:]))
|
||||||
|
req.Header.Set("Content-Type", "application/xml")
|
||||||
|
if err := a.sign(req, sha256Hex(payload), a.now().UTC()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp, err := a.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return storageClientError("storage_write_failed", "object storage lifecycle configuration failed", 0, true)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||||
|
if readErr != nil {
|
||||||
|
return storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
||||||
|
}
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) bucketLifecycleURL() (string, error) {
|
||||||
|
endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/"))
|
||||||
|
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil {
|
||||||
|
return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false)
|
||||||
|
}
|
||||||
|
bucket := objectStorageConfigString(a.channel.Config, "bucket")
|
||||||
|
hostname := endpoint.Hostname()
|
||||||
|
forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle")
|
||||||
|
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||||
|
if forcePathStyle {
|
||||||
|
endpoint.Path = path.Join(endpoint.Path, bucket)
|
||||||
|
} else if !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
||||||
|
if port := endpoint.Port(); port != "" {
|
||||||
|
endpoint.Host = bucket + "." + hostname + ":" + port
|
||||||
|
} else {
|
||||||
|
endpoint.Host = bucket + "." + endpoint.Host
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
||||||
|
if port := endpoint.Port(); port != "" {
|
||||||
|
endpoint.Host = bucket + "." + hostname + ":" + port
|
||||||
|
} else {
|
||||||
|
endpoint.Host = bucket + "." + endpoint.Host
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if endpoint.Path == "" {
|
||||||
|
endpoint.Path = "/"
|
||||||
|
}
|
||||||
|
endpoint.RawQuery = "lifecycle"
|
||||||
|
endpoint.Fragment = ""
|
||||||
|
return endpoint.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawLifecycleConfiguration struct {
|
||||||
|
Rules []rawLifecycleRule `xml:"Rule"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawLifecycleRule struct {
|
||||||
|
ID string `xml:"ID"`
|
||||||
|
Status string `xml:"Status"`
|
||||||
|
Tag rawLifecycleTag `xml:"Tag"`
|
||||||
|
Filter rawLifecycleFilter `xml:"Filter"`
|
||||||
|
Expiration rawLifecycleExpiration `xml:"Expiration"`
|
||||||
|
InnerXML string `xml:",innerxml"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawLifecycleFilter struct {
|
||||||
|
Tag rawLifecycleTag `xml:"Tag"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawLifecycleTag struct {
|
||||||
|
Key string `xml:"Key"`
|
||||||
|
Value string `xml:"Value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawLifecycleExpiration struct {
|
||||||
|
Days int `xml:"Days"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeObjectStorageLifecycle(provider string, current []byte) ([]byte, bool, error) {
|
||||||
|
configuration := rawLifecycleConfiguration{}
|
||||||
|
if len(bytes.TrimSpace(current)) > 0 {
|
||||||
|
if err := xml.Unmarshal(current, &configuration); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
managed := map[string]objectStorageLifecycleDefinition{}
|
||||||
|
for _, definition := range objectStorageLifecycleDefinitions {
|
||||||
|
managed[definition.ID] = definition
|
||||||
|
}
|
||||||
|
allManagedReady := true
|
||||||
|
for _, definition := range objectStorageLifecycleDefinitions {
|
||||||
|
matched := false
|
||||||
|
for _, rule := range configuration.Rules {
|
||||||
|
if rule.ID == definition.ID {
|
||||||
|
matched = lifecycleRuleMatches(rule, definition)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !matched {
|
||||||
|
allManagedReady = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if allManagedReady {
|
||||||
|
return current, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var output strings.Builder
|
||||||
|
output.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
|
||||||
|
if strings.EqualFold(strings.TrimSpace(provider), "s3") {
|
||||||
|
output.WriteString(`<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`)
|
||||||
|
} else {
|
||||||
|
output.WriteString(`<LifecycleConfiguration>`)
|
||||||
|
}
|
||||||
|
for _, rule := range configuration.Rules {
|
||||||
|
if _, isManaged := managed[rule.ID]; isManaged {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
output.WriteString("<Rule>")
|
||||||
|
output.WriteString(rule.InnerXML)
|
||||||
|
output.WriteString("</Rule>")
|
||||||
|
}
|
||||||
|
for _, definition := range objectStorageLifecycleDefinitions {
|
||||||
|
output.WriteString(lifecycleRuleXML(provider, definition))
|
||||||
|
}
|
||||||
|
output.WriteString(`</LifecycleConfiguration>`)
|
||||||
|
return []byte(output.String()), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func lifecycleRuleMatches(rule rawLifecycleRule, definition objectStorageLifecycleDefinition) bool {
|
||||||
|
tag := rule.Tag
|
||||||
|
if tag.Key == "" {
|
||||||
|
tag = rule.Filter.Tag
|
||||||
|
}
|
||||||
|
return strings.EqualFold(strings.TrimSpace(rule.Status), "Enabled") &&
|
||||||
|
tag.Key == "ExpiresIn" && tag.Value == definition.TagValue &&
|
||||||
|
rule.Expiration.Days == definition.Days
|
||||||
|
}
|
||||||
|
|
||||||
|
func lifecycleRuleXML(provider string, definition objectStorageLifecycleDefinition) string {
|
||||||
|
if strings.EqualFold(strings.TrimSpace(provider), "s3") {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"<Rule><ID>%s</ID><Filter><Tag><Key>ExpiresIn</Key><Value>%s</Value></Tag></Filter><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule>",
|
||||||
|
definition.ID,
|
||||||
|
definition.TagValue,
|
||||||
|
definition.Days,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"<Rule><ID>%s</ID><Prefix></Prefix><Tag><Key>ExpiresIn</Key><Value>%s</Value></Tag><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule>",
|
||||||
|
definition.ID,
|
||||||
|
definition.TagValue,
|
||||||
|
definition.Days,
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -80,6 +81,7 @@ func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
channel := testObjectStorageChannel("s3", server.URL, "s3-private")
|
channel := testObjectStorageChannel("s3", server.URL, "s3-private")
|
||||||
channel.Config["publicBaseUrl"] = "https://cdn.example"
|
channel.Config["publicBaseUrl"] = "https://cdn.example"
|
||||||
|
channel.Config["accessScope"] = "public"
|
||||||
adapter, err := newObjectStorageAdapter(channel)
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -101,6 +103,112 @@ func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrivateSignedURLUsesConfiguredTTL(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
channel := testObjectStorageChannel("s3", server.URL, "s3-private-ttl")
|
||||||
|
channel.Config["signedUrlExpiresSeconds"] = 3600
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
|
||||||
|
|
||||||
|
result, err := adapter.put(t.Context(), FileUploadPayload{
|
||||||
|
Bytes: []byte("private input"), ContentType: "image/png", FileName: "input.png", Scene: store.FileStorageSceneRequestAsset,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(stringFromAny(result["url"]))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := parsed.Query().Get("X-Amz-Expires"); got != "3600" {
|
||||||
|
t.Fatalf("X-Amz-Expires=%q, want 3600", got)
|
||||||
|
}
|
||||||
|
if got := stringFromAny(result["urlExpiresAt"]); got != "2026-08-04T01:00:00Z" {
|
||||||
|
t.Fatalf("urlExpiresAt=%q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTemporaryObjectEnsuresS3LifecycleAndAddsTagging(t *testing.T) {
|
||||||
|
var methods []string
|
||||||
|
var lifecyclePayload string
|
||||||
|
var tagging string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if _, lifecycle := r.URL.Query()["lifecycle"]; lifecycle {
|
||||||
|
methods = append(methods, r.Method+" lifecycle")
|
||||||
|
if r.Method == http.MethodGet {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
lifecyclePayload = string(body)
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
methods = append(methods, r.Method+" object")
|
||||||
|
tagging = r.Header.Get("x-amz-tagging")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
channel := testObjectStorageChannel("s3", server.URL, "s3-expiring")
|
||||||
|
channel.Config["temporaryFileExpirePolicy"] = "1m"
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
|
||||||
|
result, err := adapter.put(t.Context(), FileUploadPayload{
|
||||||
|
Bytes: []byte("temporary image"), ContentType: "image/png", Scene: store.FileStorageSceneImageResult,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := strings.Join(methods, ","); got != "GET lifecycle,PUT lifecycle,PUT object" {
|
||||||
|
t.Fatalf("methods=%s", got)
|
||||||
|
}
|
||||||
|
for _, fragment := range []string{"DeleteTempFiles-1d", "DeleteTempFiles-1m", "DeleteTempFiles-3m", "DeleteTempFiles-6m", "<Value>1m</Value>"} {
|
||||||
|
if !strings.Contains(lifecyclePayload, fragment) {
|
||||||
|
t.Fatalf("lifecycle payload missing %q: %s", fragment, lifecyclePayload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
values, err := url.ParseQuery(tagging)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if values.Get("TempFile") != "true" || values.Get("ExpiresIn") != "1m" || values.Get("ExpiresAt") != "1788393600" {
|
||||||
|
t.Fatalf("unexpected tagging: %q", tagging)
|
||||||
|
}
|
||||||
|
if got := stringFromAny(result["objectExpiresAt"]); got != "2026-09-03T00:00:00Z" {
|
||||||
|
t.Fatalf("objectExpiresAt=%q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeObjectStorageLifecyclePreservesUnrelatedRules(t *testing.T) {
|
||||||
|
current := []byte(`<?xml version="1.0"?><LifecycleConfiguration><Rule><ID>KeepArchive</ID><Prefix>archive/</Prefix><Status>Enabled</Status><Expiration><Days>365</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||||
|
merged, changed, err := mergeObjectStorageLifecycle("aliyun_oss", current)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !changed {
|
||||||
|
t.Fatal("lifecycle merge unexpectedly reported no change")
|
||||||
|
}
|
||||||
|
value := string(merged)
|
||||||
|
if !strings.Contains(value, "<ID>KeepArchive</ID>") || !strings.Contains(value, "<Prefix>archive/</Prefix>") {
|
||||||
|
t.Fatalf("unrelated lifecycle rule was not preserved: %s", value)
|
||||||
|
}
|
||||||
|
for _, definition := range objectStorageLifecycleDefinitions {
|
||||||
|
if !strings.Contains(value, "<ID>"+definition.ID+"</ID>") {
|
||||||
|
t.Fatalf("managed lifecycle rule %s missing", definition.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) {
|
func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) {
|
||||||
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
||||||
channel.Config["forcePathStyle"] = false
|
channel.Config["forcePathStyle"] = false
|
||||||
@@ -138,6 +246,37 @@ func TestAliyunOSSRequestUsesSignatureV1(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAliyunOSSSignatureIncludesLifecycleTaggingHeader(t *testing.T) {
|
||||||
|
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-tagged")
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request, err := http.NewRequest(http.MethodPut, "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/image_result/object.png", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request.Header.Set("Content-Type", "image/png")
|
||||||
|
request.Header.Set("x-oss-tagging", "ExpiresAt=1788393600&ExpiresIn=1m&TempFile=true")
|
||||||
|
now := time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC)
|
||||||
|
adapter.signOSSRequest(request, now)
|
||||||
|
stringToSign := "PUT\n\nimage/png\nTue, 04 Aug 2026 01:02:03 GMT\n" +
|
||||||
|
"x-oss-tagging:ExpiresAt=1788393600&ExpiresIn=1m&TempFile=true\n" +
|
||||||
|
"/bucket/media/image_result/object.png"
|
||||||
|
want := "OSS access-id:" + hmacSHA1Base64([]byte("access-secret"), stringToSign)
|
||||||
|
if got := request.Header.Get("Authorization"); got != want {
|
||||||
|
t.Fatalf("authorization=%q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTemporaryRetentionDoesNotApplyToPersistentUploadScene(t *testing.T) {
|
||||||
|
config := map[string]any{"temporaryFileExpirePolicy": "1d"}
|
||||||
|
retention := objectStorageRetentionFor(config, store.FileStorageSceneUpload, time.Now())
|
||||||
|
if retention.Enabled || retention.Policy != objectStorageExpirationNever {
|
||||||
|
t.Fatalf("persistent upload unexpectedly expires: %+v", retention)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) {
|
func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) {
|
||||||
var primaryCalls atomic.Int64
|
var primaryCalls atomic.Int64
|
||||||
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
|||||||
@@ -1191,7 +1191,10 @@ export function App() {
|
|||||||
const refreshed = await listFileStorageChannels(token);
|
const refreshed = await listFileStorageChannels(token);
|
||||||
setFileStorageChannels(refreshed.items);
|
setFileStorageChannels(refreshed.items);
|
||||||
setCoreState('ready');
|
setCoreState('ready');
|
||||||
setCoreMessage(`对象存储连接测试通过,Put / Head / Delete 均成功,耗时 ${result.durationMs}ms。`);
|
const lifecycleMessage = result.expirationPolicy !== 'never'
|
||||||
|
? `生命周期 ${result.expirationPolicy} 已就绪,`
|
||||||
|
: '';
|
||||||
|
setCoreMessage(`对象存储连接测试通过,${lifecycleMessage}Put / Head / Delete 均成功,耗时 ${result.durationMs}ms。`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setCoreState('error');
|
setCoreState('error');
|
||||||
setCoreMessage(err instanceof Error ? err.message : '对象存储连接测试失败');
|
setCoreMessage(err instanceof Error ? err.message : '对象存储连接测试失败');
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import type { FileStorageChannel } from '@easyai-ai-gateway/contracts';
|
||||||
|
import { channelToForm, formToPayload } from './SystemSettingsPanel';
|
||||||
|
|
||||||
|
describe('SystemSettingsPanel object storage form', () => {
|
||||||
|
it('promotes legacy CDN and expiration config while preserving unknown extensions', () => {
|
||||||
|
const channel: FileStorageChannel = {
|
||||||
|
id: 'channel-1',
|
||||||
|
channelKey: 'oss-primary',
|
||||||
|
name: 'OSS primary',
|
||||||
|
provider: 'aliyun_oss',
|
||||||
|
credentialsPreview: {},
|
||||||
|
scenes: ['upload', 'image_result', 'request_asset'],
|
||||||
|
config: {
|
||||||
|
endpoint: 'https://oss-cn-shanghai.aliyuncs.com',
|
||||||
|
region: 'cn-shanghai',
|
||||||
|
bucket: 'media',
|
||||||
|
cdnDomain: 'https://cdn.example.com/',
|
||||||
|
apiReturnExpirePolicy: '3m',
|
||||||
|
signedUrlExpiresSeconds: 3600,
|
||||||
|
customHeader: 'retained',
|
||||||
|
},
|
||||||
|
retryPolicy: {},
|
||||||
|
priority: 100,
|
||||||
|
status: 'enabled',
|
||||||
|
createdAt: '2026-08-04T00:00:00Z',
|
||||||
|
updatedAt: '2026-08-04T00:00:00Z',
|
||||||
|
};
|
||||||
|
|
||||||
|
const form = channelToForm(channel);
|
||||||
|
expect(form.publicBaseUrl).toBe('https://cdn.example.com/');
|
||||||
|
expect(form.temporaryFileExpirePolicy).toBe('3m');
|
||||||
|
expect(form.signedUrlExpiresMinutes).toBe('60');
|
||||||
|
expect(JSON.parse(form.configJson)).toEqual({ customHeader: 'retained' });
|
||||||
|
|
||||||
|
const payload = formToPayload({
|
||||||
|
...form,
|
||||||
|
objectPrefix: '/gateway/',
|
||||||
|
temporaryFileExpirePolicy: '1m',
|
||||||
|
});
|
||||||
|
expect(payload.config).toEqual({
|
||||||
|
endpoint: 'https://oss-cn-shanghai.aliyuncs.com',
|
||||||
|
region: 'cn-shanghai',
|
||||||
|
bucket: 'media',
|
||||||
|
publicBaseUrl: 'https://cdn.example.com',
|
||||||
|
objectPrefix: 'gateway',
|
||||||
|
temporaryFileExpirePolicy: '1m',
|
||||||
|
signedUrlExpiresSeconds: 3600,
|
||||||
|
customHeader: 'retained',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -21,16 +21,24 @@ type FileStorageChannelForm = {
|
|||||||
accessKeySecretPreview: string;
|
accessKeySecretPreview: string;
|
||||||
apiKey: string;
|
apiKey: string;
|
||||||
apiKeyPreview: string;
|
apiKeyPreview: string;
|
||||||
|
bucket: string;
|
||||||
channelKey: string;
|
channelKey: string;
|
||||||
configJson: string;
|
configJson: string;
|
||||||
|
endpoint: string;
|
||||||
|
forcePathStyle: boolean;
|
||||||
name: string;
|
name: string;
|
||||||
|
objectPrefix: string;
|
||||||
priority: string;
|
priority: string;
|
||||||
provider: string;
|
provider: string;
|
||||||
|
publicBaseUrl: string;
|
||||||
|
region: string;
|
||||||
retryPolicyJson: string;
|
retryPolicyJson: string;
|
||||||
sessionToken: string;
|
sessionToken: string;
|
||||||
sessionTokenPreview: string;
|
sessionTokenPreview: string;
|
||||||
|
signedUrlExpiresMinutes: string;
|
||||||
scenes: string[];
|
scenes: string[];
|
||||||
status: string;
|
status: string;
|
||||||
|
temporaryFileExpirePolicy: string;
|
||||||
uploadUrl: string;
|
uploadUrl: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -55,6 +63,14 @@ const providerOptions = [
|
|||||||
{ value: 's3', label: 'S3 / S3 兼容存储' },
|
{ value: 's3', label: 'S3 / S3 兼容存储' },
|
||||||
];
|
];
|
||||||
|
|
||||||
|
const expirationPolicyOptions = [
|
||||||
|
{ value: 'never', label: '永久保留' },
|
||||||
|
{ value: '1d', label: '1 天' },
|
||||||
|
{ value: '1m', label: '1 个月(30 天)' },
|
||||||
|
{ value: '3m', label: '3 个月(90 天)' },
|
||||||
|
{ value: '6m', label: '6 个月(180 天)' },
|
||||||
|
];
|
||||||
|
|
||||||
const defaultScenes = ['upload', 'image_result', 'request_asset'];
|
const defaultScenes = ['upload', 'image_result', 'request_asset'];
|
||||||
const sceneOptions = [
|
const sceneOptions = [
|
||||||
{ value: 'upload', label: '上传', description: 'OpenAPI / 管理端主动上传文件' },
|
{ value: 'upload', label: '上传', description: 'OpenAPI / 管理端主动上传文件' },
|
||||||
@@ -126,6 +142,15 @@ export function SystemSettingsPanel(props: {
|
|||||||
setLocalError('请至少选择一个适用场景。');
|
setLocalError('请至少选择一个适用场景。');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (form.provider !== 'server_main_openapi' && (!form.endpoint.trim() || !form.region.trim() || !form.bucket.trim())) {
|
||||||
|
setLocalError('对象存储渠道必须填写 Endpoint、Region 和 Bucket。');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const signedUrlExpiresMinutes = Number(form.signedUrlExpiresMinutes);
|
||||||
|
if (form.provider !== 'server_main_openapi' && (!Number.isInteger(signedUrlExpiresMinutes) || signedUrlExpiresMinutes < 1 || signedUrlExpiresMinutes > 10080)) {
|
||||||
|
setLocalError('私有链接有效期必须是 1 至 10080 分钟之间的整数。');
|
||||||
|
return;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
await props.onSaveFileStorageChannel(formToPayload(form), editingChannel?.id);
|
await props.onSaveFileStorageChannel(formToPayload(form), editingChannel?.id);
|
||||||
closeDialog();
|
closeDialog();
|
||||||
@@ -245,6 +270,10 @@ export function SystemSettingsPanel(props: {
|
|||||||
<span>场景: {sceneSummary(channel.scenes)}</span>
|
<span>场景: {sceneSummary(channel.scenes)}</span>
|
||||||
<span>优先级: {channel.priority}</span>
|
<span>优先级: {channel.priority}</span>
|
||||||
<span>重试: {retryPolicySummary(channel.retryPolicy)}</span>
|
<span>重试: {retryPolicySummary(channel.retryPolicy)}</span>
|
||||||
|
{channel.provider !== 'server_main_openapi' && configString(channel.config, 'region') && <span>Region: {configString(channel.config, 'region')}</span>}
|
||||||
|
{channel.provider !== 'server_main_openapi' && configString(channel.config, 'bucket') && <span>Bucket: {configString(channel.config, 'bucket')}</span>}
|
||||||
|
{channel.provider !== 'server_main_openapi' && publicBaseUrlFromConfig(channel.config) && <span>CDN: {publicBaseUrlFromConfig(channel.config)}</span>}
|
||||||
|
{channel.provider !== 'server_main_openapi' && <span>临时文件: {expirationPolicyLabel(expirationPolicyFromConfig(channel.config))}</span>}
|
||||||
{channel.uploadUrl && <span>上传路由: {channel.uploadUrl}</span>}
|
{channel.uploadUrl && <span>上传路由: {channel.uploadUrl}</span>}
|
||||||
{apiKeyPreview(channel) && <span>API Key: {apiKeyPreview(channel)}</span>}
|
{apiKeyPreview(channel) && <span>API Key: {apiKeyPreview(channel)}</span>}
|
||||||
{channel.lastError && <span>最近错误: {channel.lastError}</span>}
|
{channel.lastError && <span>最近错误: {channel.lastError}</span>}
|
||||||
@@ -348,6 +377,7 @@ export function SystemSettingsPanel(props: {
|
|||||||
const provider = event.target.value;
|
const provider = event.target.value;
|
||||||
setForm({
|
setForm({
|
||||||
...form,
|
...form,
|
||||||
|
name: editingChannel ? form.name : providerLabel(provider),
|
||||||
provider,
|
provider,
|
||||||
retryPolicyJson: editingChannel
|
retryPolicyJson: editingChannel
|
||||||
? form.retryPolicyJson
|
? form.retryPolicyJson
|
||||||
@@ -388,6 +418,35 @@ export function SystemSettingsPanel(props: {
|
|||||||
<small>保持脱敏值不变表示不修改;填写新值表示覆盖;清空后保存表示清除。</small>
|
<small>保持脱敏值不变表示不修改;填写新值表示覆盖;清空后保存表示清除。</small>
|
||||||
</Label>}
|
</Label>}
|
||||||
{form.provider !== 'server_main_openapi' && <>
|
{form.provider !== 'server_main_openapi' && <>
|
||||||
|
<Label className="spanTwo">
|
||||||
|
上传 Endpoint
|
||||||
|
<Input value={form.endpoint} onChange={(event) => setForm({ ...form, endpoint: event.target.value })} placeholder={form.provider === 'aliyun_oss' ? 'https://oss-cn-shanghai.aliyuncs.com' : 'https://s3.example.com'} />
|
||||||
|
<small>服务端写入对象使用的地址,可填写同地域内网 Endpoint;不要填写 CDN 域名。</small>
|
||||||
|
</Label>
|
||||||
|
<Label>
|
||||||
|
Region
|
||||||
|
<Input value={form.region} onChange={(event) => setForm({ ...form, region: event.target.value })} placeholder={form.provider === 'aliyun_oss' ? 'cn-shanghai' : 'us-east-1'} />
|
||||||
|
</Label>
|
||||||
|
<Label>
|
||||||
|
Bucket
|
||||||
|
<Input value={form.bucket} onChange={(event) => setForm({ ...form, bucket: event.target.value })} placeholder="easyai-media" />
|
||||||
|
</Label>
|
||||||
|
<Label className="spanTwo">
|
||||||
|
CDN / 公网访问域名(可选)
|
||||||
|
<Input value={form.publicBaseUrl} onChange={(event) => setForm({ ...form, publicBaseUrl: event.target.value })} placeholder="https://cdn.example.com" />
|
||||||
|
<small>生成媒体将返回此域名;请求素材仍使用短期私有签名 URL。</small>
|
||||||
|
</Label>
|
||||||
|
<Label>
|
||||||
|
对象路径前缀(可选)
|
||||||
|
<Input value={form.objectPrefix} onChange={(event) => setForm({ ...form, objectPrefix: event.target.value })} placeholder="gateway" />
|
||||||
|
</Label>
|
||||||
|
<Label>
|
||||||
|
临时文件保留时间
|
||||||
|
<Select value={form.temporaryFileExpirePolicy} onChange={(event) => setForm({ ...form, temporaryFileExpirePolicy: event.target.value })}>
|
||||||
|
{expirationPolicyOptions.map((item) => <option value={item.value} key={item.value}>{item.label}</option>)}
|
||||||
|
</Select>
|
||||||
|
<small>仅用于生成媒体和请求素材;云端生命周期按天异步删除。</small>
|
||||||
|
</Label>
|
||||||
<Label>
|
<Label>
|
||||||
Access Key ID
|
Access Key ID
|
||||||
<Input value={form.accessKeyId} onChange={(event) => setForm({ ...form, accessKeyId: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeyIdPreview)} />
|
<Input value={form.accessKeyId} onChange={(event) => setForm({ ...form, accessKeyId: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeyIdPreview)} />
|
||||||
@@ -396,23 +455,43 @@ export function SystemSettingsPanel(props: {
|
|||||||
Access Key Secret
|
Access Key Secret
|
||||||
<Input type="password" value={form.accessKeySecret} onChange={(event) => setForm({ ...form, accessKeySecret: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeySecretPreview)} />
|
<Input type="password" value={form.accessKeySecret} onChange={(event) => setForm({ ...form, accessKeySecret: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeySecretPreview)} />
|
||||||
</Label>
|
</Label>
|
||||||
<Label className="spanTwo">
|
|
||||||
Session Token(可选)
|
|
||||||
<Input type="password" value={form.sessionToken} onChange={(event) => setForm({ ...form, sessionToken: event.target.value })} placeholder={credentialInputPlaceholder(form.sessionTokenPreview)} />
|
|
||||||
</Label>
|
|
||||||
</>}
|
</>}
|
||||||
<Label>
|
<Label>
|
||||||
优先级
|
优先级
|
||||||
<Input type="number" min={1} value={form.priority} onChange={(event) => setForm({ ...form, priority: event.target.value })} />
|
<Input type="number" min={1} value={form.priority} onChange={(event) => setForm({ ...form, priority: event.target.value })} />
|
||||||
</Label>
|
</Label>
|
||||||
<Label className="spanTwo">
|
<details className="fileStorageAdvanced spanTwo">
|
||||||
重试策略 JSON
|
<summary>高级配置</summary>
|
||||||
<Textarea value={form.retryPolicyJson} onChange={(event) => setForm({ ...form, retryPolicyJson: event.target.value })} />
|
<div className="fileStorageAdvancedGrid">
|
||||||
</Label>
|
{form.provider !== 'server_main_openapi' && <>
|
||||||
<Label className="spanTwo">
|
<Label className="spanTwo">
|
||||||
扩展配置 JSON
|
Session Token(可选)
|
||||||
<Textarea value={form.configJson} onChange={(event) => setForm({ ...form, configJson: event.target.value })} />
|
<Input type="password" value={form.sessionToken} onChange={(event) => setForm({ ...form, sessionToken: event.target.value })} placeholder={credentialInputPlaceholder(form.sessionTokenPreview)} />
|
||||||
</Label>
|
</Label>
|
||||||
|
<Label>
|
||||||
|
私有链接有效期(分钟)
|
||||||
|
<Input type="number" min={1} max={10080} value={form.signedUrlExpiresMinutes} onChange={(event) => setForm({ ...form, signedUrlExpiresMinutes: event.target.value })} />
|
||||||
|
<small>只控制签名 URL,不会删除对象。</small>
|
||||||
|
</Label>
|
||||||
|
{form.provider === 's3' && <Label>
|
||||||
|
Bucket 寻址方式
|
||||||
|
<Select value={form.forcePathStyle ? 'path' : 'virtual'} onChange={(event) => setForm({ ...form, forcePathStyle: event.target.value === 'path' })}>
|
||||||
|
<option value="virtual">Virtual Host(推荐)</option>
|
||||||
|
<option value="path">Path Style</option>
|
||||||
|
</Select>
|
||||||
|
</Label>}
|
||||||
|
</>}
|
||||||
|
<Label className="spanTwo">
|
||||||
|
重试策略 JSON
|
||||||
|
<Textarea value={form.retryPolicyJson} onChange={(event) => setForm({ ...form, retryPolicyJson: event.target.value })} />
|
||||||
|
</Label>
|
||||||
|
<Label className="spanTwo">
|
||||||
|
其他扩展配置 JSON
|
||||||
|
<Textarea value={form.configJson} onChange={(event) => setForm({ ...form, configJson: event.target.value })} />
|
||||||
|
<small>仅保存未被上方结构化字段管理的扩展项。</small>
|
||||||
|
</Label>
|
||||||
|
</div>
|
||||||
|
</details>
|
||||||
</FormDialog>
|
</FormDialog>
|
||||||
|
|
||||||
<ConfirmDialog
|
<ConfirmDialog
|
||||||
@@ -464,22 +543,31 @@ function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
|
|||||||
accessKeySecretPreview: '',
|
accessKeySecretPreview: '',
|
||||||
apiKey: '',
|
apiKey: '',
|
||||||
apiKeyPreview: '',
|
apiKeyPreview: '',
|
||||||
|
bucket: '',
|
||||||
channelKey,
|
channelKey,
|
||||||
configJson: '{}',
|
configJson: '{}',
|
||||||
|
endpoint: '',
|
||||||
|
forcePathStyle: false,
|
||||||
name: 'server-main OpenAPI',
|
name: 'server-main OpenAPI',
|
||||||
|
objectPrefix: '',
|
||||||
priority: '100',
|
priority: '100',
|
||||||
provider: 'server_main_openapi',
|
provider: 'server_main_openapi',
|
||||||
|
publicBaseUrl: '',
|
||||||
|
region: '',
|
||||||
retryPolicyJson: stringifyJson(defaultServerMainRetryPolicy),
|
retryPolicyJson: stringifyJson(defaultServerMainRetryPolicy),
|
||||||
sessionToken: '',
|
sessionToken: '',
|
||||||
sessionTokenPreview: '',
|
sessionTokenPreview: '',
|
||||||
|
signedUrlExpiresMinutes: '15',
|
||||||
scenes: defaultScenes,
|
scenes: defaultScenes,
|
||||||
status: 'disabled',
|
status: 'disabled',
|
||||||
|
temporaryFileExpirePolicy: 'never',
|
||||||
uploadUrl: defaultUploadUrl,
|
uploadUrl: defaultUploadUrl,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
export function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
||||||
const preview = apiKeyPreview(channel);
|
const preview = apiKeyPreview(channel);
|
||||||
|
const config = channel.config ?? {};
|
||||||
return {
|
return {
|
||||||
accessKeyId: credentialPreview(channel, 'accessKeyId'),
|
accessKeyId: credentialPreview(channel, 'accessKeyId'),
|
||||||
accessKeyIdPreview: credentialPreview(channel, 'accessKeyId'),
|
accessKeyIdPreview: credentialPreview(channel, 'accessKeyId'),
|
||||||
@@ -487,27 +575,36 @@ function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
|||||||
accessKeySecretPreview: credentialPreview(channel, 'accessKeySecret'),
|
accessKeySecretPreview: credentialPreview(channel, 'accessKeySecret'),
|
||||||
apiKey: preview,
|
apiKey: preview,
|
||||||
apiKeyPreview: preview,
|
apiKeyPreview: preview,
|
||||||
|
bucket: configString(config, 'bucket'),
|
||||||
channelKey: channel.channelKey,
|
channelKey: channel.channelKey,
|
||||||
configJson: stringifyJson(channel.config ?? {}),
|
configJson: stringifyJson(channel.provider === 'server_main_openapi' ? config : unmanagedObjectStorageConfig(config)),
|
||||||
|
endpoint: configString(config, 'endpoint'),
|
||||||
|
forcePathStyle: configBoolean(config, 'forcePathStyle'),
|
||||||
name: channel.name,
|
name: channel.name,
|
||||||
|
objectPrefix: configString(config, 'objectPrefix'),
|
||||||
priority: String(channel.priority || 100),
|
priority: String(channel.priority || 100),
|
||||||
provider: channel.provider || 'server_main_openapi',
|
provider: channel.provider || 'server_main_openapi',
|
||||||
|
publicBaseUrl: publicBaseUrlFromConfig(config),
|
||||||
|
region: configString(config, 'region'),
|
||||||
retryPolicyJson: stringifyJson(channel.retryPolicy ?? defaultRetryPolicyForProvider(channel.provider)),
|
retryPolicyJson: stringifyJson(channel.retryPolicy ?? defaultRetryPolicyForProvider(channel.provider)),
|
||||||
sessionToken: credentialPreview(channel, 'sessionToken'),
|
sessionToken: credentialPreview(channel, 'sessionToken'),
|
||||||
sessionTokenPreview: credentialPreview(channel, 'sessionToken'),
|
sessionTokenPreview: credentialPreview(channel, 'sessionToken'),
|
||||||
|
signedUrlExpiresMinutes: String(Math.max(1, Math.round(configNumber(config, 'signedUrlExpiresSeconds', 900) / 60))),
|
||||||
scenes: normalizeScenes(channel.scenes),
|
scenes: normalizeScenes(channel.scenes),
|
||||||
status: channel.status || 'disabled',
|
status: channel.status || 'disabled',
|
||||||
|
temporaryFileExpirePolicy: expirationPolicyFromConfig(config),
|
||||||
uploadUrl: channel.uploadUrl || defaultUploadUrl,
|
uploadUrl: channel.uploadUrl || defaultUploadUrl,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRequest {
|
export function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRequest {
|
||||||
|
const config = parseJsonObject(form.configJson, '其他扩展配置 JSON');
|
||||||
return {
|
return {
|
||||||
accessKeyId: credentialPayloadValue(form.accessKeyId, form.accessKeyIdPreview),
|
accessKeyId: credentialPayloadValue(form.accessKeyId, form.accessKeyIdPreview),
|
||||||
accessKeySecret: credentialPayloadValue(form.accessKeySecret, form.accessKeySecretPreview),
|
accessKeySecret: credentialPayloadValue(form.accessKeySecret, form.accessKeySecretPreview),
|
||||||
apiKey: apiKeyPayloadValue(form),
|
apiKey: apiKeyPayloadValue(form),
|
||||||
channelKey: form.channelKey.trim(),
|
channelKey: form.channelKey.trim(),
|
||||||
config: parseJsonObject(form.configJson, '扩展配置 JSON'),
|
config: form.provider === 'server_main_openapi' ? config : objectStorageConfigPayload(form, config),
|
||||||
name: form.name.trim(),
|
name: form.name.trim(),
|
||||||
priority: Number(form.priority) || 100,
|
priority: Number(form.priority) || 100,
|
||||||
provider: form.provider,
|
provider: form.provider,
|
||||||
@@ -519,6 +616,73 @@ function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRe
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const managedObjectStorageConfigKeys = new Set([
|
||||||
|
'apiReturnExpirePolicy',
|
||||||
|
'bucket',
|
||||||
|
'cdnDomain',
|
||||||
|
'endpoint',
|
||||||
|
'forcePathStyle',
|
||||||
|
'objectPrefix',
|
||||||
|
'publicBaseURL',
|
||||||
|
'publicBaseUrl',
|
||||||
|
'publicDomain',
|
||||||
|
'region',
|
||||||
|
'signedUrlExpiresSeconds',
|
||||||
|
'temporaryFileExpirePolicy',
|
||||||
|
]);
|
||||||
|
|
||||||
|
function unmanagedObjectStorageConfig(config: Record<string, unknown>) {
|
||||||
|
return Object.fromEntries(Object.entries(config).filter(([key]) => !managedObjectStorageConfigKeys.has(key)));
|
||||||
|
}
|
||||||
|
|
||||||
|
function objectStorageConfigPayload(form: FileStorageChannelForm, extras: Record<string, unknown>) {
|
||||||
|
const config: Record<string, unknown> = { ...unmanagedObjectStorageConfig(extras) };
|
||||||
|
config.endpoint = form.endpoint.trim();
|
||||||
|
config.region = form.region.trim();
|
||||||
|
config.bucket = form.bucket.trim();
|
||||||
|
config.temporaryFileExpirePolicy = normalizeExpirationPolicy(form.temporaryFileExpirePolicy);
|
||||||
|
config.signedUrlExpiresSeconds = Math.max(1, Math.round(Number(form.signedUrlExpiresMinutes) || 15)) * 60;
|
||||||
|
if (form.publicBaseUrl.trim()) config.publicBaseUrl = form.publicBaseUrl.trim().replace(/\/+$/, '');
|
||||||
|
if (form.objectPrefix.trim()) config.objectPrefix = form.objectPrefix.trim().replace(/^\/+|\/+$/g, '');
|
||||||
|
if (form.provider === 's3') config.forcePathStyle = form.forcePathStyle;
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
function configString(config: Record<string, unknown> | undefined, key: string) {
|
||||||
|
const value = config?.[key];
|
||||||
|
return typeof value === 'string' ? value.trim() : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function configBoolean(config: Record<string, unknown> | undefined, key: string) {
|
||||||
|
return config?.[key] === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function configNumber(config: Record<string, unknown> | undefined, key: string, fallback: number) {
|
||||||
|
const value = Number(config?.[key]);
|
||||||
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function publicBaseUrlFromConfig(config: Record<string, unknown> | undefined) {
|
||||||
|
return configString(config, 'publicBaseUrl')
|
||||||
|
|| configString(config, 'publicBaseURL')
|
||||||
|
|| configString(config, 'cdnDomain')
|
||||||
|
|| configString(config, 'publicDomain');
|
||||||
|
}
|
||||||
|
|
||||||
|
function expirationPolicyFromConfig(config: Record<string, unknown> | undefined) {
|
||||||
|
return normalizeExpirationPolicy(configString(config, 'temporaryFileExpirePolicy') || configString(config, 'apiReturnExpirePolicy'));
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeExpirationPolicy(value: string) {
|
||||||
|
const normalized = value.trim().toLowerCase();
|
||||||
|
return expirationPolicyOptions.some((item) => item.value === normalized) ? normalized : 'never';
|
||||||
|
}
|
||||||
|
|
||||||
|
function expirationPolicyLabel(value: string) {
|
||||||
|
const normalized = normalizeExpirationPolicy(value);
|
||||||
|
return expirationPolicyOptions.find((item) => item.value === normalized)?.label ?? '永久保留';
|
||||||
|
}
|
||||||
|
|
||||||
function parseJsonObject(value: string, label: string) {
|
function parseJsonObject(value: string, label: string) {
|
||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(value || '{}') as unknown;
|
const parsed = JSON.parse(value || '{}') as unknown;
|
||||||
|
|||||||
@@ -2441,6 +2441,27 @@
|
|||||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.fileStorageAdvanced {
|
||||||
|
padding: 0.75rem;
|
||||||
|
border: 1px solid var(--border-subtle);
|
||||||
|
border-radius: 0.625rem;
|
||||||
|
background: var(--surface-subtle);
|
||||||
|
}
|
||||||
|
|
||||||
|
.fileStorageAdvanced summary {
|
||||||
|
cursor: pointer;
|
||||||
|
color: var(--text-strong);
|
||||||
|
font-size: 0.875rem;
|
||||||
|
font-weight: var(--font-weight-semibold);
|
||||||
|
}
|
||||||
|
|
||||||
|
.fileStorageAdvancedGrid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
gap: 0.875rem;
|
||||||
|
padding-top: 0.875rem;
|
||||||
|
}
|
||||||
|
|
||||||
.fileStorageSceneGrid {
|
.fileStorageSceneGrid {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
@@ -2822,6 +2843,7 @@
|
|||||||
.fileStorageGrid,
|
.fileStorageGrid,
|
||||||
.fileStorageSettingsCard,
|
.fileStorageSettingsCard,
|
||||||
.fileStorageSceneGrid,
|
.fileStorageSceneGrid,
|
||||||
|
.fileStorageAdvancedGrid,
|
||||||
.runtimePolicyFormBody,
|
.runtimePolicyFormBody,
|
||||||
.fileStorageDialogBody,
|
.fileStorageDialogBody,
|
||||||
.runtimePolicyRows,
|
.runtimePolicyRows,
|
||||||
|
|||||||
@@ -1077,6 +1077,8 @@ export interface FileStorageChannelTestResult {
|
|||||||
putSucceeded: boolean;
|
putSucceeded: boolean;
|
||||||
headSucceeded: boolean;
|
headSucceeded: boolean;
|
||||||
deleteSucceeded: boolean;
|
deleteSucceeded: boolean;
|
||||||
|
lifecycleReady: boolean;
|
||||||
|
expirationPolicy: 'never' | '1d' | '1m' | '3m' | '6m' | string;
|
||||||
durationMs: number;
|
durationMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+275
@@ -0,0 +1,275 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
import { createHmac } from 'node:crypto';
|
||||||
|
|
||||||
|
const requiredEnvironment = [
|
||||||
|
'AI_GATEWAY_TEST_OSS_ENDPOINT',
|
||||||
|
'AI_GATEWAY_TEST_OSS_REGION',
|
||||||
|
'AI_GATEWAY_TEST_OSS_ACCESS_KEY_ID',
|
||||||
|
'AI_GATEWAY_TEST_OSS_ACCESS_KEY_SECRET',
|
||||||
|
'AI_GATEWAY_TEST_OSS_BUCKET',
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const key of requiredEnvironment) {
|
||||||
|
if (!String(process.env[key] ?? '').trim()) {
|
||||||
|
throw new Error(`missing required environment: ${key}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const gatewayBaseURL = String(process.env.AI_GATEWAY_TEST_BASE_URL ?? 'http://127.0.0.1:8088').replace(/\/$/, '');
|
||||||
|
const endpoint = new URL(process.env.AI_GATEWAY_TEST_OSS_ENDPOINT);
|
||||||
|
const region = process.env.AI_GATEWAY_TEST_OSS_REGION.trim();
|
||||||
|
const accessKeyID = process.env.AI_GATEWAY_TEST_OSS_ACCESS_KEY_ID.trim();
|
||||||
|
const accessKeySecret = process.env.AI_GATEWAY_TEST_OSS_ACCESS_KEY_SECRET.trim();
|
||||||
|
const bucket = process.env.AI_GATEWAY_TEST_OSS_BUCKET.trim();
|
||||||
|
const expirationPolicy = String(process.env.AI_GATEWAY_TEST_OSS_EXPIRATION_POLICY ?? '1d').trim();
|
||||||
|
const signedURLExpiresSeconds = Number(process.env.AI_GATEWAY_TEST_OSS_SIGNED_URL_EXPIRES_SECONDS ?? 900);
|
||||||
|
const channelKey = String(
|
||||||
|
process.env.AI_GATEWAY_TEST_OSS_CHANNEL_KEY ?? `aliyun-oss-${region}-${bucket}-local`,
|
||||||
|
).trim();
|
||||||
|
const channelName = String(
|
||||||
|
process.env.AI_GATEWAY_TEST_OSS_CHANNEL_NAME ?? `Aliyun OSS ${region} ${bucket} 本地验收`,
|
||||||
|
).trim();
|
||||||
|
const managedLifecycleRuleIDs = [
|
||||||
|
'DeleteTempFiles-1d',
|
||||||
|
'DeleteTempFiles-1m',
|
||||||
|
'DeleteTempFiles-3m',
|
||||||
|
'DeleteTempFiles-6m',
|
||||||
|
];
|
||||||
|
|
||||||
|
function base64URL(value) {
|
||||||
|
return Buffer.from(value).toString('base64url');
|
||||||
|
}
|
||||||
|
|
||||||
|
function managerToken() {
|
||||||
|
if (!String(process.env.CONFIG_JWT_SECRET ?? '').trim()) {
|
||||||
|
throw new Error('missing required environment: CONFIG_JWT_SECRET');
|
||||||
|
}
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const header = base64URL(JSON.stringify({ alg: 'HS256', typ: 'JWT' }));
|
||||||
|
const payload = base64URL(JSON.stringify({
|
||||||
|
sub: 'local-oss-acceptance-manager',
|
||||||
|
username: 'local-oss-acceptance-manager',
|
||||||
|
role: ['manager'],
|
||||||
|
source: 'gateway',
|
||||||
|
gatewayUserId: 'local-oss-acceptance-manager',
|
||||||
|
tokenPurpose: 'local_break_glass_manager',
|
||||||
|
iat: now,
|
||||||
|
exp: now + 15 * 60,
|
||||||
|
}));
|
||||||
|
const signature = createHmac('sha256', process.env.CONFIG_JWT_SECRET)
|
||||||
|
.update(`${header}.${payload}`)
|
||||||
|
.digest('base64url');
|
||||||
|
return `${header}.${payload}.${signature}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function gatewayAuthorization() {
|
||||||
|
const account = String(process.env.AI_GATEWAY_ONLINE_ACCOUNT ?? '').trim();
|
||||||
|
const password = String(process.env.AI_GATEWAY_ONLINE_PASSWORD ?? '');
|
||||||
|
const gatewayHostname = new URL(gatewayBaseURL).hostname;
|
||||||
|
const localGateway = gatewayHostname === '127.0.0.1' || gatewayHostname === 'localhost' || gatewayHostname === '::1';
|
||||||
|
if (account && password && !localGateway) {
|
||||||
|
const response = await fetch(`${gatewayBaseURL}/api/v1/auth/login`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ account, password }),
|
||||||
|
});
|
||||||
|
const payload = await response.json().catch(() => ({}));
|
||||||
|
if (!response.ok || !payload.accessToken) {
|
||||||
|
throw new Error(`Gateway online login failed: HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
return `Bearer ${payload.accessToken}`;
|
||||||
|
}
|
||||||
|
return `Bearer ${managerToken()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const authorization = await gatewayAuthorization();
|
||||||
|
|
||||||
|
async function gatewayJSON(path, init = {}) {
|
||||||
|
const response = await fetch(`${gatewayBaseURL}${path}`, {
|
||||||
|
...init,
|
||||||
|
headers: {
|
||||||
|
Authorization: authorization,
|
||||||
|
...(init.body && !(init.body instanceof FormData) ? { 'Content-Type': 'application/json' } : {}),
|
||||||
|
...init.headers,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const text = await response.text();
|
||||||
|
let payload = {};
|
||||||
|
if (text) {
|
||||||
|
try {
|
||||||
|
payload = JSON.parse(text);
|
||||||
|
} catch {
|
||||||
|
payload = { message: 'non-JSON response' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!response.ok) {
|
||||||
|
const code = payload.code ?? payload.error?.code ?? 'unknown';
|
||||||
|
const message = payload.message ?? payload.error?.message ?? 'request failed';
|
||||||
|
throw new Error(`gateway ${init.method ?? 'GET'} ${path} failed: HTTP ${response.status} ${code} ${message}`);
|
||||||
|
}
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
function objectURL(objectKey = '', lifecycle = false) {
|
||||||
|
const url = new URL(endpoint.toString());
|
||||||
|
url.hostname = `${bucket}.${url.hostname}`;
|
||||||
|
url.pathname = objectKey
|
||||||
|
? `/${objectKey.split('/').map((part) => encodeURIComponent(part)).join('/')}`
|
||||||
|
: '/';
|
||||||
|
url.search = lifecycle ? '?lifecycle' : '';
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function signedOSSRequest(method, objectKey = '', lifecycle = false) {
|
||||||
|
const date = new Date().toUTCString();
|
||||||
|
const canonicalResource = lifecycle
|
||||||
|
? `/${bucket}/?lifecycle`
|
||||||
|
: `/${bucket}/${objectKey}`;
|
||||||
|
const stringToSign = `${method}\n\n\n${date}\n${canonicalResource}`;
|
||||||
|
const signature = createHmac('sha1', accessKeySecret).update(stringToSign).digest('base64');
|
||||||
|
return fetch(objectURL(objectKey, lifecycle), {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
Authorization: `OSS ${accessKeyID}:${signature}`,
|
||||||
|
Date: date,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function channelInput() {
|
||||||
|
return {
|
||||||
|
channelKey,
|
||||||
|
name: channelName,
|
||||||
|
provider: 'aliyun_oss',
|
||||||
|
accessKeyId: accessKeyID,
|
||||||
|
accessKeySecret,
|
||||||
|
scenes: ['upload', 'image_result', 'request_asset'],
|
||||||
|
config: {
|
||||||
|
endpoint: endpoint.toString().replace(/\/$/, ''),
|
||||||
|
region,
|
||||||
|
bucket,
|
||||||
|
objectKeyPrefix: 'easyai-ai-gateway/live-acceptance',
|
||||||
|
accessScope: 'private',
|
||||||
|
temporaryFileExpirePolicy: expirationPolicy,
|
||||||
|
signedUrlExpiresSeconds: signedURLExpiresSeconds,
|
||||||
|
},
|
||||||
|
retryPolicy: {
|
||||||
|
enabled: true,
|
||||||
|
maxRetries: 1,
|
||||||
|
backoffSeconds: [0.25],
|
||||||
|
},
|
||||||
|
priority: 10,
|
||||||
|
status: 'enabled',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const health = await gatewayJSON('/api/v1/healthz');
|
||||||
|
const ready = await gatewayJSON('/api/v1/readyz');
|
||||||
|
if (health.ok !== true || ready.ok !== true) {
|
||||||
|
throw new Error('gateway health or readiness check failed');
|
||||||
|
}
|
||||||
|
|
||||||
|
const listed = await gatewayJSON('/api/admin/system/file-storage/channels');
|
||||||
|
const existing = Array.isArray(listed.items)
|
||||||
|
? listed.items.find((item) => item.channelKey === channelKey)
|
||||||
|
: undefined;
|
||||||
|
const channel = existing
|
||||||
|
? await gatewayJSON(`/api/admin/system/file-storage/channels/${encodeURIComponent(existing.id)}`, {
|
||||||
|
method: 'PATCH',
|
||||||
|
body: JSON.stringify(channelInput()),
|
||||||
|
})
|
||||||
|
: await gatewayJSON('/api/admin/system/file-storage/channels', {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(channelInput()),
|
||||||
|
});
|
||||||
|
|
||||||
|
const connectionTest = await gatewayJSON(
|
||||||
|
`/api/admin/system/file-storage/channels/${encodeURIComponent(channel.id)}/test`,
|
||||||
|
{ method: 'POST' },
|
||||||
|
);
|
||||||
|
for (const key of ['putSucceeded', 'headSucceeded', 'deleteSucceeded', 'lifecycleReady']) {
|
||||||
|
if (connectionTest[key] !== true) {
|
||||||
|
throw new Error(`object storage connection test did not prove ${key}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (connectionTest.expirationPolicy !== expirationPolicy) {
|
||||||
|
throw new Error('object storage connection test returned an unexpected expiration policy');
|
||||||
|
}
|
||||||
|
|
||||||
|
const lifecycleResponse = await signedOSSRequest('GET', '', true);
|
||||||
|
const lifecycleXML = await lifecycleResponse.text();
|
||||||
|
if (!lifecycleResponse.ok) {
|
||||||
|
throw new Error(`OSS lifecycle read failed: HTTP ${lifecycleResponse.status}`);
|
||||||
|
}
|
||||||
|
const lifecycleRuleIDs = [...lifecycleXML.matchAll(/<ID>([^<]+)<\/ID>/g)].map((match) => match[1]);
|
||||||
|
const missingLifecycleRules = managedLifecycleRuleIDs.filter((id) => !lifecycleRuleIDs.includes(id));
|
||||||
|
if (missingLifecycleRules.length > 0) {
|
||||||
|
throw new Error(`OSS lifecycle rules missing: ${missingLifecycleRules.join(', ')}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const expectedContent = `easyai-oss-live-acceptance:${new Date().toISOString()}`;
|
||||||
|
const form = new FormData();
|
||||||
|
form.append('file', new Blob([expectedContent], { type: 'text/plain' }), 'oss-live-acceptance.txt');
|
||||||
|
form.append('source', 'local-oss-live-acceptance');
|
||||||
|
const upload = await gatewayJSON('/api/v1/files/upload', { method: 'POST', body: form });
|
||||||
|
if (!upload.url || !upload.objectKey || upload.accessScope !== 'private') {
|
||||||
|
throw new Error('Gateway upload response did not include a private OSS object and signed URL');
|
||||||
|
}
|
||||||
|
if (upload.storageChannel?.channelKey !== channelKey) {
|
||||||
|
throw new Error(`Gateway upload did not select the expected OSS channel: ${upload.storageChannel?.channelKey ?? 'none'}`);
|
||||||
|
}
|
||||||
|
if (!upload.urlExpiresAt || upload.objectExpiresAt || upload.expirationPolicy) {
|
||||||
|
throw new Error('normal upload must have a signed URL TTL but remain exempt from temporary-file expiration');
|
||||||
|
}
|
||||||
|
|
||||||
|
const downloadResponse = await fetch(upload.url);
|
||||||
|
const downloadedContent = await downloadResponse.text();
|
||||||
|
if (!downloadResponse.ok || downloadedContent !== expectedContent) {
|
||||||
|
throw new Error(`signed URL download failed: HTTP ${downloadResponse.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const deleteResponse = await signedOSSRequest('DELETE', upload.objectKey);
|
||||||
|
if (!deleteResponse.ok) {
|
||||||
|
throw new Error(`OSS cleanup failed: HTTP ${deleteResponse.status}`);
|
||||||
|
}
|
||||||
|
const afterDeleteResponse = await fetch(upload.url);
|
||||||
|
if (afterDeleteResponse.status !== 404) {
|
||||||
|
throw new Error(`deleted OSS object remained readable: HTTP ${afterDeleteResponse.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const signedURL = new URL(upload.url);
|
||||||
|
console.log(JSON.stringify({
|
||||||
|
gateway: {
|
||||||
|
health: health.ok,
|
||||||
|
ready: ready.ok,
|
||||||
|
},
|
||||||
|
channel: {
|
||||||
|
action: existing ? 'updated' : 'created',
|
||||||
|
id: channel.id,
|
||||||
|
channelKey: channel.channelKey,
|
||||||
|
provider: channel.provider,
|
||||||
|
status: channel.status,
|
||||||
|
region: channel.config?.region,
|
||||||
|
bucket: channel.config?.bucket,
|
||||||
|
expirationPolicy: channel.config?.temporaryFileExpirePolicy,
|
||||||
|
credentialsConfigured: Boolean(channel.credentialsPreview?.accessKeyId)
|
||||||
|
&& Boolean(channel.credentialsPreview?.accessKeySecret),
|
||||||
|
},
|
||||||
|
connectionTest,
|
||||||
|
lifecycle: {
|
||||||
|
status: lifecycleResponse.status,
|
||||||
|
managedRuleIDs: managedLifecycleRuleIDs,
|
||||||
|
},
|
||||||
|
upload: {
|
||||||
|
status: 'success',
|
||||||
|
bytes: Buffer.byteLength(expectedContent),
|
||||||
|
accessScope: upload.accessScope,
|
||||||
|
signedURLHost: signedURL.host,
|
||||||
|
signedURLExpiresAt: upload.urlExpiresAt,
|
||||||
|
normalUploadIsPermanent: !upload.objectExpiresAt,
|
||||||
|
downloadStatus: downloadResponse.status,
|
||||||
|
deleteStatus: deleteResponse.status,
|
||||||
|
afterDeleteStatus: afterDeleteResponse.status,
|
||||||
|
},
|
||||||
|
}, null, 2));
|
||||||
Reference in New Issue
Block a user