BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(storage): 完善对象存储配置与过期策略

Browse Source 
补齐 OSS/S3 的 Endpoint、Region、Bucket、CDN、对象前缀和签名有效期配置，并为生成结果与请求素材自动维护分级生命周期规则。普通上传继续保持永久，私有资源按配置生成限时签名 URL，管理端连接测试覆盖生命周期、上传、读取和删除。\n\n新增可重复的真实 OSS 验收脚本，凭据仅从本地环境读取，接口响应继续保持脱敏。\n\n验证：Go 全量测试、迁移安全检查、pnpm lint、pnpm test、pnpm build、本地阿里云 OSS 真实上传下载删除验收。
This commit is contained in:
wangbo
2026-08-04 12:44:24 +08:00
parent fe56aa46b9
commit 6a64d3936a
16 changed files with 1274 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/runner/object_storage.go
+86 -20
View File
@@ -47,11 +47,13 @@ type objectStorageAdapter struct {
}
type FileStorageChannelTestResult struct {
Provider string `json:"provider"`
PutSucceeded bool `json:"putSucceeded"`
HeadSucceeded bool `json:"headSucceeded"`
DeleteSucceeded bool `json:"deleteSucceeded"`
DurationMS int64 `json:"durationMs"`
Provider string `json:"provider"`
PutSucceeded bool `json:"putSucceeded"`
HeadSucceeded bool `json:"headSucceeded"`
DeleteSucceeded bool `json:"deleteSucceeded"`
LifecycleReady bool `json:"lifecycleReady"`
ExpirationPolicy string `json:"expirationPolicy"`
DurationMS int64 `json:"durationMs"`
}
// TestFileStorageChannel performs an isolated write, metadata read and cleanup
@@ -59,16 +61,29 @@ type FileStorageChannelTestResult struct {
// content-addressed business object and no object key or credential is exposed.
func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) {
startedAt := time.Now()
result := FileStorageChannelTestResult{Provider: strings.ToLower(strings.TrimSpace(channel.Provider))}
result := FileStorageChannelTestResult{
Provider: strings.ToLower(strings.TrimSpace(channel.Provider)),
ExpirationPolicy: objectStorageExpirationPolicy(channel.Config),
}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return result, err
}
if result.ExpirationPolicy != objectStorageExpirationNever {
if err := adapter.ensureLifecycle(ctx); err != nil {
return result, err
}
result.LifecycleReady = true
}
probeScene := store.FileStorageSceneUpload
if result.LifecycleReady {
probeScene = store.FileStorageSceneImageResult
}
payload := FileUploadPayload{
Bytes: []byte("easyai-storage-probe:" + uuid.NewString()),
ContentType: "application/octet-stream",
FileName: "probe.bin",
Scene: store.FileStorageSceneUpload,
Scene: probeScene,
Source: "admin-connection-test",
}
upload, err := adapter.put(ctx, payload)
@@ -112,6 +127,12 @@ func newObjectStorageAdapter(channel store.FileStorageChannel) (*objectStorageAd
}
func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
retention := objectStorageRetentionFor(a.channel.Config, payload.Scene, a.now().UTC())
if retention.Enabled {
if err := a.ensureLifecycle(ctx); err != nil {
return nil, err
}
}
objectKey := a.objectKey(payload)
requestURL, err := a.objectURL(objectKey)
if err != nil {
@@ -126,6 +147,14 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
}
req.Header.Set("Content-Type", contentType)
if retention.Enabled {
tagging := retention.taggingHeader()
if strings.EqualFold(a.channel.Provider, "s3") {
req.Header.Set("x-amz-tagging", tagging)
} else {
req.Header.Set("x-oss-tagging", tagging)
}
}
if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil {
return nil, err
}
@@ -148,11 +177,12 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
}
urlExpiresAt := ""
if publicURL == "" {
publicURL, err = a.presignGet(objectKey, objectStorageSignedURLTTL)
signedURLTTL := objectStorageSignedURLTTLForConfig(a.channel.Config)
publicURL, err = a.presignGet(objectKey, signedURLTTL)
if err != nil {
return nil, err
}
urlExpiresAt = a.now().UTC().Add(objectStorageSignedURLTTL).Format(time.RFC3339)
urlExpiresAt = a.now().UTC().Add(signedURLTTL).Format(time.RFC3339)
}
digest := sha256.Sum256(payload.Bytes)
result := map[string]any{
@@ -173,6 +203,10 @@ func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayloa
if urlExpiresAt != "" {
result["urlExpiresAt"] = urlExpiresAt
}
if retention.Enabled {
result["objectExpiresAt"] = retention.ExpiresAt.Format(time.RFC3339)
result["expirationPolicy"] = retention.Policy
}
return result, nil
}
@@ -301,10 +335,7 @@ func (a *objectStorageAdapter) objectURL(objectKey string) (string, error) {
}
func (a *objectStorageAdapter) publicURL(objectKey string) string {
baseURL := strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseUrl"), "/")
if baseURL == "" {
baseURL = strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseURL"), "/")
}
baseURL := strings.TrimRight(objectStoragePublicBaseURL(a.channel.Config), "/")
if baseURL == "" {
return ""
}
@@ -322,19 +353,45 @@ func (a *objectStorageAdapter) sign(req *http.Request, payloadHash string, now t
func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) {
date := now.UTC().Format(http.TimeFormat)
contentMD5 := req.Header.Get("Content-MD5")
contentType := req.Header.Get("Content-Type")
canonicalHeaders := ""
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
req.Header.Set("x-oss-security-token", token)
canonicalHeaders = "x-oss-security-token:" + token + "\n"
}
canonicalHeaders := canonicalOSSHeaders(req)
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath()
stringToSign := req.Method + "\n\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
if _, ok := req.URL.Query()["lifecycle"]; ok {
canonicalResource += "?lifecycle"
}
stringToSign := req.Method + "\n" + contentMD5 + "\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign)
req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature)
req.Header.Set("Date", date)
}
func canonicalOSSHeaders(req *http.Request) string {
headers := map[string]string{}
for key, values := range req.Header {
lower := strings.ToLower(strings.TrimSpace(key))
if strings.HasPrefix(lower, "x-oss-") {
headers[lower] = strings.Join(values, ",")
}
}
keys := make([]string, 0, len(headers))
for key := range headers {
keys = append(keys, key)
}
sort.Strings(keys)
var canonical strings.Builder
for _, key := range keys {
canonical.WriteString(key)
canonical.WriteByte(':')
canonical.WriteString(strings.Join(strings.Fields(headers[key]), " "))
canonical.WriteByte('\n')
}
return canonical.String()
}
func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) {
parsed, err := url.Parse(requestURL)
if err != nil {
@@ -487,18 +544,27 @@ func objectStorageConfigBool(config map[string]any, key string) bool {
}
func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string {
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
return scope
}
if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset {
return "private"
}
if strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseUrl")) != "" || strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseURL")) != "" {
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
return scope
}
if objectStoragePublicBaseURL(channel.Config) != "" {
return "public"
}
return "private"
}
func objectStoragePublicBaseURL(config map[string]any) string {
return firstNonEmptyString(
objectStorageConfigString(config, "publicBaseUrl"),
objectStorageConfigString(config, "publicBaseURL"),
objectStorageConfigString(config, "cdnDomain"),
objectStorageConfigString(config, "publicDomain"),
)
}
func escapeObjectKey(objectKey string) string {
parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/")
for index, part := range parts {