feat(storage): 完善对象存储配置与过期策略
补齐 OSS/S3 的 Endpoint、Region、Bucket、CDN、对象前缀和签名有效期配置，并为生成结果与请求素材自动维护分级生命周期规则。普通上传继续保持永久，私有资源按配置生成限时签名 URL，管理端连接测试覆盖生命周期、上传、读取和删除。\n\n新增可重复的真实 OSS 验收脚本，凭据仅从本地环境读取，接口响应继续保持脱敏。\n\n验证：Go 全量测试、迁移安全检查、pnpm lint、pnpm test、pnpm build、本地阿里云 OSS 真实上传下载删除验收。
This commit is contained in:
@@ -0,0 +1,378 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
objectStorageExpirationNever = "never"
|
||||
objectStorageExpiration1Day = "1d"
|
||||
objectStorageExpiration1Month = "1m"
|
||||
objectStorageExpiration3Months = "3m"
|
||||
objectStorageExpiration6Months = "6m"
|
||||
|
||||
objectStorageSignedURLTTLMin = time.Minute
|
||||
objectStorageSignedURLTTLMax = 7 * 24 * time.Hour
|
||||
)
|
||||
|
||||
type objectStorageLifecycleDefinition struct {
|
||||
ID string
|
||||
TagValue string
|
||||
Days int
|
||||
}
|
||||
|
||||
var objectStorageLifecycleDefinitions = []objectStorageLifecycleDefinition{
|
||||
{ID: "DeleteTempFiles-1d", TagValue: objectStorageExpiration1Day, Days: 1},
|
||||
{ID: "DeleteTempFiles-1m", TagValue: objectStorageExpiration1Month, Days: 30},
|
||||
{ID: "DeleteTempFiles-3m", TagValue: objectStorageExpiration3Months, Days: 90},
|
||||
{ID: "DeleteTempFiles-6m", TagValue: objectStorageExpiration6Months, Days: 180},
|
||||
}
|
||||
|
||||
type objectStorageRetention struct {
|
||||
Enabled bool
|
||||
Policy string
|
||||
TagValue string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
func (r objectStorageRetention) taggingHeader() string {
|
||||
values := url.Values{}
|
||||
values.Set("TempFile", "true")
|
||||
values.Set("ExpiresAt", strconv.FormatInt(r.ExpiresAt.Unix(), 10))
|
||||
values.Set("ExpiresIn", r.TagValue)
|
||||
return values.Encode()
|
||||
}
|
||||
|
||||
func objectStorageExpirationPolicy(config map[string]any) string {
|
||||
policy := strings.ToLower(firstNonEmptyString(
|
||||
objectStorageConfigString(config, "temporaryFileExpirePolicy"),
|
||||
objectStorageConfigString(config, "apiReturnExpirePolicy"),
|
||||
))
|
||||
switch policy {
|
||||
case objectStorageExpiration1Day,
|
||||
objectStorageExpiration1Month,
|
||||
objectStorageExpiration3Months,
|
||||
objectStorageExpiration6Months:
|
||||
return policy
|
||||
default:
|
||||
return objectStorageExpirationNever
|
||||
}
|
||||
}
|
||||
|
||||
func objectStorageRetentionFor(config map[string]any, scene string, now time.Time) objectStorageRetention {
|
||||
if strings.TrimSpace(scene) != "image_result" && strings.TrimSpace(scene) != "request_asset" {
|
||||
return objectStorageRetention{Policy: objectStorageExpirationNever}
|
||||
}
|
||||
policy := objectStorageExpirationPolicy(config)
|
||||
duration := objectStorageExpirationDuration(policy)
|
||||
if duration <= 0 {
|
||||
return objectStorageRetention{Policy: objectStorageExpirationNever}
|
||||
}
|
||||
return objectStorageRetention{
|
||||
Enabled: true,
|
||||
Policy: policy,
|
||||
TagValue: policy,
|
||||
ExpiresAt: now.Add(duration),
|
||||
}
|
||||
}
|
||||
|
||||
func objectStorageExpirationDuration(policy string) time.Duration {
|
||||
switch policy {
|
||||
case objectStorageExpiration1Day:
|
||||
return 24 * time.Hour
|
||||
case objectStorageExpiration1Month:
|
||||
return 30 * 24 * time.Hour
|
||||
case objectStorageExpiration3Months:
|
||||
return 90 * 24 * time.Hour
|
||||
case objectStorageExpiration6Months:
|
||||
return 180 * 24 * time.Hour
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func objectStorageSignedURLTTLForConfig(config map[string]any) time.Duration {
|
||||
seconds := int64FromAny(config["signedUrlExpiresSeconds"])
|
||||
if seconds <= 0 {
|
||||
return objectStorageSignedURLTTL
|
||||
}
|
||||
ttl := time.Duration(seconds) * time.Second
|
||||
if ttl < objectStorageSignedURLTTLMin {
|
||||
return objectStorageSignedURLTTLMin
|
||||
}
|
||||
if ttl > objectStorageSignedURLTTLMax {
|
||||
return objectStorageSignedURLTTLMax
|
||||
}
|
||||
return ttl
|
||||
}
|
||||
|
||||
func int64FromAny(value any) int64 {
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return int64(typed)
|
||||
case int32:
|
||||
return int64(typed)
|
||||
case int64:
|
||||
return typed
|
||||
case float32:
|
||||
return int64(typed)
|
||||
case float64:
|
||||
return int64(typed)
|
||||
case string:
|
||||
parsed, _ := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||
return parsed
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
objectStorageLifecycleReady sync.Map
|
||||
objectStorageLifecycleLocks sync.Map
|
||||
)
|
||||
|
||||
func (a *objectStorageAdapter) ensureLifecycle(ctx context.Context) error {
|
||||
cacheKey := strings.Join([]string{
|
||||
strings.ToLower(strings.TrimSpace(a.channel.Provider)),
|
||||
objectStorageConfigString(a.channel.Config, "endpoint"),
|
||||
objectStorageConfigString(a.channel.Config, "bucket"),
|
||||
}, "\x00")
|
||||
if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
lockValue, _ := objectStorageLifecycleLocks.LoadOrStore(cacheKey, &sync.Mutex{})
|
||||
lifecycleLock := lockValue.(*sync.Mutex)
|
||||
lifecycleLock.Lock()
|
||||
defer lifecycleLock.Unlock()
|
||||
if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
current, err := a.readLifecycle(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next, changed, err := mergeObjectStorageLifecycle(a.channel.Provider, current)
|
||||
if err != nil {
|
||||
return storageClientError("storage_config_invalid", "object storage lifecycle configuration is invalid", 0, false)
|
||||
}
|
||||
if changed {
|
||||
if err := a.writeLifecycle(ctx, next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
objectStorageLifecycleReady.Store(cacheKey, struct{}{})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) readLifecycle(ctx context.Context) ([]byte, error) {
|
||||
requestURL, err := a.bucketLifecycleURL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_write_failed", "object storage lifecycle check failed", 0, true)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
|
||||
if readErr != nil {
|
||||
return nil, storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) writeLifecycle(ctx context.Context, payload []byte) error {
|
||||
requestURL, err := a.bucketLifecycleURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
digest := md5.Sum(payload)
|
||||
req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(digest[:]))
|
||||
req.Header.Set("Content-Type", "application/xml")
|
||||
if err := a.sign(req, sha256Hex(payload), a.now().UTC()); err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return storageClientError("storage_write_failed", "object storage lifecycle configuration failed", 0, true)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
if readErr != nil {
|
||||
return storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) bucketLifecycleURL() (string, error) {
|
||||
endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/"))
|
||||
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil {
|
||||
return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false)
|
||||
}
|
||||
bucket := objectStorageConfigString(a.channel.Config, "bucket")
|
||||
hostname := endpoint.Hostname()
|
||||
forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle")
|
||||
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||
if forcePathStyle {
|
||||
endpoint.Path = path.Join(endpoint.Path, bucket)
|
||||
} else if !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
||||
if port := endpoint.Port(); port != "" {
|
||||
endpoint.Host = bucket + "." + hostname + ":" + port
|
||||
} else {
|
||||
endpoint.Host = bucket + "." + endpoint.Host
|
||||
}
|
||||
}
|
||||
} else if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
||||
if port := endpoint.Port(); port != "" {
|
||||
endpoint.Host = bucket + "." + hostname + ":" + port
|
||||
} else {
|
||||
endpoint.Host = bucket + "." + endpoint.Host
|
||||
}
|
||||
}
|
||||
if endpoint.Path == "" {
|
||||
endpoint.Path = "/"
|
||||
}
|
||||
endpoint.RawQuery = "lifecycle"
|
||||
endpoint.Fragment = ""
|
||||
return endpoint.String(), nil
|
||||
}
|
||||
|
||||
type rawLifecycleConfiguration struct {
|
||||
Rules []rawLifecycleRule `xml:"Rule"`
|
||||
}
|
||||
|
||||
type rawLifecycleRule struct {
|
||||
ID string `xml:"ID"`
|
||||
Status string `xml:"Status"`
|
||||
Tag rawLifecycleTag `xml:"Tag"`
|
||||
Filter rawLifecycleFilter `xml:"Filter"`
|
||||
Expiration rawLifecycleExpiration `xml:"Expiration"`
|
||||
InnerXML string `xml:",innerxml"`
|
||||
}
|
||||
|
||||
type rawLifecycleFilter struct {
|
||||
Tag rawLifecycleTag `xml:"Tag"`
|
||||
}
|
||||
|
||||
type rawLifecycleTag struct {
|
||||
Key string `xml:"Key"`
|
||||
Value string `xml:"Value"`
|
||||
}
|
||||
|
||||
type rawLifecycleExpiration struct {
|
||||
Days int `xml:"Days"`
|
||||
}
|
||||
|
||||
func mergeObjectStorageLifecycle(provider string, current []byte) ([]byte, bool, error) {
|
||||
configuration := rawLifecycleConfiguration{}
|
||||
if len(bytes.TrimSpace(current)) > 0 {
|
||||
if err := xml.Unmarshal(current, &configuration); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
managed := map[string]objectStorageLifecycleDefinition{}
|
||||
for _, definition := range objectStorageLifecycleDefinitions {
|
||||
managed[definition.ID] = definition
|
||||
}
|
||||
allManagedReady := true
|
||||
for _, definition := range objectStorageLifecycleDefinitions {
|
||||
matched := false
|
||||
for _, rule := range configuration.Rules {
|
||||
if rule.ID == definition.ID {
|
||||
matched = lifecycleRuleMatches(rule, definition)
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
allManagedReady = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allManagedReady {
|
||||
return current, false, nil
|
||||
}
|
||||
|
||||
var output strings.Builder
|
||||
output.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
|
||||
if strings.EqualFold(strings.TrimSpace(provider), "s3") {
|
||||
output.WriteString(`<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`)
|
||||
} else {
|
||||
output.WriteString(`<LifecycleConfiguration>`)
|
||||
}
|
||||
for _, rule := range configuration.Rules {
|
||||
if _, isManaged := managed[rule.ID]; isManaged {
|
||||
continue
|
||||
}
|
||||
output.WriteString("<Rule>")
|
||||
output.WriteString(rule.InnerXML)
|
||||
output.WriteString("</Rule>")
|
||||
}
|
||||
for _, definition := range objectStorageLifecycleDefinitions {
|
||||
output.WriteString(lifecycleRuleXML(provider, definition))
|
||||
}
|
||||
output.WriteString(`</LifecycleConfiguration>`)
|
||||
return []byte(output.String()), true, nil
|
||||
}
|
||||
|
||||
func lifecycleRuleMatches(rule rawLifecycleRule, definition objectStorageLifecycleDefinition) bool {
|
||||
tag := rule.Tag
|
||||
if tag.Key == "" {
|
||||
tag = rule.Filter.Tag
|
||||
}
|
||||
return strings.EqualFold(strings.TrimSpace(rule.Status), "Enabled") &&
|
||||
tag.Key == "ExpiresIn" && tag.Value == definition.TagValue &&
|
||||
rule.Expiration.Days == definition.Days
|
||||
}
|
||||
|
||||
func lifecycleRuleXML(provider string, definition objectStorageLifecycleDefinition) string {
|
||||
if strings.EqualFold(strings.TrimSpace(provider), "s3") {
|
||||
return fmt.Sprintf(
|
||||
"<Rule><ID>%s</ID><Filter><Tag><Key>ExpiresIn</Key><Value>%s</Value></Tag></Filter><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule>",
|
||||
definition.ID,
|
||||
definition.TagValue,
|
||||
definition.Days,
|
||||
)
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"<Rule><ID>%s</ID><Prefix></Prefix><Tag><Key>ExpiresIn</Key><Value>%s</Value></Tag><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule>",
|
||||
definition.ID,
|
||||
definition.TagValue,
|
||||
definition.Days,
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user