wangbo
6a64d3936a
补齐 OSS/S3 的 Endpoint、Region、Bucket、CDN、对象前缀和签名有效期配置，并为生成结果与请求素材自动维护分级生命周期规则。普通上传继续保持永久，私有资源按配置生成限时签名 URL，管理端连接测试覆盖生命周期、上传、读取和删除。\n\n新增可重复的真实 OSS 验收脚本，凭据仅从本地环境读取，接口响应继续保持脱敏。\n\n验证：Go 全量测试、迁移安全检查、pnpm lint、pnpm test、pnpm build、本地阿里云 OSS 真实上传下载删除验收。
379 lines
11 KiB
Go
379 lines
11 KiB
Go
package runner
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/md5"
|
|
"encoding/base64"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
objectStorageExpirationNever = "never"
|
|
objectStorageExpiration1Day = "1d"
|
|
objectStorageExpiration1Month = "1m"
|
|
objectStorageExpiration3Months = "3m"
|
|
objectStorageExpiration6Months = "6m"
|
|
|
|
objectStorageSignedURLTTLMin = time.Minute
|
|
objectStorageSignedURLTTLMax = 7 * 24 * time.Hour
|
|
)
|
|
|
|
type objectStorageLifecycleDefinition struct {
|
|
ID string
|
|
TagValue string
|
|
Days int
|
|
}
|
|
|
|
var objectStorageLifecycleDefinitions = []objectStorageLifecycleDefinition{
|
|
{ID: "DeleteTempFiles-1d", TagValue: objectStorageExpiration1Day, Days: 1},
|
|
{ID: "DeleteTempFiles-1m", TagValue: objectStorageExpiration1Month, Days: 30},
|
|
{ID: "DeleteTempFiles-3m", TagValue: objectStorageExpiration3Months, Days: 90},
|
|
{ID: "DeleteTempFiles-6m", TagValue: objectStorageExpiration6Months, Days: 180},
|
|
}
|
|
|
|
type objectStorageRetention struct {
|
|
Enabled bool
|
|
Policy string
|
|
TagValue string
|
|
ExpiresAt time.Time
|
|
}
|
|
|
|
func (r objectStorageRetention) taggingHeader() string {
|
|
values := url.Values{}
|
|
values.Set("TempFile", "true")
|
|
values.Set("ExpiresAt", strconv.FormatInt(r.ExpiresAt.Unix(), 10))
|
|
values.Set("ExpiresIn", r.TagValue)
|
|
return values.Encode()
|
|
}
|
|
|
|
func objectStorageExpirationPolicy(config map[string]any) string {
|
|
policy := strings.ToLower(firstNonEmptyString(
|
|
objectStorageConfigString(config, "temporaryFileExpirePolicy"),
|
|
objectStorageConfigString(config, "apiReturnExpirePolicy"),
|
|
))
|
|
switch policy {
|
|
case objectStorageExpiration1Day,
|
|
objectStorageExpiration1Month,
|
|
objectStorageExpiration3Months,
|
|
objectStorageExpiration6Months:
|
|
return policy
|
|
default:
|
|
return objectStorageExpirationNever
|
|
}
|
|
}
|
|
|
|
func objectStorageRetentionFor(config map[string]any, scene string, now time.Time) objectStorageRetention {
|
|
if strings.TrimSpace(scene) != "image_result" && strings.TrimSpace(scene) != "request_asset" {
|
|
return objectStorageRetention{Policy: objectStorageExpirationNever}
|
|
}
|
|
policy := objectStorageExpirationPolicy(config)
|
|
duration := objectStorageExpirationDuration(policy)
|
|
if duration <= 0 {
|
|
return objectStorageRetention{Policy: objectStorageExpirationNever}
|
|
}
|
|
return objectStorageRetention{
|
|
Enabled: true,
|
|
Policy: policy,
|
|
TagValue: policy,
|
|
ExpiresAt: now.Add(duration),
|
|
}
|
|
}
|
|
|
|
func objectStorageExpirationDuration(policy string) time.Duration {
|
|
switch policy {
|
|
case objectStorageExpiration1Day:
|
|
return 24 * time.Hour
|
|
case objectStorageExpiration1Month:
|
|
return 30 * 24 * time.Hour
|
|
case objectStorageExpiration3Months:
|
|
return 90 * 24 * time.Hour
|
|
case objectStorageExpiration6Months:
|
|
return 180 * 24 * time.Hour
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
func objectStorageSignedURLTTLForConfig(config map[string]any) time.Duration {
|
|
seconds := int64FromAny(config["signedUrlExpiresSeconds"])
|
|
if seconds <= 0 {
|
|
return objectStorageSignedURLTTL
|
|
}
|
|
ttl := time.Duration(seconds) * time.Second
|
|
if ttl < objectStorageSignedURLTTLMin {
|
|
return objectStorageSignedURLTTLMin
|
|
}
|
|
if ttl > objectStorageSignedURLTTLMax {
|
|
return objectStorageSignedURLTTLMax
|
|
}
|
|
return ttl
|
|
}
|
|
|
|
func int64FromAny(value any) int64 {
|
|
switch typed := value.(type) {
|
|
case int:
|
|
return int64(typed)
|
|
case int32:
|
|
return int64(typed)
|
|
case int64:
|
|
return typed
|
|
case float32:
|
|
return int64(typed)
|
|
case float64:
|
|
return int64(typed)
|
|
case string:
|
|
parsed, _ := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
|
return parsed
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
var (
|
|
objectStorageLifecycleReady sync.Map
|
|
objectStorageLifecycleLocks sync.Map
|
|
)
|
|
|
|
func (a *objectStorageAdapter) ensureLifecycle(ctx context.Context) error {
|
|
cacheKey := strings.Join([]string{
|
|
strings.ToLower(strings.TrimSpace(a.channel.Provider)),
|
|
objectStorageConfigString(a.channel.Config, "endpoint"),
|
|
objectStorageConfigString(a.channel.Config, "bucket"),
|
|
}, "\x00")
|
|
if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok {
|
|
return nil
|
|
}
|
|
|
|
lockValue, _ := objectStorageLifecycleLocks.LoadOrStore(cacheKey, &sync.Mutex{})
|
|
lifecycleLock := lockValue.(*sync.Mutex)
|
|
lifecycleLock.Lock()
|
|
defer lifecycleLock.Unlock()
|
|
if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok {
|
|
return nil
|
|
}
|
|
|
|
current, err := a.readLifecycle(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
next, changed, err := mergeObjectStorageLifecycle(a.channel.Provider, current)
|
|
if err != nil {
|
|
return storageClientError("storage_config_invalid", "object storage lifecycle configuration is invalid", 0, false)
|
|
}
|
|
if changed {
|
|
if err := a.writeLifecycle(ctx, next); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
objectStorageLifecycleReady.Store(cacheKey, struct{}{})
|
|
return nil
|
|
}
|
|
|
|
func (a *objectStorageAdapter) readLifecycle(ctx context.Context) ([]byte, error) {
|
|
requestURL, err := a.bucketLifecycleURL()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
|
if err != nil {
|
|
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
|
}
|
|
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := a.client.Do(req)
|
|
if err != nil {
|
|
return nil, storageClientError("storage_write_failed", "object storage lifecycle check failed", 0, true)
|
|
}
|
|
defer resp.Body.Close()
|
|
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
|
|
if readErr != nil {
|
|
return nil, storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
|
}
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return nil, nil
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return nil, objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body)
|
|
}
|
|
return body, nil
|
|
}
|
|
|
|
func (a *objectStorageAdapter) writeLifecycle(ctx context.Context, payload []byte) error {
|
|
requestURL, err := a.bucketLifecycleURL()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload))
|
|
if err != nil {
|
|
return storageClientError("storage_config_invalid", err.Error(), 0, false)
|
|
}
|
|
digest := md5.Sum(payload)
|
|
req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(digest[:]))
|
|
req.Header.Set("Content-Type", "application/xml")
|
|
if err := a.sign(req, sha256Hex(payload), a.now().UTC()); err != nil {
|
|
return err
|
|
}
|
|
resp, err := a.client.Do(req)
|
|
if err != nil {
|
|
return storageClientError("storage_write_failed", "object storage lifecycle configuration failed", 0, true)
|
|
}
|
|
defer resp.Body.Close()
|
|
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
|
if readErr != nil {
|
|
return storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *objectStorageAdapter) bucketLifecycleURL() (string, error) {
|
|
endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/"))
|
|
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil {
|
|
return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false)
|
|
}
|
|
bucket := objectStorageConfigString(a.channel.Config, "bucket")
|
|
hostname := endpoint.Hostname()
|
|
forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle")
|
|
if strings.EqualFold(a.channel.Provider, "s3") {
|
|
if forcePathStyle {
|
|
endpoint.Path = path.Join(endpoint.Path, bucket)
|
|
} else if !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
|
if port := endpoint.Port(); port != "" {
|
|
endpoint.Host = bucket + "." + hostname + ":" + port
|
|
} else {
|
|
endpoint.Host = bucket + "." + endpoint.Host
|
|
}
|
|
}
|
|
} else if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
|
if port := endpoint.Port(); port != "" {
|
|
endpoint.Host = bucket + "." + hostname + ":" + port
|
|
} else {
|
|
endpoint.Host = bucket + "." + endpoint.Host
|
|
}
|
|
}
|
|
if endpoint.Path == "" {
|
|
endpoint.Path = "/"
|
|
}
|
|
endpoint.RawQuery = "lifecycle"
|
|
endpoint.Fragment = ""
|
|
return endpoint.String(), nil
|
|
}
|
|
|
|
type rawLifecycleConfiguration struct {
|
|
Rules []rawLifecycleRule `xml:"Rule"`
|
|
}
|
|
|
|
type rawLifecycleRule struct {
|
|
ID string `xml:"ID"`
|
|
Status string `xml:"Status"`
|
|
Tag rawLifecycleTag `xml:"Tag"`
|
|
Filter rawLifecycleFilter `xml:"Filter"`
|
|
Expiration rawLifecycleExpiration `xml:"Expiration"`
|
|
InnerXML string `xml:",innerxml"`
|
|
}
|
|
|
|
type rawLifecycleFilter struct {
|
|
Tag rawLifecycleTag `xml:"Tag"`
|
|
}
|
|
|
|
type rawLifecycleTag struct {
|
|
Key string `xml:"Key"`
|
|
Value string `xml:"Value"`
|
|
}
|
|
|
|
type rawLifecycleExpiration struct {
|
|
Days int `xml:"Days"`
|
|
}
|
|
|
|
func mergeObjectStorageLifecycle(provider string, current []byte) ([]byte, bool, error) {
|
|
configuration := rawLifecycleConfiguration{}
|
|
if len(bytes.TrimSpace(current)) > 0 {
|
|
if err := xml.Unmarshal(current, &configuration); err != nil {
|
|
return nil, false, err
|
|
}
|
|
}
|
|
managed := map[string]objectStorageLifecycleDefinition{}
|
|
for _, definition := range objectStorageLifecycleDefinitions {
|
|
managed[definition.ID] = definition
|
|
}
|
|
allManagedReady := true
|
|
for _, definition := range objectStorageLifecycleDefinitions {
|
|
matched := false
|
|
for _, rule := range configuration.Rules {
|
|
if rule.ID == definition.ID {
|
|
matched = lifecycleRuleMatches(rule, definition)
|
|
break
|
|
}
|
|
}
|
|
if !matched {
|
|
allManagedReady = false
|
|
break
|
|
}
|
|
}
|
|
if allManagedReady {
|
|
return current, false, nil
|
|
}
|
|
|
|
var output strings.Builder
|
|
output.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
|
|
if strings.EqualFold(strings.TrimSpace(provider), "s3") {
|
|
output.WriteString(`<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`)
|
|
} else {
|
|
output.WriteString(`<LifecycleConfiguration>`)
|
|
}
|
|
for _, rule := range configuration.Rules {
|
|
if _, isManaged := managed[rule.ID]; isManaged {
|
|
continue
|
|
}
|
|
output.WriteString("<Rule>")
|
|
output.WriteString(rule.InnerXML)
|
|
output.WriteString("</Rule>")
|
|
}
|
|
for _, definition := range objectStorageLifecycleDefinitions {
|
|
output.WriteString(lifecycleRuleXML(provider, definition))
|
|
}
|
|
output.WriteString(`</LifecycleConfiguration>`)
|
|
return []byte(output.String()), true, nil
|
|
}
|
|
|
|
func lifecycleRuleMatches(rule rawLifecycleRule, definition objectStorageLifecycleDefinition) bool {
|
|
tag := rule.Tag
|
|
if tag.Key == "" {
|
|
tag = rule.Filter.Tag
|
|
}
|
|
return strings.EqualFold(strings.TrimSpace(rule.Status), "Enabled") &&
|
|
tag.Key == "ExpiresIn" && tag.Value == definition.TagValue &&
|
|
rule.Expiration.Days == definition.Days
|
|
}
|
|
|
|
func lifecycleRuleXML(provider string, definition objectStorageLifecycleDefinition) string {
|
|
if strings.EqualFold(strings.TrimSpace(provider), "s3") {
|
|
return fmt.Sprintf(
|
|
"<Rule><ID>%s</ID><Filter><Tag><Key>ExpiresIn</Key><Value>%s</Value></Tag></Filter><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule>",
|
|
definition.ID,
|
|
definition.TagValue,
|
|
definition.Days,
|
|
)
|
|
}
|
|
return fmt.Sprintf(
|
|
"<Rule><ID>%s</ID><Prefix></Prefix><Tag><Key>ExpiresIn</Key><Value>%s</Value></Tag><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule>",
|
|
definition.ID,
|
|
definition.TagValue,
|
|
definition.Days,
|
|
)
|
|
}
|