package runner import ( "bytes" "context" "crypto/md5" "encoding/base64" "encoding/xml" "fmt" "io" "net" "net/http" "net/url" "path" "strconv" "strings" "sync" "time" ) const ( objectStorageExpirationNever = "never" objectStorageExpiration1Day = "1d" objectStorageExpiration1Month = "1m" objectStorageExpiration3Months = "3m" objectStorageExpiration6Months = "6m" objectStorageSignedURLTTLMin = time.Minute objectStorageSignedURLTTLMax = 7 * 24 * time.Hour ) type objectStorageLifecycleDefinition struct { ID string TagValue string Days int } var objectStorageLifecycleDefinitions = []objectStorageLifecycleDefinition{ {ID: "DeleteTempFiles-1d", TagValue: objectStorageExpiration1Day, Days: 1}, {ID: "DeleteTempFiles-1m", TagValue: objectStorageExpiration1Month, Days: 30}, {ID: "DeleteTempFiles-3m", TagValue: objectStorageExpiration3Months, Days: 90}, {ID: "DeleteTempFiles-6m", TagValue: objectStorageExpiration6Months, Days: 180}, } type objectStorageRetention struct { Enabled bool Policy string TagValue string ExpiresAt time.Time } func (r objectStorageRetention) taggingHeader() string { values := url.Values{} values.Set("TempFile", "true") values.Set("ExpiresAt", strconv.FormatInt(r.ExpiresAt.Unix(), 10)) values.Set("ExpiresIn", r.TagValue) return values.Encode() } func objectStorageExpirationPolicy(config map[string]any) string { policy := strings.ToLower(firstNonEmptyString( objectStorageConfigString(config, "temporaryFileExpirePolicy"), objectStorageConfigString(config, "apiReturnExpirePolicy"), )) switch policy { case objectStorageExpiration1Day, objectStorageExpiration1Month, objectStorageExpiration3Months, objectStorageExpiration6Months: return policy default: return objectStorageExpirationNever } } func objectStorageRetentionFor(config map[string]any, scene string, now time.Time) objectStorageRetention { if strings.TrimSpace(scene) != "image_result" && strings.TrimSpace(scene) != "request_asset" { return objectStorageRetention{Policy: objectStorageExpirationNever} } policy := objectStorageExpirationPolicy(config) duration := objectStorageExpirationDuration(policy) if duration <= 0 { return objectStorageRetention{Policy: objectStorageExpirationNever} } return objectStorageRetention{ Enabled: true, Policy: policy, TagValue: policy, ExpiresAt: now.Add(duration), } } func objectStorageExpirationDuration(policy string) time.Duration { switch policy { case objectStorageExpiration1Day: return 24 * time.Hour case objectStorageExpiration1Month: return 30 * 24 * time.Hour case objectStorageExpiration3Months: return 90 * 24 * time.Hour case objectStorageExpiration6Months: return 180 * 24 * time.Hour default: return 0 } } func objectStorageSignedURLTTLForConfig(config map[string]any) time.Duration { seconds := int64FromAny(config["signedUrlExpiresSeconds"]) if seconds <= 0 { return objectStorageSignedURLTTL } ttl := time.Duration(seconds) * time.Second if ttl < objectStorageSignedURLTTLMin { return objectStorageSignedURLTTLMin } if ttl > objectStorageSignedURLTTLMax { return objectStorageSignedURLTTLMax } return ttl } func int64FromAny(value any) int64 { switch typed := value.(type) { case int: return int64(typed) case int32: return int64(typed) case int64: return typed case float32: return int64(typed) case float64: return int64(typed) case string: parsed, _ := strconv.ParseInt(strings.TrimSpace(typed), 10, 64) return parsed default: return 0 } } var ( objectStorageLifecycleReady sync.Map objectStorageLifecycleLocks sync.Map ) func (a *objectStorageAdapter) ensureLifecycle(ctx context.Context) error { cacheKey := strings.Join([]string{ strings.ToLower(strings.TrimSpace(a.channel.Provider)), objectStorageConfigString(a.channel.Config, "endpoint"), objectStorageConfigString(a.channel.Config, "bucket"), }, "\x00") if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok { return nil } lockValue, _ := objectStorageLifecycleLocks.LoadOrStore(cacheKey, &sync.Mutex{}) lifecycleLock := lockValue.(*sync.Mutex) lifecycleLock.Lock() defer lifecycleLock.Unlock() if _, ok := objectStorageLifecycleReady.Load(cacheKey); ok { return nil } current, err := a.readLifecycle(ctx) if err != nil { return err } next, changed, err := mergeObjectStorageLifecycle(a.channel.Provider, current) if err != nil { return storageClientError("storage_config_invalid", "object storage lifecycle configuration is invalid", 0, false) } if changed { if err := a.writeLifecycle(ctx, next); err != nil { return err } } objectStorageLifecycleReady.Store(cacheKey, struct{}{}) return nil } func (a *objectStorageAdapter) readLifecycle(ctx context.Context) ([]byte, error) { requestURL, err := a.bucketLifecycleURL() if err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) if err != nil { return nil, storageClientError("storage_config_invalid", err.Error(), 0, false) } if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil { return nil, err } resp, err := a.client.Do(req) if err != nil { return nil, storageClientError("storage_write_failed", "object storage lifecycle check failed", 0, true) } defer resp.Body.Close() body, readErr := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) if readErr != nil { return nil, storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode)) } if resp.StatusCode == http.StatusNotFound { return nil, nil } if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body) } return body, nil } func (a *objectStorageAdapter) writeLifecycle(ctx context.Context, payload []byte) error { requestURL, err := a.bucketLifecycleURL() if err != nil { return err } req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload)) if err != nil { return storageClientError("storage_config_invalid", err.Error(), 0, false) } digest := md5.Sum(payload) req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(digest[:])) req.Header.Set("Content-Type", "application/xml") if err := a.sign(req, sha256Hex(payload), a.now().UTC()); err != nil { return err } resp, err := a.client.Do(req) if err != nil { return storageClientError("storage_write_failed", "object storage lifecycle configuration failed", 0, true) } defer resp.Body.Close() body, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) if readErr != nil { return storageClientError("storage_write_failed", "object storage lifecycle response could not be read", resp.StatusCode, storageStatusRetryable(resp.StatusCode)) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { return objectStorageHTTPError("storage_config_invalid", resp.StatusCode, body) } return nil } func (a *objectStorageAdapter) bucketLifecycleURL() (string, error) { endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/")) if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil { return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false) } bucket := objectStorageConfigString(a.channel.Config, "bucket") hostname := endpoint.Hostname() forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle") if strings.EqualFold(a.channel.Provider, "s3") { if forcePathStyle { endpoint.Path = path.Join(endpoint.Path, bucket) } else if !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") { if port := endpoint.Port(); port != "" { endpoint.Host = bucket + "." + hostname + ":" + port } else { endpoint.Host = bucket + "." + endpoint.Host } } } else if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") { if port := endpoint.Port(); port != "" { endpoint.Host = bucket + "." + hostname + ":" + port } else { endpoint.Host = bucket + "." + endpoint.Host } } if endpoint.Path == "" { endpoint.Path = "/" } endpoint.RawQuery = "lifecycle" endpoint.Fragment = "" return endpoint.String(), nil } type rawLifecycleConfiguration struct { Rules []rawLifecycleRule `xml:"Rule"` } type rawLifecycleRule struct { ID string `xml:"ID"` Status string `xml:"Status"` Tag rawLifecycleTag `xml:"Tag"` Filter rawLifecycleFilter `xml:"Filter"` Expiration rawLifecycleExpiration `xml:"Expiration"` InnerXML string `xml:",innerxml"` } type rawLifecycleFilter struct { Tag rawLifecycleTag `xml:"Tag"` } type rawLifecycleTag struct { Key string `xml:"Key"` Value string `xml:"Value"` } type rawLifecycleExpiration struct { Days int `xml:"Days"` } func mergeObjectStorageLifecycle(provider string, current []byte) ([]byte, bool, error) { configuration := rawLifecycleConfiguration{} if len(bytes.TrimSpace(current)) > 0 { if err := xml.Unmarshal(current, &configuration); err != nil { return nil, false, err } } managed := map[string]objectStorageLifecycleDefinition{} for _, definition := range objectStorageLifecycleDefinitions { managed[definition.ID] = definition } allManagedReady := true for _, definition := range objectStorageLifecycleDefinitions { matched := false for _, rule := range configuration.Rules { if rule.ID == definition.ID { matched = lifecycleRuleMatches(rule, definition) break } } if !matched { allManagedReady = false break } } if allManagedReady { return current, false, nil } var output strings.Builder output.WriteString(``) if strings.EqualFold(strings.TrimSpace(provider), "s3") { output.WriteString(``) } else { output.WriteString(``) } for _, rule := range configuration.Rules { if _, isManaged := managed[rule.ID]; isManaged { continue } output.WriteString("") output.WriteString(rule.InnerXML) output.WriteString("") } for _, definition := range objectStorageLifecycleDefinitions { output.WriteString(lifecycleRuleXML(provider, definition)) } output.WriteString(``) return []byte(output.String()), true, nil } func lifecycleRuleMatches(rule rawLifecycleRule, definition objectStorageLifecycleDefinition) bool { tag := rule.Tag if tag.Key == "" { tag = rule.Filter.Tag } return strings.EqualFold(strings.TrimSpace(rule.Status), "Enabled") && tag.Key == "ExpiresIn" && tag.Value == definition.TagValue && rule.Expiration.Days == definition.Days } func lifecycleRuleXML(provider string, definition objectStorageLifecycleDefinition) string { if strings.EqualFold(strings.TrimSpace(provider), "s3") { return fmt.Sprintf( "%sExpiresIn%sEnabled%d", definition.ID, definition.TagValue, definition.Days, ) } return fmt.Sprintf( "%sExpiresIn%sEnabled%d", definition.ID, definition.TagValue, definition.Days, ) }