perf(media): 生产媒体直传阿里云 OSS
生产同构验收确认 server_main_openapi 上传通道在媒体高并发下产生 381 个 502,并将已创建任务的 p95 拉高到 172 秒。\n\n为 request_asset 与 image_result 增加环境配置控制的阿里云 OSS 直传,保留普通上传原通道;使用 Kubernetes Secret 注入凭据,并对签名、重试、场景隔离和生产配置校验增加测试。\n\n验证:Go 全量测试、go vet、OpenAPI、迁移安全检查、Kustomize/Compose 渲染、gofmt 与 git diff --check 均通过;真实 OSS PUT/CDN 读取探针通过且对象已清理。
This commit is contained in:
@@ -65,6 +65,13 @@ type Config struct {
|
|||||||
DatabaseLockTimeoutSeconds int
|
DatabaseLockTimeoutSeconds int
|
||||||
MediaRequestConcurrency int
|
MediaRequestConcurrency int
|
||||||
MediaMaterializationConcurrency int
|
MediaMaterializationConcurrency int
|
||||||
|
MediaOSSDirectEnabled bool
|
||||||
|
MediaOSSEndpoint string
|
||||||
|
MediaOSSBucket string
|
||||||
|
MediaOSSAccessKeyID string
|
||||||
|
MediaOSSAccessKeySecret string
|
||||||
|
MediaOSSPublicBaseURL string
|
||||||
|
MediaOSSObjectPrefix string
|
||||||
AsyncQueueWorkerEnabled bool
|
AsyncQueueWorkerEnabled bool
|
||||||
AsyncWorkerHardLimit int
|
AsyncWorkerHardLimit int
|
||||||
AsyncWorkerInstanceHardLimit int
|
AsyncWorkerInstanceHardLimit int
|
||||||
@@ -135,6 +142,13 @@ func Load() Config {
|
|||||||
|
|
||||||
MediaRequestConcurrency: envIntValidated("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY", 16),
|
MediaRequestConcurrency: envIntValidated("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY", 16),
|
||||||
MediaMaterializationConcurrency: envIntValidated("AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY", 8),
|
MediaMaterializationConcurrency: envIntValidated("AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY", 8),
|
||||||
|
MediaOSSDirectEnabled: env("AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED", "false") == "true",
|
||||||
|
MediaOSSEndpoint: strings.TrimRight(env("AI_GATEWAY_MEDIA_OSS_ENDPOINT", ""), "/"),
|
||||||
|
MediaOSSBucket: strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_BUCKET", "")),
|
||||||
|
MediaOSSAccessKeyID: strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID", "")),
|
||||||
|
MediaOSSAccessKeySecret: strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET", "")),
|
||||||
|
MediaOSSPublicBaseURL: strings.TrimRight(env("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL", ""), "/"),
|
||||||
|
MediaOSSObjectPrefix: strings.Trim(env("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX", "easyai-ai-gateway/media"), "/"),
|
||||||
AsyncQueueWorkerEnabled: env("AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED", "true") == "true",
|
AsyncQueueWorkerEnabled: env("AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED", "true") == "true",
|
||||||
AsyncWorkerHardLimit: envIntValidated("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", 2048),
|
AsyncWorkerHardLimit: envIntValidated("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", 2048),
|
||||||
AsyncWorkerInstanceHardLimit: envIntValidated("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT", 32),
|
AsyncWorkerInstanceHardLimit: envIntValidated("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT", 32),
|
||||||
@@ -170,6 +184,23 @@ func (c Config) Validate() error {
|
|||||||
if c.MediaRequestConcurrency != 0 && (c.MediaRequestConcurrency < 1 || c.MediaRequestConcurrency > 1024) {
|
if c.MediaRequestConcurrency != 0 && (c.MediaRequestConcurrency < 1 || c.MediaRequestConcurrency > 1024) {
|
||||||
return errors.New("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY must be between 1 and 1024")
|
return errors.New("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY must be between 1 and 1024")
|
||||||
}
|
}
|
||||||
|
if c.MediaOSSDirectEnabled {
|
||||||
|
if strings.TrimSpace(c.MediaOSSAccessKeyID) == "" || strings.TrimSpace(c.MediaOSSAccessKeySecret) == "" {
|
||||||
|
return errors.New("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID and AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET are required when direct media OSS is enabled")
|
||||||
|
}
|
||||||
|
if !validMediaOSSBucket(c.MediaOSSBucket) {
|
||||||
|
return errors.New("AI_GATEWAY_MEDIA_OSS_BUCKET is invalid")
|
||||||
|
}
|
||||||
|
if !validMediaOSSURL(c.MediaOSSEndpoint, c.AppEnv) {
|
||||||
|
return errors.New("AI_GATEWAY_MEDIA_OSS_ENDPOINT must be an HTTPS URL")
|
||||||
|
}
|
||||||
|
if !validMediaOSSURL(c.MediaOSSPublicBaseURL, c.AppEnv) {
|
||||||
|
return errors.New("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL must be an HTTPS URL")
|
||||||
|
}
|
||||||
|
if !validMediaOSSObjectPrefix(c.MediaOSSObjectPrefix) {
|
||||||
|
return errors.New("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX is invalid")
|
||||||
|
}
|
||||||
|
}
|
||||||
switch strings.ToLower(strings.TrimSpace(c.BillingEngineMode)) {
|
switch strings.ToLower(strings.TrimSpace(c.BillingEngineMode)) {
|
||||||
case "", "observe", "enforce", "hold":
|
case "", "observe", "enforce", "hold":
|
||||||
default:
|
default:
|
||||||
@@ -237,6 +268,48 @@ func (c Config) Validate() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validMediaOSSBucket(value string) bool {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if len(value) < 3 || len(value) > 63 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for index, item := range value {
|
||||||
|
if (item >= 'a' && item <= 'z') || (item >= '0' && item <= '9') || (item == '-' && index > 0 && index < len(value)-1) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func validMediaOSSURL(value string, appEnv string) bool {
|
||||||
|
parsed, err := url.Parse(strings.TrimSpace(value))
|
||||||
|
if err != nil || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.User != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if parsed.Scheme == "https" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
host := strings.ToLower(parsed.Hostname())
|
||||||
|
return !strings.EqualFold(strings.TrimSpace(appEnv), "production") &&
|
||||||
|
parsed.Scheme == "http" && (host == "127.0.0.1" || host == "localhost" || host == "::1")
|
||||||
|
}
|
||||||
|
|
||||||
|
func validMediaOSSObjectPrefix(value string) bool {
|
||||||
|
value = strings.Trim(value, "/")
|
||||||
|
if value == "" || len(value) > 256 || strings.Contains(value, "..") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, item := range value {
|
||||||
|
if (item >= 'a' && item <= 'z') || (item >= 'A' && item <= 'Z') || (item >= '0' && item <= '9') ||
|
||||||
|
item == '-' || item == '_' || item == '/' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func (c Config) EffectiveProcessRole() string {
|
func (c Config) EffectiveProcessRole() string {
|
||||||
role := strings.ToLower(strings.TrimSpace(c.ProcessRole))
|
role := strings.ToLower(strings.TrimSpace(c.ProcessRole))
|
||||||
if role == "" {
|
if role == "" {
|
||||||
|
|||||||
@@ -233,3 +233,37 @@ func TestValidateTaskHistorySettings(t *testing.T) {
|
|||||||
t.Fatalf("Validate() error = %v, want invalid cleanup batch", err)
|
t.Fatalf("Validate() error = %v, want invalid cleanup batch", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadAndValidateDirectMediaOSS(t *testing.T) {
|
||||||
|
t.Setenv("APP_ENV", "production")
|
||||||
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED", "true")
|
||||||
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_ENDPOINT", "https://media-bucket.oss-cn-shanghai.aliyuncs.com")
|
||||||
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_BUCKET", "media-bucket")
|
||||||
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID", "test-access-key")
|
||||||
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET", "test-access-secret")
|
||||||
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL", "https://cdn.example.com")
|
||||||
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX", "easyai-ai-gateway/production/media")
|
||||||
|
|
||||||
|
cfg := Load()
|
||||||
|
if !cfg.MediaOSSDirectEnabled || cfg.MediaOSSBucket != "media-bucket" ||
|
||||||
|
cfg.MediaOSSObjectPrefix != "easyai-ai-gateway/production/media" {
|
||||||
|
t.Fatalf("unexpected direct media OSS config: %+v", cfg)
|
||||||
|
}
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("validate direct media OSS: %v", err)
|
||||||
|
}
|
||||||
|
cfg.MediaOSSAccessKeySecret = ""
|
||||||
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ACCESS_KEY_SECRET") {
|
||||||
|
t.Fatalf("Validate() error = %v, want missing direct OSS secret", err)
|
||||||
|
}
|
||||||
|
cfg.MediaOSSAccessKeySecret = "test-access-secret"
|
||||||
|
cfg.MediaOSSEndpoint = "http://media-bucket.example.com"
|
||||||
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "HTTPS URL") {
|
||||||
|
t.Fatalf("Validate() error = %v, want insecure production endpoint", err)
|
||||||
|
}
|
||||||
|
cfg.MediaOSSEndpoint = "https://media-bucket.example.com"
|
||||||
|
cfg.MediaOSSObjectPrefix = "../outside"
|
||||||
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "OBJECT_PREFIX") {
|
||||||
|
t.Fatalf("Validate() error = %v, want invalid object prefix", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,175 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha1"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
const directOSSUploadTimeout = 120 * time.Second
|
||||||
|
|
||||||
|
type directOSSUploader struct {
|
||||||
|
endpoint string
|
||||||
|
bucket string
|
||||||
|
accessKeyID string
|
||||||
|
accessKeySecret string
|
||||||
|
publicBaseURL string
|
||||||
|
objectPrefix string
|
||||||
|
client *http.Client
|
||||||
|
now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
|
||||||
|
if !cfg.MediaOSSDirectEnabled {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &directOSSUploader{
|
||||||
|
endpoint: strings.TrimRight(cfg.MediaOSSEndpoint, "/"),
|
||||||
|
bucket: strings.TrimSpace(cfg.MediaOSSBucket),
|
||||||
|
accessKeyID: strings.TrimSpace(cfg.MediaOSSAccessKeyID),
|
||||||
|
accessKeySecret: strings.TrimSpace(cfg.MediaOSSAccessKeySecret),
|
||||||
|
publicBaseURL: strings.TrimRight(cfg.MediaOSSPublicBaseURL, "/"),
|
||||||
|
objectPrefix: strings.Trim(cfg.MediaOSSObjectPrefix, "/"),
|
||||||
|
client: &http.Client{
|
||||||
|
Timeout: directOSSUploadTimeout,
|
||||||
|
Transport: &http.Transport{
|
||||||
|
MaxIdleConns: 256,
|
||||||
|
MaxIdleConnsPerHost: 256,
|
||||||
|
MaxConnsPerHost: 256,
|
||||||
|
IdleConnTimeout: 90 * time.Second,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
now: time.Now,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func directOSSScene(scene string) bool {
|
||||||
|
switch strings.TrimSpace(scene) {
|
||||||
|
case store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *directOSSUploader) upload(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
|
||||||
|
if u == nil {
|
||||||
|
return nil, &clients.ClientError{Code: "upload_config_failed", Message: "direct OSS uploader is not configured", Retryable: false}
|
||||||
|
}
|
||||||
|
objectKey := u.objectKey(payload)
|
||||||
|
escapedKey := escapeOSSObjectKey(objectKey)
|
||||||
|
uploadURL := u.endpoint + "/" + escapedKey
|
||||||
|
contentType := strings.TrimSpace(payload.ContentType)
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastErr error
|
||||||
|
for attempt := 0; attempt < 3; attempt++ {
|
||||||
|
if attempt > 0 {
|
||||||
|
if err := sleepWithContext(ctx, time.Duration(attempt*attempt)*200*time.Millisecond); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
status, err := u.put(ctx, uploadURL, escapedKey, contentType, payload.Bytes)
|
||||||
|
if err == nil && status >= 200 && status < 300 {
|
||||||
|
publicURL := u.publicBaseURL + "/" + escapedKey
|
||||||
|
return map[string]any{
|
||||||
|
"url": publicURL,
|
||||||
|
"fileName": objectKey,
|
||||||
|
"storageChannel": map[string]any{
|
||||||
|
"channelKey": "environment-direct-oss",
|
||||||
|
"name": "Environment Direct OSS",
|
||||||
|
"provider": "aliyun_oss_direct",
|
||||||
|
},
|
||||||
|
"assetStorage": map[string]any{
|
||||||
|
"scene": payload.Scene,
|
||||||
|
"source": firstNonEmptyString(payload.Source, "ai-gateway"),
|
||||||
|
"strategy": "direct_aliyun_oss",
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
lastErr = err
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lastErr = fmt.Errorf("HTTP %d", status)
|
||||||
|
if status != http.StatusTooManyRequests && status < 500 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
message := "direct OSS upload failed"
|
||||||
|
if lastErr != nil {
|
||||||
|
message += ": " + lastErr.Error()
|
||||||
|
}
|
||||||
|
return nil, &clients.ClientError{Code: "upload_failed", Message: message, Retryable: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *directOSSUploader) put(ctx context.Context, uploadURL string, escapedKey string, contentType string, payload []byte) (int, error) {
|
||||||
|
date := u.now().UTC().Format(http.TimeFormat)
|
||||||
|
canonicalResource := "/" + u.bucket + "/" + escapedKey
|
||||||
|
stringToSign := "PUT\n\n" + contentType + "\n" + date + "\n" + canonicalResource
|
||||||
|
mac := hmac.New(sha1.New, []byte(u.accessKeySecret))
|
||||||
|
_, _ = mac.Write([]byte(stringToSign))
|
||||||
|
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(payload))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "OSS "+u.accessKeyID+":"+signature)
|
||||||
|
req.Header.Set("Content-Type", contentType)
|
||||||
|
req.Header.Set("Date", date)
|
||||||
|
resp, err := u.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||||
|
return resp.StatusCode, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *directOSSUploader) objectKey(payload FileUploadPayload) string {
|
||||||
|
now := u.now().UTC()
|
||||||
|
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
|
||||||
|
baseName := strings.TrimSuffix(path.Base(payload.FileName), path.Ext(payload.FileName))
|
||||||
|
baseName = sanitizeGeneratedAssetNamePart(baseName)
|
||||||
|
if baseName == "" {
|
||||||
|
baseName = "gateway-media"
|
||||||
|
}
|
||||||
|
if len(baseName) > 48 {
|
||||||
|
baseName = baseName[:48]
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"%s/%s/%04d/%02d/%02d/%s-%s%s",
|
||||||
|
u.objectPrefix,
|
||||||
|
strings.TrimSpace(payload.Scene),
|
||||||
|
now.Year(),
|
||||||
|
now.Month(),
|
||||||
|
now.Day(),
|
||||||
|
baseName,
|
||||||
|
randomHexSuffix(8),
|
||||||
|
extension,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func escapeOSSObjectKey(objectKey string) string {
|
||||||
|
parts := strings.Split(objectKey, "/")
|
||||||
|
for index, part := range parts {
|
||||||
|
parts[index] = url.PathEscape(part)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "/")
|
||||||
|
}
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha1"
|
||||||
|
"encoding/base64"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
|
||||||
|
payload := []byte("production-isomorphic-image")
|
||||||
|
var calls atomic.Int64
|
||||||
|
var uploadedPath string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
call := calls.Add(1)
|
||||||
|
if r.Method != http.MethodPut {
|
||||||
|
t.Errorf("method=%s", r.Method)
|
||||||
|
}
|
||||||
|
if r.Header.Get("Content-Type") != "image/png" || r.Header.Get("Date") == "" {
|
||||||
|
t.Errorf("missing OSS upload headers")
|
||||||
|
}
|
||||||
|
stringToSign := "PUT\n\nimage/png\n" + r.Header.Get("Date") + "\n/media-bucket" + r.URL.EscapedPath()
|
||||||
|
mac := hmac.New(sha1.New, []byte("access-secret"))
|
||||||
|
_, _ = mac.Write([]byte(stringToSign))
|
||||||
|
expectedAuthorization := "OSS access-key:" + base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||||
|
if r.Header.Get("Authorization") != expectedAuthorization {
|
||||||
|
t.Errorf("invalid OSS request signature")
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(r.Body)
|
||||||
|
if err != nil || !bytes.Equal(body, payload) {
|
||||||
|
t.Errorf("uploaded payload differs: err=%v", err)
|
||||||
|
}
|
||||||
|
uploadedPath = r.URL.EscapedPath()
|
||||||
|
if call == 1 {
|
||||||
|
http.Error(w, "retry", http.StatusBadGateway)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
cfg := config.Config{
|
||||||
|
MediaOSSDirectEnabled: true,
|
||||||
|
MediaOSSEndpoint: server.URL,
|
||||||
|
MediaOSSBucket: "media-bucket",
|
||||||
|
MediaOSSAccessKeyID: "access-key",
|
||||||
|
MediaOSSAccessKeySecret: "access-secret",
|
||||||
|
MediaOSSPublicBaseURL: "https://cdn.example.com",
|
||||||
|
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
|
||||||
|
}
|
||||||
|
uploader := newDirectOSSUploader(cfg)
|
||||||
|
uploader.now = func() time.Time {
|
||||||
|
return time.Date(2026, time.July, 30, 12, 0, 0, 0, time.UTC)
|
||||||
|
}
|
||||||
|
service := &Service{cfg: cfg, directOSS: uploader}
|
||||||
|
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
|
||||||
|
Bytes: payload,
|
||||||
|
ContentType: "image/png",
|
||||||
|
FileName: "input.png",
|
||||||
|
Scene: store.FileStorageSceneRequestAsset,
|
||||||
|
Source: "acceptance",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("direct OSS upload: %v", err)
|
||||||
|
}
|
||||||
|
if calls.Load() != 2 {
|
||||||
|
t.Fatalf("upload calls=%d, want one retry", calls.Load())
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/2026/07/30/input-") ||
|
||||||
|
!strings.HasSuffix(uploadedPath, ".png") {
|
||||||
|
t.Fatalf("unexpected object path=%q", uploadedPath)
|
||||||
|
}
|
||||||
|
if got := stringFromAny(uploaded["url"]); got != "https://cdn.example.com"+uploadedPath {
|
||||||
|
t.Fatalf("public URL=%q", got)
|
||||||
|
}
|
||||||
|
channel, _ := uploaded["storageChannel"].(map[string]any)
|
||||||
|
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
|
||||||
|
t.Fatalf("storage channel=%+v", channel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
|
||||||
|
var calls atomic.Int64
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
storageDir := t.TempDir()
|
||||||
|
cfg := config.Config{
|
||||||
|
LocalUploadedStorageDir: storageDir,
|
||||||
|
MediaOSSDirectEnabled: true,
|
||||||
|
MediaOSSEndpoint: server.URL,
|
||||||
|
MediaOSSBucket: "media-bucket",
|
||||||
|
MediaOSSAccessKeyID: "access-key",
|
||||||
|
MediaOSSAccessKeySecret: "access-secret",
|
||||||
|
MediaOSSPublicBaseURL: "https://cdn.example.com",
|
||||||
|
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
|
||||||
|
}
|
||||||
|
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
|
||||||
|
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
|
||||||
|
Bytes: []byte("general-upload"),
|
||||||
|
ContentType: "text/plain",
|
||||||
|
FileName: "general.txt",
|
||||||
|
Scene: store.FileStorageSceneUpload,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("general upload: %v", err)
|
||||||
|
}
|
||||||
|
if calls.Load() != 0 {
|
||||||
|
t.Fatalf("direct OSS calls=%d for general upload scene", calls.Load())
|
||||||
|
}
|
||||||
|
channel, _ := uploaded["storageChannel"].(map[string]any)
|
||||||
|
if stringFromAny(channel["provider"]) != "local_static" {
|
||||||
|
t.Fatalf("general upload channel=%+v", channel)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -45,6 +45,7 @@ type Service struct {
|
|||||||
admissionListener sync.Once
|
admissionListener sync.Once
|
||||||
asyncClientFactory func(int) (asyncExecutionClient, error)
|
asyncClientFactory func(int) (asyncExecutionClient, error)
|
||||||
mediaResultSlots chan struct{}
|
mediaResultSlots chan struct{}
|
||||||
|
directOSS *directOSSUploader
|
||||||
billingMetrics billingMetricsObserver
|
billingMetrics billingMetricsObserver
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -154,6 +155,7 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
|
|||||||
asyncAdmissionWake: make(chan struct{}, 1),
|
asyncAdmissionWake: make(chan struct{}, 1),
|
||||||
admissionTaskWaiters: map[string]*admissionTaskWaiter{},
|
admissionTaskWaiters: map[string]*admissionTaskWaiter{},
|
||||||
mediaResultSlots: make(chan struct{}, cfg.MediaMaterializationConcurrency),
|
mediaResultSlots: make(chan struct{}, cfg.MediaMaterializationConcurrency),
|
||||||
|
directOSS: newDirectOSSUploader(cfg),
|
||||||
}
|
}
|
||||||
if len(observers) > 0 {
|
if len(observers) > 0 {
|
||||||
service.billingMetrics = observers[0]
|
service.billingMetrics = observers[0]
|
||||||
|
|||||||
@@ -935,6 +935,9 @@ func (s *Service) UploadFile(ctx context.Context, payload FileUploadPayload) (ma
|
|||||||
if strings.TrimSpace(payload.Scene) == "" {
|
if strings.TrimSpace(payload.Scene) == "" {
|
||||||
payload.Scene = store.FileStorageSceneUpload
|
payload.Scene = store.FileStorageSceneUpload
|
||||||
}
|
}
|
||||||
|
if s.directOSS != nil && directOSSScene(payload.Scene) {
|
||||||
|
return s.directOSS.upload(ctx, payload)
|
||||||
|
}
|
||||||
channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
|
channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
||||||
|
|||||||
@@ -36,6 +36,11 @@ data:
|
|||||||
AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES: "10737418240"
|
AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES: "10737418240"
|
||||||
AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: "268435456"
|
AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: "268435456"
|
||||||
AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: "536870912"
|
AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: "536870912"
|
||||||
|
AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED: "true"
|
||||||
|
AI_GATEWAY_MEDIA_OSS_ENDPOINT: https://wangbo0808.oss-cn-shanghai.aliyuncs.com
|
||||||
|
AI_GATEWAY_MEDIA_OSS_BUCKET: wangbo0808
|
||||||
|
AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL: https://aliyuncdn.51easyai.com
|
||||||
|
AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX: easyai-ai-gateway/production/media
|
||||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"
|
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"
|
||||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "48"
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "48"
|
||||||
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
|
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
|
||||||
|
|||||||
@@ -62,6 +62,16 @@ spec:
|
|||||||
value: "16"
|
value: "16"
|
||||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
value: "8"
|
value: "8"
|
||||||
|
- name: AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: easyai-oss-backup
|
||||||
|
key: ACCESS_KEY_ID
|
||||||
|
- name: AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: easyai-oss-backup
|
||||||
|
key: ACCESS_SECRET_KEY
|
||||||
- name: POD_NAMESPACE
|
- name: POD_NAMESPACE
|
||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
@@ -195,6 +205,16 @@ spec:
|
|||||||
value: "16"
|
value: "16"
|
||||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
value: "8"
|
value: "8"
|
||||||
|
- name: AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: easyai-oss-backup
|
||||||
|
key: ACCESS_KEY_ID
|
||||||
|
- name: AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: easyai-oss-backup
|
||||||
|
key: ACCESS_SECRET_KEY
|
||||||
- name: POD_NAMESPACE
|
- name: POD_NAMESPACE
|
||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
@@ -330,6 +350,16 @@ spec:
|
|||||||
value: "24"
|
value: "24"
|
||||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||||
value: "24"
|
value: "24"
|
||||||
|
- name: AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: easyai-oss-backup
|
||||||
|
key: ACCESS_KEY_ID
|
||||||
|
- name: AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: easyai-oss-backup
|
||||||
|
key: ACCESS_SECRET_KEY
|
||||||
- name: POD_NAMESPACE
|
- name: POD_NAMESPACE
|
||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
@@ -465,6 +495,16 @@ spec:
|
|||||||
value: "24"
|
value: "24"
|
||||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||||
value: "24"
|
value: "24"
|
||||||
|
- name: AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: easyai-oss-backup
|
||||||
|
key: ACCESS_KEY_ID
|
||||||
|
- name: AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: easyai-oss-backup
|
||||||
|
key: ACCESS_SECRET_KEY
|
||||||
- name: POD_NAMESPACE
|
- name: POD_NAMESPACE
|
||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
|
|||||||
Reference in New Issue
Block a user