BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(media): 生产媒体直传阿里云 OSS

Browse Source 
生产同构验收确认 server_main_openapi 上传通道在媒体高并发下产生 381 个 502，并将已创建任务的 p95 拉高到 172 秒。\n\n为 request_asset 与 image_result 增加环境配置控制的阿里云 OSS 直传，保留普通上传原通道；使用 Kubernetes Secret 注入凭据，并对签名、重试、场景隔离和生产配置校验增加测试。\n\n验证：Go 全量测试、go vet、OpenAPI、迁移安全检查、Kustomize/Compose 渲染、gofmt 与 git diff --check 均通过；真实 OSS PUT/CDN 读取探针通过且对象已清理。
This commit is contained in:
wangbo
2026-07-31 05:20:27 +08:00
parent 7b2ab786a2
commit 352e23e099
8 changed files with 460 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/config/config.go
+73
View File
@@ -65,6 +65,13 @@ type Config struct {
DatabaseLockTimeoutSeconds int
MediaRequestConcurrency int
MediaMaterializationConcurrency int
MediaOSSDirectEnabled bool
MediaOSSEndpoint string
MediaOSSBucket string
MediaOSSAccessKeyID string
MediaOSSAccessKeySecret string
MediaOSSPublicBaseURL string
MediaOSSObjectPrefix string
AsyncQueueWorkerEnabled bool
AsyncWorkerHardLimit int
AsyncWorkerInstanceHardLimit int
@@ -135,6 +142,13 @@ func Load() Config {
MediaRequestConcurrency: envIntValidated("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY", 16),
MediaMaterializationConcurrency: envIntValidated("AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY", 8),
MediaOSSDirectEnabled: env("AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED", "false") == "true",
MediaOSSEndpoint: strings.TrimRight(env("AI_GATEWAY_MEDIA_OSS_ENDPOINT", ""), "/"),
MediaOSSBucket: strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_BUCKET", "")),
MediaOSSAccessKeyID: strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID", "")),
MediaOSSAccessKeySecret: strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET", "")),
MediaOSSPublicBaseURL: strings.TrimRight(env("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL", ""), "/"),
MediaOSSObjectPrefix: strings.Trim(env("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX", "easyai-ai-gateway/media"), "/"),
AsyncQueueWorkerEnabled: env("AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED", "true") == "true",
AsyncWorkerHardLimit: envIntValidated("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", 2048),
AsyncWorkerInstanceHardLimit: envIntValidated("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT", 32),
@@ -170,6 +184,23 @@ func (c Config) Validate() error {
if c.MediaRequestConcurrency != 0 && (c.MediaRequestConcurrency < 1 || c.MediaRequestConcurrency > 1024) {
return errors.New("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY must be between 1 and 1024")
}
if c.MediaOSSDirectEnabled {
if strings.TrimSpace(c.MediaOSSAccessKeyID) == "" || strings.TrimSpace(c.MediaOSSAccessKeySecret) == "" {
return errors.New("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID and AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET are required when direct media OSS is enabled")
}
if !validMediaOSSBucket(c.MediaOSSBucket) {
return errors.New("AI_GATEWAY_MEDIA_OSS_BUCKET is invalid")
}
if !validMediaOSSURL(c.MediaOSSEndpoint, c.AppEnv) {
return errors.New("AI_GATEWAY_MEDIA_OSS_ENDPOINT must be an HTTPS URL")
}
if !validMediaOSSURL(c.MediaOSSPublicBaseURL, c.AppEnv) {
return errors.New("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL must be an HTTPS URL")
}
if !validMediaOSSObjectPrefix(c.MediaOSSObjectPrefix) {
return errors.New("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX is invalid")
}
}
switch strings.ToLower(strings.TrimSpace(c.BillingEngineMode)) {
case "", "observe", "enforce", "hold":
default:
@@ -237,6 +268,48 @@ func (c Config) Validate() error {
return nil
}
func validMediaOSSBucket(value string) bool {
value = strings.TrimSpace(value)
if len(value) < 3 || len(value) > 63 {
return false
}
for index, item := range value {
if (item >= 'a' && item <= 'z') || (item >= '0' && item <= '9') || (item == '-' && index > 0 && index < len(value)-1) {
continue
}
return false
}
return true
}
func validMediaOSSURL(value string, appEnv string) bool {
parsed, err := url.Parse(strings.TrimSpace(value))
if err != nil || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.User != nil {
return false
}
if parsed.Scheme == "https" {
return true
}
host := strings.ToLower(parsed.Hostname())
return !strings.EqualFold(strings.TrimSpace(appEnv), "production") &&
parsed.Scheme == "http" && (host == "127.0.0.1" || host == "localhost" || host == "::1")
}
func validMediaOSSObjectPrefix(value string) bool {
value = strings.Trim(value, "/")
if value == "" || len(value) > 256 || strings.Contains(value, "..") {
return false
}
for _, item := range value {
if (item >= 'a' && item <= 'z') || (item >= 'A' && item <= 'Z') || (item >= '0' && item <= '9') ||
item == '-' || item == '_' || item == '/' {
continue
}
return false
}
return true
}
func (c Config) EffectiveProcessRole() string {
role := strings.ToLower(strings.TrimSpace(c.ProcessRole))
if role == "" {
apps/api/internal/config/config_test.go
+34
View File
@@ -233,3 +233,37 @@ func TestValidateTaskHistorySettings(t *testing.T) {
t.Fatalf("Validate() error = %v, want invalid cleanup batch", err)
}
}
func TestLoadAndValidateDirectMediaOSS(t *testing.T) {
t.Setenv("APP_ENV", "production")
t.Setenv("AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED", "true")
t.Setenv("AI_GATEWAY_MEDIA_OSS_ENDPOINT", "https://media-bucket.oss-cn-shanghai.aliyuncs.com")
t.Setenv("AI_GATEWAY_MEDIA_OSS_BUCKET", "media-bucket")
t.Setenv("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID", "test-access-key")
t.Setenv("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET", "test-access-secret")
t.Setenv("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL", "https://cdn.example.com")
t.Setenv("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX", "easyai-ai-gateway/production/media")
cfg := Load()
if !cfg.MediaOSSDirectEnabled || cfg.MediaOSSBucket != "media-bucket" ||
cfg.MediaOSSObjectPrefix != "easyai-ai-gateway/production/media" {
t.Fatalf("unexpected direct media OSS config: %+v", cfg)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("validate direct media OSS: %v", err)
}
cfg.MediaOSSAccessKeySecret = ""
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ACCESS_KEY_SECRET") {
t.Fatalf("Validate() error = %v, want missing direct OSS secret", err)
}
cfg.MediaOSSAccessKeySecret = "test-access-secret"
cfg.MediaOSSEndpoint = "http://media-bucket.example.com"
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "HTTPS URL") {
t.Fatalf("Validate() error = %v, want insecure production endpoint", err)
}
cfg.MediaOSSEndpoint = "https://media-bucket.example.com"
cfg.MediaOSSObjectPrefix = "../outside"
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "OBJECT_PREFIX") {
t.Fatalf("Validate() error = %v, want invalid object prefix", err)
}
}
apps/api/internal/runner/direct_oss.go
+175
View File
@@ -0,0 +1,175 @@
package runner
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const directOSSUploadTimeout = 120 * time.Second
type directOSSUploader struct {
endpoint string
bucket string
accessKeyID string
accessKeySecret string
publicBaseURL string
objectPrefix string
client *http.Client
now func() time.Time
}
func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
if !cfg.MediaOSSDirectEnabled {
return nil
}
return &directOSSUploader{
endpoint: strings.TrimRight(cfg.MediaOSSEndpoint, "/"),
bucket: strings.TrimSpace(cfg.MediaOSSBucket),
accessKeyID: strings.TrimSpace(cfg.MediaOSSAccessKeyID),
accessKeySecret: strings.TrimSpace(cfg.MediaOSSAccessKeySecret),
publicBaseURL: strings.TrimRight(cfg.MediaOSSPublicBaseURL, "/"),
objectPrefix: strings.Trim(cfg.MediaOSSObjectPrefix, "/"),
client: &http.Client{
Timeout: directOSSUploadTimeout,
Transport: &http.Transport{
MaxIdleConns: 256,
MaxIdleConnsPerHost: 256,
MaxConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
},
},
now: time.Now,
}
}
func directOSSScene(scene string) bool {
switch strings.TrimSpace(scene) {
case store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
return true
default:
return false
}
}
func (u *directOSSUploader) upload(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
if u == nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: "direct OSS uploader is not configured", Retryable: false}
}
objectKey := u.objectKey(payload)
escapedKey := escapeOSSObjectKey(objectKey)
uploadURL := u.endpoint + "/" + escapedKey
contentType := strings.TrimSpace(payload.ContentType)
if contentType == "" {
contentType = "application/octet-stream"
}
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
if err := sleepWithContext(ctx, time.Duration(attempt*attempt)*200*time.Millisecond); err != nil {
return nil, err
}
}
status, err := u.put(ctx, uploadURL, escapedKey, contentType, payload.Bytes)
if err == nil && status >= 200 && status < 300 {
publicURL := u.publicBaseURL + "/" + escapedKey
return map[string]any{
"url": publicURL,
"fileName": objectKey,
"storageChannel": map[string]any{
"channelKey": "environment-direct-oss",
"name": "Environment Direct OSS",
"provider": "aliyun_oss_direct",
},
"assetStorage": map[string]any{
"scene": payload.Scene,
"source": firstNonEmptyString(payload.Source, "ai-gateway"),
"strategy": "direct_aliyun_oss",
},
}, nil
}
if err != nil {
lastErr = err
continue
}
lastErr = fmt.Errorf("HTTP %d", status)
if status != http.StatusTooManyRequests && status < 500 {
break
}
}
message := "direct OSS upload failed"
if lastErr != nil {
message += ": " + lastErr.Error()
}
return nil, &clients.ClientError{Code: "upload_failed", Message: message, Retryable: true}
}
func (u *directOSSUploader) put(ctx context.Context, uploadURL string, escapedKey string, contentType string, payload []byte) (int, error) {
date := u.now().UTC().Format(http.TimeFormat)
canonicalResource := "/" + u.bucket + "/" + escapedKey
stringToSign := "PUT\n\n" + contentType + "\n" + date + "\n" + canonicalResource
mac := hmac.New(sha1.New, []byte(u.accessKeySecret))
_, _ = mac.Write([]byte(stringToSign))
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(payload))
if err != nil {
return 0, err
}
req.Header.Set("Authorization", "OSS "+u.accessKeyID+":"+signature)
req.Header.Set("Content-Type", contentType)
req.Header.Set("Date", date)
resp, err := u.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
return resp.StatusCode, nil
}
func (u *directOSSUploader) objectKey(payload FileUploadPayload) string {
now := u.now().UTC()
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
baseName := strings.TrimSuffix(path.Base(payload.FileName), path.Ext(payload.FileName))
baseName = sanitizeGeneratedAssetNamePart(baseName)
if baseName == "" {
baseName = "gateway-media"
}
if len(baseName) > 48 {
baseName = baseName[:48]
}
return fmt.Sprintf(
"%s/%s/%04d/%02d/%02d/%s-%s%s",
u.objectPrefix,
strings.TrimSpace(payload.Scene),
now.Year(),
now.Month(),
now.Day(),
baseName,
randomHexSuffix(8),
extension,
)
}
func escapeOSSObjectKey(objectKey string) string {
parts := strings.Split(objectKey, "/")
for index, part := range parts {
parts[index] = url.PathEscape(part)
}
return strings.Join(parts, "/")
}
apps/api/internal/runner/direct_oss_test.go
+128
View File
@@ -0,0 +1,128 @@
package runner
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
payload := []byte("production-isomorphic-image")
var calls atomic.Int64
var uploadedPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
call := calls.Add(1)
if r.Method != http.MethodPut {
t.Errorf("method=%s", r.Method)
}
if r.Header.Get("Content-Type") != "image/png" || r.Header.Get("Date") == "" {
t.Errorf("missing OSS upload headers")
}
stringToSign := "PUT\n\nimage/png\n" + r.Header.Get("Date") + "\n/media-bucket" + r.URL.EscapedPath()
mac := hmac.New(sha1.New, []byte("access-secret"))
_, _ = mac.Write([]byte(stringToSign))
expectedAuthorization := "OSS access-key:" + base64.StdEncoding.EncodeToString(mac.Sum(nil))
if r.Header.Get("Authorization") != expectedAuthorization {
t.Errorf("invalid OSS request signature")
}
body, err := io.ReadAll(r.Body)
if err != nil || !bytes.Equal(body, payload) {
t.Errorf("uploaded payload differs: err=%v", err)
}
uploadedPath = r.URL.EscapedPath()
if call == 1 {
http.Error(w, "retry", http.StatusBadGateway)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
cfg := config.Config{
MediaOSSDirectEnabled: true,
MediaOSSEndpoint: server.URL,
MediaOSSBucket: "media-bucket",
MediaOSSAccessKeyID: "access-key",
MediaOSSAccessKeySecret: "access-secret",
MediaOSSPublicBaseURL: "https://cdn.example.com",
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
}
uploader := newDirectOSSUploader(cfg)
uploader.now = func() time.Time {
return time.Date(2026, time.July, 30, 12, 0, 0, 0, time.UTC)
}
service := &Service{cfg: cfg, directOSS: uploader}
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
Bytes: payload,
ContentType: "image/png",
FileName: "input.png",
Scene: store.FileStorageSceneRequestAsset,
Source: "acceptance",
})
if err != nil {
t.Fatalf("direct OSS upload: %v", err)
}
if calls.Load() != 2 {
t.Fatalf("upload calls=%d, want one retry", calls.Load())
}
if !strings.HasPrefix(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/2026/07/30/input-") ||
!strings.HasSuffix(uploadedPath, ".png") {
t.Fatalf("unexpected object path=%q", uploadedPath)
}
if got := stringFromAny(uploaded["url"]); got != "https://cdn.example.com"+uploadedPath {
t.Fatalf("public URL=%q", got)
}
channel, _ := uploaded["storageChannel"].(map[string]any)
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
t.Fatalf("storage channel=%+v", channel)
}
}
func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
storageDir := t.TempDir()
cfg := config.Config{
LocalUploadedStorageDir: storageDir,
MediaOSSDirectEnabled: true,
MediaOSSEndpoint: server.URL,
MediaOSSBucket: "media-bucket",
MediaOSSAccessKeyID: "access-key",
MediaOSSAccessKeySecret: "access-secret",
MediaOSSPublicBaseURL: "https://cdn.example.com",
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
}
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
Bytes: []byte("general-upload"),
ContentType: "text/plain",
FileName: "general.txt",
Scene: store.FileStorageSceneUpload,
})
if err != nil {
t.Fatalf("general upload: %v", err)
}
if calls.Load() != 0 {
t.Fatalf("direct OSS calls=%d for general upload scene", calls.Load())
}
channel, _ := uploaded["storageChannel"].(map[string]any)
if stringFromAny(channel["provider"]) != "local_static" {
t.Fatalf("general upload channel=%+v", channel)
}
}
apps/api/internal/runner/service.go
+2
View File
@@ -45,6 +45,7 @@ type Service struct {
admissionListener sync.Once
asyncClientFactory func(int) (asyncExecutionClient, error)
mediaResultSlots chan struct{}
directOSS *directOSSUploader
billingMetrics billingMetricsObserver
}
@@ -154,6 +155,7 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
asyncAdmissionWake: make(chan struct{}, 1),
admissionTaskWaiters: map[string]*admissionTaskWaiter{},
mediaResultSlots: make(chan struct{}, cfg.MediaMaterializationConcurrency),
directOSS: newDirectOSSUploader(cfg),
}
if len(observers) > 0 {
service.billingMetrics = observers[0]
apps/api/internal/runner/upload.go
+3
View File
@@ -935,6 +935,9 @@ func (s *Service) UploadFile(ctx context.Context, payload FileUploadPayload) (ma
if strings.TrimSpace(payload.Scene) == "" {
payload.Scene = store.FileStorageSceneUpload
}
if s.directOSS != nil && directOSSScene(payload.Scene) {
return s.directOSS.upload(ctx, payload)
}
channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
if err != nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}