perf(media): 生产媒体直传阿里云 OSS

生产同构验收确认 server_main_openapi 上传通道在媒体高并发下产生 381 个 502，并将已创建任务的 p95 拉高到 172 秒。\n\n为 request_asset 与 image_result 增加环境配置控制的阿里云 OSS 直传，保留普通上传原通道；使用 Kubernetes Secret 注入凭据，并对签名、重试、场景隔离和生产配置校验增加测试。\n\n验证：Go 全量测试、go vet、OpenAPI、迁移安全检查、Kustomize/Compose 渲染、gofmt 与 git diff --check 均通过；真实 OSS PUT/CDN 读取探针通过且对象已清理。
2026-07-31 05:20:27 +08:00
parent 7b2ab786a2
commit 352e23e099
8 changed files with 460 additions and 0 deletions
@@ -65,6 +65,13 @@ type Config struct {
 	DatabaseLockTimeoutSeconds                    int
 	MediaRequestConcurrency                       int
 	MediaMaterializationConcurrency               int
+	MediaOSSDirectEnabled                         bool
+	MediaOSSEndpoint                              string
+	MediaOSSBucket                                string
+	MediaOSSAccessKeyID                           string
+	MediaOSSAccessKeySecret                       string
+	MediaOSSPublicBaseURL                         string
+	MediaOSSObjectPrefix                          string
 	AsyncQueueWorkerEnabled                       bool
 	AsyncWorkerHardLimit                          int
 	AsyncWorkerInstanceHardLimit                  int
@@ -135,6 +142,13 @@ func Load() Config {

 		MediaRequestConcurrency:           envIntValidated("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY", 16),
 		MediaMaterializationConcurrency:   envIntValidated("AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY", 8),
+		MediaOSSDirectEnabled:             env("AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED", "false") == "true",
+		MediaOSSEndpoint:                  strings.TrimRight(env("AI_GATEWAY_MEDIA_OSS_ENDPOINT", ""), "/"),
+		MediaOSSBucket:                    strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_BUCKET", "")),
+		MediaOSSAccessKeyID:               strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID", "")),
+		MediaOSSAccessKeySecret:           strings.TrimSpace(env("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET", "")),
+		MediaOSSPublicBaseURL:             strings.TrimRight(env("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL", ""), "/"),
+		MediaOSSObjectPrefix:              strings.Trim(env("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX", "easyai-ai-gateway/media"), "/"),
 		AsyncQueueWorkerEnabled:           env("AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED", "true") == "true",
 		AsyncWorkerHardLimit:              envIntValidated("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", 2048),
 		AsyncWorkerInstanceHardLimit:      envIntValidated("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT", 32),
@@ -170,6 +184,23 @@ func (c Config) Validate() error {
 	if c.MediaRequestConcurrency != 0 && (c.MediaRequestConcurrency < 1 || c.MediaRequestConcurrency > 1024) {
 		return errors.New("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY must be between 1 and 1024")
 	}
+	if c.MediaOSSDirectEnabled {
+		if strings.TrimSpace(c.MediaOSSAccessKeyID) == "" || strings.TrimSpace(c.MediaOSSAccessKeySecret) == "" {
+			return errors.New("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID and AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET are required when direct media OSS is enabled")
+		}
+		if !validMediaOSSBucket(c.MediaOSSBucket) {
+			return errors.New("AI_GATEWAY_MEDIA_OSS_BUCKET is invalid")
+		}
+		if !validMediaOSSURL(c.MediaOSSEndpoint, c.AppEnv) {
+			return errors.New("AI_GATEWAY_MEDIA_OSS_ENDPOINT must be an HTTPS URL")
+		}
+		if !validMediaOSSURL(c.MediaOSSPublicBaseURL, c.AppEnv) {
+			return errors.New("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL must be an HTTPS URL")
+		}
+		if !validMediaOSSObjectPrefix(c.MediaOSSObjectPrefix) {
+			return errors.New("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX is invalid")
+		}
+	}
 	switch strings.ToLower(strings.TrimSpace(c.BillingEngineMode)) {
 	case "", "observe", "enforce", "hold":
 	default:
@@ -237,6 +268,48 @@ func (c Config) Validate() error {
 	return nil
 }

+func validMediaOSSBucket(value string) bool {
+	value = strings.TrimSpace(value)
+	if len(value) < 3 || len(value) > 63 {
+		return false
+	}
+	for index, item := range value {
+		if (item >= 'a' && item <= 'z') || (item >= '0' && item <= '9') || (item == '-' && index > 0 && index < len(value)-1) {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+func validMediaOSSURL(value string, appEnv string) bool {
+	parsed, err := url.Parse(strings.TrimSpace(value))
+	if err != nil || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.User != nil {
+		return false
+	}
+	if parsed.Scheme == "https" {
+		return true
+	}
+	host := strings.ToLower(parsed.Hostname())
+	return !strings.EqualFold(strings.TrimSpace(appEnv), "production") &&
+		parsed.Scheme == "http" && (host == "127.0.0.1" || host == "localhost" || host == "::1")
+}
+
+func validMediaOSSObjectPrefix(value string) bool {
+	value = strings.Trim(value, "/")
+	if value == "" || len(value) > 256 || strings.Contains(value, "..") {
+		return false
+	}
+	for _, item := range value {
+		if (item >= 'a' && item <= 'z') || (item >= 'A' && item <= 'Z') || (item >= '0' && item <= '9') ||
+			item == '-' || item == '_' || item == '/' {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
 func (c Config) EffectiveProcessRole() string {
 	role := strings.ToLower(strings.TrimSpace(c.ProcessRole))
 	if role == "" {
@@ -233,3 +233,37 @@ func TestValidateTaskHistorySettings(t *testing.T) {
 		t.Fatalf("Validate() error = %v, want invalid cleanup batch", err)
 	}
 }
+
+func TestLoadAndValidateDirectMediaOSS(t *testing.T) {
+	t.Setenv("APP_ENV", "production")
+	t.Setenv("AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED", "true")
+	t.Setenv("AI_GATEWAY_MEDIA_OSS_ENDPOINT", "https://media-bucket.oss-cn-shanghai.aliyuncs.com")
+	t.Setenv("AI_GATEWAY_MEDIA_OSS_BUCKET", "media-bucket")
+	t.Setenv("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID", "test-access-key")
+	t.Setenv("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET", "test-access-secret")
+	t.Setenv("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL", "https://cdn.example.com")
+	t.Setenv("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX", "easyai-ai-gateway/production/media")
+
+	cfg := Load()
+	if !cfg.MediaOSSDirectEnabled || cfg.MediaOSSBucket != "media-bucket" ||
+		cfg.MediaOSSObjectPrefix != "easyai-ai-gateway/production/media" {
+		t.Fatalf("unexpected direct media OSS config: %+v", cfg)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("validate direct media OSS: %v", err)
+	}
+	cfg.MediaOSSAccessKeySecret = ""
+	if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ACCESS_KEY_SECRET") {
+		t.Fatalf("Validate() error = %v, want missing direct OSS secret", err)
+	}
+	cfg.MediaOSSAccessKeySecret = "test-access-secret"
+	cfg.MediaOSSEndpoint = "http://media-bucket.example.com"
+	if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "HTTPS URL") {
+		t.Fatalf("Validate() error = %v, want insecure production endpoint", err)
+	}
+	cfg.MediaOSSEndpoint = "https://media-bucket.example.com"
+	cfg.MediaOSSObjectPrefix = "../outside"
+	if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "OBJECT_PREFIX") {
+		t.Fatalf("Validate() error = %v, want invalid object prefix", err)
+	}
+}