feat(queue): 增加非文本模型分布式准入队列

使用 PostgreSQL 统一同步与异步非文本任务的并发准入、持久化等待和 Worker 容量分配，并将生产 API 与独立 Worker 角色拆分。

补充策略管理、共享契约、OpenAPI、Kubernetes 双节点 Worker 清单及跨节点验收脚本；未默认启用任何生产 queue_size 策略。

已在原基线完成 Go、前端、迁移、Shell、Kustomize 与长任务容量验收；合入最新主干后将重新执行发布门禁。

apps/api/internal/httpapi/async_worker_acceptance_integration_test.go

@@ -205,6 +205,7 @@ func asyncAcceptancePlatformPayload(key, name string, concurrent int, status str
 		"status":      status,
 		"rateLimitPolicy": map[string]any{"rules": []any{
 			map[string]any{"metric": "concurrent", "limit": concurrent, "leaseTtlSeconds": 120},
+			map[string]any{"metric": "queue_size", "limit": 512, "maxWaitSeconds": 600},
 		}},
 	}
 }
@@ -222,6 +223,7 @@ func createAsyncAcceptanceModel(t *testing.T, baseURL, token, platformID, model,
 	if mode == "override" {
 		payload["rateLimitPolicy"] = map[string]any{"rules": []any{
 			map[string]any{"metric": "concurrent", "limit": concurrent, "leaseTtlSeconds": ttl},
+			map[string]any{"metric": "queue_size", "limit": 512, "maxWaitSeconds": 600},
 		}}
 	}
 	var response struct {

apps/api/internal/httpapi/core_flow_integration_test.go

@@ -2085,6 +2085,15 @@ func assertRuntimeRecoveryReleasesPendingRateReservations(t *testing.T, ctx cont
 	}}); err != nil {
 		t.Fatalf("reserve recovery rate limit: %v", err)
 	}
+	if _, err := db.Pool().Exec(ctx, `
+UPDATE gateway_tasks
+SET status = 'failed',
+    error_code = 'integration_test',
+    error_message = 'terminal recovery fixture',
+    finished_at = now()
+WHERE id = $1::uuid`, task.ID); err != nil {
+		t.Fatalf("mark recovery reservation task terminal: %v", err)
+	}
 	recovery, err := db.RecoverInterruptedRuntimeState(ctx)
 	if err != nil {
 		t.Fatalf("recover interrupted runtime state with pending reservation: %v", err)

apps/api/internal/httpapi/handlers.go

@@ -345,6 +345,10 @@ func (s *Server) createPlatform(w http.ResponseWriter, r *http.Request) {
 	input.Config = config
 	platform, err := s.store.CreatePlatform(r.Context(), input)
 	if err != nil {
+		if store.IsInvalidRateLimitPolicy(err) {
+			writeError(w, http.StatusBadRequest, err.Error(), "invalid_parameter")
+			return
+		}
 		s.logger.Error("create platform failed", "error", err)
 		writeError(w, http.StatusInternalServerError, "create platform failed")
 		return
@@ -398,6 +402,10 @@ func (s *Server) updatePlatform(w http.ResponseWriter, r *http.Request) {
 	input.Config = config
 	platform, err := s.store.UpdatePlatform(r.Context(), r.PathValue("platformID"), input)
 	if err != nil {
+		if store.IsInvalidRateLimitPolicy(err) {
+			writeError(w, http.StatusBadRequest, err.Error(), "invalid_parameter")
+			return
+		}
 		if store.IsNotFound(err) {
 			writeError(w, http.StatusNotFound, "platform not found")
 			return
@@ -471,7 +479,7 @@ func (s *Server) createPlatformModel(w http.ResponseWriter, r *http.Request) {
 	}
 	model, err := s.store.CreatePlatformModel(r.Context(), input)
 	if err != nil {
-		if errors.Is(err, store.ErrInvalidPlatformModelConfiguration) {
+		if errors.Is(err, store.ErrInvalidPlatformModelConfiguration) || store.IsInvalidRateLimitPolicy(err) {
 			writeError(w, http.StatusBadRequest, err.Error(), "invalid_parameter")
 			return
 		}
@@ -519,7 +527,7 @@ func (s *Server) replacePlatformModels(w http.ResponseWriter, r *http.Request) {
 
 	models, err := s.store.ReplacePlatformModels(r.Context(), platformID, input.Models)
 	if err != nil {
-		if errors.Is(err, store.ErrInvalidPlatformModelConfiguration) {
+		if errors.Is(err, store.ErrInvalidPlatformModelConfiguration) || store.IsInvalidRateLimitPolicy(err) {
 			writeError(w, http.StatusBadRequest, err.Error(), "invalid_parameter")
 			return
 		}
@@ -1206,8 +1214,9 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
 			return
 		}
 		if responsePlan.asyncMode {
-			if err := s.runner.EnqueueAsyncTask(r.Context(), task); err != nil {
-				writeTaskError(http.StatusInternalServerError, err.Error(), nil, "enqueue_failed")
+			if err := s.runner.SubmitAsyncTask(r.Context(), task); err != nil {
+				applyRunErrorHeaders(w, err)
+				writeTaskError(statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), runErrorCode(err))
 				return
 			}
 			writeTaskAccepted(w, task)
@@ -1307,7 +1316,7 @@ func openAIEmbeddingsDoc() {}
 
 // openAIImagesDoc godoc
 // @Summary 创建或编辑 OpenAI Images
-// @Description 默认同步返回 OpenAI-compatible Images 响应；设置 X-Async=true 时异步创建任务并返回 202。
+// @Description 默认同步返回 OpenAI-compatible Images 响应；非文本模型并发饱和且配置队列时等待准入，超过最长等待返回 504 queue_timeout；设置 X-Async=true 时异步创建任务并返回 202。
 // @Tags images
 // @Accept json
 // @Produce json
@@ -1319,6 +1328,7 @@ func openAIEmbeddingsDoc() {}
 // @Failure 400 {object} OpenAIErrorEnvelope
 // @Failure 401 {object} OpenAIErrorEnvelope
 // @Failure 429 {object} OpenAIErrorEnvelope
+// @Failure 504 {object} OpenAIErrorEnvelope
 // @Failure 502 {object} OpenAIErrorEnvelope
 // @Router /api/v1/images/generations [post]
 // @Router /api/v1/images/edits [post]
@@ -1326,7 +1336,7 @@ func openAIImagesDoc() {}
 
 // easyAIMediaTasksDoc godoc
 // @Summary 创建 EasyAI 媒体任务
-// @Description 默认同步返回 EasyAI GeneratedResponse；设置 X-Async=true 时返回同时兼容 Gateway 与 server-main EasyAIClient 的异步提交结构。
+// @Description 默认同步返回 EasyAI GeneratedResponse；非文本模型并发饱和且配置队列时等待准入，超过最长等待返回 504 queue_timeout；设置 X-Async=true 时返回同时兼容 Gateway 与 server-main EasyAIClient 的异步提交结构。
 // @Tags media
 // @Accept json
 // @Produce json
@@ -1342,6 +1352,7 @@ func openAIImagesDoc() {}
 // @Failure 403 {object} ErrorEnvelope
 // @Failure 404 {object} ErrorEnvelope
 // @Failure 429 {object} ErrorEnvelope
+// @Failure 504 {object} ErrorEnvelope
 // @Failure 502 {object} ErrorEnvelope
 // @Router /api/v1/videos/generations [post]
 // @Router /api/v1/song/generations [post]
@@ -1351,19 +1362,10 @@ func openAIImagesDoc() {}
 func easyAIMediaTasksDoc() {}
 
 func (s *Server) requestExecutionContext(r *http.Request) (context.Context, context.CancelFunc) {
-	base := context.WithoutCancel(r.Context())
 	if s.ctx == nil {
-		return base, func() {}
+		return runner.NewRequestExecutionContext(r.Context(), context.Background())
 	}
-	ctx, cancel := context.WithCancel(base)
-	go func() {
-		select {
-		case <-s.ctx.Done():
-			cancel()
-		case <-ctx.Done():
-		}
-	}()
-	return ctx, cancel
+	return runner.NewRequestExecutionContext(r.Context(), s.ctx)
 }
 
 func requestStillConnected(r *http.Request) bool {
@@ -1663,6 +1665,8 @@ func statusFromRunError(err error) int {
 		return http.StatusNotFound
 	case store.ModelCandidateErrorCode(err) == "platform_cooling_down" || store.ModelCandidateErrorCode(err) == "model_cooling_down":
 		return http.StatusTooManyRequests
+	case errors.Is(err, store.ErrQueueTimeout) || clients.ErrorCode(err) == "queue_timeout":
+		return http.StatusGatewayTimeout
 	case errors.Is(err, store.ErrNoModelCandidate):
 		return http.StatusNotFound
 	case errors.Is(err, store.ErrRateLimited):
@@ -1710,7 +1714,11 @@ func runErrorDetails(err error) map[string]any {
 }
 
 func applyRunErrorHeaders(w http.ResponseWriter, err error) {
-	if retryAfter := store.ModelCandidateRetryAfter(err); retryAfter > 0 {
+	retryAfter := store.ModelCandidateRetryAfter(err)
+	if limitRetryAfter := store.RateLimitRetryAfter(err); limitRetryAfter > 0 {
+		retryAfter = limitRetryAfter
+	}
+	if retryAfter > 0 {
 		seconds := int((retryAfter + time.Second - 1) / time.Second)
 		if seconds < 1 {
 			seconds = 1
@@ -1783,6 +1791,13 @@ func rateLimitErrorDetail(err error) map[string]any {
 			"limit":     limitErr.Limit,
 		},
 	}
+	if limitErr.Reason != "" {
+		detail["reason"] = limitErr.Reason
+	}
+	if limitErr.QueueLimit > 0 {
+		detail["queueDepth"] = limitErr.QueueDepth
+		detail["queueLimit"] = limitErr.QueueLimit
+	}
 	if limitErr.RetryAfter > 0 {
 		detail["retryAfterMs"] = limitErr.RetryAfter.Milliseconds()
 	}

apps/api/internal/httpapi/identity_admin_handlers.go

@@ -259,6 +259,10 @@ func (s *Server) createUserGroup(w http.ResponseWriter, r *http.Request) {
 	}
 	item, err := s.store.CreateUserGroup(r.Context(), input)
 	if err != nil {
+		if store.IsInvalidRateLimitPolicy(err) {
+			writeError(w, http.StatusBadRequest, err.Error(), "invalid_parameter")
+			return
+		}
 		if store.IsUniqueViolation(err) {
 			writeError(w, http.StatusConflict, "user group key already exists")
 			return
@@ -299,6 +303,10 @@ func (s *Server) updateUserGroup(w http.ResponseWriter, r *http.Request) {
 	}
 	item, err := s.store.UpdateUserGroup(r.Context(), r.PathValue("groupID"), input)
 	if err != nil {
+		if store.IsInvalidRateLimitPolicy(err) {
+			writeError(w, http.StatusBadRequest, err.Error(), "invalid_parameter")
+			return
+		}
 		if store.IsNotFound(err) {
 			writeError(w, http.StatusNotFound, "user group not found")
 			return

apps/api/internal/httpapi/keling_compat_handlers.go

@@ -192,9 +192,10 @@ func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
 		writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusInternalServerError, 5000, "create compatibility metadata failed"))
 		return
 	}
-	if err := s.runner.EnqueueAsyncTask(r.Context(), task); err != nil {
+	if err := s.runner.SubmitAsyncTask(r.Context(), task); err != nil {
 		s.logger.Error("enqueue Kling-compatible task failed", "taskId", task.ID, "error", err)
-		writeKelingCompatError(w, requestID, newKelingCompatError(http.StatusServiceUnavailable, 5001, "video task queue is unavailable"))
+		applyRunErrorHeaders(w, err)
+		writeKelingCompatError(w, requestID, kelingCompatGatewayError(err))
 		return
 	}
 	task, createErr = s.waitForCompatibilitySubmission(r, task)

apps/api/internal/httpapi/kling_compat.go

@@ -162,8 +162,9 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
 			writeKlingCompatError(w, http.StatusInternalServerError, "create compatibility metadata failed", "task_create_failed")
 			return
 		}
-		if err := s.runner.EnqueueAsyncTask(r.Context(), task); err != nil {
-			writeKlingCompatError(w, http.StatusInternalServerError, err.Error(), "enqueue_failed")
+		if err := s.runner.SubmitAsyncTask(r.Context(), task); err != nil {
+			applyRunErrorHeaders(w, err)
+			writeKlingCompatError(w, statusFromRunError(err), runErrorMessage(err), runErrorCode(err))
 			return
 		}
 		task, err = s.waitForCompatibilitySubmission(r, task)

apps/api/internal/httpapi/rate_limit_error_detail_test.go

@@ -1,6 +1,8 @@
 package httpapi
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
@@ -48,6 +50,40 @@ func TestRateLimitErrorDetailIncludesUserGroupAndExceededMetric(t *testing.T) {
 	}
 }
 
+func TestQueueFullAndTimeoutPublicContracts(t *testing.T) {
+	queueErr := &store.RateLimitExceededError{
+		ScopeType:  "platform_model",
+		ScopeKey:   "model-1",
+		Metric:     "queue_size",
+		Reason:     "queue_full",
+		Limit:      2,
+		Current:    2,
+		Projected:  3,
+		QueueDepth: 2,
+		QueueLimit: 2,
+		RetryAfter: 1500 * time.Millisecond,
+		Retryable:  true,
+	}
+	recorder := httptest.NewRecorder()
+	applyRunErrorHeaders(recorder, queueErr)
+	if statusFromRunError(queueErr) != http.StatusTooManyRequests || runErrorCode(queueErr) != "rate_limit" {
+		t.Fatalf("queue full contract status=%d code=%s", statusFromRunError(queueErr), runErrorCode(queueErr))
+	}
+	if recorder.Header().Get("Retry-After") != "2" {
+		t.Fatalf("Retry-After = %q, want 2", recorder.Header().Get("Retry-After"))
+	}
+	details := runErrorDetails(queueErr)
+	detail, _ := details["rateLimit"].(map[string]any)
+	if detail["reason"] != "queue_full" || detail["queueDepth"] != 2 || detail["queueLimit"] != 2 {
+		t.Fatalf("queue full details = %+v", details)
+	}
+
+	timeoutErr := &store.QueueTimeoutError{TaskID: "task-1"}
+	if statusFromRunError(timeoutErr) != http.StatusGatewayTimeout || runErrorCode(timeoutErr) != "queue_timeout" {
+		t.Fatalf("queue timeout contract status=%d code=%s", statusFromRunError(timeoutErr), runErrorCode(timeoutErr))
+	}
+}
+
 func TestRunErrorMessageIncludesRateLimitSummary(t *testing.T) {
 	message := runErrorMessage(&store.RateLimitExceededError{
 		ScopeType:     "user_group",

apps/api/internal/httpapi/runtime_policy_handlers.go

@@ -276,6 +276,10 @@ func (s *Server) createRuntimePolicySet(w http.ResponseWriter, r *http.Request)
 	}
 	item, err := s.store.CreateRuntimePolicySet(r.Context(), input)
 	if err != nil {
+		if store.IsInvalidRateLimitPolicy(err) {
+			writeError(w, http.StatusBadRequest, err.Error(), "invalid_parameter")
+			return
+		}
 		if store.IsUniqueViolation(err) {
 			writeError(w, http.StatusConflict, "runtime policy key already exists")
 			return
@@ -316,6 +320,10 @@ func (s *Server) updateRuntimePolicySet(w http.ResponseWriter, r *http.Request)
 	}
 	item, err := s.store.UpdateRuntimePolicySet(r.Context(), r.PathValue("policySetID"), input)
 	if err != nil {
+		if store.IsInvalidRateLimitPolicy(err) {
+			writeError(w, http.StatusBadRequest, err.Error(), "invalid_parameter")
+			return
+		}
 		if store.IsNotFound(err) {
 			writeError(w, http.StatusNotFound, "runtime policy set not found")
 			return

apps/api/internal/httpapi/server.go

@@ -120,7 +120,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
 		return server.identityRuntime.LegacyJWTEnabled()
 	}
 	server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey
-	if cfg.AsyncQueueWorkerEnabled {
+	server.runner.StartAdmissionNotifier(ctx)
+	if cfg.RunsAsyncExecutionWorker() {
 		server.runner.StartAsyncQueueWorker(ctx)
 	} else {
 		server.runner.StartAsyncQueueClient(ctx)
@@ -128,10 +129,12 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
 			logger.Info("asynchronous queue worker disabled for this process")
 		}
 	}
-	server.runner.StartBillingSettlementWorker(ctx)
-	server.runner.StartTaskHistoryWorkers(ctx)
-	server.startLocalTempAssetCleanup(ctx)
-	server.startOIDCSessionCleanup(ctx)
+	if cfg.RunsBackgroundWorkers() {
+		server.runner.StartBillingSettlementWorker(ctx)
+		server.runner.StartTaskHistoryWorkers(ctx)
+		server.startLocalTempAssetCleanup(ctx)
+		server.startOIDCSessionCleanup(ctx)
+	}
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /healthz", server.health)
@@ -139,6 +142,9 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
 	mux.HandleFunc("GET /api/v1/healthz", server.health)
 	mux.HandleFunc("GET /api/v1/readyz", server.ready)
 	mux.Handle("GET /metrics", securityEventMetrics.DynamicHandler(db))
+	if !cfg.RunsPublicHTTP() {
+		return server.recover(mux)
+	}
 	mux.HandleFunc("GET /static/simulation/{asset}", serveSimulationAsset)
 	mux.HandleFunc("GET /static/generated/{asset}", server.serveGeneratedStaticAsset)
 	mux.HandleFunc("GET /static/uploaded/{asset}", server.serveUploadedStaticAsset)

apps/api/internal/httpapi/volces_compat_handlers.go

@@ -236,8 +236,8 @@ func (s *Server) createVolcesCompatibleTask(r *http.Request, user *auth.User, bo
 		return store.GatewayTask{}, err
 	}
 	task.CompatibilityProtocol = clients.ProtocolVolcesContents
-	if err := s.runner.EnqueueAsyncTask(r.Context(), task); err != nil {
-		return store.GatewayTask{}, &clients.ClientError{Code: "enqueue_failed", Message: err.Error(), StatusCode: http.StatusServiceUnavailable, Retryable: true}
+	if err := s.runner.SubmitAsyncTask(r.Context(), task); err != nil {
+		return store.GatewayTask{}, err
 	}
 	return s.waitForCompatibilitySubmission(r, task)
 }