feat(routing): 引入多执行池智能调度

将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。

实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。

新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。

apps/api/internal/httpapi/admin_execution_pools.go

@@ -0,0 +1,143 @@
+package httpapi
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
+)
+
+type AdminExecutionPoolWorker struct {
+	WorkerID        string         `json:"workerId"`
+	InstanceID      string         `json:"instanceId"`
+	Revision        string         `json:"revision,omitempty"`
+	ProtocolVersion string         `json:"protocolVersion"`
+	Capabilities    map[string]any `json:"capabilities"`
+	Allocated       int            `json:"allocated"`
+	SafeCapacity    int            `json:"safeCapacity"`
+	ActiveTasks     int            `json:"activeTasks"`
+	PressureState   string         `json:"pressureState"`
+	HeartbeatAt     time.Time      `json:"heartbeatAt"`
+}
+
+type AdminExecutionPool struct {
+	PoolID       string                      `json:"poolId"`
+	Labels       map[string]string           `json:"labels"`
+	Capabilities map[string]any              `json:"capabilities"`
+	State        string                      `json:"state"`
+	Capacity     *AdminExecutionPoolCapacity `json:"capacity,omitempty"`
+	Workers      []AdminExecutionPoolWorker  `json:"workers"`
+}
+
+type AdminExecutionPoolCapacity struct {
+	WorkerCount             int       `json:"workerCount"`
+	Allocated               int       `json:"allocated"`
+	SafeCapacity            int       `json:"safeCapacity"`
+	ActiveTasks             int       `json:"activeTasks"`
+	HeavyCapacity           int       `json:"heavyCapacity"`
+	HeavyTasks              int       `json:"heavyTasks"`
+	QueueDepth              int       `json:"queueDepth"`
+	EstimatedQueueWaitMilli int64     `json:"estimatedQueueWaitMs"`
+	ResourceHeadroom        float64   `json:"resourceHeadroom"`
+	Stability               float64   `json:"stability"`
+	Critical                bool      `json:"critical"`
+	SampledAt               time.Time `json:"sampledAt"`
+}
+
+type AdminRouteHealth struct {
+	PoolID               string    `json:"poolId"`
+	RouteProfileKey      string    `json:"routeProfileKey"`
+	State                string    `json:"state"`
+	SuccessRate          float64   `json:"successRate"`
+	ConnectTLSP95Milli   int64     `json:"connectTlsP95Ms"`
+	FirstByteP95Milli    int64     `json:"firstByteP95Ms"`
+	UploadBytesPerSecond float64   `json:"uploadBytesPerSecond"`
+	JitterP95Milli       int64     `json:"jitterP95Ms"`
+	ConsecutiveFailures  int       `json:"consecutiveFailures"`
+	ConsecutiveSuccesses int       `json:"consecutiveSuccesses"`
+	SampleCount          int       `json:"sampleCount"`
+	SampledAt            time.Time `json:"sampledAt"`
+	ExpiresAt            time.Time `json:"expiresAt"`
+}
+
+type AdminExecutionPoolResponse struct {
+	Pools       []AdminExecutionPool `json:"pools"`
+	RouteHealth []AdminRouteHealth   `json:"routeHealth"`
+	CapturedAt  time.Time            `json:"capturedAt"`
+}
+
+// listExecutionPools godoc
+// @Summary 查询逻辑执行池、Worker 容量和上游路由健康状态
+// @Tags Admin Runtime
+// @Produce json
+// @Success 200 {object} AdminExecutionPoolResponse
+// @Failure 500 {object} ErrorEnvelope
+// @Security BearerAuth
+// @Router /api/admin/runtime/execution-pools [get]
+func (s *Server) listExecutionPools(w http.ResponseWriter, r *http.Request) {
+	now := time.Now()
+	pools, err := s.coordinationStore.ListExecutionPools(r.Context())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	workers, err := s.coordinationStore.ListWorkers(r.Context(), now)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	capacities, err := s.coordinationStore.ListCapacity(r.Context(), now)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	health, err := s.coordinationStore.ListAllRouteHealth(r.Context())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	capacityByPool := make(map[string]executionpool.CapacitySnapshot, len(capacities))
+	for _, capacity := range capacities {
+		capacityByPool[capacity.PoolID] = capacity
+	}
+	workersByPool := make(map[string][]AdminExecutionPoolWorker)
+	for _, worker := range workers {
+		workersByPool[worker.PoolID] = append(workersByPool[worker.PoolID], AdminExecutionPoolWorker{
+			WorkerID: worker.WorkerID, InstanceID: worker.InstanceID, Revision: worker.Revision,
+			ProtocolVersion: worker.ProtocolVersion, Capabilities: worker.Capabilities,
+			Allocated: worker.Allocated, SafeCapacity: worker.SafeCapacity,
+			ActiveTasks: worker.ActiveTasks, PressureState: worker.PressureState,
+			HeartbeatAt: worker.HeartbeatAt,
+		})
+	}
+	response := AdminExecutionPoolResponse{
+		Pools: make([]AdminExecutionPool, 0, len(pools)), RouteHealth: make([]AdminRouteHealth, 0, len(health)), CapturedAt: now,
+	}
+	for _, item := range health {
+		response.RouteHealth = append(response.RouteHealth, AdminRouteHealth{
+			PoolID: item.PoolID, RouteProfileKey: item.RouteProfileKey, State: string(item.State),
+			SuccessRate: item.SuccessRate, ConnectTLSP95Milli: item.ConnectTLSP95.Milliseconds(),
+			FirstByteP95Milli: item.FirstByteP95.Milliseconds(), UploadBytesPerSecond: item.UploadBytesPerSecond,
+			JitterP95Milli: item.JitterP95.Milliseconds(), ConsecutiveFailures: item.ConsecutiveFailures,
+			ConsecutiveSuccesses: item.ConsecutiveSuccesses, SampleCount: item.SampleCount,
+			SampledAt: item.SampledAt, ExpiresAt: item.ExpiresAt,
+		})
+	}
+	for _, pool := range pools {
+		item := AdminExecutionPool{
+			PoolID: pool.ID, Labels: pool.Labels, Capabilities: pool.Capabilities,
+			State: string(pool.State), Workers: workersByPool[pool.ID],
+		}
+		if capacity, ok := capacityByPool[pool.ID]; ok {
+			item.Capacity = &AdminExecutionPoolCapacity{
+				WorkerCount: capacity.WorkerCount, Allocated: capacity.Allocated, SafeCapacity: capacity.SafeCapacity,
+				ActiveTasks: capacity.ActiveTasks, HeavyCapacity: capacity.HeavyCapacity, HeavyTasks: capacity.HeavyTasks,
+				QueueDepth: capacity.QueueDepth, EstimatedQueueWaitMilli: capacity.EstimatedQueueWait.Milliseconds(),
+				ResourceHeadroom: capacity.ResourceHeadroom, Stability: capacity.Stability,
+				Critical: capacity.Critical, SampledAt: capacity.SampledAt,
+			}
+		}
+		response.Pools = append(response.Pools, item)
+	}
+	writeJSON(w, http.StatusOK, response)
+}

apps/api/internal/httpapi/compat_protocol_test.go

@@ -125,10 +125,10 @@ func TestProtocolErrorsUseCompatibleShapesWithStandardPublicErrors(t *testing.T)
 			},
 		},
 		{
-			name: "volces", protocol: clients.ProtocolVolcesContents, status: http.StatusBadGateway,
+			name: "volces", protocol: clients.ProtocolVolcesContents, status: http.StatusGatewayTimeout,
 			assertBody: func(t *testing.T, body map[string]any) {
 				errorBody := requireObject(t, body["error"])
-				if errorBody["code"] != "upstream_submission_unknown" || errorBody["httpStatus"] != float64(http.StatusBadGateway) || errorBody["retryable"] != true {
+				if errorBody["code"] != "upstream_timeout" || errorBody["httpStatus"] != float64(http.StatusGatewayTimeout) || errorBody["retryable"] != true {
 					t.Fatalf("unexpected Volces error: %+v", body)
 				}
 				assertNoKeys(t, errorBody, "status", "taskId", "gateway_status")
@@ -151,7 +151,7 @@ func TestProtocolErrorsUseCompatibleShapesWithStandardPublicErrors(t *testing.T)
 			if test.name == "gemini" {
 				code = "rate_limit"
 			} else if test.name == "volces" {
-				code = "upstream_submission_unknown"
+				code = "upstream_timeout"
 			} else if test.name == "keling" {
 				code = "gateway_rate_limited"
 			}

apps/api/internal/httpapi/handlers.go

@@ -1892,6 +1892,11 @@ func applyRunErrorHeaders(w http.ResponseWriter, err error) {
 	if limitRetryAfter := store.RateLimitRetryAfter(err); limitRetryAfter > 0 {
 		retryAfter = limitRetryAfter
 	}
+	if retryAfter <= 0 {
+		if seconds := retryAfterSecondsFromDetails(clients.ErrorDetails(err)); seconds > 0 {
+			retryAfter = time.Duration(seconds) * time.Second
+		}
+	}
 	if retryAfter > 0 {
 		seconds := int((retryAfter + time.Second - 1) / time.Second)
 		if seconds < 1 {
@@ -1901,6 +1906,24 @@ func applyRunErrorHeaders(w http.ResponseWriter, err error) {
 	}
 }
 
+func retryAfterSecondsFromDetails(details map[string]any) int {
+	if details == nil {
+		return 0
+	}
+	switch value := details["retryAfterSeconds"].(type) {
+	case int:
+		return value
+	case int32:
+		return int(value)
+	case int64:
+		return int(value)
+	case float64:
+		return int(value)
+	default:
+		return 0
+	}
+}
+
 func rateLimitErrorSummary(err error) string {
 	var limitErr *store.RateLimitExceededError
 	if !errors.As(err, &limitErr) {

apps/api/internal/httpapi/internal_execution.go

@@ -0,0 +1,50 @@
+package httpapi
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
+)
+
+const maxInternalExecutionRequestBytes = 64 * 1024
+
+func (s *Server) internalExecution(w http.ResponseWriter, r *http.Request) {
+	body := http.MaxBytesReader(w, r.Body, maxInternalExecutionRequestBytes)
+	defer body.Close()
+	var input runner.InternalExecutionRequest
+	decoder := json.NewDecoder(body)
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&input); err != nil {
+		http.Error(w, "invalid internal execution request", http.StatusBadRequest)
+		return
+	}
+	var trailing any
+	if err := decoder.Decode(&trailing); err != io.EOF {
+		http.Error(w, "invalid internal execution request", http.StatusBadRequest)
+		return
+	}
+	w.Header().Set("Content-Type", "application/x-ndjson")
+	w.Header().Set("Cache-Control", "no-store")
+	encoder := json.NewEncoder(w)
+	flusher, _ := w.(http.Flusher)
+	writeFrame := func(frame runner.InternalExecutionFrame) error {
+		if err := encoder.Encode(frame); err != nil {
+			return err
+		}
+		if flusher != nil {
+			flusher.Flush()
+		}
+		return nil
+	}
+	result, err := s.runner.ExecuteInternal(r.Context(), r.Header.Get("Authorization"), input, func(delta clients.StreamDeltaEvent) error {
+		return writeFrame(runner.InternalExecutionFrame{Type: "delta", Delta: &delta})
+	})
+	if err != nil {
+		_ = writeFrame(runner.InternalExecutionFrameForError(err))
+		return
+	}
+	_ = writeFrame(runner.InternalExecutionFrame{Type: "result", Output: result.Output, Wire: result.Wire})
+}

apps/api/internal/httpapi/route_unavailable_error_test.go

@@ -0,0 +1,18 @@
+package httpapi
+
+import (
+	"net/http/httptest"
+	"testing"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
+)
+
+func TestApplyRunErrorHeadersUsesRouteRetryAfter(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	applyRunErrorHeaders(recorder, &clients.ClientError{
+		Code: "upstream_route_unavailable", Details: map[string]any{"retryAfterSeconds": 10},
+	})
+	if got := recorder.Header().Get("Retry-After"); got != "10" {
+		t.Fatalf("Retry-After=%q, want 10", got)
+	}
+}

apps/api/internal/httpapi/server.go

@@ -138,6 +138,7 @@ func NewServerWithStores(
 			logger.Info("asynchronous queue worker disabled for this process")
 		}
 	}
+	server.runner.StartRouteHealthProber(ctx)
 	if cfg.RunsAsyncAdmissionDispatcher() {
 		server.runner.StartAsyncAdmissionDispatcher(ctx)
 	}
@@ -156,6 +157,9 @@ func NewServerWithStores(
 	mux.Handle("GET /metrics", securityEventMetrics.DynamicHandler(postgresMetricsProvider{
 		Store: db, critical: coordinationDB, river: riverDB,
 	}))
+	if cfg.RunsAsyncExecutionWorker() {
+		mux.HandleFunc("POST /internal/v1/executions", server.internalExecution)
+	}
 	if !cfg.RunsPublicHTTP() {
 		return server.recover(mux)
 	}
@@ -247,6 +251,7 @@ func NewServerWithStores(
 	mux.Handle("PATCH /api/admin/runtime/policy-sets/{policySetID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateRuntimePolicySet)))
 	mux.Handle("DELETE /api/admin/runtime/policy-sets/{policySetID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteRuntimePolicySet)))
 	mux.Handle("GET /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getRunnerPolicy)))
+	mux.Handle("GET /api/admin/runtime/execution-pools", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listExecutionPools)))
 	mux.Handle("PATCH /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateRunnerPolicy)))
 	mux.Handle("GET /api/admin/runtime/billing-settlements", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listBillingSettlements)))
 	mux.Handle("POST /api/admin/runtime/billing-settlements/{settlementId}/retry", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.retryBillingSettlement)))