feat(routing): 引入多执行池智能调度

将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
2026-08-05 22:25:37 +08:00
parent 03c0873649
commit 7786692d32
58 changed files with 4510 additions and 348 deletions
@@ -12,6 +12,7 @@ import (
 	"time"

 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
 	"github.com/jackc/pgx/v5"
 )
@@ -25,16 +26,18 @@ const (
 type capacityStore interface {
 	TryAcquireCapacityControllerLeadership(context.Context) (store.Leadership, bool, error)
 	WorkerQueueRuntime(context.Context) (store.WorkerQueueRuntime, error)
+	ListPoolQueueRuntime(context.Context) ([]store.PoolQueueRuntime, error)
 	ListWorkerInstanceRuntime(context.Context) ([]store.WorkerInstanceRuntime, error)
 	CapacityDatabaseHealth(context.Context) (store.CapacityDatabaseHealth, error)
 	MarkWorkerDraining(context.Context, string) error
 	ReactivateWorkerInstance(context.Context, string) error
+	PublishDesiredCapacity(context.Context, executionpool.DesiredCapacity) error
 }

-type capacityKubernetes interface {
-	SiteState(context.Context, string) (KubernetesSiteState, error)
-	ScaleWorkerDeployment(context.Context, string, int) error
-	SetPodDeletionCost(context.Context, string, int) error
+type OrchestratorAdapter interface {
+	PoolState(context.Context, string, string) (PoolInfrastructureState, error)
+	ScalePool(context.Context, string, string, int) error
+	SetInstanceTerminationPriority(context.Context, string, int) error
 }

 type Status struct {
@@ -49,7 +52,8 @@ type Status struct {
 type Controller struct {
 	cfg              config.Config
 	store            capacityStore
-	kubernetes       capacityKubernetes
+	orchestrator     OrchestratorAdapter
+	pools            []config.ExecutionPoolCapacityConfig
 	logger           *slog.Logger
 	expectedRevision string
 	now              func() time.Time
@@ -62,16 +66,22 @@ type Controller struct {
 func New(
 	cfg config.Config,
 	db capacityStore,
-	kubernetes capacityKubernetes,
+	orchestrator OrchestratorAdapter,
 	logger *slog.Logger,
 ) *Controller {
 	return &Controller{
-		cfg: cfg, store: db, kubernetes: kubernetes, logger: logger,
+		cfg: cfg, store: db, orchestrator: orchestrator, logger: logger,
 		expectedRevision: strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
 		now:              time.Now,
+		pools:            mustCapacityPools(cfg),
 	}
 }

+func mustCapacityPools(cfg config.Config) []config.ExecutionPoolCapacityConfig {
+	pools, _ := cfg.CapacityPools()
+	return pools
+}
+
 func (controller *Controller) Run(ctx context.Context) {
 	for ctx.Err() == nil {
 		leadership, acquired, err := controller.store.TryAcquireCapacityControllerLeadership(ctx)
@@ -128,6 +138,14 @@ func (controller *Controller) reconcile(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	poolQueues, err := controller.store.ListPoolQueueRuntime(ctx)
+	if err != nil {
+		return err
+	}
+	demandByPool := make(map[string]int, len(poolQueues))
+	for _, poolQueue := range poolQueues {
+		demandByPool[poolQueue.PoolID] = poolQueue.Queued + poolQueue.Running
+	}
 	instances, err := controller.store.ListWorkerInstanceRuntime(ctx)
 	if err != nil {
 		return err
@@ -137,21 +155,21 @@ func (controller *Controller) reconcile(ctx context.Context) error {
 		return err
 	}
 	now := controller.now()
-	sites := make([]SiteResources, 0, 2)
-	kubernetesStates := make(map[string]KubernetesSiteState, 2)
-	for _, site := range []string{"ningbo", "hongkong"} {
-		minReplicas, maxReplicas := controller.siteReplicaBounds(site)
-		if maxReplicas == 0 {
+	poolResources := make([]PoolResources, 0, len(controller.pools))
+	infrastructureStates := make(map[string]PoolInfrastructureState, len(controller.pools))
+	for _, pool := range controller.pools {
+		if pool.MaxReplicas == 0 {
 			continue
 		}
-		state, stateErr := controller.kubernetes.SiteState(ctx, site)
+		state, stateErr := controller.orchestrator.PoolState(ctx, pool.ID, pool.AdapterRef)
 		if stateErr != nil {
 			return stateErr
 		}
-		kubernetesStates[site] = state
-		sites = append(sites, SiteResources{
-			Site: site, CurrentReplicas: state.CurrentReplicas,
-			MinReplicas: minReplicas, MaxReplicas: maxReplicas,
+		infrastructureStates[pool.ID] = state
+		poolResources = append(poolResources, PoolResources{
+			PoolID: pool.ID, CurrentReplicas: state.CurrentReplicas,
+			MinReplicas: pool.MinReplicas, MaxReplicas: pool.MaxReplicas,
+			Demand:                   demandByPool[pool.ID],
 			AllocatableMemoryBytes:   state.AllocatableMemoryBytes,
 			UsedMemoryBytes:          state.UsedMemoryBytes,
 			WorkerRequestMemoryBytes: state.WorkerRequestMemoryBytes,
@@ -163,8 +181,8 @@ func (controller *Controller) reconcile(ctx context.Context) error {
 		})
 	}
 	currentTotal := 0
-	for _, site := range sites {
-		currentTotal += site.CurrentReplicas
+	for _, pool := range poolResources {
+		currentTotal += pool.CurrentReplicas
 	}
 	target := controller.cfg.WorkerTargetOutstandingPerReplica
 	if target < 1 {
@@ -185,7 +203,7 @@ func (controller *Controller) reconcile(ctx context.Context) error {
 		controller.highSince = time.Time{}
 		controller.lowSince = time.Time{}
 	}
-	revisionHealthy := controller.revisionsMatch(instances, kubernetesStates)
+	revisionHealthy := controller.revisionsMatch(instances, infrastructureStates)
 	scaleUpEligible := !controller.highSince.IsZero() &&
 		now.Sub(controller.highSince) >= time.Duration(controller.cfg.WorkerScaleUpWindowSeconds)*time.Second &&
 		revisionHealthy
@@ -206,21 +224,33 @@ func (controller *Controller) reconcile(ctx context.Context) error {
 		SynchronousDatabasePeers:    database.SynchronousPeers,
 		ScaleUpEligible:             scaleUpEligible,
 		ScaleDownEligible:           scaleDownEligible,
-		Sites:                       sites,
+		Pools:                       poolResources,
 	})
 	if !revisionHealthy && plan.FrozenReason == "" {
 		plan.FrozenReason = "release_revision_mismatch"
 	}
+	for _, poolPlan := range plan.Pools {
+		reason := "capacity_plan"
+		if plan.FrozenReason != "" {
+			reason = "frozen:" + plan.FrozenReason
+		}
+		if err := controller.store.PublishDesiredCapacity(ctx, executionpool.DesiredCapacity{
+			PoolID: poolPlan.PoolID, Desired: poolPlan.DesiredReplicas,
+			Reason: reason, ValidUntil: now.Add(2 * controllerReconcileInterval),
+		}); err != nil {
+			return err
+		}
+	}
 	if controller.cfg.WorkerAutoscalingEnabled {
-		for _, sitePlan := range plan.Sites {
+		for _, poolPlan := range plan.Pools {
 			switch {
-			case sitePlan.DesiredReplicas > sitePlan.CurrentReplicas:
-				if err := controller.kubernetes.ScaleWorkerDeployment(ctx, sitePlan.Site, sitePlan.DesiredReplicas); err != nil {
+			case poolPlan.DesiredReplicas > poolPlan.CurrentReplicas:
+				if err := controller.orchestrator.ScalePool(ctx, poolPlan.PoolID, controller.adapterRef(poolPlan.PoolID), poolPlan.DesiredReplicas); err != nil {
 					return err
 				}
-				controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.DesiredReplicas, "scale_up")
-			case sitePlan.DesiredReplicas < sitePlan.CurrentReplicas:
-				if err := controller.reconcileScaleDown(ctx, now, sitePlan, instances); err != nil {
+				controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.DesiredReplicas, "scale_up")
+			case poolPlan.DesiredReplicas < poolPlan.CurrentReplicas:
+				if err := controller.reconcileScaleDown(ctx, now, poolPlan, instances); err != nil {
 					return err
 				}
 			}
@@ -256,21 +286,24 @@ func nodeResources(states []KubernetesNodeState) []NodeResources {
 func (controller *Controller) reconcileScaleDown(
 	ctx context.Context,
 	now time.Time,
-	sitePlan SitePlan,
+	poolPlan PoolPlan,
 	instances []store.WorkerInstanceRuntime,
 ) error {
 	for _, instance := range instances {
-		if instance.Site != sitePlan.Site || instance.Status != "draining" {
+		if workerPoolID(instance) != poolPlan.PoolID || instance.Status != "draining" {
 			continue
 		}
 		if instance.RunningTasks == 0 && instance.ActiveLeases == 0 {
-			if err := controller.kubernetes.SetPodDeletionCost(ctx, instance.PodName, controllerDeletionCost); err != nil {
+			if strings.TrimSpace(instance.OrchestratorInstanceRef) == "" {
+				return errors.New("worker orchestrator instance reference is required for scale down")
+			}
+			if err := controller.orchestrator.SetInstanceTerminationPriority(ctx, instance.OrchestratorInstanceRef, controllerDeletionCost); err != nil {
 				return err
 			}
-			if err := controller.kubernetes.ScaleWorkerDeployment(ctx, sitePlan.Site, sitePlan.CurrentReplicas-1); err != nil {
+			if err := controller.orchestrator.ScalePool(ctx, poolPlan.PoolID, controller.adapterRef(poolPlan.PoolID), poolPlan.CurrentReplicas-1); err != nil {
 				return err
 			}
-			controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas-1, "drained_scale_down")
+			controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas-1, "drained_scale_down")
 			return nil
 		}
 		if instance.DrainingAt != nil &&
@@ -278,14 +311,14 @@ func (controller *Controller) reconcileScaleDown(
 			if err := controller.store.ReactivateWorkerInstance(ctx, instance.InstanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
 				return err
 			}
-			controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas, "drain_timeout")
+			controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas, "drain_timeout")
 		}
 		return nil
 	}
 	var candidate *store.WorkerInstanceRuntime
 	for index := range instances {
 		instance := &instances[index]
-		if instance.Site != sitePlan.Site || instance.Status != "active" {
+		if workerPoolID(*instance) != poolPlan.PoolID || instance.Status != "active" {
 			continue
 		}
 		if candidate == nil ||
@@ -299,24 +332,29 @@ func (controller *Controller) reconcileScaleDown(
 	if err := controller.store.MarkWorkerDraining(ctx, candidate.InstanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
 		return err
 	}
-	controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas, "drain_started")
+	controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas, "drain_started")
 	return nil
 }

-func (controller *Controller) siteReplicaBounds(site string) (int, int) {
-	switch site {
-	case "ningbo":
-		return controller.cfg.WorkerMinReplicasNingbo, controller.cfg.WorkerMaxReplicasNingbo
-	case "hongkong":
-		return controller.cfg.WorkerMinReplicasHongkong, controller.cfg.WorkerMaxReplicasHongkong
-	default:
-		return 0, 0
+func workerPoolID(instance store.WorkerInstanceRuntime) string {
+	if strings.TrimSpace(instance.PoolID) != "" {
+		return instance.PoolID
 	}
+	return instance.Site
+}
+
+func (controller *Controller) adapterRef(poolID string) string {
+	for _, pool := range controller.pools {
+		if pool.ID == poolID {
+			return pool.AdapterRef
+		}
+	}
+	return poolID
 }

 func (controller *Controller) revisionsMatch(
 	instances []store.WorkerInstanceRuntime,
-	states map[string]KubernetesSiteState,
+	states map[string]PoolInfrastructureState,
 ) bool {
 	if controller.expectedRevision == "" {
 		return true