feat(routing): 引入多执行池智能调度

将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口,新增 Kubernetes 和静态容量适配器,并以 shadow 模式接入生产配置。

实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌,以及提交状态不明时禁止重复分配的安全语义。

新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
2026-08-05 22:25:37 +08:00
parent 03c0873649
commit 7786692d32
58 changed files with 4510 additions and 348 deletions
@@ -0,0 +1,40 @@
package capacitycontroller
import (
"errors"
"os"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
)
func NewConfiguredAdapter(cfg config.Config) (OrchestratorAdapter, error) {
switch strings.ToLower(strings.TrimSpace(cfg.CapacityOrchestratorAdapter)) {
case "static":
pools, err := cfg.CapacityPools()
if err != nil {
return nil, err
}
initial := make(map[string]int, len(pools))
for _, pool := range pools {
initial[pool.ID] = pool.BootstrapReplicas
}
return NewStaticAdapter(initial), nil
case "", "kubernetes":
return NewKubernetesClient(KubernetesConfig{
Namespace: adapterEnv("AI_GATEWAY_CAPACITY_CONTROLLER_NAMESPACE", adapterEnv("POD_NAMESPACE", "easyai")),
APIServer: adapterEnv("AI_GATEWAY_CAPACITY_CONTROLLER_API_SERVER", "https://kubernetes.default.svc"),
TokenFile: adapterEnv("AI_GATEWAY_CAPACITY_CONTROLLER_TOKEN_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/token"),
CAFile: adapterEnv("AI_GATEWAY_CAPACITY_CONTROLLER_CA_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"),
})
default:
return nil, errors.New("unsupported capacity orchestrator adapter")
}
}
func adapterEnv(name, fallback string) string {
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
return value
}
return fallback
}
@@ -12,6 +12,7 @@ import (
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/jackc/pgx/v5"
)
@@ -25,16 +26,18 @@ const (
type capacityStore interface {
TryAcquireCapacityControllerLeadership(context.Context) (store.Leadership, bool, error)
WorkerQueueRuntime(context.Context) (store.WorkerQueueRuntime, error)
ListPoolQueueRuntime(context.Context) ([]store.PoolQueueRuntime, error)
ListWorkerInstanceRuntime(context.Context) ([]store.WorkerInstanceRuntime, error)
CapacityDatabaseHealth(context.Context) (store.CapacityDatabaseHealth, error)
MarkWorkerDraining(context.Context, string) error
ReactivateWorkerInstance(context.Context, string) error
PublishDesiredCapacity(context.Context, executionpool.DesiredCapacity) error
}
type capacityKubernetes interface {
SiteState(context.Context, string) (KubernetesSiteState, error)
ScaleWorkerDeployment(context.Context, string, int) error
SetPodDeletionCost(context.Context, string, int) error
type OrchestratorAdapter interface {
PoolState(context.Context, string, string) (PoolInfrastructureState, error)
ScalePool(context.Context, string, string, int) error
SetInstanceTerminationPriority(context.Context, string, int) error
}
type Status struct {
@@ -49,7 +52,8 @@ type Status struct {
type Controller struct {
cfg config.Config
store capacityStore
kubernetes capacityKubernetes
orchestrator OrchestratorAdapter
pools []config.ExecutionPoolCapacityConfig
logger *slog.Logger
expectedRevision string
now func() time.Time
@@ -62,16 +66,22 @@ type Controller struct {
func New(
cfg config.Config,
db capacityStore,
kubernetes capacityKubernetes,
orchestrator OrchestratorAdapter,
logger *slog.Logger,
) *Controller {
return &Controller{
cfg: cfg, store: db, kubernetes: kubernetes, logger: logger,
cfg: cfg, store: db, orchestrator: orchestrator, logger: logger,
expectedRevision: strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
now: time.Now,
pools: mustCapacityPools(cfg),
}
}
func mustCapacityPools(cfg config.Config) []config.ExecutionPoolCapacityConfig {
pools, _ := cfg.CapacityPools()
return pools
}
func (controller *Controller) Run(ctx context.Context) {
for ctx.Err() == nil {
leadership, acquired, err := controller.store.TryAcquireCapacityControllerLeadership(ctx)
@@ -128,6 +138,14 @@ func (controller *Controller) reconcile(ctx context.Context) error {
if err != nil {
return err
}
poolQueues, err := controller.store.ListPoolQueueRuntime(ctx)
if err != nil {
return err
}
demandByPool := make(map[string]int, len(poolQueues))
for _, poolQueue := range poolQueues {
demandByPool[poolQueue.PoolID] = poolQueue.Queued + poolQueue.Running
}
instances, err := controller.store.ListWorkerInstanceRuntime(ctx)
if err != nil {
return err
@@ -137,21 +155,21 @@ func (controller *Controller) reconcile(ctx context.Context) error {
return err
}
now := controller.now()
sites := make([]SiteResources, 0, 2)
kubernetesStates := make(map[string]KubernetesSiteState, 2)
for _, site := range []string{"ningbo", "hongkong"} {
minReplicas, maxReplicas := controller.siteReplicaBounds(site)
if maxReplicas == 0 {
poolResources := make([]PoolResources, 0, len(controller.pools))
infrastructureStates := make(map[string]PoolInfrastructureState, len(controller.pools))
for _, pool := range controller.pools {
if pool.MaxReplicas == 0 {
continue
}
state, stateErr := controller.kubernetes.SiteState(ctx, site)
state, stateErr := controller.orchestrator.PoolState(ctx, pool.ID, pool.AdapterRef)
if stateErr != nil {
return stateErr
}
kubernetesStates[site] = state
sites = append(sites, SiteResources{
Site: site, CurrentReplicas: state.CurrentReplicas,
MinReplicas: minReplicas, MaxReplicas: maxReplicas,
infrastructureStates[pool.ID] = state
poolResources = append(poolResources, PoolResources{
PoolID: pool.ID, CurrentReplicas: state.CurrentReplicas,
MinReplicas: pool.MinReplicas, MaxReplicas: pool.MaxReplicas,
Demand: demandByPool[pool.ID],
AllocatableMemoryBytes: state.AllocatableMemoryBytes,
UsedMemoryBytes: state.UsedMemoryBytes,
WorkerRequestMemoryBytes: state.WorkerRequestMemoryBytes,
@@ -163,8 +181,8 @@ func (controller *Controller) reconcile(ctx context.Context) error {
})
}
currentTotal := 0
for _, site := range sites {
currentTotal += site.CurrentReplicas
for _, pool := range poolResources {
currentTotal += pool.CurrentReplicas
}
target := controller.cfg.WorkerTargetOutstandingPerReplica
if target < 1 {
@@ -185,7 +203,7 @@ func (controller *Controller) reconcile(ctx context.Context) error {
controller.highSince = time.Time{}
controller.lowSince = time.Time{}
}
revisionHealthy := controller.revisionsMatch(instances, kubernetesStates)
revisionHealthy := controller.revisionsMatch(instances, infrastructureStates)
scaleUpEligible := !controller.highSince.IsZero() &&
now.Sub(controller.highSince) >= time.Duration(controller.cfg.WorkerScaleUpWindowSeconds)*time.Second &&
revisionHealthy
@@ -206,21 +224,33 @@ func (controller *Controller) reconcile(ctx context.Context) error {
SynchronousDatabasePeers: database.SynchronousPeers,
ScaleUpEligible: scaleUpEligible,
ScaleDownEligible: scaleDownEligible,
Sites: sites,
Pools: poolResources,
})
if !revisionHealthy && plan.FrozenReason == "" {
plan.FrozenReason = "release_revision_mismatch"
}
for _, poolPlan := range plan.Pools {
reason := "capacity_plan"
if plan.FrozenReason != "" {
reason = "frozen:" + plan.FrozenReason
}
if err := controller.store.PublishDesiredCapacity(ctx, executionpool.DesiredCapacity{
PoolID: poolPlan.PoolID, Desired: poolPlan.DesiredReplicas,
Reason: reason, ValidUntil: now.Add(2 * controllerReconcileInterval),
}); err != nil {
return err
}
}
if controller.cfg.WorkerAutoscalingEnabled {
for _, sitePlan := range plan.Sites {
for _, poolPlan := range plan.Pools {
switch {
case sitePlan.DesiredReplicas > sitePlan.CurrentReplicas:
if err := controller.kubernetes.ScaleWorkerDeployment(ctx, sitePlan.Site, sitePlan.DesiredReplicas); err != nil {
case poolPlan.DesiredReplicas > poolPlan.CurrentReplicas:
if err := controller.orchestrator.ScalePool(ctx, poolPlan.PoolID, controller.adapterRef(poolPlan.PoolID), poolPlan.DesiredReplicas); err != nil {
return err
}
controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.DesiredReplicas, "scale_up")
case sitePlan.DesiredReplicas < sitePlan.CurrentReplicas:
if err := controller.reconcileScaleDown(ctx, now, sitePlan, instances); err != nil {
controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.DesiredReplicas, "scale_up")
case poolPlan.DesiredReplicas < poolPlan.CurrentReplicas:
if err := controller.reconcileScaleDown(ctx, now, poolPlan, instances); err != nil {
return err
}
}
@@ -256,21 +286,24 @@ func nodeResources(states []KubernetesNodeState) []NodeResources {
func (controller *Controller) reconcileScaleDown(
ctx context.Context,
now time.Time,
sitePlan SitePlan,
poolPlan PoolPlan,
instances []store.WorkerInstanceRuntime,
) error {
for _, instance := range instances {
if instance.Site != sitePlan.Site || instance.Status != "draining" {
if workerPoolID(instance) != poolPlan.PoolID || instance.Status != "draining" {
continue
}
if instance.RunningTasks == 0 && instance.ActiveLeases == 0 {
if err := controller.kubernetes.SetPodDeletionCost(ctx, instance.PodName, controllerDeletionCost); err != nil {
if strings.TrimSpace(instance.OrchestratorInstanceRef) == "" {
return errors.New("worker orchestrator instance reference is required for scale down")
}
if err := controller.orchestrator.SetInstanceTerminationPriority(ctx, instance.OrchestratorInstanceRef, controllerDeletionCost); err != nil {
return err
}
if err := controller.kubernetes.ScaleWorkerDeployment(ctx, sitePlan.Site, sitePlan.CurrentReplicas-1); err != nil {
if err := controller.orchestrator.ScalePool(ctx, poolPlan.PoolID, controller.adapterRef(poolPlan.PoolID), poolPlan.CurrentReplicas-1); err != nil {
return err
}
controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas-1, "drained_scale_down")
controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas-1, "drained_scale_down")
return nil
}
if instance.DrainingAt != nil &&
@@ -278,14 +311,14 @@ func (controller *Controller) reconcileScaleDown(
if err := controller.store.ReactivateWorkerInstance(ctx, instance.InstanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
return err
}
controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas, "drain_timeout")
controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas, "drain_timeout")
}
return nil
}
var candidate *store.WorkerInstanceRuntime
for index := range instances {
instance := &instances[index]
if instance.Site != sitePlan.Site || instance.Status != "active" {
if workerPoolID(*instance) != poolPlan.PoolID || instance.Status != "active" {
continue
}
if candidate == nil ||
@@ -299,24 +332,29 @@ func (controller *Controller) reconcileScaleDown(
if err := controller.store.MarkWorkerDraining(ctx, candidate.InstanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
return err
}
controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas, "drain_started")
controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas, "drain_started")
return nil
}
func (controller *Controller) siteReplicaBounds(site string) (int, int) {
switch site {
case "ningbo":
return controller.cfg.WorkerMinReplicasNingbo, controller.cfg.WorkerMaxReplicasNingbo
case "hongkong":
return controller.cfg.WorkerMinReplicasHongkong, controller.cfg.WorkerMaxReplicasHongkong
default:
return 0, 0
func workerPoolID(instance store.WorkerInstanceRuntime) string {
if strings.TrimSpace(instance.PoolID) != "" {
return instance.PoolID
}
return instance.Site
}
func (controller *Controller) adapterRef(poolID string) string {
for _, pool := range controller.pools {
if pool.ID == poolID {
return pool.AdapterRef
}
}
return poolID
}
func (controller *Controller) revisionsMatch(
instances []store.WorkerInstanceRuntime,
states map[string]KubernetesSiteState,
states map[string]PoolInfrastructureState,
) bool {
if controller.expectedRevision == "" {
return true
@@ -32,8 +32,8 @@ type KubernetesClient struct {
client *http.Client
}
type KubernetesSiteState struct {
Site string
type PoolInfrastructureState struct {
PoolID string
NodeName string
CurrentReplicas int
Revision string
@@ -45,10 +45,10 @@ type KubernetesSiteState struct {
UsedMilliCPU int64
WorkerRequestMilliCPU int64
MemoryPressure bool
Nodes []KubernetesNodeState
Nodes []PoolNodeState
}
type KubernetesNodeState struct {
type PoolNodeState struct {
NodeName string
CurrentReplicas int
AllocatableMemoryBytes int64
@@ -60,6 +60,25 @@ type KubernetesNodeState struct {
MemoryPressure bool
}
type KubernetesSiteState = PoolInfrastructureState
type KubernetesNodeState = PoolNodeState
var _ OrchestratorAdapter = (*KubernetesClient)(nil)
func (client *KubernetesClient) PoolState(ctx context.Context, poolID, adapterRef string) (PoolInfrastructureState, error) {
state, err := client.SiteState(ctx, adapterRef)
state.PoolID = poolID
return state, err
}
func (client *KubernetesClient) ScalePool(ctx context.Context, _, adapterRef string, replicas int) error {
return client.ScaleWorkerDeployment(ctx, adapterRef, replicas)
}
func (client *KubernetesClient) SetInstanceTerminationPriority(ctx context.Context, instanceRef string, priority int) error {
return client.SetPodDeletionCost(ctx, instanceRef, priority)
}
func NewKubernetesClient(config KubernetesConfig) (*KubernetesClient, error) {
if strings.TrimSpace(config.Namespace) == "" {
return nil, errors.New("capacity controller Kubernetes namespace is required")
@@ -144,7 +163,7 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
if err := client.getJSON(ctx, deploymentPath, &deployment); err != nil {
return KubernetesSiteState{}, err
}
state := KubernetesSiteState{Site: site, CurrentReplicas: deployment.Spec.Replicas}
state := KubernetesSiteState{PoolID: site, CurrentReplicas: deployment.Spec.Replicas}
for _, container := range deployment.Spec.Template.Spec.Containers {
if container.Name != "worker" {
continue
@@ -129,7 +129,7 @@ func TestKubernetesClientAllowsDisabledSiteWithoutWorkerNodes(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if state.Site != "ningbo" || state.CurrentReplicas != 0 || len(state.Nodes) != 0 {
if state.PoolID != "ningbo" || state.CurrentReplicas != 0 || len(state.Nodes) != 0 {
t.Fatalf("site state=%+v", state)
}
}
+75 -67
View File
@@ -5,8 +5,9 @@ import (
"sort"
)
type SiteResources struct {
Site string
type PoolResources struct {
PoolID string
Demand int
CurrentReplicas int
MinReplicas int
MaxReplicas int
@@ -47,11 +48,12 @@ type PlanInput struct {
SynchronousDatabasePeers int
ScaleUpEligible bool
ScaleDownEligible bool
Sites []SiteResources
Pools []PoolResources
}
type SitePlan struct {
Site string `json:"site"`
type PoolPlan struct {
PoolID string `json:"poolId"`
Demand int `json:"demand"`
CurrentReplicas int `json:"currentReplicas"`
DesiredReplicas int `json:"desiredReplicas"`
ResourceMax int `json:"resourceMax"`
@@ -64,7 +66,7 @@ type Plan struct {
DesiredTotal int `json:"desiredTotal"`
CurrentTotal int `json:"currentTotal"`
FrozenReason string `json:"frozenReason,omitempty"`
Sites []SitePlan `json:"sites"`
Pools []PoolPlan `json:"pools"`
}
func CalculatePlan(input PlanInput) Plan {
@@ -88,33 +90,34 @@ func CalculatePlan(input PlanInput) Plan {
plan := Plan{RawDesired: rawDesired}
minTotal := 0
resourceMaxTotal := 0
for _, site := range input.Sites {
for _, node := range site.Nodes {
for _, pool := range input.Pools {
for _, node := range pool.Nodes {
if node.MemoryPressure {
site.MemoryPressure = true
pool.MemoryPressure = true
}
}
current := max(site.CurrentReplicas, 0)
minReplicas := max(site.MinReplicas, 0)
configMax := max(site.MaxReplicas, minReplicas)
resourceMax, memoryPercent, cpuPercent := siteResourceMaximum(
site,
current := max(pool.CurrentReplicas, 0)
minReplicas := max(pool.MinReplicas, 0)
configMax := max(pool.MaxReplicas, minReplicas)
resourceMax, memoryPercent, cpuPercent := poolResourceMaximum(
pool,
input.MemoryTargetPercent,
input.CPUTargetPercent,
)
resourceMax = min(resourceMax, configMax)
if resourceMax < minReplicas && plan.FrozenReason == "" {
plan.FrozenReason = "site_resource_budget"
plan.FrozenReason = "pool_resource_budget"
}
resourceMax = max(resourceMax, minReplicas)
plan.Sites = append(plan.Sites, SitePlan{
Site: site.Site, CurrentReplicas: current, DesiredReplicas: minReplicas,
plan.Pools = append(plan.Pools, PoolPlan{
PoolID: pool.PoolID, CurrentReplicas: current, DesiredReplicas: minReplicas,
Demand: pool.Demand,
ResourceMax: resourceMax, MemoryPercent: memoryPercent, CPUPercent: cpuPercent,
})
plan.CurrentTotal += current
minTotal += minReplicas
resourceMaxTotal += resourceMax
if site.MemoryPressure || memoryPercent >= float64(input.MemoryHardPercent) {
if pool.MemoryPressure || memoryPercent >= float64(input.MemoryHardPercent) {
plan.FrozenReason = "node_memory_pressure"
}
}
@@ -123,10 +126,10 @@ func CalculatePlan(input PlanInput) Plan {
(input.DatabaseConnectionBudget-input.NonWorkerConnectionBudget)/input.WorkerDatabasePoolMax,
0,
)
capSiteResourceMaxima(plan.Sites, workerReplicaBudget)
capPoolResourceMaxima(plan.Pools, workerReplicaBudget)
resourceMaxTotal = 0
for _, site := range plan.Sites {
resourceMaxTotal += site.ResourceMax
for _, pool := range plan.Pools {
resourceMaxTotal += pool.ResourceMax
}
if workerReplicaBudget < minTotal && plan.FrozenReason == "" {
plan.FrozenReason = "database_connection_budget"
@@ -167,47 +170,47 @@ func CalculatePlan(input PlanInput) Plan {
}
desired = max(desired, minTotal)
plan.DesiredTotal = desired
distributeDesiredReplicas(plan.Sites, desired)
distributeDesiredReplicas(plan.Pools, desired)
return plan
}
func capSiteResourceMaxima(sites []SitePlan, total int) {
func capPoolResourceMaxima(pools []PoolPlan, total int) {
minimum := 0
original := make(map[string]int, len(sites))
for index := range sites {
original[sites[index].Site] = sites[index].ResourceMax
sites[index].ResourceMax = sites[index].DesiredReplicas
minimum += sites[index].ResourceMax
original := make(map[string]int, len(pools))
for index := range pools {
original[pools[index].PoolID] = pools[index].ResourceMax
pools[index].ResourceMax = pools[index].DesiredReplicas
minimum += pools[index].ResourceMax
}
target := max(total, minimum)
assigned := minimum
for assigned < target {
sort.SliceStable(sites, func(left, right int) bool {
leftRoom := original[sites[left].Site] - sites[left].ResourceMax
rightRoom := original[sites[right].Site] - sites[right].ResourceMax
sort.SliceStable(pools, func(left, right int) bool {
leftRoom := original[pools[left].PoolID] - pools[left].ResourceMax
rightRoom := original[pools[right].PoolID] - pools[right].ResourceMax
if leftRoom != rightRoom {
return leftRoom > rightRoom
}
return sites[left].Site < sites[right].Site
return pools[left].PoolID < pools[right].PoolID
})
if original[sites[0].Site] <= sites[0].ResourceMax {
if original[pools[0].PoolID] <= pools[0].ResourceMax {
break
}
sites[0].ResourceMax++
pools[0].ResourceMax++
assigned++
}
}
func siteResourceMaximum(site SiteResources, memoryTargetPercent int, cpuTargetPercent int) (int, float64, float64) {
if len(site.Nodes) > 0 {
if site.WorkerRequestMemoryBytes <= 0 || site.WorkerRequestMilliCPU <= 0 {
func poolResourceMaximum(pool PoolResources, memoryTargetPercent int, cpuTargetPercent int) (int, float64, float64) {
if len(pool.Nodes) > 0 {
if pool.WorkerRequestMemoryBytes <= 0 || pool.WorkerRequestMilliCPU <= 0 {
return 0, 0, 0
}
memoryMax := 0
cpuMax := 0
memoryPercent := float64(0)
cpuPercent := float64(0)
for _, node := range site.Nodes {
for _, node := range pool.Nodes {
nodeMemoryPercent := usagePercent(node.UsedMemoryBytes, node.AllocatableMemoryBytes)
nodeCPUPercent := usagePercent(node.UsedMilliCPU, node.AllocatableMilliCPU)
memoryPercent = max(memoryPercent, nodeMemoryPercent)
@@ -217,60 +220,65 @@ func siteResourceMaximum(site SiteResources, memoryTargetPercent int, cpuTargetP
0,
)
memoryBudget := node.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorkerMemory
memoryMax += int(max(memoryBudget, 0) / site.WorkerRequestMemoryBytes)
memoryMax += int(max(memoryBudget, 0) / pool.WorkerRequestMemoryBytes)
nonWorkerCPU := max(
node.UsedMilliCPU-node.WorkerUsedMilliCPU,
0,
)
cpuBudget := node.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorkerCPU
cpuMax += int(max(cpuBudget, 0) / site.WorkerRequestMilliCPU)
cpuMax += int(max(cpuBudget, 0) / pool.WorkerRequestMilliCPU)
}
return min(site.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
return min(pool.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
}
memoryPercent := usagePercent(site.UsedMemoryBytes, site.AllocatableMemoryBytes)
cpuPercent := usagePercent(site.UsedMilliCPU, site.AllocatableMilliCPU)
memoryMax := site.MaxReplicas
if site.AllocatableMemoryBytes > 0 && site.WorkerRequestMemoryBytes > 0 {
nonWorker := max(site.UsedMemoryBytes-int64(site.CurrentReplicas)*site.WorkerRequestMemoryBytes, 0)
budget := site.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorker
memoryMax = int(max(budget, 0) / site.WorkerRequestMemoryBytes)
memoryPercent := usagePercent(pool.UsedMemoryBytes, pool.AllocatableMemoryBytes)
cpuPercent := usagePercent(pool.UsedMilliCPU, pool.AllocatableMilliCPU)
memoryMax := pool.MaxReplicas
if pool.AllocatableMemoryBytes > 0 && pool.WorkerRequestMemoryBytes > 0 {
nonWorker := max(pool.UsedMemoryBytes-int64(pool.CurrentReplicas)*pool.WorkerRequestMemoryBytes, 0)
budget := pool.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorker
memoryMax = int(max(budget, 0) / pool.WorkerRequestMemoryBytes)
}
cpuMax := site.MaxReplicas
if site.AllocatableMilliCPU > 0 && site.WorkerRequestMilliCPU > 0 {
nonWorker := max(site.UsedMilliCPU-int64(site.CurrentReplicas)*site.WorkerRequestMilliCPU, 0)
budget := site.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorker
cpuMax = int(max(budget, 0) / site.WorkerRequestMilliCPU)
cpuMax := pool.MaxReplicas
if pool.AllocatableMilliCPU > 0 && pool.WorkerRequestMilliCPU > 0 {
nonWorker := max(pool.UsedMilliCPU-int64(pool.CurrentReplicas)*pool.WorkerRequestMilliCPU, 0)
budget := pool.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorker
cpuMax = int(max(budget, 0) / pool.WorkerRequestMilliCPU)
}
return min(site.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
return min(pool.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
}
func distributeDesiredReplicas(sites []SitePlan, desired int) {
if len(sites) == 0 {
func distributeDesiredReplicas(pools []PoolPlan, desired int) {
if len(pools) == 0 {
return
}
assigned := 0
for index := range sites {
assigned += sites[index].DesiredReplicas
for index := range pools {
assigned += pools[index].DesiredReplicas
}
for assigned < desired {
sort.SliceStable(sites, func(left, right int) bool {
leftRoom := sites[left].ResourceMax - sites[left].DesiredReplicas
rightRoom := sites[right].ResourceMax - sites[right].DesiredReplicas
sort.SliceStable(pools, func(left, right int) bool {
leftDemandPerReplica := float64(max(pools[left].Demand, 0)) / float64(max(pools[left].DesiredReplicas, 1))
rightDemandPerReplica := float64(max(pools[right].Demand, 0)) / float64(max(pools[right].DesiredReplicas, 1))
if leftDemandPerReplica != rightDemandPerReplica {
return leftDemandPerReplica > rightDemandPerReplica
}
leftRoom := pools[left].ResourceMax - pools[left].DesiredReplicas
rightRoom := pools[right].ResourceMax - pools[right].DesiredReplicas
if leftRoom != rightRoom {
return leftRoom > rightRoom
}
if sites[left].DesiredReplicas != sites[right].DesiredReplicas {
return sites[left].DesiredReplicas < sites[right].DesiredReplicas
if pools[left].DesiredReplicas != pools[right].DesiredReplicas {
return pools[left].DesiredReplicas < pools[right].DesiredReplicas
}
return sites[left].Site < sites[right].Site
return pools[left].PoolID < pools[right].PoolID
})
if sites[0].DesiredReplicas >= sites[0].ResourceMax {
if pools[0].DesiredReplicas >= pools[0].ResourceMax {
break
}
sites[0].DesiredReplicas++
pools[0].DesiredReplicas++
assigned++
}
sort.Slice(sites, func(left, right int) bool { return sites[left].Site < sites[right].Site })
sort.Slice(pools, func(left, right int) bool { return pools[left].PoolID < pools[right].PoolID })
}
func usagePercent(used int64, allocatable int64) float64 {
@@ -12,15 +12,15 @@ func TestCalculatePlanRespectsResourceDatabaseAndStepLimits(t *testing.T) {
MemoryTargetPercent: 75, MemoryHardPercent: 85, CPUTargetPercent: 70,
DatabaseConnections: 80, DatabaseConnectionBudget: 150, WorkerDatabasePoolMax: 24,
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
Sites: []SiteResources{
Pools: []PoolResources{
{
Site: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
PoolID: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
AllocatableMemoryBytes: 8 << 30, UsedMemoryBytes: 6 << 30,
WorkerRequestMemoryBytes: 1 << 30,
AllocatableMilliCPU: 4000, UsedMilliCPU: 2000, WorkerRequestMilliCPU: 500,
},
{
Site: "hongkong", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
PoolID: "hongkong", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
AllocatableMemoryBytes: 8 << 30, UsedMemoryBytes: 3 << 30,
WorkerRequestMemoryBytes: 1 << 30,
AllocatableMilliCPU: 4000, UsedMilliCPU: 500, WorkerRequestMilliCPU: 500,
@@ -34,8 +34,8 @@ func TestCalculatePlanRespectsResourceDatabaseAndStepLimits(t *testing.T) {
if plan.DesiredTotal != 4 {
t.Fatalf("desired total=%d, want step-limited 4: %+v", plan.DesiredTotal, plan)
}
if plan.Sites[0].Site != "hongkong" || plan.Sites[0].DesiredReplicas != 3 {
t.Fatalf("site allocation=%+v, want hongkong=3 ningbo=1", plan.Sites)
if plan.Pools[0].PoolID != "hongkong" || plan.Pools[0].DesiredReplicas != 3 {
t.Fatalf("pool allocation=%+v, want hongkong=3 ningbo=1", plan.Pools)
}
}
@@ -44,8 +44,8 @@ func TestCalculatePlanFreezesScaleUpOnHardMemoryPressure(t *testing.T) {
Queued: 200, InstanceSlots: 24,
DatabaseConnections: 10, DatabaseConnectionBudget: 150, WorkerDatabasePoolMax: 24,
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
Sites: []SiteResources{{
Site: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
Pools: []PoolResources{{
PoolID: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
AllocatableMemoryBytes: 8 << 30, UsedMemoryBytes: 7 << 30,
WorkerRequestMemoryBytes: 1 << 30,
}},
@@ -60,14 +60,14 @@ func TestCalculatePlanReportsConfiguredMinimumOutsideResourceBudget(t *testing.T
Queued: 200, InstanceSlots: 24,
DatabaseConnections: 10, DatabaseConnectionBudget: 150, WorkerDatabasePoolMax: 24,
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
Sites: []SiteResources{{
Site: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
Pools: []PoolResources{{
PoolID: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
AllocatableMemoryBytes: 8 << 30, UsedMemoryBytes: 13 << 29,
WorkerRequestMemoryBytes: 2 << 30,
AllocatableMilliCPU: 4000, UsedMilliCPU: 2800, WorkerRequestMilliCPU: 500,
}},
})
if plan.DesiredTotal != 1 || plan.FrozenReason != "site_resource_budget" {
if plan.DesiredTotal != 1 || plan.FrozenReason != "pool_resource_budget" {
t.Fatalf("plan=%+v, want the existing minimum preserved and expansion frozen", plan)
}
}
@@ -77,9 +77,9 @@ func TestCalculatePlanWaitsForSafeScaleDownWindow(t *testing.T) {
InstanceSlots: 24, DatabaseConnections: 20, DatabaseConnectionBudget: 150,
WorkerDatabasePoolMax: 24, SynchronousDatabasePeers: 1,
ScaleDownEligible: false,
Sites: []SiteResources{
{Site: "ningbo", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 4},
{Site: "hongkong", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 4},
Pools: []PoolResources{
{PoolID: "ningbo", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 4},
{PoolID: "hongkong", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 4},
},
}
plan := CalculatePlan(input)
@@ -99,8 +99,8 @@ func TestCalculatePlanAddsCapacityAcrossLabeledSiteNodes(t *testing.T) {
MemoryTargetPercent: 75, MemoryHardPercent: 85, CPUTargetPercent: 70,
DatabaseConnections: 20, DatabaseConnectionBudget: 150, WorkerDatabasePoolMax: 20,
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
Sites: []SiteResources{{
Site: "hongkong", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 8,
Pools: []PoolResources{{
PoolID: "hongkong", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 8,
WorkerRequestMemoryBytes: 1 << 30, WorkerRequestMilliCPU: 500,
Nodes: []NodeResources{
{
@@ -116,8 +116,8 @@ func TestCalculatePlanAddsCapacityAcrossLabeledSiteNodes(t *testing.T) {
},
}},
})
if plan.Sites[0].ResourceMax != 7 {
t.Fatalf("resource max=%d, want database-budgeted multi-node maximum: %+v", plan.Sites[0].ResourceMax, plan)
if plan.Pools[0].ResourceMax != 7 {
t.Fatalf("resource max=%d, want database-budgeted multi-node maximum: %+v", plan.Pools[0].ResourceMax, plan)
}
if plan.DesiredTotal != 4 {
t.Fatalf("desired=%d, want two-wave step from 2 to 4", plan.DesiredTotal)
@@ -130,17 +130,17 @@ func TestCalculatePlanCapsReplicaMaximumByDeclaredDatabasePools(t *testing.T) {
DatabaseConnections: 20, DatabaseConnectionBudget: 150,
NonWorkerConnectionBudget: 72, WorkerDatabasePoolMax: 32,
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
Sites: []SiteResources{
{Site: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4},
{Site: "hongkong", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4},
Pools: []PoolResources{
{PoolID: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4},
{PoolID: "hongkong", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4},
},
})
if plan.DesiredTotal != 2 {
t.Fatalf("desired=%d, want the current 1+1 database-safe topology: %+v", plan.DesiredTotal, plan)
}
resourceMax := 0
for _, site := range plan.Sites {
resourceMax += site.ResourceMax
for _, pool := range plan.Pools {
resourceMax += pool.ResourceMax
}
if resourceMax != 2 {
t.Fatalf("resource maximum=%d, want floor((150-72)/32)=2: %+v", resourceMax, plan)
@@ -150,13 +150,13 @@ func TestCalculatePlanCapsReplicaMaximumByDeclaredDatabasePools(t *testing.T) {
func TestPlanJSONUsesAcceptanceContractFieldNames(t *testing.T) {
payload, err := json.Marshal(Plan{
RawDesired: 3,
Sites: []SitePlan{{Site: "hongkong", ResourceMax: 2}},
Pools: []PoolPlan{{PoolID: "hongkong", ResourceMax: 2}},
})
if err != nil {
t.Fatal(err)
}
text := string(payload)
for _, field := range []string{`"rawDesired":3`, `"sites"`, `"site":"hongkong"`, `"resourceMax":2`} {
for _, field := range []string{`"rawDesired":3`, `"pools"`, `"poolId":"hongkong"`, `"resourceMax":2`} {
if !strings.Contains(text, field) {
t.Fatalf("plan JSON %s does not contain %s", text, field)
}
@@ -0,0 +1,42 @@
package capacitycontroller
import (
"context"
"sync"
)
// StaticAdapter proves that capacity coordination does not require Kubernetes.
// It reports configured fixed replicas and intentionally ignores scale writes.
type StaticAdapter struct {
mu sync.RWMutex
replicas map[string]int
}
var _ OrchestratorAdapter = (*StaticAdapter)(nil)
func NewStaticAdapter(initial map[string]int) *StaticAdapter {
copyOfInitial := make(map[string]int, len(initial))
for poolID, replicas := range initial {
copyOfInitial[poolID] = replicas
}
return &StaticAdapter{replicas: copyOfInitial}
}
func (adapter *StaticAdapter) PoolState(_ context.Context, poolID, _ string) (PoolInfrastructureState, error) {
adapter.mu.RLock()
replicas := adapter.replicas[poolID]
adapter.mu.RUnlock()
return PoolInfrastructureState{
PoolID: poolID, CurrentReplicas: replicas,
AllocatableMemoryBytes: 1 << 60, WorkerRequestMemoryBytes: 1,
AllocatableMilliCPU: 1 << 50, WorkerRequestMilliCPU: 1,
}, nil
}
func (adapter *StaticAdapter) ScalePool(context.Context, string, string, int) error {
return nil
}
func (adapter *StaticAdapter) SetInstanceTerminationPriority(context.Context, string, int) error {
return nil
}
@@ -0,0 +1,24 @@
package capacitycontroller
import (
"context"
"testing"
)
func TestStaticAdapterDoesNotRequireKubernetes(t *testing.T) {
adapter := NewStaticAdapter(map[string]int{"pool-a": 2})
state, err := adapter.PoolState(context.Background(), "pool-a", "ignored")
if err != nil {
t.Fatal(err)
}
if state.PoolID != "pool-a" || state.CurrentReplicas != 2 {
t.Fatalf("state=%+v", state)
}
if err := adapter.ScalePool(context.Background(), "pool-a", "ignored", 4); err != nil {
t.Fatal(err)
}
state, _ = adapter.PoolState(context.Background(), "pool-a", "ignored")
if state.CurrentReplicas != 2 {
t.Fatalf("static adapter scaled to %d", state.CurrentReplicas)
}
}