feat(routing): 引入多执行池智能调度
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口,新增 Kubernetes 和静态容量适配器,并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌,以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
package capacitycontroller
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
)
|
||||
|
||||
func NewConfiguredAdapter(cfg config.Config) (OrchestratorAdapter, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(cfg.CapacityOrchestratorAdapter)) {
|
||||
case "static":
|
||||
pools, err := cfg.CapacityPools()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
initial := make(map[string]int, len(pools))
|
||||
for _, pool := range pools {
|
||||
initial[pool.ID] = pool.BootstrapReplicas
|
||||
}
|
||||
return NewStaticAdapter(initial), nil
|
||||
case "", "kubernetes":
|
||||
return NewKubernetesClient(KubernetesConfig{
|
||||
Namespace: adapterEnv("AI_GATEWAY_CAPACITY_CONTROLLER_NAMESPACE", adapterEnv("POD_NAMESPACE", "easyai")),
|
||||
APIServer: adapterEnv("AI_GATEWAY_CAPACITY_CONTROLLER_API_SERVER", "https://kubernetes.default.svc"),
|
||||
TokenFile: adapterEnv("AI_GATEWAY_CAPACITY_CONTROLLER_TOKEN_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/token"),
|
||||
CAFile: adapterEnv("AI_GATEWAY_CAPACITY_CONTROLLER_CA_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"),
|
||||
})
|
||||
default:
|
||||
return nil, errors.New("unsupported capacity orchestrator adapter")
|
||||
}
|
||||
}
|
||||
|
||||
func adapterEnv(name, fallback string) string {
|
||||
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
@@ -25,16 +26,18 @@ const (
|
||||
type capacityStore interface {
|
||||
TryAcquireCapacityControllerLeadership(context.Context) (store.Leadership, bool, error)
|
||||
WorkerQueueRuntime(context.Context) (store.WorkerQueueRuntime, error)
|
||||
ListPoolQueueRuntime(context.Context) ([]store.PoolQueueRuntime, error)
|
||||
ListWorkerInstanceRuntime(context.Context) ([]store.WorkerInstanceRuntime, error)
|
||||
CapacityDatabaseHealth(context.Context) (store.CapacityDatabaseHealth, error)
|
||||
MarkWorkerDraining(context.Context, string) error
|
||||
ReactivateWorkerInstance(context.Context, string) error
|
||||
PublishDesiredCapacity(context.Context, executionpool.DesiredCapacity) error
|
||||
}
|
||||
|
||||
type capacityKubernetes interface {
|
||||
SiteState(context.Context, string) (KubernetesSiteState, error)
|
||||
ScaleWorkerDeployment(context.Context, string, int) error
|
||||
SetPodDeletionCost(context.Context, string, int) error
|
||||
type OrchestratorAdapter interface {
|
||||
PoolState(context.Context, string, string) (PoolInfrastructureState, error)
|
||||
ScalePool(context.Context, string, string, int) error
|
||||
SetInstanceTerminationPriority(context.Context, string, int) error
|
||||
}
|
||||
|
||||
type Status struct {
|
||||
@@ -49,7 +52,8 @@ type Status struct {
|
||||
type Controller struct {
|
||||
cfg config.Config
|
||||
store capacityStore
|
||||
kubernetes capacityKubernetes
|
||||
orchestrator OrchestratorAdapter
|
||||
pools []config.ExecutionPoolCapacityConfig
|
||||
logger *slog.Logger
|
||||
expectedRevision string
|
||||
now func() time.Time
|
||||
@@ -62,16 +66,22 @@ type Controller struct {
|
||||
func New(
|
||||
cfg config.Config,
|
||||
db capacityStore,
|
||||
kubernetes capacityKubernetes,
|
||||
orchestrator OrchestratorAdapter,
|
||||
logger *slog.Logger,
|
||||
) *Controller {
|
||||
return &Controller{
|
||||
cfg: cfg, store: db, kubernetes: kubernetes, logger: logger,
|
||||
cfg: cfg, store: db, orchestrator: orchestrator, logger: logger,
|
||||
expectedRevision: strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
|
||||
now: time.Now,
|
||||
pools: mustCapacityPools(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
func mustCapacityPools(cfg config.Config) []config.ExecutionPoolCapacityConfig {
|
||||
pools, _ := cfg.CapacityPools()
|
||||
return pools
|
||||
}
|
||||
|
||||
func (controller *Controller) Run(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
leadership, acquired, err := controller.store.TryAcquireCapacityControllerLeadership(ctx)
|
||||
@@ -128,6 +138,14 @@ func (controller *Controller) reconcile(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
poolQueues, err := controller.store.ListPoolQueueRuntime(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
demandByPool := make(map[string]int, len(poolQueues))
|
||||
for _, poolQueue := range poolQueues {
|
||||
demandByPool[poolQueue.PoolID] = poolQueue.Queued + poolQueue.Running
|
||||
}
|
||||
instances, err := controller.store.ListWorkerInstanceRuntime(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -137,21 +155,21 @@ func (controller *Controller) reconcile(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
now := controller.now()
|
||||
sites := make([]SiteResources, 0, 2)
|
||||
kubernetesStates := make(map[string]KubernetesSiteState, 2)
|
||||
for _, site := range []string{"ningbo", "hongkong"} {
|
||||
minReplicas, maxReplicas := controller.siteReplicaBounds(site)
|
||||
if maxReplicas == 0 {
|
||||
poolResources := make([]PoolResources, 0, len(controller.pools))
|
||||
infrastructureStates := make(map[string]PoolInfrastructureState, len(controller.pools))
|
||||
for _, pool := range controller.pools {
|
||||
if pool.MaxReplicas == 0 {
|
||||
continue
|
||||
}
|
||||
state, stateErr := controller.kubernetes.SiteState(ctx, site)
|
||||
state, stateErr := controller.orchestrator.PoolState(ctx, pool.ID, pool.AdapterRef)
|
||||
if stateErr != nil {
|
||||
return stateErr
|
||||
}
|
||||
kubernetesStates[site] = state
|
||||
sites = append(sites, SiteResources{
|
||||
Site: site, CurrentReplicas: state.CurrentReplicas,
|
||||
MinReplicas: minReplicas, MaxReplicas: maxReplicas,
|
||||
infrastructureStates[pool.ID] = state
|
||||
poolResources = append(poolResources, PoolResources{
|
||||
PoolID: pool.ID, CurrentReplicas: state.CurrentReplicas,
|
||||
MinReplicas: pool.MinReplicas, MaxReplicas: pool.MaxReplicas,
|
||||
Demand: demandByPool[pool.ID],
|
||||
AllocatableMemoryBytes: state.AllocatableMemoryBytes,
|
||||
UsedMemoryBytes: state.UsedMemoryBytes,
|
||||
WorkerRequestMemoryBytes: state.WorkerRequestMemoryBytes,
|
||||
@@ -163,8 +181,8 @@ func (controller *Controller) reconcile(ctx context.Context) error {
|
||||
})
|
||||
}
|
||||
currentTotal := 0
|
||||
for _, site := range sites {
|
||||
currentTotal += site.CurrentReplicas
|
||||
for _, pool := range poolResources {
|
||||
currentTotal += pool.CurrentReplicas
|
||||
}
|
||||
target := controller.cfg.WorkerTargetOutstandingPerReplica
|
||||
if target < 1 {
|
||||
@@ -185,7 +203,7 @@ func (controller *Controller) reconcile(ctx context.Context) error {
|
||||
controller.highSince = time.Time{}
|
||||
controller.lowSince = time.Time{}
|
||||
}
|
||||
revisionHealthy := controller.revisionsMatch(instances, kubernetesStates)
|
||||
revisionHealthy := controller.revisionsMatch(instances, infrastructureStates)
|
||||
scaleUpEligible := !controller.highSince.IsZero() &&
|
||||
now.Sub(controller.highSince) >= time.Duration(controller.cfg.WorkerScaleUpWindowSeconds)*time.Second &&
|
||||
revisionHealthy
|
||||
@@ -206,21 +224,33 @@ func (controller *Controller) reconcile(ctx context.Context) error {
|
||||
SynchronousDatabasePeers: database.SynchronousPeers,
|
||||
ScaleUpEligible: scaleUpEligible,
|
||||
ScaleDownEligible: scaleDownEligible,
|
||||
Sites: sites,
|
||||
Pools: poolResources,
|
||||
})
|
||||
if !revisionHealthy && plan.FrozenReason == "" {
|
||||
plan.FrozenReason = "release_revision_mismatch"
|
||||
}
|
||||
for _, poolPlan := range plan.Pools {
|
||||
reason := "capacity_plan"
|
||||
if plan.FrozenReason != "" {
|
||||
reason = "frozen:" + plan.FrozenReason
|
||||
}
|
||||
if err := controller.store.PublishDesiredCapacity(ctx, executionpool.DesiredCapacity{
|
||||
PoolID: poolPlan.PoolID, Desired: poolPlan.DesiredReplicas,
|
||||
Reason: reason, ValidUntil: now.Add(2 * controllerReconcileInterval),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if controller.cfg.WorkerAutoscalingEnabled {
|
||||
for _, sitePlan := range plan.Sites {
|
||||
for _, poolPlan := range plan.Pools {
|
||||
switch {
|
||||
case sitePlan.DesiredReplicas > sitePlan.CurrentReplicas:
|
||||
if err := controller.kubernetes.ScaleWorkerDeployment(ctx, sitePlan.Site, sitePlan.DesiredReplicas); err != nil {
|
||||
case poolPlan.DesiredReplicas > poolPlan.CurrentReplicas:
|
||||
if err := controller.orchestrator.ScalePool(ctx, poolPlan.PoolID, controller.adapterRef(poolPlan.PoolID), poolPlan.DesiredReplicas); err != nil {
|
||||
return err
|
||||
}
|
||||
controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.DesiredReplicas, "scale_up")
|
||||
case sitePlan.DesiredReplicas < sitePlan.CurrentReplicas:
|
||||
if err := controller.reconcileScaleDown(ctx, now, sitePlan, instances); err != nil {
|
||||
controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.DesiredReplicas, "scale_up")
|
||||
case poolPlan.DesiredReplicas < poolPlan.CurrentReplicas:
|
||||
if err := controller.reconcileScaleDown(ctx, now, poolPlan, instances); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -256,21 +286,24 @@ func nodeResources(states []KubernetesNodeState) []NodeResources {
|
||||
func (controller *Controller) reconcileScaleDown(
|
||||
ctx context.Context,
|
||||
now time.Time,
|
||||
sitePlan SitePlan,
|
||||
poolPlan PoolPlan,
|
||||
instances []store.WorkerInstanceRuntime,
|
||||
) error {
|
||||
for _, instance := range instances {
|
||||
if instance.Site != sitePlan.Site || instance.Status != "draining" {
|
||||
if workerPoolID(instance) != poolPlan.PoolID || instance.Status != "draining" {
|
||||
continue
|
||||
}
|
||||
if instance.RunningTasks == 0 && instance.ActiveLeases == 0 {
|
||||
if err := controller.kubernetes.SetPodDeletionCost(ctx, instance.PodName, controllerDeletionCost); err != nil {
|
||||
if strings.TrimSpace(instance.OrchestratorInstanceRef) == "" {
|
||||
return errors.New("worker orchestrator instance reference is required for scale down")
|
||||
}
|
||||
if err := controller.orchestrator.SetInstanceTerminationPriority(ctx, instance.OrchestratorInstanceRef, controllerDeletionCost); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := controller.kubernetes.ScaleWorkerDeployment(ctx, sitePlan.Site, sitePlan.CurrentReplicas-1); err != nil {
|
||||
if err := controller.orchestrator.ScalePool(ctx, poolPlan.PoolID, controller.adapterRef(poolPlan.PoolID), poolPlan.CurrentReplicas-1); err != nil {
|
||||
return err
|
||||
}
|
||||
controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas-1, "drained_scale_down")
|
||||
controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas-1, "drained_scale_down")
|
||||
return nil
|
||||
}
|
||||
if instance.DrainingAt != nil &&
|
||||
@@ -278,14 +311,14 @@ func (controller *Controller) reconcileScaleDown(
|
||||
if err := controller.store.ReactivateWorkerInstance(ctx, instance.InstanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas, "drain_timeout")
|
||||
controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas, "drain_timeout")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var candidate *store.WorkerInstanceRuntime
|
||||
for index := range instances {
|
||||
instance := &instances[index]
|
||||
if instance.Site != sitePlan.Site || instance.Status != "active" {
|
||||
if workerPoolID(*instance) != poolPlan.PoolID || instance.Status != "active" {
|
||||
continue
|
||||
}
|
||||
if candidate == nil ||
|
||||
@@ -299,24 +332,29 @@ func (controller *Controller) reconcileScaleDown(
|
||||
if err := controller.store.MarkWorkerDraining(ctx, candidate.InstanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
controller.observeScale(sitePlan.Site, sitePlan.CurrentReplicas, sitePlan.CurrentReplicas, "drain_started")
|
||||
controller.observeScale(poolPlan.PoolID, poolPlan.CurrentReplicas, poolPlan.CurrentReplicas, "drain_started")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (controller *Controller) siteReplicaBounds(site string) (int, int) {
|
||||
switch site {
|
||||
case "ningbo":
|
||||
return controller.cfg.WorkerMinReplicasNingbo, controller.cfg.WorkerMaxReplicasNingbo
|
||||
case "hongkong":
|
||||
return controller.cfg.WorkerMinReplicasHongkong, controller.cfg.WorkerMaxReplicasHongkong
|
||||
default:
|
||||
return 0, 0
|
||||
func workerPoolID(instance store.WorkerInstanceRuntime) string {
|
||||
if strings.TrimSpace(instance.PoolID) != "" {
|
||||
return instance.PoolID
|
||||
}
|
||||
return instance.Site
|
||||
}
|
||||
|
||||
func (controller *Controller) adapterRef(poolID string) string {
|
||||
for _, pool := range controller.pools {
|
||||
if pool.ID == poolID {
|
||||
return pool.AdapterRef
|
||||
}
|
||||
}
|
||||
return poolID
|
||||
}
|
||||
|
||||
func (controller *Controller) revisionsMatch(
|
||||
instances []store.WorkerInstanceRuntime,
|
||||
states map[string]KubernetesSiteState,
|
||||
states map[string]PoolInfrastructureState,
|
||||
) bool {
|
||||
if controller.expectedRevision == "" {
|
||||
return true
|
||||
|
||||
@@ -32,8 +32,8 @@ type KubernetesClient struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
type KubernetesSiteState struct {
|
||||
Site string
|
||||
type PoolInfrastructureState struct {
|
||||
PoolID string
|
||||
NodeName string
|
||||
CurrentReplicas int
|
||||
Revision string
|
||||
@@ -45,10 +45,10 @@ type KubernetesSiteState struct {
|
||||
UsedMilliCPU int64
|
||||
WorkerRequestMilliCPU int64
|
||||
MemoryPressure bool
|
||||
Nodes []KubernetesNodeState
|
||||
Nodes []PoolNodeState
|
||||
}
|
||||
|
||||
type KubernetesNodeState struct {
|
||||
type PoolNodeState struct {
|
||||
NodeName string
|
||||
CurrentReplicas int
|
||||
AllocatableMemoryBytes int64
|
||||
@@ -60,6 +60,25 @@ type KubernetesNodeState struct {
|
||||
MemoryPressure bool
|
||||
}
|
||||
|
||||
type KubernetesSiteState = PoolInfrastructureState
|
||||
type KubernetesNodeState = PoolNodeState
|
||||
|
||||
var _ OrchestratorAdapter = (*KubernetesClient)(nil)
|
||||
|
||||
func (client *KubernetesClient) PoolState(ctx context.Context, poolID, adapterRef string) (PoolInfrastructureState, error) {
|
||||
state, err := client.SiteState(ctx, adapterRef)
|
||||
state.PoolID = poolID
|
||||
return state, err
|
||||
}
|
||||
|
||||
func (client *KubernetesClient) ScalePool(ctx context.Context, _, adapterRef string, replicas int) error {
|
||||
return client.ScaleWorkerDeployment(ctx, adapterRef, replicas)
|
||||
}
|
||||
|
||||
func (client *KubernetesClient) SetInstanceTerminationPriority(ctx context.Context, instanceRef string, priority int) error {
|
||||
return client.SetPodDeletionCost(ctx, instanceRef, priority)
|
||||
}
|
||||
|
||||
func NewKubernetesClient(config KubernetesConfig) (*KubernetesClient, error) {
|
||||
if strings.TrimSpace(config.Namespace) == "" {
|
||||
return nil, errors.New("capacity controller Kubernetes namespace is required")
|
||||
@@ -144,7 +163,7 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
|
||||
if err := client.getJSON(ctx, deploymentPath, &deployment); err != nil {
|
||||
return KubernetesSiteState{}, err
|
||||
}
|
||||
state := KubernetesSiteState{Site: site, CurrentReplicas: deployment.Spec.Replicas}
|
||||
state := KubernetesSiteState{PoolID: site, CurrentReplicas: deployment.Spec.Replicas}
|
||||
for _, container := range deployment.Spec.Template.Spec.Containers {
|
||||
if container.Name != "worker" {
|
||||
continue
|
||||
|
||||
@@ -129,7 +129,7 @@ func TestKubernetesClientAllowsDisabledSiteWithoutWorkerNodes(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.Site != "ningbo" || state.CurrentReplicas != 0 || len(state.Nodes) != 0 {
|
||||
if state.PoolID != "ningbo" || state.CurrentReplicas != 0 || len(state.Nodes) != 0 {
|
||||
t.Fatalf("site state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,9 @@ import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
type SiteResources struct {
|
||||
Site string
|
||||
type PoolResources struct {
|
||||
PoolID string
|
||||
Demand int
|
||||
CurrentReplicas int
|
||||
MinReplicas int
|
||||
MaxReplicas int
|
||||
@@ -47,11 +48,12 @@ type PlanInput struct {
|
||||
SynchronousDatabasePeers int
|
||||
ScaleUpEligible bool
|
||||
ScaleDownEligible bool
|
||||
Sites []SiteResources
|
||||
Pools []PoolResources
|
||||
}
|
||||
|
||||
type SitePlan struct {
|
||||
Site string `json:"site"`
|
||||
type PoolPlan struct {
|
||||
PoolID string `json:"poolId"`
|
||||
Demand int `json:"demand"`
|
||||
CurrentReplicas int `json:"currentReplicas"`
|
||||
DesiredReplicas int `json:"desiredReplicas"`
|
||||
ResourceMax int `json:"resourceMax"`
|
||||
@@ -64,7 +66,7 @@ type Plan struct {
|
||||
DesiredTotal int `json:"desiredTotal"`
|
||||
CurrentTotal int `json:"currentTotal"`
|
||||
FrozenReason string `json:"frozenReason,omitempty"`
|
||||
Sites []SitePlan `json:"sites"`
|
||||
Pools []PoolPlan `json:"pools"`
|
||||
}
|
||||
|
||||
func CalculatePlan(input PlanInput) Plan {
|
||||
@@ -88,33 +90,34 @@ func CalculatePlan(input PlanInput) Plan {
|
||||
plan := Plan{RawDesired: rawDesired}
|
||||
minTotal := 0
|
||||
resourceMaxTotal := 0
|
||||
for _, site := range input.Sites {
|
||||
for _, node := range site.Nodes {
|
||||
for _, pool := range input.Pools {
|
||||
for _, node := range pool.Nodes {
|
||||
if node.MemoryPressure {
|
||||
site.MemoryPressure = true
|
||||
pool.MemoryPressure = true
|
||||
}
|
||||
}
|
||||
current := max(site.CurrentReplicas, 0)
|
||||
minReplicas := max(site.MinReplicas, 0)
|
||||
configMax := max(site.MaxReplicas, minReplicas)
|
||||
resourceMax, memoryPercent, cpuPercent := siteResourceMaximum(
|
||||
site,
|
||||
current := max(pool.CurrentReplicas, 0)
|
||||
minReplicas := max(pool.MinReplicas, 0)
|
||||
configMax := max(pool.MaxReplicas, minReplicas)
|
||||
resourceMax, memoryPercent, cpuPercent := poolResourceMaximum(
|
||||
pool,
|
||||
input.MemoryTargetPercent,
|
||||
input.CPUTargetPercent,
|
||||
)
|
||||
resourceMax = min(resourceMax, configMax)
|
||||
if resourceMax < minReplicas && plan.FrozenReason == "" {
|
||||
plan.FrozenReason = "site_resource_budget"
|
||||
plan.FrozenReason = "pool_resource_budget"
|
||||
}
|
||||
resourceMax = max(resourceMax, minReplicas)
|
||||
plan.Sites = append(plan.Sites, SitePlan{
|
||||
Site: site.Site, CurrentReplicas: current, DesiredReplicas: minReplicas,
|
||||
plan.Pools = append(plan.Pools, PoolPlan{
|
||||
PoolID: pool.PoolID, CurrentReplicas: current, DesiredReplicas: minReplicas,
|
||||
Demand: pool.Demand,
|
||||
ResourceMax: resourceMax, MemoryPercent: memoryPercent, CPUPercent: cpuPercent,
|
||||
})
|
||||
plan.CurrentTotal += current
|
||||
minTotal += minReplicas
|
||||
resourceMaxTotal += resourceMax
|
||||
if site.MemoryPressure || memoryPercent >= float64(input.MemoryHardPercent) {
|
||||
if pool.MemoryPressure || memoryPercent >= float64(input.MemoryHardPercent) {
|
||||
plan.FrozenReason = "node_memory_pressure"
|
||||
}
|
||||
}
|
||||
@@ -123,10 +126,10 @@ func CalculatePlan(input PlanInput) Plan {
|
||||
(input.DatabaseConnectionBudget-input.NonWorkerConnectionBudget)/input.WorkerDatabasePoolMax,
|
||||
0,
|
||||
)
|
||||
capSiteResourceMaxima(plan.Sites, workerReplicaBudget)
|
||||
capPoolResourceMaxima(plan.Pools, workerReplicaBudget)
|
||||
resourceMaxTotal = 0
|
||||
for _, site := range plan.Sites {
|
||||
resourceMaxTotal += site.ResourceMax
|
||||
for _, pool := range plan.Pools {
|
||||
resourceMaxTotal += pool.ResourceMax
|
||||
}
|
||||
if workerReplicaBudget < minTotal && plan.FrozenReason == "" {
|
||||
plan.FrozenReason = "database_connection_budget"
|
||||
@@ -167,47 +170,47 @@ func CalculatePlan(input PlanInput) Plan {
|
||||
}
|
||||
desired = max(desired, minTotal)
|
||||
plan.DesiredTotal = desired
|
||||
distributeDesiredReplicas(plan.Sites, desired)
|
||||
distributeDesiredReplicas(plan.Pools, desired)
|
||||
return plan
|
||||
}
|
||||
|
||||
func capSiteResourceMaxima(sites []SitePlan, total int) {
|
||||
func capPoolResourceMaxima(pools []PoolPlan, total int) {
|
||||
minimum := 0
|
||||
original := make(map[string]int, len(sites))
|
||||
for index := range sites {
|
||||
original[sites[index].Site] = sites[index].ResourceMax
|
||||
sites[index].ResourceMax = sites[index].DesiredReplicas
|
||||
minimum += sites[index].ResourceMax
|
||||
original := make(map[string]int, len(pools))
|
||||
for index := range pools {
|
||||
original[pools[index].PoolID] = pools[index].ResourceMax
|
||||
pools[index].ResourceMax = pools[index].DesiredReplicas
|
||||
minimum += pools[index].ResourceMax
|
||||
}
|
||||
target := max(total, minimum)
|
||||
assigned := minimum
|
||||
for assigned < target {
|
||||
sort.SliceStable(sites, func(left, right int) bool {
|
||||
leftRoom := original[sites[left].Site] - sites[left].ResourceMax
|
||||
rightRoom := original[sites[right].Site] - sites[right].ResourceMax
|
||||
sort.SliceStable(pools, func(left, right int) bool {
|
||||
leftRoom := original[pools[left].PoolID] - pools[left].ResourceMax
|
||||
rightRoom := original[pools[right].PoolID] - pools[right].ResourceMax
|
||||
if leftRoom != rightRoom {
|
||||
return leftRoom > rightRoom
|
||||
}
|
||||
return sites[left].Site < sites[right].Site
|
||||
return pools[left].PoolID < pools[right].PoolID
|
||||
})
|
||||
if original[sites[0].Site] <= sites[0].ResourceMax {
|
||||
if original[pools[0].PoolID] <= pools[0].ResourceMax {
|
||||
break
|
||||
}
|
||||
sites[0].ResourceMax++
|
||||
pools[0].ResourceMax++
|
||||
assigned++
|
||||
}
|
||||
}
|
||||
|
||||
func siteResourceMaximum(site SiteResources, memoryTargetPercent int, cpuTargetPercent int) (int, float64, float64) {
|
||||
if len(site.Nodes) > 0 {
|
||||
if site.WorkerRequestMemoryBytes <= 0 || site.WorkerRequestMilliCPU <= 0 {
|
||||
func poolResourceMaximum(pool PoolResources, memoryTargetPercent int, cpuTargetPercent int) (int, float64, float64) {
|
||||
if len(pool.Nodes) > 0 {
|
||||
if pool.WorkerRequestMemoryBytes <= 0 || pool.WorkerRequestMilliCPU <= 0 {
|
||||
return 0, 0, 0
|
||||
}
|
||||
memoryMax := 0
|
||||
cpuMax := 0
|
||||
memoryPercent := float64(0)
|
||||
cpuPercent := float64(0)
|
||||
for _, node := range site.Nodes {
|
||||
for _, node := range pool.Nodes {
|
||||
nodeMemoryPercent := usagePercent(node.UsedMemoryBytes, node.AllocatableMemoryBytes)
|
||||
nodeCPUPercent := usagePercent(node.UsedMilliCPU, node.AllocatableMilliCPU)
|
||||
memoryPercent = max(memoryPercent, nodeMemoryPercent)
|
||||
@@ -217,60 +220,65 @@ func siteResourceMaximum(site SiteResources, memoryTargetPercent int, cpuTargetP
|
||||
0,
|
||||
)
|
||||
memoryBudget := node.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorkerMemory
|
||||
memoryMax += int(max(memoryBudget, 0) / site.WorkerRequestMemoryBytes)
|
||||
memoryMax += int(max(memoryBudget, 0) / pool.WorkerRequestMemoryBytes)
|
||||
nonWorkerCPU := max(
|
||||
node.UsedMilliCPU-node.WorkerUsedMilliCPU,
|
||||
0,
|
||||
)
|
||||
cpuBudget := node.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorkerCPU
|
||||
cpuMax += int(max(cpuBudget, 0) / site.WorkerRequestMilliCPU)
|
||||
cpuMax += int(max(cpuBudget, 0) / pool.WorkerRequestMilliCPU)
|
||||
}
|
||||
return min(site.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
|
||||
return min(pool.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
|
||||
}
|
||||
memoryPercent := usagePercent(site.UsedMemoryBytes, site.AllocatableMemoryBytes)
|
||||
cpuPercent := usagePercent(site.UsedMilliCPU, site.AllocatableMilliCPU)
|
||||
memoryMax := site.MaxReplicas
|
||||
if site.AllocatableMemoryBytes > 0 && site.WorkerRequestMemoryBytes > 0 {
|
||||
nonWorker := max(site.UsedMemoryBytes-int64(site.CurrentReplicas)*site.WorkerRequestMemoryBytes, 0)
|
||||
budget := site.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorker
|
||||
memoryMax = int(max(budget, 0) / site.WorkerRequestMemoryBytes)
|
||||
memoryPercent := usagePercent(pool.UsedMemoryBytes, pool.AllocatableMemoryBytes)
|
||||
cpuPercent := usagePercent(pool.UsedMilliCPU, pool.AllocatableMilliCPU)
|
||||
memoryMax := pool.MaxReplicas
|
||||
if pool.AllocatableMemoryBytes > 0 && pool.WorkerRequestMemoryBytes > 0 {
|
||||
nonWorker := max(pool.UsedMemoryBytes-int64(pool.CurrentReplicas)*pool.WorkerRequestMemoryBytes, 0)
|
||||
budget := pool.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorker
|
||||
memoryMax = int(max(budget, 0) / pool.WorkerRequestMemoryBytes)
|
||||
}
|
||||
cpuMax := site.MaxReplicas
|
||||
if site.AllocatableMilliCPU > 0 && site.WorkerRequestMilliCPU > 0 {
|
||||
nonWorker := max(site.UsedMilliCPU-int64(site.CurrentReplicas)*site.WorkerRequestMilliCPU, 0)
|
||||
budget := site.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorker
|
||||
cpuMax = int(max(budget, 0) / site.WorkerRequestMilliCPU)
|
||||
cpuMax := pool.MaxReplicas
|
||||
if pool.AllocatableMilliCPU > 0 && pool.WorkerRequestMilliCPU > 0 {
|
||||
nonWorker := max(pool.UsedMilliCPU-int64(pool.CurrentReplicas)*pool.WorkerRequestMilliCPU, 0)
|
||||
budget := pool.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorker
|
||||
cpuMax = int(max(budget, 0) / pool.WorkerRequestMilliCPU)
|
||||
}
|
||||
return min(site.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
|
||||
return min(pool.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
|
||||
}
|
||||
|
||||
func distributeDesiredReplicas(sites []SitePlan, desired int) {
|
||||
if len(sites) == 0 {
|
||||
func distributeDesiredReplicas(pools []PoolPlan, desired int) {
|
||||
if len(pools) == 0 {
|
||||
return
|
||||
}
|
||||
assigned := 0
|
||||
for index := range sites {
|
||||
assigned += sites[index].DesiredReplicas
|
||||
for index := range pools {
|
||||
assigned += pools[index].DesiredReplicas
|
||||
}
|
||||
for assigned < desired {
|
||||
sort.SliceStable(sites, func(left, right int) bool {
|
||||
leftRoom := sites[left].ResourceMax - sites[left].DesiredReplicas
|
||||
rightRoom := sites[right].ResourceMax - sites[right].DesiredReplicas
|
||||
sort.SliceStable(pools, func(left, right int) bool {
|
||||
leftDemandPerReplica := float64(max(pools[left].Demand, 0)) / float64(max(pools[left].DesiredReplicas, 1))
|
||||
rightDemandPerReplica := float64(max(pools[right].Demand, 0)) / float64(max(pools[right].DesiredReplicas, 1))
|
||||
if leftDemandPerReplica != rightDemandPerReplica {
|
||||
return leftDemandPerReplica > rightDemandPerReplica
|
||||
}
|
||||
leftRoom := pools[left].ResourceMax - pools[left].DesiredReplicas
|
||||
rightRoom := pools[right].ResourceMax - pools[right].DesiredReplicas
|
||||
if leftRoom != rightRoom {
|
||||
return leftRoom > rightRoom
|
||||
}
|
||||
if sites[left].DesiredReplicas != sites[right].DesiredReplicas {
|
||||
return sites[left].DesiredReplicas < sites[right].DesiredReplicas
|
||||
if pools[left].DesiredReplicas != pools[right].DesiredReplicas {
|
||||
return pools[left].DesiredReplicas < pools[right].DesiredReplicas
|
||||
}
|
||||
return sites[left].Site < sites[right].Site
|
||||
return pools[left].PoolID < pools[right].PoolID
|
||||
})
|
||||
if sites[0].DesiredReplicas >= sites[0].ResourceMax {
|
||||
if pools[0].DesiredReplicas >= pools[0].ResourceMax {
|
||||
break
|
||||
}
|
||||
sites[0].DesiredReplicas++
|
||||
pools[0].DesiredReplicas++
|
||||
assigned++
|
||||
}
|
||||
sort.Slice(sites, func(left, right int) bool { return sites[left].Site < sites[right].Site })
|
||||
sort.Slice(pools, func(left, right int) bool { return pools[left].PoolID < pools[right].PoolID })
|
||||
}
|
||||
|
||||
func usagePercent(used int64, allocatable int64) float64 {
|
||||
|
||||
@@ -12,15 +12,15 @@ func TestCalculatePlanRespectsResourceDatabaseAndStepLimits(t *testing.T) {
|
||||
MemoryTargetPercent: 75, MemoryHardPercent: 85, CPUTargetPercent: 70,
|
||||
DatabaseConnections: 80, DatabaseConnectionBudget: 150, WorkerDatabasePoolMax: 24,
|
||||
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
|
||||
Sites: []SiteResources{
|
||||
Pools: []PoolResources{
|
||||
{
|
||||
Site: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
|
||||
PoolID: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
|
||||
AllocatableMemoryBytes: 8 << 30, UsedMemoryBytes: 6 << 30,
|
||||
WorkerRequestMemoryBytes: 1 << 30,
|
||||
AllocatableMilliCPU: 4000, UsedMilliCPU: 2000, WorkerRequestMilliCPU: 500,
|
||||
},
|
||||
{
|
||||
Site: "hongkong", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
|
||||
PoolID: "hongkong", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
|
||||
AllocatableMemoryBytes: 8 << 30, UsedMemoryBytes: 3 << 30,
|
||||
WorkerRequestMemoryBytes: 1 << 30,
|
||||
AllocatableMilliCPU: 4000, UsedMilliCPU: 500, WorkerRequestMilliCPU: 500,
|
||||
@@ -34,8 +34,8 @@ func TestCalculatePlanRespectsResourceDatabaseAndStepLimits(t *testing.T) {
|
||||
if plan.DesiredTotal != 4 {
|
||||
t.Fatalf("desired total=%d, want step-limited 4: %+v", plan.DesiredTotal, plan)
|
||||
}
|
||||
if plan.Sites[0].Site != "hongkong" || plan.Sites[0].DesiredReplicas != 3 {
|
||||
t.Fatalf("site allocation=%+v, want hongkong=3 ningbo=1", plan.Sites)
|
||||
if plan.Pools[0].PoolID != "hongkong" || plan.Pools[0].DesiredReplicas != 3 {
|
||||
t.Fatalf("pool allocation=%+v, want hongkong=3 ningbo=1", plan.Pools)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,8 +44,8 @@ func TestCalculatePlanFreezesScaleUpOnHardMemoryPressure(t *testing.T) {
|
||||
Queued: 200, InstanceSlots: 24,
|
||||
DatabaseConnections: 10, DatabaseConnectionBudget: 150, WorkerDatabasePoolMax: 24,
|
||||
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
|
||||
Sites: []SiteResources{{
|
||||
Site: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
|
||||
Pools: []PoolResources{{
|
||||
PoolID: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
|
||||
AllocatableMemoryBytes: 8 << 30, UsedMemoryBytes: 7 << 30,
|
||||
WorkerRequestMemoryBytes: 1 << 30,
|
||||
}},
|
||||
@@ -60,14 +60,14 @@ func TestCalculatePlanReportsConfiguredMinimumOutsideResourceBudget(t *testing.T
|
||||
Queued: 200, InstanceSlots: 24,
|
||||
DatabaseConnections: 10, DatabaseConnectionBudget: 150, WorkerDatabasePoolMax: 24,
|
||||
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
|
||||
Sites: []SiteResources{{
|
||||
Site: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
|
||||
Pools: []PoolResources{{
|
||||
PoolID: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4,
|
||||
AllocatableMemoryBytes: 8 << 30, UsedMemoryBytes: 13 << 29,
|
||||
WorkerRequestMemoryBytes: 2 << 30,
|
||||
AllocatableMilliCPU: 4000, UsedMilliCPU: 2800, WorkerRequestMilliCPU: 500,
|
||||
}},
|
||||
})
|
||||
if plan.DesiredTotal != 1 || plan.FrozenReason != "site_resource_budget" {
|
||||
if plan.DesiredTotal != 1 || plan.FrozenReason != "pool_resource_budget" {
|
||||
t.Fatalf("plan=%+v, want the existing minimum preserved and expansion frozen", plan)
|
||||
}
|
||||
}
|
||||
@@ -77,9 +77,9 @@ func TestCalculatePlanWaitsForSafeScaleDownWindow(t *testing.T) {
|
||||
InstanceSlots: 24, DatabaseConnections: 20, DatabaseConnectionBudget: 150,
|
||||
WorkerDatabasePoolMax: 24, SynchronousDatabasePeers: 1,
|
||||
ScaleDownEligible: false,
|
||||
Sites: []SiteResources{
|
||||
{Site: "ningbo", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 4},
|
||||
{Site: "hongkong", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 4},
|
||||
Pools: []PoolResources{
|
||||
{PoolID: "ningbo", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 4},
|
||||
{PoolID: "hongkong", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 4},
|
||||
},
|
||||
}
|
||||
plan := CalculatePlan(input)
|
||||
@@ -99,8 +99,8 @@ func TestCalculatePlanAddsCapacityAcrossLabeledSiteNodes(t *testing.T) {
|
||||
MemoryTargetPercent: 75, MemoryHardPercent: 85, CPUTargetPercent: 70,
|
||||
DatabaseConnections: 20, DatabaseConnectionBudget: 150, WorkerDatabasePoolMax: 20,
|
||||
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
|
||||
Sites: []SiteResources{{
|
||||
Site: "hongkong", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 8,
|
||||
Pools: []PoolResources{{
|
||||
PoolID: "hongkong", CurrentReplicas: 2, MinReplicas: 1, MaxReplicas: 8,
|
||||
WorkerRequestMemoryBytes: 1 << 30, WorkerRequestMilliCPU: 500,
|
||||
Nodes: []NodeResources{
|
||||
{
|
||||
@@ -116,8 +116,8 @@ func TestCalculatePlanAddsCapacityAcrossLabeledSiteNodes(t *testing.T) {
|
||||
},
|
||||
}},
|
||||
})
|
||||
if plan.Sites[0].ResourceMax != 7 {
|
||||
t.Fatalf("resource max=%d, want database-budgeted multi-node maximum: %+v", plan.Sites[0].ResourceMax, plan)
|
||||
if plan.Pools[0].ResourceMax != 7 {
|
||||
t.Fatalf("resource max=%d, want database-budgeted multi-node maximum: %+v", plan.Pools[0].ResourceMax, plan)
|
||||
}
|
||||
if plan.DesiredTotal != 4 {
|
||||
t.Fatalf("desired=%d, want two-wave step from 2 to 4", plan.DesiredTotal)
|
||||
@@ -130,17 +130,17 @@ func TestCalculatePlanCapsReplicaMaximumByDeclaredDatabasePools(t *testing.T) {
|
||||
DatabaseConnections: 20, DatabaseConnectionBudget: 150,
|
||||
NonWorkerConnectionBudget: 72, WorkerDatabasePoolMax: 32,
|
||||
SynchronousDatabasePeers: 1, ScaleUpEligible: true,
|
||||
Sites: []SiteResources{
|
||||
{Site: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4},
|
||||
{Site: "hongkong", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4},
|
||||
Pools: []PoolResources{
|
||||
{PoolID: "ningbo", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4},
|
||||
{PoolID: "hongkong", CurrentReplicas: 1, MinReplicas: 1, MaxReplicas: 4},
|
||||
},
|
||||
})
|
||||
if plan.DesiredTotal != 2 {
|
||||
t.Fatalf("desired=%d, want the current 1+1 database-safe topology: %+v", plan.DesiredTotal, plan)
|
||||
}
|
||||
resourceMax := 0
|
||||
for _, site := range plan.Sites {
|
||||
resourceMax += site.ResourceMax
|
||||
for _, pool := range plan.Pools {
|
||||
resourceMax += pool.ResourceMax
|
||||
}
|
||||
if resourceMax != 2 {
|
||||
t.Fatalf("resource maximum=%d, want floor((150-72)/32)=2: %+v", resourceMax, plan)
|
||||
@@ -150,13 +150,13 @@ func TestCalculatePlanCapsReplicaMaximumByDeclaredDatabasePools(t *testing.T) {
|
||||
func TestPlanJSONUsesAcceptanceContractFieldNames(t *testing.T) {
|
||||
payload, err := json.Marshal(Plan{
|
||||
RawDesired: 3,
|
||||
Sites: []SitePlan{{Site: "hongkong", ResourceMax: 2}},
|
||||
Pools: []PoolPlan{{PoolID: "hongkong", ResourceMax: 2}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(payload)
|
||||
for _, field := range []string{`"rawDesired":3`, `"sites"`, `"site":"hongkong"`, `"resourceMax":2`} {
|
||||
for _, field := range []string{`"rawDesired":3`, `"pools"`, `"poolId":"hongkong"`, `"resourceMax":2`} {
|
||||
if !strings.Contains(text, field) {
|
||||
t.Fatalf("plan JSON %s does not contain %s", text, field)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package capacitycontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// StaticAdapter proves that capacity coordination does not require Kubernetes.
|
||||
// It reports configured fixed replicas and intentionally ignores scale writes.
|
||||
type StaticAdapter struct {
|
||||
mu sync.RWMutex
|
||||
replicas map[string]int
|
||||
}
|
||||
|
||||
var _ OrchestratorAdapter = (*StaticAdapter)(nil)
|
||||
|
||||
func NewStaticAdapter(initial map[string]int) *StaticAdapter {
|
||||
copyOfInitial := make(map[string]int, len(initial))
|
||||
for poolID, replicas := range initial {
|
||||
copyOfInitial[poolID] = replicas
|
||||
}
|
||||
return &StaticAdapter{replicas: copyOfInitial}
|
||||
}
|
||||
|
||||
func (adapter *StaticAdapter) PoolState(_ context.Context, poolID, _ string) (PoolInfrastructureState, error) {
|
||||
adapter.mu.RLock()
|
||||
replicas := adapter.replicas[poolID]
|
||||
adapter.mu.RUnlock()
|
||||
return PoolInfrastructureState{
|
||||
PoolID: poolID, CurrentReplicas: replicas,
|
||||
AllocatableMemoryBytes: 1 << 60, WorkerRequestMemoryBytes: 1,
|
||||
AllocatableMilliCPU: 1 << 50, WorkerRequestMilliCPU: 1,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (adapter *StaticAdapter) ScalePool(context.Context, string, string, int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (adapter *StaticAdapter) SetInstanceTerminationPriority(context.Context, string, int) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package capacitycontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStaticAdapterDoesNotRequireKubernetes(t *testing.T) {
|
||||
adapter := NewStaticAdapter(map[string]int{"pool-a": 2})
|
||||
state, err := adapter.PoolState(context.Background(), "pool-a", "ignored")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.PoolID != "pool-a" || state.CurrentReplicas != 2 {
|
||||
t.Fatalf("state=%+v", state)
|
||||
}
|
||||
if err := adapter.ScalePool(context.Background(), "pool-a", "ignored", 4); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, _ = adapter.PoolState(context.Background(), "pool-a", "ignored")
|
||||
if state.CurrentReplicas != 2 {
|
||||
t.Fatalf("static adapter scaled to %d", state.CurrentReplicas)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user