wangbo
7786692d32
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
290 lines
9.5 KiB
Go
290 lines
9.5 KiB
Go
package capacitycontroller
|
|
|
|
import (
|
|
"math"
|
|
"sort"
|
|
)
|
|
|
|
type PoolResources struct {
|
|
PoolID string
|
|
Demand int
|
|
CurrentReplicas int
|
|
MinReplicas int
|
|
MaxReplicas int
|
|
AllocatableMemoryBytes int64
|
|
UsedMemoryBytes int64
|
|
WorkerRequestMemoryBytes int64
|
|
AllocatableMilliCPU int64
|
|
UsedMilliCPU int64
|
|
WorkerRequestMilliCPU int64
|
|
MemoryPressure bool
|
|
Nodes []NodeResources
|
|
}
|
|
|
|
type NodeResources struct {
|
|
NodeName string
|
|
CurrentReplicas int
|
|
AllocatableMemoryBytes int64
|
|
UsedMemoryBytes int64
|
|
WorkerUsedMemoryBytes int64
|
|
AllocatableMilliCPU int64
|
|
UsedMilliCPU int64
|
|
WorkerUsedMilliCPU int64
|
|
MemoryPressure bool
|
|
}
|
|
|
|
type PlanInput struct {
|
|
Queued int
|
|
Running int
|
|
InstanceSlots int
|
|
TargetOutstandingPerReplica int
|
|
MemoryTargetPercent int
|
|
MemoryHardPercent int
|
|
CPUTargetPercent int
|
|
DatabaseConnections int
|
|
DatabaseConnectionBudget int
|
|
NonWorkerConnectionBudget int
|
|
WorkerDatabasePoolMax int
|
|
SynchronousDatabasePeers int
|
|
ScaleUpEligible bool
|
|
ScaleDownEligible bool
|
|
Pools []PoolResources
|
|
}
|
|
|
|
type PoolPlan struct {
|
|
PoolID string `json:"poolId"`
|
|
Demand int `json:"demand"`
|
|
CurrentReplicas int `json:"currentReplicas"`
|
|
DesiredReplicas int `json:"desiredReplicas"`
|
|
ResourceMax int `json:"resourceMax"`
|
|
MemoryPercent float64 `json:"memoryPercent"`
|
|
CPUPercent float64 `json:"cpuPercent"`
|
|
}
|
|
|
|
type Plan struct {
|
|
RawDesired int `json:"rawDesired"`
|
|
DesiredTotal int `json:"desiredTotal"`
|
|
CurrentTotal int `json:"currentTotal"`
|
|
FrozenReason string `json:"frozenReason,omitempty"`
|
|
Pools []PoolPlan `json:"pools"`
|
|
}
|
|
|
|
func CalculatePlan(input PlanInput) Plan {
|
|
if input.InstanceSlots < 1 {
|
|
input.InstanceSlots = 1
|
|
}
|
|
target := input.TargetOutstandingPerReplica
|
|
if target < 1 {
|
|
target = 2 * input.InstanceSlots
|
|
}
|
|
if input.MemoryTargetPercent < 1 {
|
|
input.MemoryTargetPercent = 75
|
|
}
|
|
if input.MemoryHardPercent < 1 {
|
|
input.MemoryHardPercent = 85
|
|
}
|
|
if input.CPUTargetPercent < 1 {
|
|
input.CPUTargetPercent = 70
|
|
}
|
|
rawDesired := int(math.Ceil(float64(max(input.Queued+input.Running, 0)) / float64(target)))
|
|
plan := Plan{RawDesired: rawDesired}
|
|
minTotal := 0
|
|
resourceMaxTotal := 0
|
|
for _, pool := range input.Pools {
|
|
for _, node := range pool.Nodes {
|
|
if node.MemoryPressure {
|
|
pool.MemoryPressure = true
|
|
}
|
|
}
|
|
current := max(pool.CurrentReplicas, 0)
|
|
minReplicas := max(pool.MinReplicas, 0)
|
|
configMax := max(pool.MaxReplicas, minReplicas)
|
|
resourceMax, memoryPercent, cpuPercent := poolResourceMaximum(
|
|
pool,
|
|
input.MemoryTargetPercent,
|
|
input.CPUTargetPercent,
|
|
)
|
|
resourceMax = min(resourceMax, configMax)
|
|
if resourceMax < minReplicas && plan.FrozenReason == "" {
|
|
plan.FrozenReason = "pool_resource_budget"
|
|
}
|
|
resourceMax = max(resourceMax, minReplicas)
|
|
plan.Pools = append(plan.Pools, PoolPlan{
|
|
PoolID: pool.PoolID, CurrentReplicas: current, DesiredReplicas: minReplicas,
|
|
Demand: pool.Demand,
|
|
ResourceMax: resourceMax, MemoryPercent: memoryPercent, CPUPercent: cpuPercent,
|
|
})
|
|
plan.CurrentTotal += current
|
|
minTotal += minReplicas
|
|
resourceMaxTotal += resourceMax
|
|
if pool.MemoryPressure || memoryPercent >= float64(input.MemoryHardPercent) {
|
|
plan.FrozenReason = "node_memory_pressure"
|
|
}
|
|
}
|
|
if input.DatabaseConnectionBudget > 0 && input.WorkerDatabasePoolMax > 0 {
|
|
workerReplicaBudget := max(
|
|
(input.DatabaseConnectionBudget-input.NonWorkerConnectionBudget)/input.WorkerDatabasePoolMax,
|
|
0,
|
|
)
|
|
capPoolResourceMaxima(plan.Pools, workerReplicaBudget)
|
|
resourceMaxTotal = 0
|
|
for _, pool := range plan.Pools {
|
|
resourceMaxTotal += pool.ResourceMax
|
|
}
|
|
if workerReplicaBudget < minTotal && plan.FrozenReason == "" {
|
|
plan.FrozenReason = "database_connection_budget"
|
|
}
|
|
}
|
|
desired := max(rawDesired, minTotal)
|
|
desired = min(desired, resourceMaxTotal)
|
|
|
|
if input.SynchronousDatabasePeers < 1 && plan.FrozenReason == "" {
|
|
plan.FrozenReason = "database_not_synchronous"
|
|
}
|
|
if desired > plan.CurrentTotal {
|
|
if input.DatabaseConnectionBudget > 0 && input.WorkerDatabasePoolMax > 0 {
|
|
additional := max(
|
|
(input.DatabaseConnectionBudget-input.DatabaseConnections)/input.WorkerDatabasePoolMax,
|
|
0,
|
|
)
|
|
desired = min(desired, plan.CurrentTotal+additional)
|
|
}
|
|
if !input.ScaleUpEligible || plan.FrozenReason != "" ||
|
|
(input.DatabaseConnectionBudget > 0 && input.DatabaseConnections >= input.DatabaseConnectionBudget) {
|
|
desired = plan.CurrentTotal
|
|
if plan.FrozenReason == "" {
|
|
if !input.ScaleUpEligible {
|
|
plan.FrozenReason = "scale_up_window"
|
|
} else {
|
|
plan.FrozenReason = "database_connection_budget"
|
|
}
|
|
}
|
|
} else {
|
|
desired = min(desired, max(plan.CurrentTotal*2, plan.CurrentTotal+2))
|
|
}
|
|
} else if desired < plan.CurrentTotal && !input.ScaleDownEligible {
|
|
desired = plan.CurrentTotal
|
|
if plan.FrozenReason == "" {
|
|
plan.FrozenReason = "scale_down_stabilization"
|
|
}
|
|
}
|
|
desired = max(desired, minTotal)
|
|
plan.DesiredTotal = desired
|
|
distributeDesiredReplicas(plan.Pools, desired)
|
|
return plan
|
|
}
|
|
|
|
func capPoolResourceMaxima(pools []PoolPlan, total int) {
|
|
minimum := 0
|
|
original := make(map[string]int, len(pools))
|
|
for index := range pools {
|
|
original[pools[index].PoolID] = pools[index].ResourceMax
|
|
pools[index].ResourceMax = pools[index].DesiredReplicas
|
|
minimum += pools[index].ResourceMax
|
|
}
|
|
target := max(total, minimum)
|
|
assigned := minimum
|
|
for assigned < target {
|
|
sort.SliceStable(pools, func(left, right int) bool {
|
|
leftRoom := original[pools[left].PoolID] - pools[left].ResourceMax
|
|
rightRoom := original[pools[right].PoolID] - pools[right].ResourceMax
|
|
if leftRoom != rightRoom {
|
|
return leftRoom > rightRoom
|
|
}
|
|
return pools[left].PoolID < pools[right].PoolID
|
|
})
|
|
if original[pools[0].PoolID] <= pools[0].ResourceMax {
|
|
break
|
|
}
|
|
pools[0].ResourceMax++
|
|
assigned++
|
|
}
|
|
}
|
|
|
|
func poolResourceMaximum(pool PoolResources, memoryTargetPercent int, cpuTargetPercent int) (int, float64, float64) {
|
|
if len(pool.Nodes) > 0 {
|
|
if pool.WorkerRequestMemoryBytes <= 0 || pool.WorkerRequestMilliCPU <= 0 {
|
|
return 0, 0, 0
|
|
}
|
|
memoryMax := 0
|
|
cpuMax := 0
|
|
memoryPercent := float64(0)
|
|
cpuPercent := float64(0)
|
|
for _, node := range pool.Nodes {
|
|
nodeMemoryPercent := usagePercent(node.UsedMemoryBytes, node.AllocatableMemoryBytes)
|
|
nodeCPUPercent := usagePercent(node.UsedMilliCPU, node.AllocatableMilliCPU)
|
|
memoryPercent = max(memoryPercent, nodeMemoryPercent)
|
|
cpuPercent = max(cpuPercent, nodeCPUPercent)
|
|
nonWorkerMemory := max(
|
|
node.UsedMemoryBytes-node.WorkerUsedMemoryBytes,
|
|
0,
|
|
)
|
|
memoryBudget := node.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorkerMemory
|
|
memoryMax += int(max(memoryBudget, 0) / pool.WorkerRequestMemoryBytes)
|
|
nonWorkerCPU := max(
|
|
node.UsedMilliCPU-node.WorkerUsedMilliCPU,
|
|
0,
|
|
)
|
|
cpuBudget := node.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorkerCPU
|
|
cpuMax += int(max(cpuBudget, 0) / pool.WorkerRequestMilliCPU)
|
|
}
|
|
return min(pool.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
|
|
}
|
|
memoryPercent := usagePercent(pool.UsedMemoryBytes, pool.AllocatableMemoryBytes)
|
|
cpuPercent := usagePercent(pool.UsedMilliCPU, pool.AllocatableMilliCPU)
|
|
memoryMax := pool.MaxReplicas
|
|
if pool.AllocatableMemoryBytes > 0 && pool.WorkerRequestMemoryBytes > 0 {
|
|
nonWorker := max(pool.UsedMemoryBytes-int64(pool.CurrentReplicas)*pool.WorkerRequestMemoryBytes, 0)
|
|
budget := pool.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorker
|
|
memoryMax = int(max(budget, 0) / pool.WorkerRequestMemoryBytes)
|
|
}
|
|
cpuMax := pool.MaxReplicas
|
|
if pool.AllocatableMilliCPU > 0 && pool.WorkerRequestMilliCPU > 0 {
|
|
nonWorker := max(pool.UsedMilliCPU-int64(pool.CurrentReplicas)*pool.WorkerRequestMilliCPU, 0)
|
|
budget := pool.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorker
|
|
cpuMax = int(max(budget, 0) / pool.WorkerRequestMilliCPU)
|
|
}
|
|
return min(pool.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
|
|
}
|
|
|
|
func distributeDesiredReplicas(pools []PoolPlan, desired int) {
|
|
if len(pools) == 0 {
|
|
return
|
|
}
|
|
assigned := 0
|
|
for index := range pools {
|
|
assigned += pools[index].DesiredReplicas
|
|
}
|
|
for assigned < desired {
|
|
sort.SliceStable(pools, func(left, right int) bool {
|
|
leftDemandPerReplica := float64(max(pools[left].Demand, 0)) / float64(max(pools[left].DesiredReplicas, 1))
|
|
rightDemandPerReplica := float64(max(pools[right].Demand, 0)) / float64(max(pools[right].DesiredReplicas, 1))
|
|
if leftDemandPerReplica != rightDemandPerReplica {
|
|
return leftDemandPerReplica > rightDemandPerReplica
|
|
}
|
|
leftRoom := pools[left].ResourceMax - pools[left].DesiredReplicas
|
|
rightRoom := pools[right].ResourceMax - pools[right].DesiredReplicas
|
|
if leftRoom != rightRoom {
|
|
return leftRoom > rightRoom
|
|
}
|
|
if pools[left].DesiredReplicas != pools[right].DesiredReplicas {
|
|
return pools[left].DesiredReplicas < pools[right].DesiredReplicas
|
|
}
|
|
return pools[left].PoolID < pools[right].PoolID
|
|
})
|
|
if pools[0].DesiredReplicas >= pools[0].ResourceMax {
|
|
break
|
|
}
|
|
pools[0].DesiredReplicas++
|
|
assigned++
|
|
}
|
|
sort.Slice(pools, func(left, right int) bool { return pools[left].PoolID < pools[right].PoolID })
|
|
}
|
|
|
|
func usagePercent(used int64, allocatable int64) float64 {
|
|
if allocatable <= 0 {
|
|
return 0
|
|
}
|
|
return float64(used) * 100 / float64(allocatable)
|
|
}
|