wangbo
e05922b0f4
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
282 lines
9.0 KiB
Go
282 lines
9.0 KiB
Go
package capacitycontroller
|
|
|
|
import (
|
|
"math"
|
|
"sort"
|
|
)
|
|
|
|
type SiteResources struct {
|
|
Site string
|
|
CurrentReplicas int
|
|
MinReplicas int
|
|
MaxReplicas int
|
|
AllocatableMemoryBytes int64
|
|
UsedMemoryBytes int64
|
|
WorkerRequestMemoryBytes int64
|
|
AllocatableMilliCPU int64
|
|
UsedMilliCPU int64
|
|
WorkerRequestMilliCPU int64
|
|
MemoryPressure bool
|
|
Nodes []NodeResources
|
|
}
|
|
|
|
type NodeResources struct {
|
|
NodeName string
|
|
CurrentReplicas int
|
|
AllocatableMemoryBytes int64
|
|
UsedMemoryBytes int64
|
|
WorkerUsedMemoryBytes int64
|
|
AllocatableMilliCPU int64
|
|
UsedMilliCPU int64
|
|
WorkerUsedMilliCPU int64
|
|
MemoryPressure bool
|
|
}
|
|
|
|
type PlanInput struct {
|
|
Queued int
|
|
Running int
|
|
InstanceSlots int
|
|
TargetOutstandingPerReplica int
|
|
MemoryTargetPercent int
|
|
MemoryHardPercent int
|
|
CPUTargetPercent int
|
|
DatabaseConnections int
|
|
DatabaseConnectionBudget int
|
|
NonWorkerConnectionBudget int
|
|
WorkerDatabasePoolMax int
|
|
SynchronousDatabasePeers int
|
|
ScaleUpEligible bool
|
|
ScaleDownEligible bool
|
|
Sites []SiteResources
|
|
}
|
|
|
|
type SitePlan struct {
|
|
Site string `json:"site"`
|
|
CurrentReplicas int `json:"currentReplicas"`
|
|
DesiredReplicas int `json:"desiredReplicas"`
|
|
ResourceMax int `json:"resourceMax"`
|
|
MemoryPercent float64 `json:"memoryPercent"`
|
|
CPUPercent float64 `json:"cpuPercent"`
|
|
}
|
|
|
|
type Plan struct {
|
|
RawDesired int `json:"rawDesired"`
|
|
DesiredTotal int `json:"desiredTotal"`
|
|
CurrentTotal int `json:"currentTotal"`
|
|
FrozenReason string `json:"frozenReason,omitempty"`
|
|
Sites []SitePlan `json:"sites"`
|
|
}
|
|
|
|
func CalculatePlan(input PlanInput) Plan {
|
|
if input.InstanceSlots < 1 {
|
|
input.InstanceSlots = 1
|
|
}
|
|
target := input.TargetOutstandingPerReplica
|
|
if target < 1 {
|
|
target = 2 * input.InstanceSlots
|
|
}
|
|
if input.MemoryTargetPercent < 1 {
|
|
input.MemoryTargetPercent = 75
|
|
}
|
|
if input.MemoryHardPercent < 1 {
|
|
input.MemoryHardPercent = 85
|
|
}
|
|
if input.CPUTargetPercent < 1 {
|
|
input.CPUTargetPercent = 70
|
|
}
|
|
rawDesired := int(math.Ceil(float64(max(input.Queued+input.Running, 0)) / float64(target)))
|
|
plan := Plan{RawDesired: rawDesired}
|
|
minTotal := 0
|
|
resourceMaxTotal := 0
|
|
for _, site := range input.Sites {
|
|
for _, node := range site.Nodes {
|
|
if node.MemoryPressure {
|
|
site.MemoryPressure = true
|
|
}
|
|
}
|
|
current := max(site.CurrentReplicas, 0)
|
|
minReplicas := max(site.MinReplicas, 0)
|
|
configMax := max(site.MaxReplicas, minReplicas)
|
|
resourceMax, memoryPercent, cpuPercent := siteResourceMaximum(
|
|
site,
|
|
input.MemoryTargetPercent,
|
|
input.CPUTargetPercent,
|
|
)
|
|
resourceMax = min(resourceMax, configMax)
|
|
if resourceMax < minReplicas && plan.FrozenReason == "" {
|
|
plan.FrozenReason = "site_resource_budget"
|
|
}
|
|
resourceMax = max(resourceMax, minReplicas)
|
|
plan.Sites = append(plan.Sites, SitePlan{
|
|
Site: site.Site, CurrentReplicas: current, DesiredReplicas: minReplicas,
|
|
ResourceMax: resourceMax, MemoryPercent: memoryPercent, CPUPercent: cpuPercent,
|
|
})
|
|
plan.CurrentTotal += current
|
|
minTotal += minReplicas
|
|
resourceMaxTotal += resourceMax
|
|
if site.MemoryPressure || memoryPercent >= float64(input.MemoryHardPercent) {
|
|
plan.FrozenReason = "node_memory_pressure"
|
|
}
|
|
}
|
|
if input.DatabaseConnectionBudget > 0 && input.WorkerDatabasePoolMax > 0 {
|
|
workerReplicaBudget := max(
|
|
(input.DatabaseConnectionBudget-input.NonWorkerConnectionBudget)/input.WorkerDatabasePoolMax,
|
|
0,
|
|
)
|
|
capSiteResourceMaxima(plan.Sites, workerReplicaBudget)
|
|
resourceMaxTotal = 0
|
|
for _, site := range plan.Sites {
|
|
resourceMaxTotal += site.ResourceMax
|
|
}
|
|
if workerReplicaBudget < minTotal && plan.FrozenReason == "" {
|
|
plan.FrozenReason = "database_connection_budget"
|
|
}
|
|
}
|
|
desired := max(rawDesired, minTotal)
|
|
desired = min(desired, resourceMaxTotal)
|
|
|
|
if input.SynchronousDatabasePeers < 1 && plan.FrozenReason == "" {
|
|
plan.FrozenReason = "database_not_synchronous"
|
|
}
|
|
if desired > plan.CurrentTotal {
|
|
if input.DatabaseConnectionBudget > 0 && input.WorkerDatabasePoolMax > 0 {
|
|
additional := max(
|
|
(input.DatabaseConnectionBudget-input.DatabaseConnections)/input.WorkerDatabasePoolMax,
|
|
0,
|
|
)
|
|
desired = min(desired, plan.CurrentTotal+additional)
|
|
}
|
|
if !input.ScaleUpEligible || plan.FrozenReason != "" ||
|
|
(input.DatabaseConnectionBudget > 0 && input.DatabaseConnections >= input.DatabaseConnectionBudget) {
|
|
desired = plan.CurrentTotal
|
|
if plan.FrozenReason == "" {
|
|
if !input.ScaleUpEligible {
|
|
plan.FrozenReason = "scale_up_window"
|
|
} else {
|
|
plan.FrozenReason = "database_connection_budget"
|
|
}
|
|
}
|
|
} else {
|
|
desired = min(desired, max(plan.CurrentTotal*2, plan.CurrentTotal+2))
|
|
}
|
|
} else if desired < plan.CurrentTotal && !input.ScaleDownEligible {
|
|
desired = plan.CurrentTotal
|
|
if plan.FrozenReason == "" {
|
|
plan.FrozenReason = "scale_down_stabilization"
|
|
}
|
|
}
|
|
desired = max(desired, minTotal)
|
|
plan.DesiredTotal = desired
|
|
distributeDesiredReplicas(plan.Sites, desired)
|
|
return plan
|
|
}
|
|
|
|
func capSiteResourceMaxima(sites []SitePlan, total int) {
|
|
minimum := 0
|
|
original := make(map[string]int, len(sites))
|
|
for index := range sites {
|
|
original[sites[index].Site] = sites[index].ResourceMax
|
|
sites[index].ResourceMax = sites[index].DesiredReplicas
|
|
minimum += sites[index].ResourceMax
|
|
}
|
|
target := max(total, minimum)
|
|
assigned := minimum
|
|
for assigned < target {
|
|
sort.SliceStable(sites, func(left, right int) bool {
|
|
leftRoom := original[sites[left].Site] - sites[left].ResourceMax
|
|
rightRoom := original[sites[right].Site] - sites[right].ResourceMax
|
|
if leftRoom != rightRoom {
|
|
return leftRoom > rightRoom
|
|
}
|
|
return sites[left].Site < sites[right].Site
|
|
})
|
|
if original[sites[0].Site] <= sites[0].ResourceMax {
|
|
break
|
|
}
|
|
sites[0].ResourceMax++
|
|
assigned++
|
|
}
|
|
}
|
|
|
|
func siteResourceMaximum(site SiteResources, memoryTargetPercent int, cpuTargetPercent int) (int, float64, float64) {
|
|
if len(site.Nodes) > 0 {
|
|
if site.WorkerRequestMemoryBytes <= 0 || site.WorkerRequestMilliCPU <= 0 {
|
|
return 0, 0, 0
|
|
}
|
|
memoryMax := 0
|
|
cpuMax := 0
|
|
memoryPercent := float64(0)
|
|
cpuPercent := float64(0)
|
|
for _, node := range site.Nodes {
|
|
nodeMemoryPercent := usagePercent(node.UsedMemoryBytes, node.AllocatableMemoryBytes)
|
|
nodeCPUPercent := usagePercent(node.UsedMilliCPU, node.AllocatableMilliCPU)
|
|
memoryPercent = max(memoryPercent, nodeMemoryPercent)
|
|
cpuPercent = max(cpuPercent, nodeCPUPercent)
|
|
nonWorkerMemory := max(
|
|
node.UsedMemoryBytes-node.WorkerUsedMemoryBytes,
|
|
0,
|
|
)
|
|
memoryBudget := node.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorkerMemory
|
|
memoryMax += int(max(memoryBudget, 0) / site.WorkerRequestMemoryBytes)
|
|
nonWorkerCPU := max(
|
|
node.UsedMilliCPU-node.WorkerUsedMilliCPU,
|
|
0,
|
|
)
|
|
cpuBudget := node.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorkerCPU
|
|
cpuMax += int(max(cpuBudget, 0) / site.WorkerRequestMilliCPU)
|
|
}
|
|
return min(site.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
|
|
}
|
|
memoryPercent := usagePercent(site.UsedMemoryBytes, site.AllocatableMemoryBytes)
|
|
cpuPercent := usagePercent(site.UsedMilliCPU, site.AllocatableMilliCPU)
|
|
memoryMax := site.MaxReplicas
|
|
if site.AllocatableMemoryBytes > 0 && site.WorkerRequestMemoryBytes > 0 {
|
|
nonWorker := max(site.UsedMemoryBytes-int64(site.CurrentReplicas)*site.WorkerRequestMemoryBytes, 0)
|
|
budget := site.AllocatableMemoryBytes*int64(memoryTargetPercent)/100 - nonWorker
|
|
memoryMax = int(max(budget, 0) / site.WorkerRequestMemoryBytes)
|
|
}
|
|
cpuMax := site.MaxReplicas
|
|
if site.AllocatableMilliCPU > 0 && site.WorkerRequestMilliCPU > 0 {
|
|
nonWorker := max(site.UsedMilliCPU-int64(site.CurrentReplicas)*site.WorkerRequestMilliCPU, 0)
|
|
budget := site.AllocatableMilliCPU*int64(cpuTargetPercent)/100 - nonWorker
|
|
cpuMax = int(max(budget, 0) / site.WorkerRequestMilliCPU)
|
|
}
|
|
return min(site.MaxReplicas, min(memoryMax, cpuMax)), memoryPercent, cpuPercent
|
|
}
|
|
|
|
func distributeDesiredReplicas(sites []SitePlan, desired int) {
|
|
if len(sites) == 0 {
|
|
return
|
|
}
|
|
assigned := 0
|
|
for index := range sites {
|
|
assigned += sites[index].DesiredReplicas
|
|
}
|
|
for assigned < desired {
|
|
sort.SliceStable(sites, func(left, right int) bool {
|
|
leftRoom := sites[left].ResourceMax - sites[left].DesiredReplicas
|
|
rightRoom := sites[right].ResourceMax - sites[right].DesiredReplicas
|
|
if leftRoom != rightRoom {
|
|
return leftRoom > rightRoom
|
|
}
|
|
if sites[left].DesiredReplicas != sites[right].DesiredReplicas {
|
|
return sites[left].DesiredReplicas < sites[right].DesiredReplicas
|
|
}
|
|
return sites[left].Site < sites[right].Site
|
|
})
|
|
if sites[0].DesiredReplicas >= sites[0].ResourceMax {
|
|
break
|
|
}
|
|
sites[0].DesiredReplicas++
|
|
assigned++
|
|
}
|
|
sort.Slice(sites, func(left, right int) bool { return sites[left].Site < sites[right].Site })
|
|
}
|
|
|
|
func usagePercent(used int64, allocatable int64) float64 {
|
|
if allocatable <= 0 {
|
|
return 0
|
|
}
|
|
return float64(used) * 100 / float64(allocatable)
|
|
}
|