perf(worker): 按实例内存上限扩展异步容量
将策略推导出的集群目标与单 Worker 内存安全上限分离,避免失活实例的全部容量转移到单个 2 GiB Pod。 生产环境单实例上限和媒体物化并发设为 24;两实例正常时总容量由 16 提升至 48,后续可通过增加 Worker 节点继续扩展。新增 0093 迁移保存实例容量上限,并兼容滚动发布中的旧实例。 风险:更高媒体并发会增加 Worker 和节点内存压力,保留单实例 24 的硬边界并由持续监控观察 RSS、OOM、队列与节点余量。 验证:go test ./... -count=1;go vet ./...;真实 PostgreSQL 分配与故障转移测试;迁移安全检查;kubectl kustomize;gofmt -l;git diff --check。
This commit is contained in:
@@ -18,6 +18,7 @@ type WorkerRegistrationInput struct {
|
||||
Site string
|
||||
Revision string
|
||||
DesiredCapacity int
|
||||
CapacityLimit int
|
||||
HeartbeatStaleAfter time.Duration
|
||||
}
|
||||
|
||||
@@ -25,10 +26,16 @@ type WorkerAllocation struct {
|
||||
InstanceID string
|
||||
DesiredCapacity int
|
||||
Allocated int
|
||||
GlobalAllocated int
|
||||
ActiveInstances int
|
||||
HeartbeatAt time.Time
|
||||
}
|
||||
|
||||
type activeWorkerCapacity struct {
|
||||
InstanceID string
|
||||
CapacityLimit int
|
||||
}
|
||||
|
||||
// ActiveWorkerCapacity returns the cluster-wide execution capacity currently
|
||||
// owned by live Worker instances.
|
||||
func (s *Store) ActiveWorkerCapacity(ctx context.Context) (int, error) {
|
||||
@@ -49,6 +56,12 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
|
||||
if input.DesiredCapacity < 0 {
|
||||
return WorkerAllocation{}, errors.New("worker desired capacity cannot be negative")
|
||||
}
|
||||
if input.CapacityLimit < 0 {
|
||||
return WorkerAllocation{}, errors.New("worker capacity limit cannot be negative")
|
||||
}
|
||||
if input.CapacityLimit == 0 {
|
||||
input.CapacityLimit = input.DesiredCapacity
|
||||
}
|
||||
staleAfter := input.HeartbeatStaleAfter
|
||||
if staleAfter < workerHeartbeatStaleAfter {
|
||||
staleAfter = workerHeartbeatStaleAfter
|
||||
@@ -70,9 +83,9 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_worker_instances (
|
||||
instance_id, pod_uid, pod_name, site, revision, status,
|
||||
desired_capacity, allocated_capacity, started_at, heartbeat_at, updated_at
|
||||
desired_capacity, capacity_limit, allocated_capacity, started_at, heartbeat_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6, 0, now(), now(), now())
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, 0, now(), now(), now())
|
||||
ON CONFLICT (instance_id) DO UPDATE
|
||||
SET pod_uid = EXCLUDED.pod_uid,
|
||||
pod_name = EXCLUDED.pod_name,
|
||||
@@ -80,6 +93,7 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
revision = EXCLUDED.revision,
|
||||
status = 'active',
|
||||
desired_capacity = EXCLUDED.desired_capacity,
|
||||
capacity_limit = EXCLUDED.capacity_limit,
|
||||
heartbeat_at = now(),
|
||||
updated_at = now()`,
|
||||
input.InstanceID,
|
||||
@@ -88,6 +102,7 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
strings.TrimSpace(input.Site),
|
||||
strings.TrimSpace(input.Revision),
|
||||
input.DesiredCapacity,
|
||||
input.CapacityLimit,
|
||||
); err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
@@ -101,7 +116,7 @@ WHERE status <> 'active'
|
||||
}
|
||||
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT instance_id
|
||||
SELECT instance_id, capacity_limit
|
||||
FROM gateway_worker_instances
|
||||
WHERE status = 'active'
|
||||
AND heartbeat_at > now() - $1::interval
|
||||
@@ -110,41 +125,37 @@ FOR UPDATE`, staleAfter.String())
|
||||
if err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
activeIDs := make([]string, 0)
|
||||
activeWorkers := make([]activeWorkerCapacity, 0)
|
||||
for rows.Next() {
|
||||
var instanceID string
|
||||
if err := rows.Scan(&instanceID); err != nil {
|
||||
var worker activeWorkerCapacity
|
||||
if err := rows.Scan(&worker.InstanceID, &worker.CapacityLimit); err != nil {
|
||||
rows.Close()
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
activeIDs = append(activeIDs, instanceID)
|
||||
activeWorkers = append(activeWorkers, worker)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
rows.Close()
|
||||
if len(activeIDs) == 0 {
|
||||
if len(activeWorkers) == 0 {
|
||||
return WorkerAllocation{}, errors.New("worker registration was not active after heartbeat")
|
||||
}
|
||||
|
||||
base := input.DesiredCapacity / len(activeIDs)
|
||||
remainder := input.DesiredCapacity % len(activeIDs)
|
||||
allocations, globalAllocated := allocateWorkerCapacities(activeWorkers, input.DesiredCapacity)
|
||||
allocated := 0
|
||||
for index, instanceID := range activeIDs {
|
||||
capacity := base
|
||||
if index < remainder {
|
||||
capacity++
|
||||
}
|
||||
for _, worker := range activeWorkers {
|
||||
capacity := allocations[worker.InstanceID]
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_worker_instances
|
||||
SET desired_capacity = $2,
|
||||
allocated_capacity = $3,
|
||||
updated_at = now()
|
||||
WHERE instance_id = $1`, instanceID, input.DesiredCapacity, capacity); err != nil {
|
||||
WHERE instance_id = $1`, worker.InstanceID, input.DesiredCapacity, capacity); err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
if instanceID == input.InstanceID {
|
||||
if worker.InstanceID == input.InstanceID {
|
||||
allocated = capacity
|
||||
}
|
||||
}
|
||||
@@ -162,11 +173,42 @@ WHERE instance_id = $1`, input.InstanceID).Scan(&heartbeatAt); err != nil {
|
||||
InstanceID: input.InstanceID,
|
||||
DesiredCapacity: input.DesiredCapacity,
|
||||
Allocated: allocated,
|
||||
ActiveInstances: len(activeIDs),
|
||||
GlobalAllocated: globalAllocated,
|
||||
ActiveInstances: len(activeWorkers),
|
||||
HeartbeatAt: heartbeatAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func allocateWorkerCapacities(workers []activeWorkerCapacity, desired int) (map[string]int, int) {
|
||||
allocations := make(map[string]int, len(workers))
|
||||
if desired <= 0 || len(workers) == 0 {
|
||||
return allocations, 0
|
||||
}
|
||||
remaining := desired
|
||||
for remaining > 0 {
|
||||
progressed := false
|
||||
for _, worker := range workers {
|
||||
limit := worker.CapacityLimit
|
||||
if limit <= 0 {
|
||||
limit = desired
|
||||
}
|
||||
if allocations[worker.InstanceID] >= limit {
|
||||
continue
|
||||
}
|
||||
allocations[worker.InstanceID]++
|
||||
remaining--
|
||||
progressed = true
|
||||
if remaining == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !progressed {
|
||||
break
|
||||
}
|
||||
}
|
||||
return allocations, desired - remaining
|
||||
}
|
||||
|
||||
func (s *Store) MarkWorkerDraining(ctx context.Context, instanceID string) error {
|
||||
result, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_worker_instances
|
||||
|
||||
Reference in New Issue
Block a user