perf(worker): 按实例内存上限扩展异步容量
将策略推导出的集群目标与单 Worker 内存安全上限分离,避免失活实例的全部容量转移到单个 2 GiB Pod。 生产环境单实例上限和媒体物化并发设为 24;两实例正常时总容量由 16 提升至 48,后续可通过增加 Worker 节点继续扩展。新增 0093 迁移保存实例容量上限,并兼容滚动发布中的旧实例。 风险:更高媒体并发会增加 Worker 和节点内存压力,保留单实例 24 的硬边界并由持续监控观察 RSS、OOM、队列与节点余量。 验证:go test ./... -count=1;go vet ./...;真实 PostgreSQL 分配与故障转移测试;迁移安全检查;kubectl kustomize;gofmt -l;git diff --check。
This commit is contained in:
@@ -625,4 +625,17 @@ WHERE instance_id = $1`, secondID, (workerHeartbeatStaleAfter + time.Second).Str
|
||||
if err != nil || first.Allocated != 5 || first.ActiveInstances != 1 {
|
||||
t.Fatalf("single-worker failover allocation = %+v, err=%v", first, err)
|
||||
}
|
||||
|
||||
first, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
|
||||
InstanceID: firstID, DesiredCapacity: 100, CapacityLimit: 3,
|
||||
})
|
||||
if err != nil || first.Allocated != 3 || first.GlobalAllocated != 3 {
|
||||
t.Fatalf("bounded single-worker allocation = %+v, err=%v", first, err)
|
||||
}
|
||||
second, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
|
||||
InstanceID: secondID, DesiredCapacity: 100, CapacityLimit: 2,
|
||||
})
|
||||
if err != nil || second.Allocated != 2 || second.GlobalAllocated != 5 || second.ActiveInstances != 2 {
|
||||
t.Fatalf("bounded two-worker allocation = %+v, err=%v", second, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ type WorkerRegistrationInput struct {
|
||||
Site string
|
||||
Revision string
|
||||
DesiredCapacity int
|
||||
CapacityLimit int
|
||||
HeartbeatStaleAfter time.Duration
|
||||
}
|
||||
|
||||
@@ -25,10 +26,16 @@ type WorkerAllocation struct {
|
||||
InstanceID string
|
||||
DesiredCapacity int
|
||||
Allocated int
|
||||
GlobalAllocated int
|
||||
ActiveInstances int
|
||||
HeartbeatAt time.Time
|
||||
}
|
||||
|
||||
type activeWorkerCapacity struct {
|
||||
InstanceID string
|
||||
CapacityLimit int
|
||||
}
|
||||
|
||||
// ActiveWorkerCapacity returns the cluster-wide execution capacity currently
|
||||
// owned by live Worker instances.
|
||||
func (s *Store) ActiveWorkerCapacity(ctx context.Context) (int, error) {
|
||||
@@ -49,6 +56,12 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
|
||||
if input.DesiredCapacity < 0 {
|
||||
return WorkerAllocation{}, errors.New("worker desired capacity cannot be negative")
|
||||
}
|
||||
if input.CapacityLimit < 0 {
|
||||
return WorkerAllocation{}, errors.New("worker capacity limit cannot be negative")
|
||||
}
|
||||
if input.CapacityLimit == 0 {
|
||||
input.CapacityLimit = input.DesiredCapacity
|
||||
}
|
||||
staleAfter := input.HeartbeatStaleAfter
|
||||
if staleAfter < workerHeartbeatStaleAfter {
|
||||
staleAfter = workerHeartbeatStaleAfter
|
||||
@@ -70,9 +83,9 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_worker_instances (
|
||||
instance_id, pod_uid, pod_name, site, revision, status,
|
||||
desired_capacity, allocated_capacity, started_at, heartbeat_at, updated_at
|
||||
desired_capacity, capacity_limit, allocated_capacity, started_at, heartbeat_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6, 0, now(), now(), now())
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, 0, now(), now(), now())
|
||||
ON CONFLICT (instance_id) DO UPDATE
|
||||
SET pod_uid = EXCLUDED.pod_uid,
|
||||
pod_name = EXCLUDED.pod_name,
|
||||
@@ -80,6 +93,7 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
revision = EXCLUDED.revision,
|
||||
status = 'active',
|
||||
desired_capacity = EXCLUDED.desired_capacity,
|
||||
capacity_limit = EXCLUDED.capacity_limit,
|
||||
heartbeat_at = now(),
|
||||
updated_at = now()`,
|
||||
input.InstanceID,
|
||||
@@ -88,6 +102,7 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
strings.TrimSpace(input.Site),
|
||||
strings.TrimSpace(input.Revision),
|
||||
input.DesiredCapacity,
|
||||
input.CapacityLimit,
|
||||
); err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
@@ -101,7 +116,7 @@ WHERE status <> 'active'
|
||||
}
|
||||
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT instance_id
|
||||
SELECT instance_id, capacity_limit
|
||||
FROM gateway_worker_instances
|
||||
WHERE status = 'active'
|
||||
AND heartbeat_at > now() - $1::interval
|
||||
@@ -110,41 +125,37 @@ FOR UPDATE`, staleAfter.String())
|
||||
if err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
activeIDs := make([]string, 0)
|
||||
activeWorkers := make([]activeWorkerCapacity, 0)
|
||||
for rows.Next() {
|
||||
var instanceID string
|
||||
if err := rows.Scan(&instanceID); err != nil {
|
||||
var worker activeWorkerCapacity
|
||||
if err := rows.Scan(&worker.InstanceID, &worker.CapacityLimit); err != nil {
|
||||
rows.Close()
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
activeIDs = append(activeIDs, instanceID)
|
||||
activeWorkers = append(activeWorkers, worker)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
rows.Close()
|
||||
if len(activeIDs) == 0 {
|
||||
if len(activeWorkers) == 0 {
|
||||
return WorkerAllocation{}, errors.New("worker registration was not active after heartbeat")
|
||||
}
|
||||
|
||||
base := input.DesiredCapacity / len(activeIDs)
|
||||
remainder := input.DesiredCapacity % len(activeIDs)
|
||||
allocations, globalAllocated := allocateWorkerCapacities(activeWorkers, input.DesiredCapacity)
|
||||
allocated := 0
|
||||
for index, instanceID := range activeIDs {
|
||||
capacity := base
|
||||
if index < remainder {
|
||||
capacity++
|
||||
}
|
||||
for _, worker := range activeWorkers {
|
||||
capacity := allocations[worker.InstanceID]
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_worker_instances
|
||||
SET desired_capacity = $2,
|
||||
allocated_capacity = $3,
|
||||
updated_at = now()
|
||||
WHERE instance_id = $1`, instanceID, input.DesiredCapacity, capacity); err != nil {
|
||||
WHERE instance_id = $1`, worker.InstanceID, input.DesiredCapacity, capacity); err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
if instanceID == input.InstanceID {
|
||||
if worker.InstanceID == input.InstanceID {
|
||||
allocated = capacity
|
||||
}
|
||||
}
|
||||
@@ -162,11 +173,42 @@ WHERE instance_id = $1`, input.InstanceID).Scan(&heartbeatAt); err != nil {
|
||||
InstanceID: input.InstanceID,
|
||||
DesiredCapacity: input.DesiredCapacity,
|
||||
Allocated: allocated,
|
||||
ActiveInstances: len(activeIDs),
|
||||
GlobalAllocated: globalAllocated,
|
||||
ActiveInstances: len(activeWorkers),
|
||||
HeartbeatAt: heartbeatAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func allocateWorkerCapacities(workers []activeWorkerCapacity, desired int) (map[string]int, int) {
|
||||
allocations := make(map[string]int, len(workers))
|
||||
if desired <= 0 || len(workers) == 0 {
|
||||
return allocations, 0
|
||||
}
|
||||
remaining := desired
|
||||
for remaining > 0 {
|
||||
progressed := false
|
||||
for _, worker := range workers {
|
||||
limit := worker.CapacityLimit
|
||||
if limit <= 0 {
|
||||
limit = desired
|
||||
}
|
||||
if allocations[worker.InstanceID] >= limit {
|
||||
continue
|
||||
}
|
||||
allocations[worker.InstanceID]++
|
||||
remaining--
|
||||
progressed = true
|
||||
if remaining == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !progressed {
|
||||
break
|
||||
}
|
||||
}
|
||||
return allocations, desired - remaining
|
||||
}
|
||||
|
||||
func (s *Store) MarkWorkerDraining(ctx context.Context, instanceID string) error {
|
||||
result, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_worker_instances
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAllocateWorkerCapacitiesHonorsInstanceLimits(t *testing.T) {
|
||||
workers := []activeWorkerCapacity{
|
||||
{InstanceID: "worker-a", CapacityLimit: 24},
|
||||
{InstanceID: "worker-b", CapacityLimit: 24},
|
||||
}
|
||||
allocations, global := allocateWorkerCapacities(workers, 310)
|
||||
if global != 48 || allocations["worker-a"] != 24 || allocations["worker-b"] != 24 {
|
||||
t.Fatalf("allocations=%v global=%d, want 24/24 and 48", allocations, global)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateWorkerCapacitiesBalancesBelowLimits(t *testing.T) {
|
||||
workers := []activeWorkerCapacity{
|
||||
{InstanceID: "worker-a", CapacityLimit: 24},
|
||||
{InstanceID: "worker-b", CapacityLimit: 24},
|
||||
}
|
||||
allocations, global := allocateWorkerCapacities(workers, 5)
|
||||
if global != 5 || allocations["worker-a"] != 3 || allocations["worker-b"] != 2 {
|
||||
t.Fatalf("allocations=%v global=%d, want 3/2 and 5", allocations, global)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateWorkerCapacitiesKeepsSurvivorBounded(t *testing.T) {
|
||||
workers := []activeWorkerCapacity{{InstanceID: "worker-a", CapacityLimit: 24}}
|
||||
allocations, global := allocateWorkerCapacities(workers, 310)
|
||||
if global != 24 || allocations["worker-a"] != 24 {
|
||||
t.Fatalf("allocations=%v global=%d, want survivor capped at 24", allocations, global)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateWorkerCapacitiesSupportsUnequalLimits(t *testing.T) {
|
||||
workers := []activeWorkerCapacity{
|
||||
{InstanceID: "worker-a", CapacityLimit: 1},
|
||||
{InstanceID: "worker-b", CapacityLimit: 4},
|
||||
}
|
||||
allocations, global := allocateWorkerCapacities(workers, 5)
|
||||
if global != 5 || allocations["worker-a"] != 1 || allocations["worker-b"] != 4 {
|
||||
t.Fatalf("allocations=%v global=%d, want 1/4 and 5", allocations, global)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user