feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -12,14 +12,24 @@ import (
|
||||
const workerHeartbeatStaleAfter = 30 * time.Second
|
||||
|
||||
type WorkerRegistrationInput struct {
|
||||
InstanceID string
|
||||
PodUID string
|
||||
PodName string
|
||||
Site string
|
||||
Revision string
|
||||
DesiredCapacity int
|
||||
CapacityLimit int
|
||||
HeartbeatStaleAfter time.Duration
|
||||
InstanceID string
|
||||
PodUID string
|
||||
PodName string
|
||||
Site string
|
||||
Revision string
|
||||
DesiredCapacity int
|
||||
CapacityLimit int
|
||||
LoadMode string
|
||||
SafeCapacity int
|
||||
HeavyCapacity int
|
||||
ActiveTasks int
|
||||
PreparingTasks int
|
||||
WaitingUpstreamTasks int
|
||||
FinalizingTasks int
|
||||
PressureState string
|
||||
PressureReason string
|
||||
LoadSampledAt time.Time
|
||||
HeartbeatStaleAfter time.Duration
|
||||
}
|
||||
|
||||
type WorkerAllocation struct {
|
||||
@@ -59,9 +69,28 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
|
||||
if input.CapacityLimit < 0 {
|
||||
return WorkerAllocation{}, errors.New("worker capacity limit cannot be negative")
|
||||
}
|
||||
if input.SafeCapacity < 0 || input.HeavyCapacity < 0 || input.ActiveTasks < 0 || input.PreparingTasks < 0 || input.WaitingUpstreamTasks < 0 || input.FinalizingTasks < 0 {
|
||||
return WorkerAllocation{}, errors.New("worker load values cannot be negative")
|
||||
}
|
||||
if input.ActiveTasks != input.PreparingTasks+input.WaitingUpstreamTasks+input.FinalizingTasks {
|
||||
return WorkerAllocation{}, errors.New("worker active task count must equal phase task counts")
|
||||
}
|
||||
if input.CapacityLimit == 0 {
|
||||
input.CapacityLimit = input.DesiredCapacity
|
||||
}
|
||||
hardCapacityLimit := input.CapacityLimit
|
||||
if strings.EqualFold(strings.TrimSpace(input.LoadMode), "adaptive") {
|
||||
input.CapacityLimit = min(input.CapacityLimit, input.SafeCapacity)
|
||||
}
|
||||
pressureState := strings.ToLower(strings.TrimSpace(input.PressureState))
|
||||
switch pressureState {
|
||||
case "normal", "busy", "critical":
|
||||
default:
|
||||
pressureState = "unknown"
|
||||
}
|
||||
if input.LoadSampledAt.IsZero() {
|
||||
input.LoadSampledAt = time.Now()
|
||||
}
|
||||
staleAfter := input.HeartbeatStaleAfter
|
||||
if staleAfter < workerHeartbeatStaleAfter {
|
||||
staleAfter = workerHeartbeatStaleAfter
|
||||
@@ -83,9 +112,16 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_worker_instances (
|
||||
instance_id, pod_uid, pod_name, site, revision, status,
|
||||
desired_capacity, capacity_limit, allocated_capacity, started_at, heartbeat_at, updated_at
|
||||
desired_capacity, capacity_limit, hard_capacity_limit, safe_capacity, heavy_capacity,
|
||||
active_tasks, preparing_tasks, waiting_upstream_tasks, finalizing_tasks,
|
||||
pressure_state, pressure_reason, load_sampled_at,
|
||||
allocated_capacity, started_at, heartbeat_at, updated_at
|
||||
)
|
||||
VALUES (
|
||||
$1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10,
|
||||
$11, $12, $13, $14, $15, $16, $17,
|
||||
0, now(), now(), now()
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, 0, now(), now(), now())
|
||||
ON CONFLICT (instance_id) DO UPDATE
|
||||
SET pod_uid = EXCLUDED.pod_uid,
|
||||
pod_name = EXCLUDED.pod_name,
|
||||
@@ -97,6 +133,16 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
END,
|
||||
desired_capacity = EXCLUDED.desired_capacity,
|
||||
capacity_limit = EXCLUDED.capacity_limit,
|
||||
hard_capacity_limit = EXCLUDED.hard_capacity_limit,
|
||||
safe_capacity = EXCLUDED.safe_capacity,
|
||||
heavy_capacity = EXCLUDED.heavy_capacity,
|
||||
active_tasks = EXCLUDED.active_tasks,
|
||||
preparing_tasks = EXCLUDED.preparing_tasks,
|
||||
waiting_upstream_tasks = EXCLUDED.waiting_upstream_tasks,
|
||||
finalizing_tasks = EXCLUDED.finalizing_tasks,
|
||||
pressure_state = EXCLUDED.pressure_state,
|
||||
pressure_reason = EXCLUDED.pressure_reason,
|
||||
load_sampled_at = EXCLUDED.load_sampled_at,
|
||||
heartbeat_at = now(),
|
||||
updated_at = now()`,
|
||||
input.InstanceID,
|
||||
@@ -106,6 +152,16 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
strings.TrimSpace(input.Revision),
|
||||
input.DesiredCapacity,
|
||||
input.CapacityLimit,
|
||||
hardCapacityLimit,
|
||||
input.SafeCapacity,
|
||||
input.HeavyCapacity,
|
||||
input.ActiveTasks,
|
||||
input.PreparingTasks,
|
||||
input.WaitingUpstreamTasks,
|
||||
input.FinalizingTasks,
|
||||
pressureState,
|
||||
strings.TrimSpace(input.PressureReason),
|
||||
input.LoadSampledAt,
|
||||
); err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
@@ -198,7 +254,7 @@ func allocateWorkerCapacities(workers []activeWorkerCapacity, desired int) (map[
|
||||
for _, worker := range workers {
|
||||
limit := worker.CapacityLimit
|
||||
if limit <= 0 {
|
||||
limit = desired
|
||||
continue
|
||||
}
|
||||
if allocations[worker.InstanceID] >= limit {
|
||||
continue
|
||||
@@ -254,24 +310,52 @@ WHERE instance_id = $1
|
||||
}
|
||||
|
||||
type WorkerInstanceRuntime struct {
|
||||
InstanceID string
|
||||
PodUID string
|
||||
PodName string
|
||||
Site string
|
||||
Revision string
|
||||
Status string
|
||||
Allocated int
|
||||
CapacityLimit int
|
||||
RunningTasks int
|
||||
ActiveLeases int
|
||||
HeartbeatAt time.Time
|
||||
DrainingAt *time.Time
|
||||
InstanceID string `json:"instanceId"`
|
||||
PodUID string `json:"podUid,omitempty"`
|
||||
PodName string `json:"podName,omitempty"`
|
||||
Site string `json:"site,omitempty"`
|
||||
Revision string `json:"revision,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Allocated int `json:"allocatedCapacity"`
|
||||
CapacityLimit int `json:"capacityLimit"`
|
||||
HardCapacityLimit int `json:"hardCapacityLimit"`
|
||||
SafeCapacity int `json:"safeCapacity"`
|
||||
HeavyCapacity int `json:"heavyCapacity"`
|
||||
ReportedActiveTasks int `json:"reportedActiveTasks"`
|
||||
PreparingTasks int `json:"preparingTasks"`
|
||||
WaitingUpstreamTasks int `json:"waitingUpstreamTasks"`
|
||||
FinalizingTasks int `json:"finalizingTasks"`
|
||||
PressureState string `json:"pressureState"`
|
||||
PressureReason string `json:"pressureReason,omitempty"`
|
||||
LoadSampledAt *time.Time `json:"loadSampledAt,omitempty"`
|
||||
RunningTasks int `json:"runningTasks"`
|
||||
ActiveLeases int `json:"activeLeases"`
|
||||
HeartbeatAt time.Time `json:"heartbeatAt"`
|
||||
DrainingAt *time.Time `json:"drainingAt,omitempty"`
|
||||
}
|
||||
|
||||
type WorkerQueueRuntime struct {
|
||||
Queued int
|
||||
Running int
|
||||
OldestWaitSeconds float64
|
||||
Queued int `json:"queued"`
|
||||
Running int `json:"running"`
|
||||
OldestWaitSeconds float64 `json:"oldestWaitSeconds"`
|
||||
}
|
||||
|
||||
type WorkerClusterRuntime struct {
|
||||
Workers []WorkerInstanceRuntime `json:"workers"`
|
||||
Queue WorkerQueueRuntime `json:"queue"`
|
||||
CapturedAt time.Time `json:"capturedAt"`
|
||||
}
|
||||
|
||||
func (s *Store) GetWorkerClusterRuntime(ctx context.Context) (WorkerClusterRuntime, error) {
|
||||
workers, err := s.ListWorkerInstanceRuntime(ctx)
|
||||
if err != nil {
|
||||
return WorkerClusterRuntime{}, err
|
||||
}
|
||||
queue, err := s.WorkerQueueRuntime(ctx)
|
||||
if err != nil {
|
||||
return WorkerClusterRuntime{}, err
|
||||
}
|
||||
return WorkerClusterRuntime{Workers: workers, Queue: queue, CapturedAt: time.Now()}, nil
|
||||
}
|
||||
|
||||
type CapacityDatabaseHealth struct {
|
||||
@@ -306,6 +390,16 @@ SELECT worker.instance_id,
|
||||
worker.status,
|
||||
worker.allocated_capacity,
|
||||
worker.capacity_limit,
|
||||
worker.hard_capacity_limit,
|
||||
worker.safe_capacity,
|
||||
worker.heavy_capacity,
|
||||
worker.active_tasks,
|
||||
worker.preparing_tasks,
|
||||
worker.waiting_upstream_tasks,
|
||||
worker.finalizing_tasks,
|
||||
worker.pressure_state,
|
||||
worker.pressure_reason,
|
||||
worker.load_sampled_at,
|
||||
count(DISTINCT task.id) FILTER (WHERE task.status = 'running')::int,
|
||||
count(DISTINCT lease.id) FILTER (WHERE lease.released_at IS NULL)::int,
|
||||
worker.heartbeat_at,
|
||||
@@ -339,6 +433,16 @@ ORDER BY worker.site ASC, worker.status DESC, worker.instance_id ASC`,
|
||||
&instance.Status,
|
||||
&instance.Allocated,
|
||||
&instance.CapacityLimit,
|
||||
&instance.HardCapacityLimit,
|
||||
&instance.SafeCapacity,
|
||||
&instance.HeavyCapacity,
|
||||
&instance.ReportedActiveTasks,
|
||||
&instance.PreparingTasks,
|
||||
&instance.WaitingUpstreamTasks,
|
||||
&instance.FinalizingTasks,
|
||||
&instance.PressureState,
|
||||
&instance.PressureReason,
|
||||
&instance.LoadSampledAt,
|
||||
&instance.RunningTasks,
|
||||
&instance.ActiveLeases,
|
||||
&instance.HeartbeatAt,
|
||||
@@ -476,7 +580,7 @@ WITH orphaned AS MATERIALIZED (
|
||||
),
|
||||
released_leases AS (
|
||||
UPDATE gateway_concurrency_leases lease
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
FROM orphaned
|
||||
WHERE lease.task_id = orphaned.task_id
|
||||
AND lease.released_at IS NULL
|
||||
|
||||
Reference in New Issue
Block a user