feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -747,14 +747,14 @@ SELECT COUNT(*)
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()`, taskID).Scan(&count)
|
||||
AND expires_at > statement_timestamp()`, taskID).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func resetTaskAdmissionToWaitingTx(ctx context.Context, tx pgx.Tx, input TaskAdmissionInput) (TaskAdmission, error) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, input.TaskID); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
@@ -862,7 +862,7 @@ func (s *Store) deleteTaskAdmissionOnce(ctx context.Context, taskID string) erro
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, taskID); err != nil {
|
||||
return err
|
||||
@@ -884,7 +884,7 @@ SELECT id::text,
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()
|
||||
AND expires_at > statement_timestamp()
|
||||
ORDER BY scope_type, scope_key, id`, taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -924,7 +924,7 @@ WHERE admission.mode = 'async'
|
||||
FROM gateway_concurrency_leases lease
|
||||
WHERE lease.task_id = admission.task_id
|
||||
AND lease.released_at IS NULL
|
||||
AND lease.expires_at > now()
|
||||
AND lease.expires_at > statement_timestamp()
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -980,7 +980,7 @@ WHERE admission.mode = 'async'
|
||||
FROM gateway_concurrency_leases lease
|
||||
WHERE lease.task_id = admission.task_id
|
||||
AND lease.released_at IS NULL
|
||||
AND lease.expires_at > now()
|
||||
AND lease.expires_at > statement_timestamp()
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -1006,6 +1006,29 @@ LIMIT $1`, limit)
|
||||
return admissions, rows.Err()
|
||||
}
|
||||
|
||||
// RequestWaitingTaskAdmissionReselect marks every queued task bound to a
|
||||
// saturated platform model so the dispatcher can route the next batch to a
|
||||
// different eligible candidate. The durable marker lets multiple dispatchers
|
||||
// observe the same decision without assigning tasks to a specific Worker.
|
||||
func (s *Store) RequestWaitingTaskAdmissionReselect(ctx context.Context, platformModelID string) (int64, error) {
|
||||
result, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_task_admissions admission
|
||||
SET reselect_requested_at = now(),
|
||||
updated_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE admission.task_id = task.id
|
||||
AND admission.platform_model_id = $1::uuid
|
||||
AND admission.mode = 'async'
|
||||
AND admission.status = 'waiting'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.reselect_requested_at IS NULL`, platformModelID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// ListWaitingTaskAdmissionIDs returns one FIFO leader for every independent
|
||||
// platform-model queue, additionally requiring the task to lead its user-group
|
||||
// queue when one exists. It is used after capacity is released so API
|
||||
@@ -1268,12 +1291,12 @@ func admissionScopeStatesTx(ctx context.Context, tx pgx.Tx, scopes []AdmissionSc
|
||||
if scope.ConcurrentLimit > 0 {
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(lease_value), 0)::float8,
|
||||
COALESCE(MIN(expires_at), now() + interval '1 second')
|
||||
COALESCE(MIN(expires_at), statement_timestamp() + interval '1 second')
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE scope_type = $1
|
||||
AND scope_key = $2
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()`, scope.ScopeType, scope.ScopeKey).Scan(&state.Active, &state.NextLeaseExpiration); err != nil {
|
||||
AND expires_at > statement_timestamp()`, scope.ScopeType, scope.ScopeKey).Scan(&state.Active, &state.NextLeaseExpiration); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
state.Saturated = state.Active+scope.Amount > scope.ConcurrentLimit
|
||||
|
||||
Reference in New Issue
Block a user