feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -184,12 +184,12 @@ func reserveConcurrencyLease(ctx context.Context, tx pgx.Tx, taskID string, atte
|
||||
var nextAvailableAt time.Time
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(lease_value), 0)::float8,
|
||||
COALESCE(MIN(expires_at), now() + ($3::int * interval '1 second'))
|
||||
COALESCE(MIN(expires_at), statement_timestamp() + ($3::int * interval '1 second'))
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE scope_type = $1
|
||||
AND scope_key = $2
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()`,
|
||||
AND expires_at > statement_timestamp()`,
|
||||
reservation.ScopeType,
|
||||
reservation.ScopeKey,
|
||||
reservation.LeaseTTLSeconds,
|
||||
@@ -218,14 +218,20 @@ WHERE scope_type = $1
|
||||
}
|
||||
var leaseID string
|
||||
if err := tx.QueryRow(ctx, `
|
||||
INSERT INTO gateway_concurrency_leases (task_id, attempt_id, scope_type, scope_key, lease_value, expires_at)
|
||||
VALUES ($1::uuid, NULLIF($2, '')::uuid, $3, $4, $5, now() + ($6::int * interval '1 second'))
|
||||
INSERT INTO gateway_concurrency_leases (
|
||||
task_id, attempt_id, scope_type, scope_key, lease_value, limit_value, acquired_at, expires_at
|
||||
)
|
||||
VALUES (
|
||||
$1::uuid, NULLIF($2, '')::uuid, $3, $4, $5, $6,
|
||||
statement_timestamp(), statement_timestamp() + ($7::int * interval '1 second')
|
||||
)
|
||||
RETURNING id::text`,
|
||||
taskID,
|
||||
attemptID,
|
||||
reservation.ScopeType,
|
||||
reservation.ScopeKey,
|
||||
reservation.Amount,
|
||||
reservation.Limit,
|
||||
reservation.LeaseTTLSeconds,
|
||||
).Scan(&leaseID); err != nil {
|
||||
return ConcurrencyLease{}, err
|
||||
@@ -381,7 +387,7 @@ func (s *Store) ReleaseConcurrencyLeases(ctx context.Context, leases []Concurren
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE id = ANY($1::uuid[]) AND released_at IS NULL`, leaseIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -411,7 +417,7 @@ func (s *Store) RenewConcurrencyLeases(ctx context.Context, leases []Concurrency
|
||||
}
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases lease
|
||||
SET expires_at = now() + (renewal.ttl_seconds * interval '1 second')
|
||||
SET expires_at = statement_timestamp() + (renewal.ttl_seconds * interval '1 second')
|
||||
FROM unnest($1::uuid[], $2::int[]) AS renewal(id, ttl_seconds)
|
||||
WHERE lease.id = renewal.id
|
||||
AND lease.released_at IS NULL
|
||||
@@ -705,7 +711,7 @@ WHERE attempt.task_id = task.id
|
||||
result.FailedAttempts = tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = ANY($1::uuid[])
|
||||
AND released_at IS NULL`, taskIDs)
|
||||
if err != nil {
|
||||
@@ -772,7 +778,7 @@ FOR UPDATE OF task SKIP LOCKED`, runtimeRecoveryBatchSize)
|
||||
var result RuntimeRecoveryResult
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = ANY($1::uuid[])
|
||||
AND released_at IS NULL`, taskIDs)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user