feat(queue): 增加非文本模型分布式准入队列
使用 PostgreSQL 统一同步与异步非文本任务的并发准入、持久化等待和 Worker 容量分配，并将生产 API 与独立 Worker 角色拆分。 补充策略管理、共享契约、OpenAPI、Kubernetes 双节点 Worker 清单及跨节点验收脚本；未默认启用任何生产 queue_size 策略。 已在原基线完成 Go、前端、迁移、Shell、Kustomize 与长任务容量验收；合入最新主干后将重新执行发布门禁。
This commit is contained in:
@@ -20,6 +20,84 @@ type RuntimeRecoveryResult struct {
|
||||
|
||||
var ErrConcurrencyLeaseLost = errors.New("concurrency lease lost")
|
||||
|
||||
// CheckRateLimits performs a non-consuming admission preflight for fixed-window
|
||||
// limits. The same limits are checked and reserved again when execution starts.
|
||||
func (s *Store) CheckRateLimits(ctx context.Context, reservations []RateLimitReservation) error {
|
||||
for _, reservation := range reservations {
|
||||
if reservation.Limit <= 0 || reservation.Amount <= 0 || reservation.Metric == "concurrent" || reservation.Metric == "queue_size" {
|
||||
continue
|
||||
}
|
||||
if reservation.Metric == "" || reservation.Amount > reservation.Limit {
|
||||
return &RateLimitExceededError{
|
||||
ScopeType: reservation.ScopeType,
|
||||
ScopeKey: reservation.ScopeKey,
|
||||
ScopeName: reservation.ScopeName,
|
||||
ScopeMetadata: reservation.ScopeMetadata,
|
||||
Metric: reservation.Metric,
|
||||
Limit: reservation.Limit,
|
||||
Amount: reservation.Amount,
|
||||
Projected: reservation.Amount,
|
||||
WindowSeconds: reservation.WindowSeconds,
|
||||
Policy: reservation.Policy,
|
||||
Message: fmt.Sprintf("rate limit exceeded: %s request amount %.0f is greater than limit %.0f", reservation.Metric, reservation.Amount, reservation.Limit),
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
windowSeconds := reservation.WindowSeconds
|
||||
if windowSeconds <= 0 {
|
||||
windowSeconds = 60
|
||||
}
|
||||
var used float64
|
||||
var reserved float64
|
||||
var resetAt time.Time
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
WITH bounds AS (
|
||||
SELECT to_timestamp(floor(extract(epoch FROM now()) / $4::int) * $4::int) AS window_start,
|
||||
to_timestamp(floor(extract(epoch FROM now()) / $4::int) * $4::int) + ($4::int * interval '1 second') AS reset_at
|
||||
)
|
||||
SELECT COALESCE(counters.used_value, 0)::float8,
|
||||
COALESCE(counters.reserved_value, 0)::float8,
|
||||
bounds.reset_at
|
||||
FROM bounds
|
||||
LEFT JOIN gateway_rate_limit_counters counters
|
||||
ON counters.scope_type = $1
|
||||
AND counters.scope_key = $2
|
||||
AND counters.metric = $3
|
||||
AND counters.window_start = bounds.window_start`,
|
||||
reservation.ScopeType,
|
||||
reservation.ScopeKey,
|
||||
reservation.Metric,
|
||||
windowSeconds,
|
||||
).Scan(&used, &reserved, &resetAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
current := used + reserved
|
||||
if current+reservation.Amount > reservation.Limit {
|
||||
return &RateLimitExceededError{
|
||||
ScopeType: reservation.ScopeType,
|
||||
ScopeKey: reservation.ScopeKey,
|
||||
ScopeName: reservation.ScopeName,
|
||||
ScopeMetadata: reservation.ScopeMetadata,
|
||||
Metric: reservation.Metric,
|
||||
Limit: reservation.Limit,
|
||||
Amount: reservation.Amount,
|
||||
Current: current,
|
||||
Used: used,
|
||||
Reserved: reserved,
|
||||
Projected: current + reservation.Amount,
|
||||
WindowSeconds: windowSeconds,
|
||||
ResetAt: resetAt,
|
||||
Policy: reservation.Policy,
|
||||
Message: fmt.Sprintf("rate limit exceeded: %s window has no remaining capacity", reservation.Metric),
|
||||
RetryAfter: retryAfterUntil(resetAt),
|
||||
Retryable: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ReserveRateLimits(ctx context.Context, taskID string, attemptID string, reservations []RateLimitReservation) (RateLimitResult, error) {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
@@ -30,6 +108,9 @@ func (s *Store) ReserveRateLimits(ctx context.Context, taskID string, attemptID
|
||||
lockKeys := make([]string, 0)
|
||||
lockKeySet := make(map[string]struct{})
|
||||
for _, reservation := range reservations {
|
||||
if reservation.Metric == "queue_size" {
|
||||
continue
|
||||
}
|
||||
if reservation.Metric != "concurrent" || reservation.Limit <= 0 || reservation.Amount <= 0 {
|
||||
continue
|
||||
}
|
||||
@@ -377,15 +458,28 @@ func (s *Store) RecoverInterruptedRuntimeState(ctx context.Context) (RuntimeReco
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('gateway-runtime-recovery', 0))`); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
|
||||
result := RuntimeRecoveryResult{}
|
||||
rows, err := tx.Query(ctx, `
|
||||
UPDATE gateway_rate_limit_reservations
|
||||
UPDATE gateway_rate_limit_reservations reservation
|
||||
SET status = 'released',
|
||||
reason = 'server_restarted',
|
||||
reason = 'execution_lease_expired',
|
||||
finalized_at = now(),
|
||||
updated_at = now()
|
||||
WHERE status = 'reserved'
|
||||
FROM gateway_tasks task
|
||||
WHERE reservation.task_id = task.id
|
||||
AND reservation.status = 'reserved'
|
||||
AND (
|
||||
task.status IN ('succeeded', 'failed', 'cancelled')
|
||||
OR (
|
||||
task.status IN ('queued', 'running')
|
||||
AND task.execution_lease_expires_at IS NOT NULL
|
||||
AND task.execution_lease_expires_at <= now()
|
||||
)
|
||||
)
|
||||
RETURNING scope_type, scope_key, metric, window_start, reserved_amount::float8`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
@@ -415,7 +509,7 @@ RETURNING scope_type, scope_key, metric, window_start, reserved_amount::float8`)
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
WHERE released_at IS NULL
|
||||
AND expires_at > now()`)
|
||||
AND expires_at <= now()`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
@@ -425,10 +519,22 @@ WHERE released_at IS NULL
|
||||
UPDATE gateway_task_attempts
|
||||
SET status = 'failed',
|
||||
retryable = true,
|
||||
error_code = 'server_restarted',
|
||||
error_message = 'attempt interrupted by service restart',
|
||||
error_code = 'execution_lease_expired',
|
||||
error_message = 'attempt execution lease expired',
|
||||
finished_at = now()
|
||||
WHERE status = 'running'`)
|
||||
WHERE status = 'running'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_tasks task
|
||||
WHERE task.id = gateway_task_attempts.task_id
|
||||
AND (
|
||||
task.status IN ('succeeded', 'failed', 'cancelled')
|
||||
OR (
|
||||
task.execution_lease_expires_at IS NOT NULL
|
||||
AND task.execution_lease_expires_at <= now()
|
||||
)
|
||||
)
|
||||
)`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
@@ -448,6 +554,8 @@ SET status = 'queued',
|
||||
updated_at = now()
|
||||
WHERE async_mode = true
|
||||
AND status = 'running'
|
||||
AND execution_lease_expires_at IS NOT NULL
|
||||
AND execution_lease_expires_at <= now()
|
||||
RETURNING id::text`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
@@ -516,6 +624,8 @@ SET status = 'failed',
|
||||
updated_at = now()
|
||||
WHERE async_mode = false
|
||||
AND status = 'running'
|
||||
AND execution_lease_expires_at IS NOT NULL
|
||||
AND execution_lease_expires_at <= now()
|
||||
RETURNING id::text`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
|
||||
Reference in New Issue
Block a user