perf(queue): 限制大媒体任务内存并消除准入惊群

将同步与异步非文本任务的准入唤醒改为按任务和全局队首推进,批量续租等待者,避免大量请求同时争抢 advisory lock 和数据库连接。

对 Base64 请求解析、素材解码、上游媒体执行和结果物化增加分层并发限制,复用请求素材并释放重复 Gemini wire 数据;生产 API 默认入口解析并发 16,媒体物化并发 8。

新增 1000 个同步 Gemini 图像编辑请求的模拟上游压力验收,10 MiB 输入和输出下全部成功,Heap 峰值增长约 2.58 GiB,并验证共享上传哈希、单次 attempt 与本地零落盘。

验证:go test ./... -count=1;go vet ./...;gofmt -l 无输出;kubectl kustomize deploy/kubernetes/production;10 MiB Gemini Base64 千任务压力测试通过。
This commit is contained in:
2026-07-30 13:13:32 +08:00
parent fd3b6bf042
commit ea58d21d03
26 changed files with 1980 additions and 74 deletions
+218 -4
View File
@@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
"math"
"sort"
"strings"
"time"
@@ -15,6 +16,7 @@ import (
const (
admissionWaiterLeaseTTL = 15 * time.Second
admissionNotifyChannel = "gateway_task_admission"
executionSlotWaitMax = 24 * time.Hour
)
var ErrQueueTimeout = errors.New("task admission queue wait timed out")
@@ -113,6 +115,111 @@ func (s *Store) TryTaskAdmissionWithAdmittedHook(
return s.tryTaskAdmission(ctx, input, onAdmitted)
}
// QueueTaskAdmissionWithHook durably registers an asynchronous FIFO waiter and
// runs onQueued in the same transaction. It intentionally does not reserve
// concurrency leases: the Worker dispatcher obtains business and execution
// capacity only when the task reaches the head of every applicable queue.
func (s *Store) QueueTaskAdmissionWithHook(
ctx context.Context,
input TaskAdmissionInput,
onQueued func(pgx.Tx) error,
) (TaskAdmission, error) {
if err := validateTaskAdmissionInput(input); err != nil {
return TaskAdmission{}, err
}
if input.Mode != "async" {
return TaskAdmission{}, errors.New("queued admission registration requires asynchronous mode")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return TaskAdmission{}, err
}
defer tx.Rollback(ctx)
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, "task-admission:"+input.TaskID); err != nil {
return TaskAdmission{}, err
}
var taskActive bool
if err := tx.QueryRow(ctx, `
SELECT status IN ('queued', 'running')
FROM gateway_tasks
WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
return TaskAdmission{}, err
}
if !taskActive {
return TaskAdmission{}, ErrTaskExecutionFinished
}
admission, found, err := loadTaskAdmissionTx(ctx, tx, input.TaskID)
if err != nil {
return TaskAdmission{}, err
}
scopes := normalizedAdmissionScopes(input.Scopes)
lockScopes := append([]AdmissionScope{}, scopes...)
if found {
lockScopes = append(lockScopes,
AdmissionScope{ScopeType: "platform_model", ScopeKey: admission.PlatformModelID},
AdmissionScope{ScopeType: "user_group", ScopeKey: admission.UserGroupID},
)
}
for _, scope := range normalizedAdmissionScopes(lockScopes) {
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, admissionLockKey(scope)); err != nil {
return TaskAdmission{}, err
}
}
if found && admission.Status == "admitted" {
activeLeases, leaseErr := activeTaskAdmissionLeaseCountTx(ctx, tx, input.TaskID)
if leaseErr != nil {
return TaskAdmission{}, leaseErr
}
if activeLeases == 0 {
admission, err = resetTaskAdmissionToWaitingTx(ctx, tx, input)
if err != nil {
return TaskAdmission{}, err
}
}
}
if found && admission.Status == "waiting" && !admission.WaitDeadlineAt.After(time.Now()) {
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
return TaskAdmission{}, err
}
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return TaskAdmission{}, err
}
if err := tx.Commit(ctx); err != nil {
return TaskAdmission{}, err
}
return TaskAdmission{}, &QueueTimeoutError{TaskID: input.TaskID}
}
if !found {
scopeStates, stateErr := admissionScopeStatesTx(ctx, tx, scopes)
if stateErr != nil {
return TaskAdmission{}, stateErr
}
if admissionErr := validateQueuedAdmissionCapacity(scopeStates); admissionErr != nil {
return TaskAdmission{}, admissionErr
}
admission, err = insertTaskAdmissionTx(ctx, tx, input, admissionDeadline(scopes))
if err != nil {
return TaskAdmission{}, err
}
found = true
}
if onQueued != nil {
if err := onQueued(tx); err != nil {
return TaskAdmission{}, err
}
}
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
return TaskAdmission{}, err
}
if err := tx.Commit(ctx); err != nil {
return TaskAdmission{}, err
}
return admission, nil
}
func (s *Store) tryTaskAdmission(
ctx context.Context,
input TaskAdmissionInput,
@@ -196,7 +303,7 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
return TaskAdmissionResult{}, err
}
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return TaskAdmissionResult{}, err
}
if err := tx.Commit(ctx); err != nil {
@@ -276,7 +383,10 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
if err != nil {
return TaskAdmissionResult{}, err
}
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
// Continue the FIFO chain after one task is admitted. Every API process
// only probes the global head it owns, so free capacity is filled without
// broadcasting an advisory-lock attempt to every waiter.
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return TaskAdmissionResult{}, err
}
if onAdmitted != nil {
@@ -312,6 +422,29 @@ func validateNewAdmissionCapacity(states []admissionScopeState) error {
return nil
}
func validateQueuedAdmissionCapacity(states []admissionScopeState) error {
// Waiting asynchronous tasks have not acquired leases yet. Count them as
// virtual capacity reservations so a burst cannot bypass concurrent and
// queue limits before the dispatcher creates River jobs.
for _, state := range states {
if state.Scope.ConcurrentLimit <= 0 {
continue
}
virtualUsage := state.Active + float64(state.Waiting)*state.Scope.Amount + state.Scope.Amount
if virtualUsage <= state.Scope.ConcurrentLimit {
continue
}
if state.Scope.QueueLimit <= 0 {
return concurrencyAdmissionError(state)
}
virtualQueueDepth := int(math.Ceil(virtualUsage - state.Scope.ConcurrentLimit))
if virtualQueueDepth > state.Scope.QueueLimit {
return queueFullAdmissionError(state)
}
}
return nil
}
// RebindWaitingTaskAdmission migrates a queued task to a newly selected
// candidate without changing its FIFO age or priority. Capacity checks happen
// while both the old and new queue locks are held.
@@ -469,6 +602,32 @@ WHERE task_id = $1::uuid
return nil
}
// RenewTaskAdmissionWaiters refreshes all synchronous waiters owned by one API
// process in a single statement. This keeps the 15-second crash-recovery lease
// without running a full advisory-lock admission transaction per waiter.
func (s *Store) RenewTaskAdmissionWaiters(ctx context.Context, taskIDs []string, waiterIDs []string) error {
if len(taskIDs) == 0 {
return nil
}
if len(taskIDs) != len(waiterIDs) {
return errors.New("task admission waiter renewal requires matching task and waiter IDs")
}
_, err := s.pool.Exec(ctx, `
UPDATE gateway_task_admissions admission
SET waiter_lease_expires_at = now() + $3::interval,
updated_at = now()
FROM unnest($1::uuid[], $2::text[]) AS owned(task_id, waiter_id)
WHERE admission.task_id = owned.task_id
AND admission.waiter_id = owned.waiter_id
AND admission.mode = 'sync'
AND admission.status = 'waiting'`,
taskIDs,
waiterIDs,
admissionWaiterLeaseTTL.String(),
)
return err
}
func (s *Store) DeleteTaskAdmission(ctx context.Context, taskID string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
@@ -488,7 +647,7 @@ WHERE task_id = $1::uuid
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
return err
}
if err := notifyTaskAdmissionTx(ctx, tx, taskID); err != nil {
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
return err
}
return tx.Commit(ctx)
@@ -546,6 +705,52 @@ LIMIT $1`, limit)
return taskIDs, rows.Err()
}
// ListWaitingTaskAdmissionIDs returns a bounded set of FIFO leaders across
// platform-model and user-group queues. It is used after capacity is released
// so API processes wake only plausible queue heads instead of every
// synchronous waiter.
func (s *Store) ListWaitingTaskAdmissionIDs(ctx context.Context, limit int) ([]string, error) {
if limit <= 0 {
limit = 256
}
rows, err := s.pool.Query(ctx, `
WITH ranked AS (
SELECT task_id,
priority,
enqueued_at,
row_number() OVER (
PARTITION BY platform_model_id
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
) AS platform_rank,
row_number() OVER (
PARTITION BY user_group_id
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
) AS group_rank
FROM gateway_task_admissions
WHERE status = 'waiting'
AND mode = 'sync'
)
SELECT task_id::text
FROM ranked
WHERE platform_rank <= $1
AND group_rank <= $1
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
LIMIT $1`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
taskIDs := make([]string, 0, limit)
for rows.Next() {
var taskID string
if err := rows.Scan(&taskID); err != nil {
return nil, err
}
taskIDs = append(taskIDs, taskID)
}
return taskIDs, rows.Err()
}
func (s *Store) GetTaskAdmission(ctx context.Context, taskID string) (TaskAdmission, error) {
admission, found, err := loadTaskAdmission(ctx, s.pool, taskID)
if err != nil {
@@ -793,6 +998,12 @@ SELECT COUNT(*)
FROM gateway_task_admissions
WHERE status = 'waiting' AND user_group_id::text = $1`, scope.ScopeKey).Scan(&waiting)
return waiting, err
case "worker_capacity":
err := tx.QueryRow(ctx, `
SELECT COUNT(*)
FROM gateway_task_admissions
WHERE status = 'waiting' AND mode = 'async'`).Scan(&waiting)
return waiting, err
default:
return 0, nil
}
@@ -813,7 +1024,10 @@ func admissionDeadline(scopes []AdmissionScope) time.Time {
}
}
if maxWait == 0 {
maxWait = DefaultQueueMaxWaitSeconds
// A task may be registered before a River execution slot is available
// even when business queueing is disabled. Keep that infrastructure wait
// separate from the policy's default 600-second queue timeout.
return time.Now().Add(executionSlotWaitMax)
}
return time.Now().Add(time.Duration(maxWait) * time.Second)
}
@@ -402,6 +402,59 @@ VALUES ($1::uuid, 1, $2::uuid, $3::uuid, $4, 'running', 'submitting')`,
t.Fatalf("post-submission cancellation changed=%v err=%v, want unchanged", changed, err)
}
queuedAtomicTask := createTask(true)
queuedHookFailure := errors.New("synthetic queued hook failure")
_, err = first.QueueTaskAdmissionWithHook(ctx, inputFor(queuedAtomicTask, 100, ""), func(pgx.Tx) error {
return queuedHookFailure
})
if !errors.Is(err, queuedHookFailure) {
t.Fatalf("queued hook failure = %v, want synthetic failure", err)
}
var queuedAdmissions, queuedLeases int
if err := first.pool.QueryRow(ctx, `
SELECT
(SELECT COUNT(*) FROM gateway_task_admissions WHERE task_id = $1::uuid),
(SELECT COUNT(*) FROM gateway_concurrency_leases WHERE task_id = $1::uuid AND released_at IS NULL)`,
queuedAtomicTask.ID,
).Scan(&queuedAdmissions, &queuedLeases); err != nil {
t.Fatalf("read rolled back queued hook state: %v", err)
}
if queuedAdmissions != 0 || queuedLeases != 0 {
t.Fatalf("failed queued hook left admissions=%d leases=%d", queuedAdmissions, queuedLeases)
}
const queuedSyntheticRiverJobID int64 = 987654320
queuedAdmission, err := first.QueueTaskAdmissionWithHook(ctx, inputFor(queuedAtomicTask, 100, ""), func(tx pgx.Tx) error {
_, hookErr := tx.Exec(ctx, `
UPDATE gateway_tasks
SET river_job_id = $2
WHERE id = $1::uuid`, queuedAtomicTask.ID, queuedSyntheticRiverJobID)
return hookErr
})
if err != nil || queuedAdmission.Status != "waiting" {
t.Fatalf("atomic queued admission=%+v err=%v, want waiting", queuedAdmission, err)
}
var riverJobID int64
if err := first.pool.QueryRow(ctx, `
SELECT
river_job_id,
(SELECT COUNT(*)
FROM gateway_concurrency_leases
WHERE task_id = $1::uuid AND released_at IS NULL)
FROM gateway_tasks
WHERE id = $1::uuid`, queuedAtomicTask.ID).Scan(&riverJobID, &queuedLeases); err != nil {
t.Fatalf("read atomic queued River marker: %v", err)
}
if riverJobID != queuedSyntheticRiverJobID || queuedLeases != 0 {
t.Fatalf("queued River marker=%d leases=%d, want %d/0", riverJobID, queuedLeases, queuedSyntheticRiverJobID)
}
result, err = second.TryTaskAdmission(ctx, inputFor(queuedAtomicTask, 100, ""))
if err != nil || !result.Admitted || len(result.Leases) != 1 {
t.Fatalf("worker-time queued admission result=%+v err=%v", result, err)
}
if err := first.DeleteTaskAdmission(ctx, queuedAtomicTask.ID); err != nil {
t.Fatalf("release queued atomic task: %v", err)
}
atomicTask := createTask(true)
hookFailure := errors.New("synthetic admitted hook failure")
_, err = first.TryTaskAdmissionWithAdmittedHook(ctx, inputFor(atomicTask, 100, ""), func(pgx.Tx) error {
@@ -433,7 +486,6 @@ WHERE id = $1::uuid`, atomicTask.ID, syntheticRiverJobID)
if err != nil || !result.Admitted {
t.Fatalf("atomic admitted hook result=%+v err=%v", result, err)
}
var riverJobID int64
if err := first.pool.QueryRow(ctx, `
SELECT river_job_id
FROM gateway_tasks
+23 -1
View File
@@ -26,6 +26,7 @@ type Store struct {
const (
postgresApplicationName = "easyai-ai-gateway"
postgresConnectTimeout = 5 * time.Second
postgresPoolWarmLimit = 64
)
func defaultAPIKeyScopes() []string {
@@ -88,7 +89,7 @@ func ConnectWithMaxConns(ctx context.Context, databaseURL string, maxConns int)
if err != nil {
return nil, err
}
if err := pool.Ping(ctx); err != nil {
if err := warmPostgresPool(ctx, pool, int(config.MinIdleConns)); err != nil {
pool.Close()
return nil, err
}
@@ -104,10 +105,31 @@ func postgresPoolConfig(databaseURL string, maxConns ...int) (*pgxpool.Config, e
config.ConnConfig.RuntimeParams["application_name"] = postgresApplicationName
if len(maxConns) > 0 && maxConns[0] > 0 {
config.MaxConns = int32(maxConns[0])
config.MinIdleConns = int32(min(maxConns[0], postgresPoolWarmLimit))
}
return config, nil
}
func warmPostgresPool(ctx context.Context, pool *pgxpool.Pool, count int) error {
if count <= 0 {
return pool.Ping(ctx)
}
connections := make([]*pgxpool.Conn, 0, count)
defer func() {
for _, connection := range connections {
connection.Release()
}
}()
for len(connections) < count {
connection, err := pool.Acquire(ctx)
if err != nil {
return err
}
connections = append(connections, connection)
}
return nil
}
func IsPostgresUnavailable(err error) bool {
if err == nil {
return false
@@ -27,6 +27,34 @@ func TestPostgresPoolConfigRejectsMalformedURL(t *testing.T) {
}
}
func TestPostgresPoolConfigWarmsBoundedIdleConnections(t *testing.T) {
config, err := postgresPoolConfig(
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
24,
)
if err != nil {
t.Fatalf("parse PostgreSQL pool config: %v", err)
}
if config.MaxConns != 24 || config.MinIdleConns != 24 {
t.Fatalf(
"pool bounds max=%d minIdle=%d, want 24/24",
config.MaxConns,
config.MinIdleConns,
)
}
small, err := postgresPoolConfig(
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
4,
)
if err != nil {
t.Fatalf("parse small PostgreSQL pool config: %v", err)
}
if small.MinIdleConns != 4 {
t.Fatalf("small pool minIdle=%d, want 4", small.MinIdleConns)
}
}
func TestIsPostgresUnavailableClassifiesConnectivityFailures(t *testing.T) {
for _, testCase := range []struct {
name string
@@ -1,6 +1,10 @@
package store
import "testing"
import (
"errors"
"testing"
"time"
)
func TestNormalizeAndValidateQueuePolicy(t *testing.T) {
t.Run("defaults maximum wait and preserves extensions", func(t *testing.T) {
@@ -227,3 +231,64 @@ func TestAsyncWorkerCapacityRespectsUserGroupCeiling(t *testing.T) {
})
}
}
func TestAdmissionDeadlineSeparatesExecutionSlotWaitFromPolicyQueueTimeout(t *testing.T) {
now := time.Now()
infrastructureDeadline := admissionDeadline([]AdmissionScope{{
ConcurrentLimit: 10,
}})
if wait := infrastructureDeadline.Sub(now); wait < 23*time.Hour || wait > executionSlotWaitMax+time.Second {
t.Fatalf("execution-slot deadline wait=%s, want about %s", wait, executionSlotWaitMax)
}
defaultPolicyDeadline := admissionDeadline([]AdmissionScope{{
ConcurrentLimit: 10,
QueueLimit: 20,
}})
if wait := defaultPolicyDeadline.Sub(now); wait < 599*time.Second || wait > 601*time.Second {
t.Fatalf("default policy deadline wait=%s, want about 600s", wait)
}
explicitPolicyDeadline := admissionDeadline([]AdmissionScope{{
ConcurrentLimit: 10,
QueueLimit: 20,
MaxWaitSeconds: 90,
}})
if wait := explicitPolicyDeadline.Sub(now); wait < 89*time.Second || wait > 91*time.Second {
t.Fatalf("explicit policy deadline wait=%s, want about 90s", wait)
}
}
func TestQueuedAdmissionCountsFIFOWaitersAsVirtualCapacity(t *testing.T) {
scope := AdmissionScope{
ScopeType: "platform_model",
ScopeKey: "model-1",
ConcurrentLimit: 1,
Amount: 1,
QueueLimit: 2,
}
for waiting := 0; waiting <= 2; waiting++ {
if err := validateQueuedAdmissionCapacity([]admissionScopeState{{
Scope: scope,
Waiting: waiting,
}}); err != nil {
t.Fatalf("waiting=%d rejected within concurrent=1 queue=2: %v", waiting, err)
}
}
err := validateQueuedAdmissionCapacity([]admissionScopeState{{
Scope: scope,
Waiting: 3,
}})
var limitErr *RateLimitExceededError
if !errors.As(err, &limitErr) || limitErr.Reason != "queue_full" {
t.Fatalf("fourth task error=%v, want queue_full", err)
}
scope.QueueLimit = 0
if err := validateQueuedAdmissionCapacity([]admissionScopeState{{
Scope: scope,
Waiting: 1,
}}); !errors.As(err, &limitErr) || limitErr.Metric != "concurrent" {
t.Fatalf("queue-disabled second task error=%v, want concurrent rate limit", err)
}
}
+2 -2
View File
@@ -684,7 +684,7 @@ WHERE task_id = $1::uuid
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
return err
}
return notifyTaskAdmissionTx(ctx, tx, taskID)
return notifyTaskAdmissionTx(ctx, tx, "*")
})
if err != nil {
return GatewayTask{}, false, err
@@ -770,7 +770,7 @@ WHERE task_id = $1::uuid
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
return err
}
return notifyTaskAdmissionTx(ctx, tx, taskID)
return notifyTaskAdmissionTx(ctx, tx, "*")
})
if err != nil {
return GatewayTask{}, false, err
@@ -29,6 +29,18 @@ type WorkerAllocation struct {
HeartbeatAt time.Time
}
// ActiveWorkerCapacity returns the cluster-wide execution capacity currently
// owned by live Worker instances.
func (s *Store) ActiveWorkerCapacity(ctx context.Context) (int, error) {
var capacity int
err := s.pool.QueryRow(ctx, `
SELECT COALESCE(SUM(allocated_capacity), 0)::int
FROM gateway_worker_instances
WHERE status = 'active'
AND heartbeat_at > now() - $1::interval`, workerHeartbeatStaleAfter.String()).Scan(&capacity)
return capacity, err
}
func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistrationInput) (WorkerAllocation, error) {
input.InstanceID = strings.TrimSpace(input.InstanceID)
if input.InstanceID == "" {