perf(queue): 限制大媒体任务内存并消除准入惊群
将同步与异步非文本任务的准入唤醒改为按任务和全局队首推进,批量续租等待者,避免大量请求同时争抢 advisory lock 和数据库连接。 对 Base64 请求解析、素材解码、上游媒体执行和结果物化增加分层并发限制,复用请求素材并释放重复 Gemini wire 数据;生产 API 默认入口解析并发 16,媒体物化并发 8。 新增 1000 个同步 Gemini 图像编辑请求的模拟上游压力验收,10 MiB 输入和输出下全部成功,Heap 峰值增长约 2.58 GiB,并验证共享上传哈希、单次 attempt 与本地零落盘。 验证:go test ./... -count=1;go vet ./...;gofmt -l 无输出;kubectl kustomize deploy/kubernetes/production;10 MiB Gemini Base64 千任务压力测试通过。
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
const (
|
||||
admissionWaiterLeaseTTL = 15 * time.Second
|
||||
admissionNotifyChannel = "gateway_task_admission"
|
||||
executionSlotWaitMax = 24 * time.Hour
|
||||
)
|
||||
|
||||
var ErrQueueTimeout = errors.New("task admission queue wait timed out")
|
||||
@@ -113,6 +115,111 @@ func (s *Store) TryTaskAdmissionWithAdmittedHook(
|
||||
return s.tryTaskAdmission(ctx, input, onAdmitted)
|
||||
}
|
||||
|
||||
// QueueTaskAdmissionWithHook durably registers an asynchronous FIFO waiter and
|
||||
// runs onQueued in the same transaction. It intentionally does not reserve
|
||||
// concurrency leases: the Worker dispatcher obtains business and execution
|
||||
// capacity only when the task reaches the head of every applicable queue.
|
||||
func (s *Store) QueueTaskAdmissionWithHook(
|
||||
ctx context.Context,
|
||||
input TaskAdmissionInput,
|
||||
onQueued func(pgx.Tx) error,
|
||||
) (TaskAdmission, error) {
|
||||
if err := validateTaskAdmissionInput(input); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
if input.Mode != "async" {
|
||||
return TaskAdmission{}, errors.New("queued admission registration requires asynchronous mode")
|
||||
}
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, "task-admission:"+input.TaskID); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
var taskActive bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT status IN ('queued', 'running')
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
if !taskActive {
|
||||
return TaskAdmission{}, ErrTaskExecutionFinished
|
||||
}
|
||||
|
||||
admission, found, err := loadTaskAdmissionTx(ctx, tx, input.TaskID)
|
||||
if err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
scopes := normalizedAdmissionScopes(input.Scopes)
|
||||
lockScopes := append([]AdmissionScope{}, scopes...)
|
||||
if found {
|
||||
lockScopes = append(lockScopes,
|
||||
AdmissionScope{ScopeType: "platform_model", ScopeKey: admission.PlatformModelID},
|
||||
AdmissionScope{ScopeType: "user_group", ScopeKey: admission.UserGroupID},
|
||||
)
|
||||
}
|
||||
for _, scope := range normalizedAdmissionScopes(lockScopes) {
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, admissionLockKey(scope)); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if found && admission.Status == "admitted" {
|
||||
activeLeases, leaseErr := activeTaskAdmissionLeaseCountTx(ctx, tx, input.TaskID)
|
||||
if leaseErr != nil {
|
||||
return TaskAdmission{}, leaseErr
|
||||
}
|
||||
if activeLeases == 0 {
|
||||
admission, err = resetTaskAdmissionToWaitingTx(ctx, tx, input)
|
||||
if err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if found && admission.Status == "waiting" && !admission.WaitDeadlineAt.After(time.Now()) {
|
||||
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
return TaskAdmission{}, &QueueTimeoutError{TaskID: input.TaskID}
|
||||
}
|
||||
if !found {
|
||||
scopeStates, stateErr := admissionScopeStatesTx(ctx, tx, scopes)
|
||||
if stateErr != nil {
|
||||
return TaskAdmission{}, stateErr
|
||||
}
|
||||
if admissionErr := validateQueuedAdmissionCapacity(scopeStates); admissionErr != nil {
|
||||
return TaskAdmission{}, admissionErr
|
||||
}
|
||||
admission, err = insertTaskAdmissionTx(ctx, tx, input, admissionDeadline(scopes))
|
||||
if err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
found = true
|
||||
}
|
||||
if onQueued != nil {
|
||||
if err := onQueued(tx); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
}
|
||||
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
}
|
||||
return admission, nil
|
||||
}
|
||||
|
||||
func (s *Store) tryTaskAdmission(
|
||||
ctx context.Context,
|
||||
input TaskAdmissionInput,
|
||||
@@ -196,7 +303,7 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
if err := expireTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
||||
return TaskAdmissionResult{}, err
|
||||
}
|
||||
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
||||
return TaskAdmissionResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
@@ -276,7 +383,10 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
if err != nil {
|
||||
return TaskAdmissionResult{}, err
|
||||
}
|
||||
if err := notifyTaskAdmissionTx(ctx, tx, input.TaskID); err != nil {
|
||||
// Continue the FIFO chain after one task is admitted. Every API process
|
||||
// only probes the global head it owns, so free capacity is filled without
|
||||
// broadcasting an advisory-lock attempt to every waiter.
|
||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
||||
return TaskAdmissionResult{}, err
|
||||
}
|
||||
if onAdmitted != nil {
|
||||
@@ -312,6 +422,29 @@ func validateNewAdmissionCapacity(states []admissionScopeState) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateQueuedAdmissionCapacity(states []admissionScopeState) error {
|
||||
// Waiting asynchronous tasks have not acquired leases yet. Count them as
|
||||
// virtual capacity reservations so a burst cannot bypass concurrent and
|
||||
// queue limits before the dispatcher creates River jobs.
|
||||
for _, state := range states {
|
||||
if state.Scope.ConcurrentLimit <= 0 {
|
||||
continue
|
||||
}
|
||||
virtualUsage := state.Active + float64(state.Waiting)*state.Scope.Amount + state.Scope.Amount
|
||||
if virtualUsage <= state.Scope.ConcurrentLimit {
|
||||
continue
|
||||
}
|
||||
if state.Scope.QueueLimit <= 0 {
|
||||
return concurrencyAdmissionError(state)
|
||||
}
|
||||
virtualQueueDepth := int(math.Ceil(virtualUsage - state.Scope.ConcurrentLimit))
|
||||
if virtualQueueDepth > state.Scope.QueueLimit {
|
||||
return queueFullAdmissionError(state)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RebindWaitingTaskAdmission migrates a queued task to a newly selected
|
||||
// candidate without changing its FIFO age or priority. Capacity checks happen
|
||||
// while both the old and new queue locks are held.
|
||||
@@ -469,6 +602,32 @@ WHERE task_id = $1::uuid
|
||||
return nil
|
||||
}
|
||||
|
||||
// RenewTaskAdmissionWaiters refreshes all synchronous waiters owned by one API
|
||||
// process in a single statement. This keeps the 15-second crash-recovery lease
|
||||
// without running a full advisory-lock admission transaction per waiter.
|
||||
func (s *Store) RenewTaskAdmissionWaiters(ctx context.Context, taskIDs []string, waiterIDs []string) error {
|
||||
if len(taskIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(taskIDs) != len(waiterIDs) {
|
||||
return errors.New("task admission waiter renewal requires matching task and waiter IDs")
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_task_admissions admission
|
||||
SET waiter_lease_expires_at = now() + $3::interval,
|
||||
updated_at = now()
|
||||
FROM unnest($1::uuid[], $2::text[]) AS owned(task_id, waiter_id)
|
||||
WHERE admission.task_id = owned.task_id
|
||||
AND admission.waiter_id = owned.waiter_id
|
||||
AND admission.mode = 'sync'
|
||||
AND admission.status = 'waiting'`,
|
||||
taskIDs,
|
||||
waiterIDs,
|
||||
admissionWaiterLeaseTTL.String(),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) DeleteTaskAdmission(ctx context.Context, taskID string) error {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
@@ -488,7 +647,7 @@ WHERE task_id = $1::uuid
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := notifyTaskAdmissionTx(ctx, tx, taskID); err != nil {
|
||||
if err := notifyTaskAdmissionTx(ctx, tx, "*"); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
@@ -546,6 +705,52 @@ LIMIT $1`, limit)
|
||||
return taskIDs, rows.Err()
|
||||
}
|
||||
|
||||
// ListWaitingTaskAdmissionIDs returns a bounded set of FIFO leaders across
|
||||
// platform-model and user-group queues. It is used after capacity is released
|
||||
// so API processes wake only plausible queue heads instead of every
|
||||
// synchronous waiter.
|
||||
func (s *Store) ListWaitingTaskAdmissionIDs(ctx context.Context, limit int) ([]string, error) {
|
||||
if limit <= 0 {
|
||||
limit = 256
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
WITH ranked AS (
|
||||
SELECT task_id,
|
||||
priority,
|
||||
enqueued_at,
|
||||
row_number() OVER (
|
||||
PARTITION BY platform_model_id
|
||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
||||
) AS platform_rank,
|
||||
row_number() OVER (
|
||||
PARTITION BY user_group_id
|
||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
||||
) AS group_rank
|
||||
FROM gateway_task_admissions
|
||||
WHERE status = 'waiting'
|
||||
AND mode = 'sync'
|
||||
)
|
||||
SELECT task_id::text
|
||||
FROM ranked
|
||||
WHERE platform_rank <= $1
|
||||
AND group_rank <= $1
|
||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
taskIDs := make([]string, 0, limit)
|
||||
for rows.Next() {
|
||||
var taskID string
|
||||
if err := rows.Scan(&taskID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskIDs = append(taskIDs, taskID)
|
||||
}
|
||||
return taskIDs, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) GetTaskAdmission(ctx context.Context, taskID string) (TaskAdmission, error) {
|
||||
admission, found, err := loadTaskAdmission(ctx, s.pool, taskID)
|
||||
if err != nil {
|
||||
@@ -793,6 +998,12 @@ SELECT COUNT(*)
|
||||
FROM gateway_task_admissions
|
||||
WHERE status = 'waiting' AND user_group_id::text = $1`, scope.ScopeKey).Scan(&waiting)
|
||||
return waiting, err
|
||||
case "worker_capacity":
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM gateway_task_admissions
|
||||
WHERE status = 'waiting' AND mode = 'async'`).Scan(&waiting)
|
||||
return waiting, err
|
||||
default:
|
||||
return 0, nil
|
||||
}
|
||||
@@ -813,7 +1024,10 @@ func admissionDeadline(scopes []AdmissionScope) time.Time {
|
||||
}
|
||||
}
|
||||
if maxWait == 0 {
|
||||
maxWait = DefaultQueueMaxWaitSeconds
|
||||
// A task may be registered before a River execution slot is available
|
||||
// even when business queueing is disabled. Keep that infrastructure wait
|
||||
// separate from the policy's default 600-second queue timeout.
|
||||
return time.Now().Add(executionSlotWaitMax)
|
||||
}
|
||||
return time.Now().Add(time.Duration(maxWait) * time.Second)
|
||||
}
|
||||
|
||||
@@ -402,6 +402,59 @@ VALUES ($1::uuid, 1, $2::uuid, $3::uuid, $4, 'running', 'submitting')`,
|
||||
t.Fatalf("post-submission cancellation changed=%v err=%v, want unchanged", changed, err)
|
||||
}
|
||||
|
||||
queuedAtomicTask := createTask(true)
|
||||
queuedHookFailure := errors.New("synthetic queued hook failure")
|
||||
_, err = first.QueueTaskAdmissionWithHook(ctx, inputFor(queuedAtomicTask, 100, ""), func(pgx.Tx) error {
|
||||
return queuedHookFailure
|
||||
})
|
||||
if !errors.Is(err, queuedHookFailure) {
|
||||
t.Fatalf("queued hook failure = %v, want synthetic failure", err)
|
||||
}
|
||||
var queuedAdmissions, queuedLeases int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM gateway_task_admissions WHERE task_id = $1::uuid),
|
||||
(SELECT COUNT(*) FROM gateway_concurrency_leases WHERE task_id = $1::uuid AND released_at IS NULL)`,
|
||||
queuedAtomicTask.ID,
|
||||
).Scan(&queuedAdmissions, &queuedLeases); err != nil {
|
||||
t.Fatalf("read rolled back queued hook state: %v", err)
|
||||
}
|
||||
if queuedAdmissions != 0 || queuedLeases != 0 {
|
||||
t.Fatalf("failed queued hook left admissions=%d leases=%d", queuedAdmissions, queuedLeases)
|
||||
}
|
||||
const queuedSyntheticRiverJobID int64 = 987654320
|
||||
queuedAdmission, err := first.QueueTaskAdmissionWithHook(ctx, inputFor(queuedAtomicTask, 100, ""), func(tx pgx.Tx) error {
|
||||
_, hookErr := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET river_job_id = $2
|
||||
WHERE id = $1::uuid`, queuedAtomicTask.ID, queuedSyntheticRiverJobID)
|
||||
return hookErr
|
||||
})
|
||||
if err != nil || queuedAdmission.Status != "waiting" {
|
||||
t.Fatalf("atomic queued admission=%+v err=%v, want waiting", queuedAdmission, err)
|
||||
}
|
||||
var riverJobID int64
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
river_job_id,
|
||||
(SELECT COUNT(*)
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE task_id = $1::uuid AND released_at IS NULL)
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid`, queuedAtomicTask.ID).Scan(&riverJobID, &queuedLeases); err != nil {
|
||||
t.Fatalf("read atomic queued River marker: %v", err)
|
||||
}
|
||||
if riverJobID != queuedSyntheticRiverJobID || queuedLeases != 0 {
|
||||
t.Fatalf("queued River marker=%d leases=%d, want %d/0", riverJobID, queuedLeases, queuedSyntheticRiverJobID)
|
||||
}
|
||||
result, err = second.TryTaskAdmission(ctx, inputFor(queuedAtomicTask, 100, ""))
|
||||
if err != nil || !result.Admitted || len(result.Leases) != 1 {
|
||||
t.Fatalf("worker-time queued admission result=%+v err=%v", result, err)
|
||||
}
|
||||
if err := first.DeleteTaskAdmission(ctx, queuedAtomicTask.ID); err != nil {
|
||||
t.Fatalf("release queued atomic task: %v", err)
|
||||
}
|
||||
|
||||
atomicTask := createTask(true)
|
||||
hookFailure := errors.New("synthetic admitted hook failure")
|
||||
_, err = first.TryTaskAdmissionWithAdmittedHook(ctx, inputFor(atomicTask, 100, ""), func(pgx.Tx) error {
|
||||
@@ -433,7 +486,6 @@ WHERE id = $1::uuid`, atomicTask.ID, syntheticRiverJobID)
|
||||
if err != nil || !result.Admitted {
|
||||
t.Fatalf("atomic admitted hook result=%+v err=%v", result, err)
|
||||
}
|
||||
var riverJobID int64
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT river_job_id
|
||||
FROM gateway_tasks
|
||||
|
||||
@@ -26,6 +26,7 @@ type Store struct {
|
||||
const (
|
||||
postgresApplicationName = "easyai-ai-gateway"
|
||||
postgresConnectTimeout = 5 * time.Second
|
||||
postgresPoolWarmLimit = 64
|
||||
)
|
||||
|
||||
func defaultAPIKeyScopes() []string {
|
||||
@@ -88,7 +89,7 @@ func ConnectWithMaxConns(ctx context.Context, databaseURL string, maxConns int)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
if err := warmPostgresPool(ctx, pool, int(config.MinIdleConns)); err != nil {
|
||||
pool.Close()
|
||||
return nil, err
|
||||
}
|
||||
@@ -104,10 +105,31 @@ func postgresPoolConfig(databaseURL string, maxConns ...int) (*pgxpool.Config, e
|
||||
config.ConnConfig.RuntimeParams["application_name"] = postgresApplicationName
|
||||
if len(maxConns) > 0 && maxConns[0] > 0 {
|
||||
config.MaxConns = int32(maxConns[0])
|
||||
config.MinIdleConns = int32(min(maxConns[0], postgresPoolWarmLimit))
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func warmPostgresPool(ctx context.Context, pool *pgxpool.Pool, count int) error {
|
||||
if count <= 0 {
|
||||
return pool.Ping(ctx)
|
||||
}
|
||||
connections := make([]*pgxpool.Conn, 0, count)
|
||||
defer func() {
|
||||
for _, connection := range connections {
|
||||
connection.Release()
|
||||
}
|
||||
}()
|
||||
for len(connections) < count {
|
||||
connection, err := pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
connections = append(connections, connection)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func IsPostgresUnavailable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
|
||||
@@ -27,6 +27,34 @@ func TestPostgresPoolConfigRejectsMalformedURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresPoolConfigWarmsBoundedIdleConnections(t *testing.T) {
|
||||
config, err := postgresPoolConfig(
|
||||
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
|
||||
24,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("parse PostgreSQL pool config: %v", err)
|
||||
}
|
||||
if config.MaxConns != 24 || config.MinIdleConns != 24 {
|
||||
t.Fatalf(
|
||||
"pool bounds max=%d minIdle=%d, want 24/24",
|
||||
config.MaxConns,
|
||||
config.MinIdleConns,
|
||||
)
|
||||
}
|
||||
|
||||
small, err := postgresPoolConfig(
|
||||
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
|
||||
4,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("parse small PostgreSQL pool config: %v", err)
|
||||
}
|
||||
if small.MinIdleConns != 4 {
|
||||
t.Fatalf("small pool minIdle=%d, want 4", small.MinIdleConns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPostgresUnavailableClassifiesConnectivityFailures(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNormalizeAndValidateQueuePolicy(t *testing.T) {
|
||||
t.Run("defaults maximum wait and preserves extensions", func(t *testing.T) {
|
||||
@@ -227,3 +231,64 @@ func TestAsyncWorkerCapacityRespectsUserGroupCeiling(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdmissionDeadlineSeparatesExecutionSlotWaitFromPolicyQueueTimeout(t *testing.T) {
|
||||
now := time.Now()
|
||||
infrastructureDeadline := admissionDeadline([]AdmissionScope{{
|
||||
ConcurrentLimit: 10,
|
||||
}})
|
||||
if wait := infrastructureDeadline.Sub(now); wait < 23*time.Hour || wait > executionSlotWaitMax+time.Second {
|
||||
t.Fatalf("execution-slot deadline wait=%s, want about %s", wait, executionSlotWaitMax)
|
||||
}
|
||||
|
||||
defaultPolicyDeadline := admissionDeadline([]AdmissionScope{{
|
||||
ConcurrentLimit: 10,
|
||||
QueueLimit: 20,
|
||||
}})
|
||||
if wait := defaultPolicyDeadline.Sub(now); wait < 599*time.Second || wait > 601*time.Second {
|
||||
t.Fatalf("default policy deadline wait=%s, want about 600s", wait)
|
||||
}
|
||||
|
||||
explicitPolicyDeadline := admissionDeadline([]AdmissionScope{{
|
||||
ConcurrentLimit: 10,
|
||||
QueueLimit: 20,
|
||||
MaxWaitSeconds: 90,
|
||||
}})
|
||||
if wait := explicitPolicyDeadline.Sub(now); wait < 89*time.Second || wait > 91*time.Second {
|
||||
t.Fatalf("explicit policy deadline wait=%s, want about 90s", wait)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedAdmissionCountsFIFOWaitersAsVirtualCapacity(t *testing.T) {
|
||||
scope := AdmissionScope{
|
||||
ScopeType: "platform_model",
|
||||
ScopeKey: "model-1",
|
||||
ConcurrentLimit: 1,
|
||||
Amount: 1,
|
||||
QueueLimit: 2,
|
||||
}
|
||||
for waiting := 0; waiting <= 2; waiting++ {
|
||||
if err := validateQueuedAdmissionCapacity([]admissionScopeState{{
|
||||
Scope: scope,
|
||||
Waiting: waiting,
|
||||
}}); err != nil {
|
||||
t.Fatalf("waiting=%d rejected within concurrent=1 queue=2: %v", waiting, err)
|
||||
}
|
||||
}
|
||||
err := validateQueuedAdmissionCapacity([]admissionScopeState{{
|
||||
Scope: scope,
|
||||
Waiting: 3,
|
||||
}})
|
||||
var limitErr *RateLimitExceededError
|
||||
if !errors.As(err, &limitErr) || limitErr.Reason != "queue_full" {
|
||||
t.Fatalf("fourth task error=%v, want queue_full", err)
|
||||
}
|
||||
|
||||
scope.QueueLimit = 0
|
||||
if err := validateQueuedAdmissionCapacity([]admissionScopeState{{
|
||||
Scope: scope,
|
||||
Waiting: 1,
|
||||
}}); !errors.As(err, &limitErr) || limitErr.Metric != "concurrent" {
|
||||
t.Fatalf("queue-disabled second task error=%v, want concurrent rate limit", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -684,7 +684,7 @@ WHERE task_id = $1::uuid
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||
return err
|
||||
}
|
||||
return notifyTaskAdmissionTx(ctx, tx, taskID)
|
||||
return notifyTaskAdmissionTx(ctx, tx, "*")
|
||||
})
|
||||
if err != nil {
|
||||
return GatewayTask{}, false, err
|
||||
@@ -770,7 +770,7 @@ WHERE task_id = $1::uuid
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||
return err
|
||||
}
|
||||
return notifyTaskAdmissionTx(ctx, tx, taskID)
|
||||
return notifyTaskAdmissionTx(ctx, tx, "*")
|
||||
})
|
||||
if err != nil {
|
||||
return GatewayTask{}, false, err
|
||||
|
||||
@@ -29,6 +29,18 @@ type WorkerAllocation struct {
|
||||
HeartbeatAt time.Time
|
||||
}
|
||||
|
||||
// ActiveWorkerCapacity returns the cluster-wide execution capacity currently
|
||||
// owned by live Worker instances.
|
||||
func (s *Store) ActiveWorkerCapacity(ctx context.Context) (int, error) {
|
||||
var capacity int
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(allocated_capacity), 0)::int
|
||||
FROM gateway_worker_instances
|
||||
WHERE status = 'active'
|
||||
AND heartbeat_at > now() - $1::interval`, workerHeartbeatStaleAfter.String()).Scan(&capacity)
|
||||
return capacity, err
|
||||
}
|
||||
|
||||
func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistrationInput) (WorkerAllocation, error) {
|
||||
input.InstanceID = strings.TrimSpace(input.InstanceID)
|
||||
if input.InstanceID == "" {
|
||||
|
||||
Reference in New Issue
Block a user