feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -689,7 +689,7 @@ ON CONFLICT (task_id, event_type) DO NOTHING`, runID); err != nil {
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases lease
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
FROM gateway_tasks task
|
||||
WHERE lease.task_id = task.id
|
||||
AND lease.released_at IS NULL
|
||||
|
||||
@@ -747,14 +747,14 @@ SELECT COUNT(*)
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()`, taskID).Scan(&count)
|
||||
AND expires_at > statement_timestamp()`, taskID).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func resetTaskAdmissionToWaitingTx(ctx context.Context, tx pgx.Tx, input TaskAdmissionInput) (TaskAdmission, error) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, input.TaskID); err != nil {
|
||||
return TaskAdmission{}, err
|
||||
@@ -862,7 +862,7 @@ func (s *Store) deleteTaskAdmissionOnce(ctx context.Context, taskID string) erro
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, taskID); err != nil {
|
||||
return err
|
||||
@@ -884,7 +884,7 @@ SELECT id::text,
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()
|
||||
AND expires_at > statement_timestamp()
|
||||
ORDER BY scope_type, scope_key, id`, taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -924,7 +924,7 @@ WHERE admission.mode = 'async'
|
||||
FROM gateway_concurrency_leases lease
|
||||
WHERE lease.task_id = admission.task_id
|
||||
AND lease.released_at IS NULL
|
||||
AND lease.expires_at > now()
|
||||
AND lease.expires_at > statement_timestamp()
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -980,7 +980,7 @@ WHERE admission.mode = 'async'
|
||||
FROM gateway_concurrency_leases lease
|
||||
WHERE lease.task_id = admission.task_id
|
||||
AND lease.released_at IS NULL
|
||||
AND lease.expires_at > now()
|
||||
AND lease.expires_at > statement_timestamp()
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -1006,6 +1006,29 @@ LIMIT $1`, limit)
|
||||
return admissions, rows.Err()
|
||||
}
|
||||
|
||||
// RequestWaitingTaskAdmissionReselect marks every queued task bound to a
|
||||
// saturated platform model so the dispatcher can route the next batch to a
|
||||
// different eligible candidate. The durable marker lets multiple dispatchers
|
||||
// observe the same decision without assigning tasks to a specific Worker.
|
||||
func (s *Store) RequestWaitingTaskAdmissionReselect(ctx context.Context, platformModelID string) (int64, error) {
|
||||
result, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_task_admissions admission
|
||||
SET reselect_requested_at = now(),
|
||||
updated_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE admission.task_id = task.id
|
||||
AND admission.platform_model_id = $1::uuid
|
||||
AND admission.mode = 'async'
|
||||
AND admission.status = 'waiting'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.reselect_requested_at IS NULL`, platformModelID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// ListWaitingTaskAdmissionIDs returns one FIFO leader for every independent
|
||||
// platform-model queue, additionally requiring the task to lead its user-group
|
||||
// queue when one exists. It is used after capacity is released so API
|
||||
@@ -1268,12 +1291,12 @@ func admissionScopeStatesTx(ctx context.Context, tx pgx.Tx, scopes []AdmissionSc
|
||||
if scope.ConcurrentLimit > 0 {
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(lease_value), 0)::float8,
|
||||
COALESCE(MIN(expires_at), now() + interval '1 second')
|
||||
COALESCE(MIN(expires_at), statement_timestamp() + interval '1 second')
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE scope_type = $1
|
||||
AND scope_key = $2
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()`, scope.ScopeType, scope.ScopeKey).Scan(&state.Active, &state.NextLeaseExpiration); err != nil {
|
||||
AND expires_at > statement_timestamp()`, scope.ScopeType, scope.ScopeKey).Scan(&state.Active, &state.NextLeaseExpiration); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
state.Saturated = state.Active+scope.Amount > scope.ConcurrentLimit
|
||||
|
||||
@@ -451,6 +451,20 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID, queuedSyntheticRiverJobID)
|
||||
if listedSnapshot == nil || len(listedSnapshot.Scopes) != len(queuedAdmission.Scopes) {
|
||||
t.Fatalf("listed admission snapshot=%+v, want %d scopes", listedSnapshot, len(queuedAdmission.Scopes))
|
||||
}
|
||||
markedForReselect, err := first.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
|
||||
if err != nil {
|
||||
t.Fatalf("request waiting admission reselection: %v", err)
|
||||
}
|
||||
if markedForReselect < 1 {
|
||||
t.Fatalf("marked admissions=%d, want at least the queued task", markedForReselect)
|
||||
}
|
||||
reselectAdmission, err := first.GetTaskAdmission(ctx, queuedAtomicTask.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("read admission reselection marker: %v", err)
|
||||
}
|
||||
if reselectAdmission.ReselectRequestedAt.IsZero() {
|
||||
t.Fatal("queued admission was not marked for candidate reselection")
|
||||
}
|
||||
var riverJobID int64
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
@@ -1296,4 +1310,35 @@ WHERE instance_id = $1`, secondID, (workerHeartbeatStaleAfter + time.Second).Str
|
||||
if err != nil || second.Allocated != 2 || second.GlobalAllocated != 5 || second.ActiveInstances != 2 {
|
||||
t.Fatalf("bounded two-worker allocation = %+v, err=%v", second, err)
|
||||
}
|
||||
first, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
|
||||
InstanceID: firstID, DesiredCapacity: 100, CapacityLimit: 10,
|
||||
LoadMode: "adaptive", SafeCapacity: 0, HeavyCapacity: 1,
|
||||
ActiveTasks: 2, WaitingUpstreamTasks: 2,
|
||||
PressureState: "critical", PressureReason: "memory",
|
||||
})
|
||||
if err != nil || first.Allocated != 0 {
|
||||
t.Fatalf("critical worker allocation = %+v, err=%v", first, err)
|
||||
}
|
||||
second, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
|
||||
InstanceID: secondID, DesiredCapacity: 100, CapacityLimit: 10,
|
||||
LoadMode: "adaptive", SafeCapacity: 5, HeavyCapacity: 2,
|
||||
ActiveTasks: 3, PreparingTasks: 1, WaitingUpstreamTasks: 1, FinalizingTasks: 1,
|
||||
PressureState: "normal",
|
||||
})
|
||||
if err != nil || second.Allocated != 5 || second.GlobalAllocated != 5 {
|
||||
t.Fatalf("adaptive redistribution allocation = %+v, err=%v", second, err)
|
||||
}
|
||||
instances, err := db.ListWorkerInstanceRuntime(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list adaptive worker runtime: %v", err)
|
||||
}
|
||||
foundCritical := false
|
||||
for _, instance := range instances {
|
||||
if instance.InstanceID == firstID {
|
||||
foundCritical = instance.SafeCapacity == 0 && instance.ReportedActiveTasks == 2 && instance.WaitingUpstreamTasks == 2 && instance.PressureState == "critical"
|
||||
}
|
||||
}
|
||||
if !foundCritical {
|
||||
t.Fatalf("adaptive runtime did not expose the critical worker: %+v", instances)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,19 +6,28 @@ import (
|
||||
)
|
||||
|
||||
type AsyncWorkerCapacitySnapshot struct {
|
||||
Capacity int
|
||||
GlobalCapacity int
|
||||
Desired int
|
||||
HardLimit int
|
||||
Capped bool
|
||||
EnabledModels int
|
||||
UnlimitedModels int
|
||||
EnabledGroups int
|
||||
UnlimitedGroups int
|
||||
ModelDesired int
|
||||
GroupDesired int
|
||||
ActiveInstances int
|
||||
InstanceID string
|
||||
Capacity int
|
||||
GlobalCapacity int
|
||||
Desired int
|
||||
HardLimit int
|
||||
Capped bool
|
||||
EnabledModels int
|
||||
UnlimitedModels int
|
||||
EnabledGroups int
|
||||
UnlimitedGroups int
|
||||
ModelDesired int
|
||||
GroupDesired int
|
||||
ActiveInstances int
|
||||
InstanceID string
|
||||
LoadMode string
|
||||
LocalSafeCapacity int
|
||||
LocalHeavyCapacity int
|
||||
LocalActiveTasks int
|
||||
LocalPreparingTasks int
|
||||
LocalWaitingTasks int
|
||||
LocalFinalizingTasks int
|
||||
LocalPressureState string
|
||||
LocalPressureReason string
|
||||
}
|
||||
|
||||
func (s *Store) AsyncWorkerCapacity(ctx context.Context, hardLimit int) (AsyncWorkerCapacitySnapshot, error) {
|
||||
|
||||
@@ -100,7 +100,7 @@ LEFT JOIN (
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE scope_type = 'platform_model'
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()
|
||||
AND expires_at > statement_timestamp()
|
||||
GROUP BY scope_key
|
||||
) con ON con.scope_key = m.id::text
|
||||
LEFT JOIN (
|
||||
|
||||
@@ -169,7 +169,7 @@ LEFT JOIN (
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE scope_type = 'platform_model'
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()
|
||||
AND expires_at > statement_timestamp()
|
||||
GROUP BY scope_key
|
||||
) con ON con.scope_key = m.id::text
|
||||
LEFT JOIN (
|
||||
|
||||
@@ -184,12 +184,12 @@ func reserveConcurrencyLease(ctx context.Context, tx pgx.Tx, taskID string, atte
|
||||
var nextAvailableAt time.Time
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(lease_value), 0)::float8,
|
||||
COALESCE(MIN(expires_at), now() + ($3::int * interval '1 second'))
|
||||
COALESCE(MIN(expires_at), statement_timestamp() + ($3::int * interval '1 second'))
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE scope_type = $1
|
||||
AND scope_key = $2
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()`,
|
||||
AND expires_at > statement_timestamp()`,
|
||||
reservation.ScopeType,
|
||||
reservation.ScopeKey,
|
||||
reservation.LeaseTTLSeconds,
|
||||
@@ -218,14 +218,20 @@ WHERE scope_type = $1
|
||||
}
|
||||
var leaseID string
|
||||
if err := tx.QueryRow(ctx, `
|
||||
INSERT INTO gateway_concurrency_leases (task_id, attempt_id, scope_type, scope_key, lease_value, expires_at)
|
||||
VALUES ($1::uuid, NULLIF($2, '')::uuid, $3, $4, $5, now() + ($6::int * interval '1 second'))
|
||||
INSERT INTO gateway_concurrency_leases (
|
||||
task_id, attempt_id, scope_type, scope_key, lease_value, limit_value, acquired_at, expires_at
|
||||
)
|
||||
VALUES (
|
||||
$1::uuid, NULLIF($2, '')::uuid, $3, $4, $5, $6,
|
||||
statement_timestamp(), statement_timestamp() + ($7::int * interval '1 second')
|
||||
)
|
||||
RETURNING id::text`,
|
||||
taskID,
|
||||
attemptID,
|
||||
reservation.ScopeType,
|
||||
reservation.ScopeKey,
|
||||
reservation.Amount,
|
||||
reservation.Limit,
|
||||
reservation.LeaseTTLSeconds,
|
||||
).Scan(&leaseID); err != nil {
|
||||
return ConcurrencyLease{}, err
|
||||
@@ -381,7 +387,7 @@ func (s *Store) ReleaseConcurrencyLeases(ctx context.Context, leases []Concurren
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE id = ANY($1::uuid[]) AND released_at IS NULL`, leaseIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -411,7 +417,7 @@ func (s *Store) RenewConcurrencyLeases(ctx context.Context, leases []Concurrency
|
||||
}
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases lease
|
||||
SET expires_at = now() + (renewal.ttl_seconds * interval '1 second')
|
||||
SET expires_at = statement_timestamp() + (renewal.ttl_seconds * interval '1 second')
|
||||
FROM unnest($1::uuid[], $2::int[]) AS renewal(id, ttl_seconds)
|
||||
WHERE lease.id = renewal.id
|
||||
AND lease.released_at IS NULL
|
||||
@@ -705,7 +711,7 @@ WHERE attempt.task_id = task.id
|
||||
result.FailedAttempts = tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = ANY($1::uuid[])
|
||||
AND released_at IS NULL`, taskIDs)
|
||||
if err != nil {
|
||||
@@ -772,7 +778,7 @@ FOR UPDATE OF task SKIP LOCKED`, runtimeRecoveryBatchSize)
|
||||
var result RuntimeRecoveryResult
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = ANY($1::uuid[])
|
||||
AND released_at IS NULL`, taskIDs)
|
||||
if err != nil {
|
||||
|
||||
@@ -98,23 +98,168 @@ WHERE scope_type = 'platform_model'
|
||||
}
|
||||
|
||||
var active int64
|
||||
var storedLimit float64
|
||||
if err := first.Pool().QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
SELECT COUNT(*), COALESCE(MAX(limit_value), 0)::float8
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE scope_type = 'platform_model'
|
||||
AND scope_key = $1
|
||||
AND released_at IS NULL
|
||||
AND expires_at > now()`, scopeKey).Scan(&active); err != nil {
|
||||
AND expires_at > now()`, scopeKey).Scan(&active, &storedLimit); err != nil {
|
||||
t.Fatalf("count active leases: %v", err)
|
||||
}
|
||||
if successes.Load() != 64 || active != 64 {
|
||||
t.Fatalf("successful reservations=%d active leases=%d, want exactly 64", successes.Load(), active)
|
||||
if successes.Load() != 64 || active != 64 || storedLimit != 64 {
|
||||
t.Fatalf("successful reservations=%d active leases=%d stored limit=%.0f, want exactly 64", successes.Load(), active, storedLimit)
|
||||
}
|
||||
if peak.Load() > 64 {
|
||||
t.Fatalf("active lease peak=%d, want <=64", peak.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrencyLeaseTimestampStartsAtReservationStatement(t *testing.T) {
|
||||
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run concurrency lease PostgreSQL integration tests")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
database, err := Connect(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect store: %v", err)
|
||||
}
|
||||
defer database.Close()
|
||||
|
||||
scopeKey := "statement-clock-" + time.Now().UTC().Format("20060102150405.000000000")
|
||||
taskIDs := createLeaseTestTasks(t, ctx, database, 1, scopeKey)
|
||||
defer deleteLeaseTestTasks(t, database, taskIDs)
|
||||
|
||||
tx, err := database.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("begin reservation transaction: %v", err)
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
var transactionStartedAt time.Time
|
||||
if err := tx.QueryRow(ctx, `SELECT now()`).Scan(&transactionStartedAt); err != nil {
|
||||
t.Fatalf("read transaction start: %v", err)
|
||||
}
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
lease, err := reserveConcurrencyLease(ctx, tx, taskIDs[0], "", RateLimitReservation{
|
||||
ScopeType: "platform_model",
|
||||
ScopeKey: scopeKey,
|
||||
Metric: "concurrent",
|
||||
Limit: 1,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 30,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("reserve concurrency lease: %v", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatalf("commit reservation transaction: %v", err)
|
||||
}
|
||||
|
||||
var acquiredAt, expiresAt time.Time
|
||||
if err := database.pool.QueryRow(ctx, `
|
||||
SELECT acquired_at, expires_at
|
||||
FROM gateway_concurrency_leases
|
||||
WHERE id = $1::uuid`, lease.ID).Scan(&acquiredAt, &expiresAt); err != nil {
|
||||
t.Fatalf("read lease timestamps: %v", err)
|
||||
}
|
||||
if elapsed := acquiredAt.Sub(transactionStartedAt); elapsed < time.Second {
|
||||
t.Fatalf("lease acquired_at advanced by %s, want at least 1s after transaction start", elapsed)
|
||||
}
|
||||
if ttl := expiresAt.Sub(acquiredAt); ttl != 30*time.Second {
|
||||
t.Fatalf("lease ttl=%s, want 30s", ttl)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCounterWindowReservationIsAtomicAcrossPools(t *testing.T) {
|
||||
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run rate limit PostgreSQL integration tests")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
first, err := Connect(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect first store: %v", err)
|
||||
}
|
||||
defer first.Close()
|
||||
second, err := Connect(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect second store: %v", err)
|
||||
}
|
||||
defer second.Close()
|
||||
|
||||
tests := []struct {
|
||||
metric string
|
||||
limit float64
|
||||
amount float64
|
||||
wantSuccesses int64
|
||||
}{
|
||||
{metric: "rpm", limit: 37, amount: 1, wantSuccesses: 37},
|
||||
{metric: "tpm_total", limit: 1_000, amount: 25, wantSuccesses: 40},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.metric, func(t *testing.T) {
|
||||
scopeKey := "atomic-" + test.metric + "-" + time.Now().UTC().Format("20060102150405.000000000")
|
||||
taskIDs := createLeaseTestTasks(t, ctx, first, 128, scopeKey)
|
||||
defer deleteLeaseTestTasks(t, first, taskIDs)
|
||||
|
||||
var successes atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, len(taskIDs))
|
||||
for index, taskID := range taskIDs {
|
||||
wg.Add(1)
|
||||
go func(index int, taskID string) {
|
||||
defer wg.Done()
|
||||
target := first
|
||||
if index%2 == 1 {
|
||||
target = second
|
||||
}
|
||||
_, err := target.ReserveRateLimits(ctx, taskID, "", []RateLimitReservation{{
|
||||
ScopeType: "platform_model",
|
||||
ScopeKey: scopeKey,
|
||||
Metric: test.metric,
|
||||
Limit: test.limit,
|
||||
Amount: test.amount,
|
||||
WindowSeconds: 3600,
|
||||
}})
|
||||
if err == nil {
|
||||
successes.Add(1)
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, ErrRateLimited) {
|
||||
errs <- err
|
||||
}
|
||||
}(index, taskID)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Fatalf("unexpected reservation error: %v", err)
|
||||
}
|
||||
|
||||
var current float64
|
||||
if err := first.Pool().QueryRow(ctx, `
|
||||
SELECT COALESCE(MAX(used_value + reserved_value), 0)::float8
|
||||
FROM gateway_rate_limit_counters
|
||||
WHERE scope_type = 'platform_model'
|
||||
AND scope_key = $1
|
||||
AND metric = $2`, scopeKey, test.metric).Scan(¤t); err != nil {
|
||||
t.Fatalf("read %s counter: %v", test.metric, err)
|
||||
}
|
||||
if successes.Load() != test.wantSuccesses {
|
||||
t.Fatalf("successful %s reservations=%d, want exactly %d", test.metric, successes.Load(), test.wantSuccesses)
|
||||
}
|
||||
wantCurrent := float64(test.wantSuccesses) * test.amount
|
||||
if current != wantCurrent || current > test.limit {
|
||||
t.Fatalf("%s current=%.0f, want %.0f and <= %.0f", test.metric, current, wantCurrent, test.limit)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrencyLeaseRenewalExtendsAndReleases(t *testing.T) {
|
||||
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
|
||||
@@ -757,7 +757,7 @@ WHERE task_id = $1::uuid
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, taskID); err != nil {
|
||||
return err
|
||||
@@ -922,7 +922,7 @@ RETURNING `+gatewayTaskColumns, taskID, message))
|
||||
changed = true
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, taskID); err != nil {
|
||||
return err
|
||||
@@ -1011,7 +1011,7 @@ WHERE task_id = $1::uuid
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, taskID); err != nil {
|
||||
return err
|
||||
@@ -1720,7 +1720,7 @@ ON CONFLICT (task_id, event_type) DO NOTHING`,
|
||||
if input.FinalizeAdmission {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, input.TaskID); err != nil {
|
||||
return err
|
||||
@@ -2096,7 +2096,7 @@ ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON)
|
||||
if input.FinalizeAdmission {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, input.TaskID); err != nil {
|
||||
return err
|
||||
|
||||
@@ -12,14 +12,24 @@ import (
|
||||
const workerHeartbeatStaleAfter = 30 * time.Second
|
||||
|
||||
type WorkerRegistrationInput struct {
|
||||
InstanceID string
|
||||
PodUID string
|
||||
PodName string
|
||||
Site string
|
||||
Revision string
|
||||
DesiredCapacity int
|
||||
CapacityLimit int
|
||||
HeartbeatStaleAfter time.Duration
|
||||
InstanceID string
|
||||
PodUID string
|
||||
PodName string
|
||||
Site string
|
||||
Revision string
|
||||
DesiredCapacity int
|
||||
CapacityLimit int
|
||||
LoadMode string
|
||||
SafeCapacity int
|
||||
HeavyCapacity int
|
||||
ActiveTasks int
|
||||
PreparingTasks int
|
||||
WaitingUpstreamTasks int
|
||||
FinalizingTasks int
|
||||
PressureState string
|
||||
PressureReason string
|
||||
LoadSampledAt time.Time
|
||||
HeartbeatStaleAfter time.Duration
|
||||
}
|
||||
|
||||
type WorkerAllocation struct {
|
||||
@@ -59,9 +69,28 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
|
||||
if input.CapacityLimit < 0 {
|
||||
return WorkerAllocation{}, errors.New("worker capacity limit cannot be negative")
|
||||
}
|
||||
if input.SafeCapacity < 0 || input.HeavyCapacity < 0 || input.ActiveTasks < 0 || input.PreparingTasks < 0 || input.WaitingUpstreamTasks < 0 || input.FinalizingTasks < 0 {
|
||||
return WorkerAllocation{}, errors.New("worker load values cannot be negative")
|
||||
}
|
||||
if input.ActiveTasks != input.PreparingTasks+input.WaitingUpstreamTasks+input.FinalizingTasks {
|
||||
return WorkerAllocation{}, errors.New("worker active task count must equal phase task counts")
|
||||
}
|
||||
if input.CapacityLimit == 0 {
|
||||
input.CapacityLimit = input.DesiredCapacity
|
||||
}
|
||||
hardCapacityLimit := input.CapacityLimit
|
||||
if strings.EqualFold(strings.TrimSpace(input.LoadMode), "adaptive") {
|
||||
input.CapacityLimit = min(input.CapacityLimit, input.SafeCapacity)
|
||||
}
|
||||
pressureState := strings.ToLower(strings.TrimSpace(input.PressureState))
|
||||
switch pressureState {
|
||||
case "normal", "busy", "critical":
|
||||
default:
|
||||
pressureState = "unknown"
|
||||
}
|
||||
if input.LoadSampledAt.IsZero() {
|
||||
input.LoadSampledAt = time.Now()
|
||||
}
|
||||
staleAfter := input.HeartbeatStaleAfter
|
||||
if staleAfter < workerHeartbeatStaleAfter {
|
||||
staleAfter = workerHeartbeatStaleAfter
|
||||
@@ -83,9 +112,16 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_worker_instances (
|
||||
instance_id, pod_uid, pod_name, site, revision, status,
|
||||
desired_capacity, capacity_limit, allocated_capacity, started_at, heartbeat_at, updated_at
|
||||
desired_capacity, capacity_limit, hard_capacity_limit, safe_capacity, heavy_capacity,
|
||||
active_tasks, preparing_tasks, waiting_upstream_tasks, finalizing_tasks,
|
||||
pressure_state, pressure_reason, load_sampled_at,
|
||||
allocated_capacity, started_at, heartbeat_at, updated_at
|
||||
)
|
||||
VALUES (
|
||||
$1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10,
|
||||
$11, $12, $13, $14, $15, $16, $17,
|
||||
0, now(), now(), now()
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, 0, now(), now(), now())
|
||||
ON CONFLICT (instance_id) DO UPDATE
|
||||
SET pod_uid = EXCLUDED.pod_uid,
|
||||
pod_name = EXCLUDED.pod_name,
|
||||
@@ -97,6 +133,16 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
END,
|
||||
desired_capacity = EXCLUDED.desired_capacity,
|
||||
capacity_limit = EXCLUDED.capacity_limit,
|
||||
hard_capacity_limit = EXCLUDED.hard_capacity_limit,
|
||||
safe_capacity = EXCLUDED.safe_capacity,
|
||||
heavy_capacity = EXCLUDED.heavy_capacity,
|
||||
active_tasks = EXCLUDED.active_tasks,
|
||||
preparing_tasks = EXCLUDED.preparing_tasks,
|
||||
waiting_upstream_tasks = EXCLUDED.waiting_upstream_tasks,
|
||||
finalizing_tasks = EXCLUDED.finalizing_tasks,
|
||||
pressure_state = EXCLUDED.pressure_state,
|
||||
pressure_reason = EXCLUDED.pressure_reason,
|
||||
load_sampled_at = EXCLUDED.load_sampled_at,
|
||||
heartbeat_at = now(),
|
||||
updated_at = now()`,
|
||||
input.InstanceID,
|
||||
@@ -106,6 +152,16 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
strings.TrimSpace(input.Revision),
|
||||
input.DesiredCapacity,
|
||||
input.CapacityLimit,
|
||||
hardCapacityLimit,
|
||||
input.SafeCapacity,
|
||||
input.HeavyCapacity,
|
||||
input.ActiveTasks,
|
||||
input.PreparingTasks,
|
||||
input.WaitingUpstreamTasks,
|
||||
input.FinalizingTasks,
|
||||
pressureState,
|
||||
strings.TrimSpace(input.PressureReason),
|
||||
input.LoadSampledAt,
|
||||
); err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
@@ -198,7 +254,7 @@ func allocateWorkerCapacities(workers []activeWorkerCapacity, desired int) (map[
|
||||
for _, worker := range workers {
|
||||
limit := worker.CapacityLimit
|
||||
if limit <= 0 {
|
||||
limit = desired
|
||||
continue
|
||||
}
|
||||
if allocations[worker.InstanceID] >= limit {
|
||||
continue
|
||||
@@ -254,24 +310,52 @@ WHERE instance_id = $1
|
||||
}
|
||||
|
||||
type WorkerInstanceRuntime struct {
|
||||
InstanceID string
|
||||
PodUID string
|
||||
PodName string
|
||||
Site string
|
||||
Revision string
|
||||
Status string
|
||||
Allocated int
|
||||
CapacityLimit int
|
||||
RunningTasks int
|
||||
ActiveLeases int
|
||||
HeartbeatAt time.Time
|
||||
DrainingAt *time.Time
|
||||
InstanceID string `json:"instanceId"`
|
||||
PodUID string `json:"podUid,omitempty"`
|
||||
PodName string `json:"podName,omitempty"`
|
||||
Site string `json:"site,omitempty"`
|
||||
Revision string `json:"revision,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Allocated int `json:"allocatedCapacity"`
|
||||
CapacityLimit int `json:"capacityLimit"`
|
||||
HardCapacityLimit int `json:"hardCapacityLimit"`
|
||||
SafeCapacity int `json:"safeCapacity"`
|
||||
HeavyCapacity int `json:"heavyCapacity"`
|
||||
ReportedActiveTasks int `json:"reportedActiveTasks"`
|
||||
PreparingTasks int `json:"preparingTasks"`
|
||||
WaitingUpstreamTasks int `json:"waitingUpstreamTasks"`
|
||||
FinalizingTasks int `json:"finalizingTasks"`
|
||||
PressureState string `json:"pressureState"`
|
||||
PressureReason string `json:"pressureReason,omitempty"`
|
||||
LoadSampledAt *time.Time `json:"loadSampledAt,omitempty"`
|
||||
RunningTasks int `json:"runningTasks"`
|
||||
ActiveLeases int `json:"activeLeases"`
|
||||
HeartbeatAt time.Time `json:"heartbeatAt"`
|
||||
DrainingAt *time.Time `json:"drainingAt,omitempty"`
|
||||
}
|
||||
|
||||
type WorkerQueueRuntime struct {
|
||||
Queued int
|
||||
Running int
|
||||
OldestWaitSeconds float64
|
||||
Queued int `json:"queued"`
|
||||
Running int `json:"running"`
|
||||
OldestWaitSeconds float64 `json:"oldestWaitSeconds"`
|
||||
}
|
||||
|
||||
type WorkerClusterRuntime struct {
|
||||
Workers []WorkerInstanceRuntime `json:"workers"`
|
||||
Queue WorkerQueueRuntime `json:"queue"`
|
||||
CapturedAt time.Time `json:"capturedAt"`
|
||||
}
|
||||
|
||||
func (s *Store) GetWorkerClusterRuntime(ctx context.Context) (WorkerClusterRuntime, error) {
|
||||
workers, err := s.ListWorkerInstanceRuntime(ctx)
|
||||
if err != nil {
|
||||
return WorkerClusterRuntime{}, err
|
||||
}
|
||||
queue, err := s.WorkerQueueRuntime(ctx)
|
||||
if err != nil {
|
||||
return WorkerClusterRuntime{}, err
|
||||
}
|
||||
return WorkerClusterRuntime{Workers: workers, Queue: queue, CapturedAt: time.Now()}, nil
|
||||
}
|
||||
|
||||
type CapacityDatabaseHealth struct {
|
||||
@@ -306,6 +390,16 @@ SELECT worker.instance_id,
|
||||
worker.status,
|
||||
worker.allocated_capacity,
|
||||
worker.capacity_limit,
|
||||
worker.hard_capacity_limit,
|
||||
worker.safe_capacity,
|
||||
worker.heavy_capacity,
|
||||
worker.active_tasks,
|
||||
worker.preparing_tasks,
|
||||
worker.waiting_upstream_tasks,
|
||||
worker.finalizing_tasks,
|
||||
worker.pressure_state,
|
||||
worker.pressure_reason,
|
||||
worker.load_sampled_at,
|
||||
count(DISTINCT task.id) FILTER (WHERE task.status = 'running')::int,
|
||||
count(DISTINCT lease.id) FILTER (WHERE lease.released_at IS NULL)::int,
|
||||
worker.heartbeat_at,
|
||||
@@ -339,6 +433,16 @@ ORDER BY worker.site ASC, worker.status DESC, worker.instance_id ASC`,
|
||||
&instance.Status,
|
||||
&instance.Allocated,
|
||||
&instance.CapacityLimit,
|
||||
&instance.HardCapacityLimit,
|
||||
&instance.SafeCapacity,
|
||||
&instance.HeavyCapacity,
|
||||
&instance.ReportedActiveTasks,
|
||||
&instance.PreparingTasks,
|
||||
&instance.WaitingUpstreamTasks,
|
||||
&instance.FinalizingTasks,
|
||||
&instance.PressureState,
|
||||
&instance.PressureReason,
|
||||
&instance.LoadSampledAt,
|
||||
&instance.RunningTasks,
|
||||
&instance.ActiveLeases,
|
||||
&instance.HeartbeatAt,
|
||||
@@ -476,7 +580,7 @@ WITH orphaned AS MATERIALIZED (
|
||||
),
|
||||
released_leases AS (
|
||||
UPDATE gateway_concurrency_leases lease
|
||||
SET released_at = now()
|
||||
SET released_at = statement_timestamp()
|
||||
FROM orphaned
|
||||
WHERE lease.task_id = orphaned.task_id
|
||||
AND lease.released_at IS NULL
|
||||
|
||||
@@ -42,3 +42,15 @@ func TestAllocateWorkerCapacitiesSupportsUnequalLimits(t *testing.T) {
|
||||
t.Fatalf("allocations=%v global=%d, want 1/4 and 5", allocations, global)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateWorkerCapacitiesRedistributesFromPressuredWorker(t *testing.T) {
|
||||
workers := []activeWorkerCapacity{
|
||||
{InstanceID: "worker-a", CapacityLimit: 0},
|
||||
{InstanceID: "worker-b", CapacityLimit: 6},
|
||||
{InstanceID: "worker-c", CapacityLimit: 6},
|
||||
}
|
||||
allocations, global := allocateWorkerCapacities(workers, 8)
|
||||
if global != 8 || allocations["worker-a"] != 0 || allocations["worker-b"] != 4 || allocations["worker-c"] != 4 {
|
||||
t.Fatalf("allocations=%v global=%d, want 0/4/4 and 8", allocations, global)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user