feat(worker): 实现集群限流与自适应负载

保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
2026-08-03 00:13:46 +08:00
parent 9a01fd4657
commit c28bf74230
52 changed files with 3700 additions and 272 deletions
+1 -1
View File
@@ -689,7 +689,7 @@ ON CONFLICT (task_id, event_type) DO NOTHING`, runID); err != nil {
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases lease
SET released_at = now()
SET released_at = statement_timestamp()
FROM gateway_tasks task
WHERE lease.task_id = task.id
AND lease.released_at IS NULL
+31 -8
View File
@@ -747,14 +747,14 @@ SELECT COUNT(*)
FROM gateway_concurrency_leases
WHERE task_id = $1::uuid
AND released_at IS NULL
AND expires_at > now()`, taskID).Scan(&count)
AND expires_at > statement_timestamp()`, taskID).Scan(&count)
return count, err
}
func resetTaskAdmissionToWaitingTx(ctx context.Context, tx pgx.Tx, input TaskAdmissionInput) (TaskAdmission, error) {
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = $1::uuid
AND released_at IS NULL`, input.TaskID); err != nil {
return TaskAdmission{}, err
@@ -862,7 +862,7 @@ func (s *Store) deleteTaskAdmissionOnce(ctx context.Context, taskID string) erro
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = $1::uuid
AND released_at IS NULL`, taskID); err != nil {
return err
@@ -884,7 +884,7 @@ SELECT id::text,
FROM gateway_concurrency_leases
WHERE task_id = $1::uuid
AND released_at IS NULL
AND expires_at > now()
AND expires_at > statement_timestamp()
ORDER BY scope_type, scope_key, id`, taskID)
if err != nil {
return nil, err
@@ -924,7 +924,7 @@ WHERE admission.mode = 'async'
FROM gateway_concurrency_leases lease
WHERE lease.task_id = admission.task_id
AND lease.released_at IS NULL
AND lease.expires_at > now()
AND lease.expires_at > statement_timestamp()
)
)
)
@@ -980,7 +980,7 @@ WHERE admission.mode = 'async'
FROM gateway_concurrency_leases lease
WHERE lease.task_id = admission.task_id
AND lease.released_at IS NULL
AND lease.expires_at > now()
AND lease.expires_at > statement_timestamp()
)
)
)
@@ -1006,6 +1006,29 @@ LIMIT $1`, limit)
return admissions, rows.Err()
}
// RequestWaitingTaskAdmissionReselect marks every queued task bound to a
// saturated platform model so the dispatcher can route the next batch to a
// different eligible candidate. The durable marker lets multiple dispatchers
// observe the same decision without assigning tasks to a specific Worker.
func (s *Store) RequestWaitingTaskAdmissionReselect(ctx context.Context, platformModelID string) (int64, error) {
result, err := s.pool.Exec(ctx, `
UPDATE gateway_task_admissions admission
SET reselect_requested_at = now(),
updated_at = now()
FROM gateway_tasks task
WHERE admission.task_id = task.id
AND admission.platform_model_id = $1::uuid
AND admission.mode = 'async'
AND admission.status = 'waiting'
AND task.status = 'queued'
AND task.next_run_at <= now()
AND admission.reselect_requested_at IS NULL`, platformModelID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
// ListWaitingTaskAdmissionIDs returns one FIFO leader for every independent
// platform-model queue, additionally requiring the task to lead its user-group
// queue when one exists. It is used after capacity is released so API
@@ -1268,12 +1291,12 @@ func admissionScopeStatesTx(ctx context.Context, tx pgx.Tx, scopes []AdmissionSc
if scope.ConcurrentLimit > 0 {
if err := tx.QueryRow(ctx, `
SELECT COALESCE(SUM(lease_value), 0)::float8,
COALESCE(MIN(expires_at), now() + interval '1 second')
COALESCE(MIN(expires_at), statement_timestamp() + interval '1 second')
FROM gateway_concurrency_leases
WHERE scope_type = $1
AND scope_key = $2
AND released_at IS NULL
AND expires_at > now()`, scope.ScopeType, scope.ScopeKey).Scan(&state.Active, &state.NextLeaseExpiration); err != nil {
AND expires_at > statement_timestamp()`, scope.ScopeType, scope.ScopeKey).Scan(&state.Active, &state.NextLeaseExpiration); err != nil {
return nil, err
}
state.Saturated = state.Active+scope.Amount > scope.ConcurrentLimit
@@ -451,6 +451,20 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID, queuedSyntheticRiverJobID)
if listedSnapshot == nil || len(listedSnapshot.Scopes) != len(queuedAdmission.Scopes) {
t.Fatalf("listed admission snapshot=%+v, want %d scopes", listedSnapshot, len(queuedAdmission.Scopes))
}
markedForReselect, err := first.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
if err != nil {
t.Fatalf("request waiting admission reselection: %v", err)
}
if markedForReselect < 1 {
t.Fatalf("marked admissions=%d, want at least the queued task", markedForReselect)
}
reselectAdmission, err := first.GetTaskAdmission(ctx, queuedAtomicTask.ID)
if err != nil {
t.Fatalf("read admission reselection marker: %v", err)
}
if reselectAdmission.ReselectRequestedAt.IsZero() {
t.Fatal("queued admission was not marked for candidate reselection")
}
var riverJobID int64
if err := first.pool.QueryRow(ctx, `
SELECT
@@ -1296,4 +1310,35 @@ WHERE instance_id = $1`, secondID, (workerHeartbeatStaleAfter + time.Second).Str
if err != nil || second.Allocated != 2 || second.GlobalAllocated != 5 || second.ActiveInstances != 2 {
t.Fatalf("bounded two-worker allocation = %+v, err=%v", second, err)
}
first, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
InstanceID: firstID, DesiredCapacity: 100, CapacityLimit: 10,
LoadMode: "adaptive", SafeCapacity: 0, HeavyCapacity: 1,
ActiveTasks: 2, WaitingUpstreamTasks: 2,
PressureState: "critical", PressureReason: "memory",
})
if err != nil || first.Allocated != 0 {
t.Fatalf("critical worker allocation = %+v, err=%v", first, err)
}
second, err = db.RegisterWorkerInstance(ctx, WorkerRegistrationInput{
InstanceID: secondID, DesiredCapacity: 100, CapacityLimit: 10,
LoadMode: "adaptive", SafeCapacity: 5, HeavyCapacity: 2,
ActiveTasks: 3, PreparingTasks: 1, WaitingUpstreamTasks: 1, FinalizingTasks: 1,
PressureState: "normal",
})
if err != nil || second.Allocated != 5 || second.GlobalAllocated != 5 {
t.Fatalf("adaptive redistribution allocation = %+v, err=%v", second, err)
}
instances, err := db.ListWorkerInstanceRuntime(ctx)
if err != nil {
t.Fatalf("list adaptive worker runtime: %v", err)
}
foundCritical := false
for _, instance := range instances {
if instance.InstanceID == firstID {
foundCritical = instance.SafeCapacity == 0 && instance.ReportedActiveTasks == 2 && instance.WaitingUpstreamTasks == 2 && instance.PressureState == "critical"
}
}
if !foundCritical {
t.Fatalf("adaptive runtime did not expose the critical worker: %+v", instances)
}
}
@@ -6,19 +6,28 @@ import (
)
type AsyncWorkerCapacitySnapshot struct {
Capacity int
GlobalCapacity int
Desired int
HardLimit int
Capped bool
EnabledModels int
UnlimitedModels int
EnabledGroups int
UnlimitedGroups int
ModelDesired int
GroupDesired int
ActiveInstances int
InstanceID string
Capacity int
GlobalCapacity int
Desired int
HardLimit int
Capped bool
EnabledModels int
UnlimitedModels int
EnabledGroups int
UnlimitedGroups int
ModelDesired int
GroupDesired int
ActiveInstances int
InstanceID string
LoadMode string
LocalSafeCapacity int
LocalHeavyCapacity int
LocalActiveTasks int
LocalPreparingTasks int
LocalWaitingTasks int
LocalFinalizingTasks int
LocalPressureState string
LocalPressureReason string
}
func (s *Store) AsyncWorkerCapacity(ctx context.Context, hardLimit int) (AsyncWorkerCapacitySnapshot, error) {
+1 -1
View File
@@ -100,7 +100,7 @@ LEFT JOIN (
FROM gateway_concurrency_leases
WHERE scope_type = 'platform_model'
AND released_at IS NULL
AND expires_at > now()
AND expires_at > statement_timestamp()
GROUP BY scope_key
) con ON con.scope_key = m.id::text
LEFT JOIN (
+1 -1
View File
@@ -169,7 +169,7 @@ LEFT JOIN (
FROM gateway_concurrency_leases
WHERE scope_type = 'platform_model'
AND released_at IS NULL
AND expires_at > now()
AND expires_at > statement_timestamp()
GROUP BY scope_key
) con ON con.scope_key = m.id::text
LEFT JOIN (
+14 -8
View File
@@ -184,12 +184,12 @@ func reserveConcurrencyLease(ctx context.Context, tx pgx.Tx, taskID string, atte
var nextAvailableAt time.Time
if err := tx.QueryRow(ctx, `
SELECT COALESCE(SUM(lease_value), 0)::float8,
COALESCE(MIN(expires_at), now() + ($3::int * interval '1 second'))
COALESCE(MIN(expires_at), statement_timestamp() + ($3::int * interval '1 second'))
FROM gateway_concurrency_leases
WHERE scope_type = $1
AND scope_key = $2
AND released_at IS NULL
AND expires_at > now()`,
AND expires_at > statement_timestamp()`,
reservation.ScopeType,
reservation.ScopeKey,
reservation.LeaseTTLSeconds,
@@ -218,14 +218,20 @@ WHERE scope_type = $1
}
var leaseID string
if err := tx.QueryRow(ctx, `
INSERT INTO gateway_concurrency_leases (task_id, attempt_id, scope_type, scope_key, lease_value, expires_at)
VALUES ($1::uuid, NULLIF($2, '')::uuid, $3, $4, $5, now() + ($6::int * interval '1 second'))
INSERT INTO gateway_concurrency_leases (
task_id, attempt_id, scope_type, scope_key, lease_value, limit_value, acquired_at, expires_at
)
VALUES (
$1::uuid, NULLIF($2, '')::uuid, $3, $4, $5, $6,
statement_timestamp(), statement_timestamp() + ($7::int * interval '1 second')
)
RETURNING id::text`,
taskID,
attemptID,
reservation.ScopeType,
reservation.ScopeKey,
reservation.Amount,
reservation.Limit,
reservation.LeaseTTLSeconds,
).Scan(&leaseID); err != nil {
return ConcurrencyLease{}, err
@@ -381,7 +387,7 @@ func (s *Store) ReleaseConcurrencyLeases(ctx context.Context, leases []Concurren
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE id = ANY($1::uuid[]) AND released_at IS NULL`, leaseIDs); err != nil {
return err
}
@@ -411,7 +417,7 @@ func (s *Store) RenewConcurrencyLeases(ctx context.Context, leases []Concurrency
}
tag, err := s.pool.Exec(ctx, `
UPDATE gateway_concurrency_leases lease
SET expires_at = now() + (renewal.ttl_seconds * interval '1 second')
SET expires_at = statement_timestamp() + (renewal.ttl_seconds * interval '1 second')
FROM unnest($1::uuid[], $2::int[]) AS renewal(id, ttl_seconds)
WHERE lease.id = renewal.id
AND lease.released_at IS NULL
@@ -705,7 +711,7 @@ WHERE attempt.task_id = task.id
result.FailedAttempts = tag.RowsAffected()
tag, err = tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = ANY($1::uuid[])
AND released_at IS NULL`, taskIDs)
if err != nil {
@@ -772,7 +778,7 @@ FOR UPDATE OF task SKIP LOCKED`, runtimeRecoveryBatchSize)
var result RuntimeRecoveryResult
tag, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = ANY($1::uuid[])
AND released_at IS NULL`, taskIDs)
if err != nil {
@@ -98,23 +98,168 @@ WHERE scope_type = 'platform_model'
}
var active int64
var storedLimit float64
if err := first.Pool().QueryRow(ctx, `
SELECT COUNT(*)
SELECT COUNT(*), COALESCE(MAX(limit_value), 0)::float8
FROM gateway_concurrency_leases
WHERE scope_type = 'platform_model'
AND scope_key = $1
AND released_at IS NULL
AND expires_at > now()`, scopeKey).Scan(&active); err != nil {
AND expires_at > now()`, scopeKey).Scan(&active, &storedLimit); err != nil {
t.Fatalf("count active leases: %v", err)
}
if successes.Load() != 64 || active != 64 {
t.Fatalf("successful reservations=%d active leases=%d, want exactly 64", successes.Load(), active)
if successes.Load() != 64 || active != 64 || storedLimit != 64 {
t.Fatalf("successful reservations=%d active leases=%d stored limit=%.0f, want exactly 64", successes.Load(), active, storedLimit)
}
if peak.Load() > 64 {
t.Fatalf("active lease peak=%d, want <=64", peak.Load())
}
}
func TestConcurrencyLeaseTimestampStartsAtReservationStatement(t *testing.T) {
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run concurrency lease PostgreSQL integration tests")
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
database, err := Connect(ctx, databaseURL)
if err != nil {
t.Fatalf("connect store: %v", err)
}
defer database.Close()
scopeKey := "statement-clock-" + time.Now().UTC().Format("20060102150405.000000000")
taskIDs := createLeaseTestTasks(t, ctx, database, 1, scopeKey)
defer deleteLeaseTestTasks(t, database, taskIDs)
tx, err := database.pool.Begin(ctx)
if err != nil {
t.Fatalf("begin reservation transaction: %v", err)
}
defer rollbackTransaction(tx)
var transactionStartedAt time.Time
if err := tx.QueryRow(ctx, `SELECT now()`).Scan(&transactionStartedAt); err != nil {
t.Fatalf("read transaction start: %v", err)
}
time.Sleep(1100 * time.Millisecond)
lease, err := reserveConcurrencyLease(ctx, tx, taskIDs[0], "", RateLimitReservation{
ScopeType: "platform_model",
ScopeKey: scopeKey,
Metric: "concurrent",
Limit: 1,
Amount: 1,
LeaseTTLSeconds: 30,
})
if err != nil {
t.Fatalf("reserve concurrency lease: %v", err)
}
if err := tx.Commit(ctx); err != nil {
t.Fatalf("commit reservation transaction: %v", err)
}
var acquiredAt, expiresAt time.Time
if err := database.pool.QueryRow(ctx, `
SELECT acquired_at, expires_at
FROM gateway_concurrency_leases
WHERE id = $1::uuid`, lease.ID).Scan(&acquiredAt, &expiresAt); err != nil {
t.Fatalf("read lease timestamps: %v", err)
}
if elapsed := acquiredAt.Sub(transactionStartedAt); elapsed < time.Second {
t.Fatalf("lease acquired_at advanced by %s, want at least 1s after transaction start", elapsed)
}
if ttl := expiresAt.Sub(acquiredAt); ttl != 30*time.Second {
t.Fatalf("lease ttl=%s, want 30s", ttl)
}
}
func TestCounterWindowReservationIsAtomicAcrossPools(t *testing.T) {
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run rate limit PostgreSQL integration tests")
}
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
first, err := Connect(ctx, databaseURL)
if err != nil {
t.Fatalf("connect first store: %v", err)
}
defer first.Close()
second, err := Connect(ctx, databaseURL)
if err != nil {
t.Fatalf("connect second store: %v", err)
}
defer second.Close()
tests := []struct {
metric string
limit float64
amount float64
wantSuccesses int64
}{
{metric: "rpm", limit: 37, amount: 1, wantSuccesses: 37},
{metric: "tpm_total", limit: 1_000, amount: 25, wantSuccesses: 40},
}
for _, test := range tests {
t.Run(test.metric, func(t *testing.T) {
scopeKey := "atomic-" + test.metric + "-" + time.Now().UTC().Format("20060102150405.000000000")
taskIDs := createLeaseTestTasks(t, ctx, first, 128, scopeKey)
defer deleteLeaseTestTasks(t, first, taskIDs)
var successes atomic.Int64
var wg sync.WaitGroup
errs := make(chan error, len(taskIDs))
for index, taskID := range taskIDs {
wg.Add(1)
go func(index int, taskID string) {
defer wg.Done()
target := first
if index%2 == 1 {
target = second
}
_, err := target.ReserveRateLimits(ctx, taskID, "", []RateLimitReservation{{
ScopeType: "platform_model",
ScopeKey: scopeKey,
Metric: test.metric,
Limit: test.limit,
Amount: test.amount,
WindowSeconds: 3600,
}})
if err == nil {
successes.Add(1)
return
}
if !errors.Is(err, ErrRateLimited) {
errs <- err
}
}(index, taskID)
}
wg.Wait()
close(errs)
for err := range errs {
t.Fatalf("unexpected reservation error: %v", err)
}
var current float64
if err := first.Pool().QueryRow(ctx, `
SELECT COALESCE(MAX(used_value + reserved_value), 0)::float8
FROM gateway_rate_limit_counters
WHERE scope_type = 'platform_model'
AND scope_key = $1
AND metric = $2`, scopeKey, test.metric).Scan(&current); err != nil {
t.Fatalf("read %s counter: %v", test.metric, err)
}
if successes.Load() != test.wantSuccesses {
t.Fatalf("successful %s reservations=%d, want exactly %d", test.metric, successes.Load(), test.wantSuccesses)
}
wantCurrent := float64(test.wantSuccesses) * test.amount
if current != wantCurrent || current > test.limit {
t.Fatalf("%s current=%.0f, want %.0f and <= %.0f", test.metric, current, wantCurrent, test.limit)
}
})
}
}
func TestConcurrencyLeaseRenewalExtendsAndReleases(t *testing.T) {
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
if databaseURL == "" {
+5 -5
View File
@@ -757,7 +757,7 @@ WHERE task_id = $1::uuid
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = $1::uuid
AND released_at IS NULL`, taskID); err != nil {
return err
@@ -922,7 +922,7 @@ RETURNING `+gatewayTaskColumns, taskID, message))
changed = true
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = $1::uuid
AND released_at IS NULL`, taskID); err != nil {
return err
@@ -1011,7 +1011,7 @@ WHERE task_id = $1::uuid
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = $1::uuid
AND released_at IS NULL`, taskID); err != nil {
return err
@@ -1720,7 +1720,7 @@ ON CONFLICT (task_id, event_type) DO NOTHING`,
if input.FinalizeAdmission {
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = $1::uuid
AND released_at IS NULL`, input.TaskID); err != nil {
return err
@@ -2096,7 +2096,7 @@ ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON)
if input.FinalizeAdmission {
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = now()
SET released_at = statement_timestamp()
WHERE task_id = $1::uuid
AND released_at IS NULL`, input.TaskID); err != nil {
return err
+131 -27
View File
@@ -12,14 +12,24 @@ import (
const workerHeartbeatStaleAfter = 30 * time.Second
type WorkerRegistrationInput struct {
InstanceID string
PodUID string
PodName string
Site string
Revision string
DesiredCapacity int
CapacityLimit int
HeartbeatStaleAfter time.Duration
InstanceID string
PodUID string
PodName string
Site string
Revision string
DesiredCapacity int
CapacityLimit int
LoadMode string
SafeCapacity int
HeavyCapacity int
ActiveTasks int
PreparingTasks int
WaitingUpstreamTasks int
FinalizingTasks int
PressureState string
PressureReason string
LoadSampledAt time.Time
HeartbeatStaleAfter time.Duration
}
type WorkerAllocation struct {
@@ -59,9 +69,28 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
if input.CapacityLimit < 0 {
return WorkerAllocation{}, errors.New("worker capacity limit cannot be negative")
}
if input.SafeCapacity < 0 || input.HeavyCapacity < 0 || input.ActiveTasks < 0 || input.PreparingTasks < 0 || input.WaitingUpstreamTasks < 0 || input.FinalizingTasks < 0 {
return WorkerAllocation{}, errors.New("worker load values cannot be negative")
}
if input.ActiveTasks != input.PreparingTasks+input.WaitingUpstreamTasks+input.FinalizingTasks {
return WorkerAllocation{}, errors.New("worker active task count must equal phase task counts")
}
if input.CapacityLimit == 0 {
input.CapacityLimit = input.DesiredCapacity
}
hardCapacityLimit := input.CapacityLimit
if strings.EqualFold(strings.TrimSpace(input.LoadMode), "adaptive") {
input.CapacityLimit = min(input.CapacityLimit, input.SafeCapacity)
}
pressureState := strings.ToLower(strings.TrimSpace(input.PressureState))
switch pressureState {
case "normal", "busy", "critical":
default:
pressureState = "unknown"
}
if input.LoadSampledAt.IsZero() {
input.LoadSampledAt = time.Now()
}
staleAfter := input.HeartbeatStaleAfter
if staleAfter < workerHeartbeatStaleAfter {
staleAfter = workerHeartbeatStaleAfter
@@ -83,9 +112,16 @@ func (s *Store) RegisterWorkerInstance(ctx context.Context, input WorkerRegistra
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_worker_instances (
instance_id, pod_uid, pod_name, site, revision, status,
desired_capacity, capacity_limit, allocated_capacity, started_at, heartbeat_at, updated_at
desired_capacity, capacity_limit, hard_capacity_limit, safe_capacity, heavy_capacity,
active_tasks, preparing_tasks, waiting_upstream_tasks, finalizing_tasks,
pressure_state, pressure_reason, load_sampled_at,
allocated_capacity, started_at, heartbeat_at, updated_at
)
VALUES (
$1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, $16, $17,
0, now(), now(), now()
)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, 0, now(), now(), now())
ON CONFLICT (instance_id) DO UPDATE
SET pod_uid = EXCLUDED.pod_uid,
pod_name = EXCLUDED.pod_name,
@@ -97,6 +133,16 @@ SET pod_uid = EXCLUDED.pod_uid,
END,
desired_capacity = EXCLUDED.desired_capacity,
capacity_limit = EXCLUDED.capacity_limit,
hard_capacity_limit = EXCLUDED.hard_capacity_limit,
safe_capacity = EXCLUDED.safe_capacity,
heavy_capacity = EXCLUDED.heavy_capacity,
active_tasks = EXCLUDED.active_tasks,
preparing_tasks = EXCLUDED.preparing_tasks,
waiting_upstream_tasks = EXCLUDED.waiting_upstream_tasks,
finalizing_tasks = EXCLUDED.finalizing_tasks,
pressure_state = EXCLUDED.pressure_state,
pressure_reason = EXCLUDED.pressure_reason,
load_sampled_at = EXCLUDED.load_sampled_at,
heartbeat_at = now(),
updated_at = now()`,
input.InstanceID,
@@ -106,6 +152,16 @@ SET pod_uid = EXCLUDED.pod_uid,
strings.TrimSpace(input.Revision),
input.DesiredCapacity,
input.CapacityLimit,
hardCapacityLimit,
input.SafeCapacity,
input.HeavyCapacity,
input.ActiveTasks,
input.PreparingTasks,
input.WaitingUpstreamTasks,
input.FinalizingTasks,
pressureState,
strings.TrimSpace(input.PressureReason),
input.LoadSampledAt,
); err != nil {
return WorkerAllocation{}, err
}
@@ -198,7 +254,7 @@ func allocateWorkerCapacities(workers []activeWorkerCapacity, desired int) (map[
for _, worker := range workers {
limit := worker.CapacityLimit
if limit <= 0 {
limit = desired
continue
}
if allocations[worker.InstanceID] >= limit {
continue
@@ -254,24 +310,52 @@ WHERE instance_id = $1
}
type WorkerInstanceRuntime struct {
InstanceID string
PodUID string
PodName string
Site string
Revision string
Status string
Allocated int
CapacityLimit int
RunningTasks int
ActiveLeases int
HeartbeatAt time.Time
DrainingAt *time.Time
InstanceID string `json:"instanceId"`
PodUID string `json:"podUid,omitempty"`
PodName string `json:"podName,omitempty"`
Site string `json:"site,omitempty"`
Revision string `json:"revision,omitempty"`
Status string `json:"status"`
Allocated int `json:"allocatedCapacity"`
CapacityLimit int `json:"capacityLimit"`
HardCapacityLimit int `json:"hardCapacityLimit"`
SafeCapacity int `json:"safeCapacity"`
HeavyCapacity int `json:"heavyCapacity"`
ReportedActiveTasks int `json:"reportedActiveTasks"`
PreparingTasks int `json:"preparingTasks"`
WaitingUpstreamTasks int `json:"waitingUpstreamTasks"`
FinalizingTasks int `json:"finalizingTasks"`
PressureState string `json:"pressureState"`
PressureReason string `json:"pressureReason,omitempty"`
LoadSampledAt *time.Time `json:"loadSampledAt,omitempty"`
RunningTasks int `json:"runningTasks"`
ActiveLeases int `json:"activeLeases"`
HeartbeatAt time.Time `json:"heartbeatAt"`
DrainingAt *time.Time `json:"drainingAt,omitempty"`
}
type WorkerQueueRuntime struct {
Queued int
Running int
OldestWaitSeconds float64
Queued int `json:"queued"`
Running int `json:"running"`
OldestWaitSeconds float64 `json:"oldestWaitSeconds"`
}
type WorkerClusterRuntime struct {
Workers []WorkerInstanceRuntime `json:"workers"`
Queue WorkerQueueRuntime `json:"queue"`
CapturedAt time.Time `json:"capturedAt"`
}
func (s *Store) GetWorkerClusterRuntime(ctx context.Context) (WorkerClusterRuntime, error) {
workers, err := s.ListWorkerInstanceRuntime(ctx)
if err != nil {
return WorkerClusterRuntime{}, err
}
queue, err := s.WorkerQueueRuntime(ctx)
if err != nil {
return WorkerClusterRuntime{}, err
}
return WorkerClusterRuntime{Workers: workers, Queue: queue, CapturedAt: time.Now()}, nil
}
type CapacityDatabaseHealth struct {
@@ -306,6 +390,16 @@ SELECT worker.instance_id,
worker.status,
worker.allocated_capacity,
worker.capacity_limit,
worker.hard_capacity_limit,
worker.safe_capacity,
worker.heavy_capacity,
worker.active_tasks,
worker.preparing_tasks,
worker.waiting_upstream_tasks,
worker.finalizing_tasks,
worker.pressure_state,
worker.pressure_reason,
worker.load_sampled_at,
count(DISTINCT task.id) FILTER (WHERE task.status = 'running')::int,
count(DISTINCT lease.id) FILTER (WHERE lease.released_at IS NULL)::int,
worker.heartbeat_at,
@@ -339,6 +433,16 @@ ORDER BY worker.site ASC, worker.status DESC, worker.instance_id ASC`,
&instance.Status,
&instance.Allocated,
&instance.CapacityLimit,
&instance.HardCapacityLimit,
&instance.SafeCapacity,
&instance.HeavyCapacity,
&instance.ReportedActiveTasks,
&instance.PreparingTasks,
&instance.WaitingUpstreamTasks,
&instance.FinalizingTasks,
&instance.PressureState,
&instance.PressureReason,
&instance.LoadSampledAt,
&instance.RunningTasks,
&instance.ActiveLeases,
&instance.HeartbeatAt,
@@ -476,7 +580,7 @@ WITH orphaned AS MATERIALIZED (
),
released_leases AS (
UPDATE gateway_concurrency_leases lease
SET released_at = now()
SET released_at = statement_timestamp()
FROM orphaned
WHERE lease.task_id = orphaned.task_id
AND lease.released_at IS NULL
@@ -42,3 +42,15 @@ func TestAllocateWorkerCapacitiesSupportsUnequalLimits(t *testing.T) {
t.Fatalf("allocations=%v global=%d, want 1/4 and 5", allocations, global)
}
}
func TestAllocateWorkerCapacitiesRedistributesFromPressuredWorker(t *testing.T) {
workers := []activeWorkerCapacity{
{InstanceID: "worker-a", CapacityLimit: 0},
{InstanceID: "worker-b", CapacityLimit: 6},
{InstanceID: "worker-c", CapacityLimit: 6},
}
allocations, global := allocateWorkerCapacities(workers, 8)
if global != 8 || allocations["worker-a"] != 0 || allocations["worker-b"] != 4 || allocations["worker-c"] != 4 {
t.Fatalf("allocations=%v global=%d, want 0/4/4 and 8", allocations, global)
}
}