feat(worker): 实现集群限流与自适应负载

保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。

apps/api/internal/runner/queue_worker.go

@@ -10,8 +10,10 @@ import (
 
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/riverqueue/river"
 	"github.com/riverqueue/river/riverdriver/riverpgxv5"
 	"github.com/riverqueue/river/rivermigrate"
@@ -45,6 +47,10 @@ type asyncTaskWorker struct {
 	service *Service
 }
 
+type workerLoadSampler interface {
+	Sample(databaseConnections, databaseMax int32) workerload.ResourceSample
+}
+
 func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs]) error {
 	task, err := w.service.store.GetTask(ctx, job.Args.TaskID)
 	if err != nil {
@@ -53,6 +59,12 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
 	if task.Status == "succeeded" || task.Status == "failed" || task.Status == "cancelled" {
 		return nil
 	}
+	loadLease, admitted := w.service.tryStartWorkerTask()
+	if !admitted {
+		return river.JobSnooze(workerLoadRetryDelay(task.ID))
+	}
+	defer loadLease.Release()
+	ctx = context.WithValue(ctx, workerLoadLeaseContextKey{}, loadLease)
 	executionToken := uuid.NewString()
 	result, runErr := w.service.executeWithToken(ctx, task, authUserFromTask(task), nil, executionToken)
 	if runErr == nil {
@@ -323,21 +335,36 @@ func (s *Service) makeAsyncExecutionClient(capacity int) (asyncExecutionClient,
 
 func (s *Service) loadAsyncWorkerCapacity(ctx context.Context) (store.AsyncWorkerCapacitySnapshot, error) {
 	if s.asyncCapacityLoader != nil {
-		return s.asyncCapacityLoader(ctx, s.cfg.AsyncWorkerHardLimit)
+		snapshot, err := s.asyncCapacityLoader(ctx, s.cfg.AsyncWorkerHardLimit)
+		if err == nil && s.workerLoad != nil {
+			s.workerLoad.SetClaimLimit(snapshot.Capacity)
+		}
+		return snapshot, err
 	}
 	snapshot, err := s.coordinationStore.AsyncWorkerCapacity(ctx, s.cfg.AsyncWorkerHardLimit)
 	if err != nil {
 		return store.AsyncWorkerCapacitySnapshot{}, err
 	}
+	loadSnapshot := s.sampleWorkerLoad()
 	allocation, err := s.coordinationStore.RegisterWorkerInstance(ctx, store.WorkerRegistrationInput{
-		InstanceID:          s.workerInstanceID,
-		PodUID:              strings.TrimSpace(os.Getenv("POD_UID")),
-		PodName:             strings.TrimSpace(os.Getenv("POD_NAME")),
-		Site:                strings.TrimSpace(os.Getenv("EASYAI_SITE")),
-		Revision:            strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
-		DesiredCapacity:     snapshot.Capacity,
-		CapacityLimit:       s.cfg.AsyncWorkerInstanceHardLimit,
-		HeartbeatStaleAfter: time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
+		InstanceID:           s.workerInstanceID,
+		PodUID:               strings.TrimSpace(os.Getenv("POD_UID")),
+		PodName:              strings.TrimSpace(os.Getenv("POD_NAME")),
+		Site:                 strings.TrimSpace(os.Getenv("EASYAI_SITE")),
+		Revision:             strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
+		DesiredCapacity:      snapshot.Capacity,
+		CapacityLimit:        s.cfg.AsyncWorkerInstanceHardLimit,
+		LoadMode:             loadSnapshot.Mode,
+		SafeCapacity:         loadSnapshot.SafeCapacity,
+		HeavyCapacity:        loadSnapshot.HeavyLimit,
+		ActiveTasks:          loadSnapshot.ActiveTasks,
+		PreparingTasks:       loadSnapshot.PreparingTasks,
+		WaitingUpstreamTasks: loadSnapshot.WaitingUpstreamTasks,
+		FinalizingTasks:      loadSnapshot.FinalizingTasks,
+		PressureState:        string(loadSnapshot.PressureState),
+		PressureReason:       loadSnapshot.PressureReason,
+		LoadSampledAt:        loadSnapshot.SampledAt,
+		HeartbeatStaleAfter:  time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
 	})
 	if err != nil {
 		return store.AsyncWorkerCapacitySnapshot{}, err
@@ -346,9 +373,60 @@ func (s *Service) loadAsyncWorkerCapacity(ctx context.Context) (store.AsyncWorke
 	snapshot.GlobalCapacity = allocation.GlobalAllocated
 	snapshot.ActiveInstances = allocation.ActiveInstances
 	snapshot.InstanceID = allocation.InstanceID
+	loadSnapshot = s.workerLoad.SetClaimLimit(allocation.Allocated)
+	snapshot.LoadMode = loadSnapshot.Mode
+	snapshot.LocalSafeCapacity = loadSnapshot.SafeCapacity
+	snapshot.LocalHeavyCapacity = loadSnapshot.HeavyLimit
+	snapshot.LocalActiveTasks = loadSnapshot.ActiveTasks
+	snapshot.LocalPreparingTasks = loadSnapshot.PreparingTasks
+	snapshot.LocalWaitingTasks = loadSnapshot.WaitingUpstreamTasks
+	snapshot.LocalFinalizingTasks = loadSnapshot.FinalizingTasks
+	snapshot.LocalPressureState = string(loadSnapshot.PressureState)
+	snapshot.LocalPressureReason = loadSnapshot.PressureReason
 	return snapshot, nil
 }
 
+func (s *Service) sampleWorkerLoad() workerload.Snapshot {
+	if s.workerLoad == nil {
+		return workerload.Snapshot{Mode: workerload.ModeLegacy, SafeCapacity: s.cfg.AsyncWorkerInstanceHardLimit, HeavyLimit: s.cfg.AsyncWorkerInstanceHardLimit}
+	}
+	if s.workerLoadSampler == nil {
+		return s.workerLoad.Snapshot()
+	}
+	var connections int32
+	var maximum int32
+	seen := make(map[*pgxpool.Pool]struct{}, 3)
+	for _, database := range []*store.Store{s.store, s.coordinationStore, s.riverStore} {
+		if database == nil || database.Pool() == nil {
+			continue
+		}
+		pool := database.Pool()
+		if _, ok := seen[pool]; ok {
+			continue
+		}
+		seen[pool] = struct{}{}
+		statistics := pool.Stat()
+		connections += statistics.AcquiredConns()
+		maximum += statistics.MaxConns()
+	}
+	return s.workerLoad.Observe(s.workerLoadSampler.Sample(connections, maximum))
+}
+
+func (s *Service) tryStartWorkerTask() (*workerload.Lease, bool) {
+	if s.workerLoad == nil {
+		return nil, true
+	}
+	return s.workerLoad.TryStart()
+}
+
+func workerLoadRetryDelay(taskID string) time.Duration {
+	var value uint32
+	for _, character := range []byte(taskID) {
+		value = value*33 + uint32(character)
+	}
+	return 250*time.Millisecond + time.Duration(value%501)*time.Millisecond
+}
+
 func (s *Service) refreshAsyncWorkerCapacity(ctx context.Context) {
 	ticker := time.NewTicker(time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * time.Second)
 	defer ticker.Stop()
@@ -427,6 +505,12 @@ func (s *Service) resizeAsyncWorkerCapacity(ctx context.Context) {
 		"modelDesired", snapshot.ModelDesired,
 		"groupDesired", snapshot.GroupDesired,
 		"capped", snapshot.Capped,
+		"loadMode", snapshot.LoadMode,
+		"localSafeCapacity", snapshot.LocalSafeCapacity,
+		"localHeavyCapacity", snapshot.LocalHeavyCapacity,
+		"localActiveTasks", snapshot.LocalActiveTasks,
+		"pressureState", snapshot.LocalPressureState,
+		"pressureReason", snapshot.LocalPressureReason,
 	)
 	if oldClient != nil {
 		go s.drainAsyncWorkerClient(oldClient)
@@ -493,6 +577,16 @@ func (s *Service) observeAsyncWorkerCapacity(snapshot store.AsyncWorkerCapacityS
 	if ok {
 		distributedObserver.SetDistributedWorkerCapacity(snapshot.ActiveInstances, snapshot.GlobalCapacity, snapshot.Capacity)
 	}
+	loadObserver, ok := s.billingMetrics.(interface {
+		SetWorkerLoad(activeLimit, heavyLimit, active, preparing, waiting, finalizing int, pressure string)
+	})
+	if ok {
+		loadObserver.SetWorkerLoad(
+			snapshot.LocalSafeCapacity, snapshot.LocalHeavyCapacity, snapshot.LocalActiveTasks,
+			snapshot.LocalPreparingTasks, snapshot.LocalWaitingTasks, snapshot.LocalFinalizingTasks,
+			snapshot.LocalPressureState,
+		)
+	}
 }
 
 func (s *Service) observeAsyncWorkerResize(outcome string) {