perf(queue): 按策略动态扩缩异步 Worker

将 River 执行容量按平台模型与用户组的有效并发策略动态调整，并为长任务续租、并发租约原子抢占和限流退避补充保护。\n\n统一平台模型限流继承语义，兼容历史 platformLimits/modelLimits，并为三个迁移图像模型建立独立并发租约。补充管理端显式继承/覆盖配置、指标、单元测试及隔离 PostgreSQL 验收。\n\n验证：go test ./... -count=1；pnpm lint；pnpm test；pnpm build；隔离 PostgreSQL 并发原子性/续租测试；128 任务与三分钟长任务动态 Worker 验收。

apps/api/internal/runner/service.go

@@ -10,6 +10,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
@@ -23,14 +24,20 @@ import (
 )
 
 type Service struct {
-	cfg            config.Config
-	store          *store.Store
-	logger         *slog.Logger
-	clients        map[string]clients.Client
-	scriptExecutor *scriptengine.Executor
-	httpClients    *httpClientCache
-	riverClient    *river.Client[pgx.Tx]
-	billingMetrics billingMetricsObserver
+	cfg                  config.Config
+	store                *store.Store
+	logger               *slog.Logger
+	clients              map[string]clients.Client
+	scriptExecutor       *scriptengine.Executor
+	httpClients          *httpClientCache
+	riverMu              sync.RWMutex
+	riverControlClient   *river.Client[pgx.Tx]
+	riverExecutionClient asyncExecutionClient
+	riverDrainingClients map[asyncExecutionClient]struct{}
+	riverWorkerCapacity  int
+	asyncCapacityLoader  func(context.Context, int) (store.AsyncWorkerCapacitySnapshot, error)
+	asyncClientFactory   func(int) (asyncExecutionClient, error)
+	billingMetrics       billingMetricsObserver
 }
 
 type billingMetricsObserver interface {
@@ -75,6 +82,12 @@ func (e *TaskQueuedError) Is(target error) bool {
 }
 
 func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...billingMetricsObserver) *Service {
+	if cfg.AsyncWorkerHardLimit == 0 {
+		cfg.AsyncWorkerHardLimit = 2048
+	}
+	if cfg.AsyncWorkerRefreshIntervalSeconds == 0 {
+		cfg.AsyncWorkerRefreshIntervalSeconds = 5
+	}
 	httpClients := newHTTPClientCache()
 	scriptExecutor := &scriptengine.Executor{Logger: logger}
 	service := &Service{
@@ -891,7 +904,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 			_ = s.store.ReleaseRateLimitReservations(context.WithoutCancel(ctx), limitResult.Reservations, "attempt_failed")
 		}
 	}()
-	defer s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.LeaseIDs)
+	defer s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.Leases)
 
 	attemptID, err := s.store.CreateTaskAttempt(ctx, store.CreateTaskAttemptInput{
 		TaskID:             task.ID,
@@ -1031,7 +1044,8 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 		return nil
 	}
 	var submissionWire *clients.WireResponse
-	response, err := client.Run(ctx, clients.Request{
+	runCtx, stopLeaseRenewal := s.startConcurrencyLeaseRenewal(ctx, task.ID, limitResult.Leases)
+	response, err := client.Run(runCtx, clients.Request{
 		Kind:              task.Kind,
 		ModelType:         candidate.ModelType,
 		Model:             task.Model,
@@ -1083,6 +1097,13 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 		UpstreamPreviousResponseID: responseExecution.UpstreamPreviousResponseID,
 		PreviousResponseTurns:      responseExecution.PreviousTurns,
 	})
+	if leaseErr := stopLeaseRenewal(); leaseErr != nil {
+		err = &clients.ClientError{
+			Code:      "concurrency_lease_lost",
+			Message:   leaseErr.Error(),
+			Retryable: true,
+		}
+	}
 	callFinishedAt := time.Now()
 	if err == nil {
 		if markErr := setSubmissionStatus("response_received"); markErr != nil {
@@ -1500,6 +1521,7 @@ func (s *Service) requeueRateLimitedTask(ctx context.Context, task store.Gateway
 	if delay <= 0 {
 		delay = 5 * time.Second
 	}
+	delay += taskRetryJitter(task.ID)
 	queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, delay, candidate.QueueKey)
 	if err != nil {
 		return store.GatewayTask{}, 0, err
@@ -1515,6 +1537,81 @@ func (s *Service) requeueRateLimitedTask(ctx context.Context, task store.Gateway
 	return queued, delay, nil
 }
 
+func taskRetryJitter(taskID string) time.Duration {
+	var sum uint32
+	for _, value := range []byte(taskID) {
+		sum = sum*33 + uint32(value)
+	}
+	return time.Duration(sum%251) * time.Millisecond
+}
+
+func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID string, leases []store.ConcurrencyLease) (context.Context, func() error) {
+	if len(leases) == 0 {
+		return ctx, func() error { return nil }
+	}
+	interval := 30 * time.Second
+	for _, lease := range leases {
+		ttl := lease.TTL
+		if ttl <= 0 {
+			ttl = 120 * time.Second
+		}
+		candidate := ttl / 3
+		if candidate < time.Second {
+			candidate = time.Second
+		}
+		if candidate < interval {
+			interval = candidate
+		}
+	}
+	runCtx, cancelRun := context.WithCancel(ctx)
+	renewCtx, cancelRenew := context.WithCancel(ctx)
+	done := make(chan error, 1)
+	go func() {
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-renewCtx.Done():
+				done <- nil
+				return
+			case <-ticker.C:
+				if err := s.store.RenewConcurrencyLeases(renewCtx, leases); err != nil {
+					if renewCtx.Err() != nil {
+						done <- nil
+						return
+					}
+					outcome := "failure"
+					if errors.Is(err, store.ErrConcurrencyLeaseLost) {
+						outcome = "lost"
+					}
+					s.observeConcurrencyLeaseRenewal(outcome)
+					s.logger.Error("concurrency lease renewal failed; cancelling upstream execution",
+						"taskID", taskID, "leaseCount", len(leases), "outcome", outcome, "error", err)
+					done <- err
+					cancelRun()
+					return
+				}
+				s.observeConcurrencyLeaseRenewal("success")
+			}
+		}
+	}()
+	return runCtx, func() error {
+		cancelRenew()
+		renewalErr := <-done
+		cancelRun()
+		return renewalErr
+	}
+}
+
+func (s *Service) observeConcurrencyLeaseRenewal(outcome string) {
+	observer, ok := s.billingMetrics.(interface {
+		ObserveConcurrencyLeaseRenewal(string)
+	})
+	if ok {
+		observer.ObserveConcurrencyLeaseRenewal(outcome)
+	}
+}
+
 func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) {
 	queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, 0, "")
 	if err != nil {