feat(worker): 实现集群限流与自适应负载

保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。

apps/api/internal/runner/admission.go

@@ -205,23 +205,12 @@ func (s *Service) taskAdmissionScopes(
 }
 
 func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionScope) []store.AdmissionScope {
-	if task.RunMode != "acceptance" && task.RunMode != "acceptance_canary" {
+	if task.RunMode != "acceptance" {
 		return scopes
 	}
 	out := append([]store.AdmissionScope(nil), scopes...)
 	for index := range out {
-		// Protocol-emulated acceptance measures Gateway and Worker capacity, so
-		// the isolated Run is bounded by the worker_capacity scope instead of a
-		// production supplier quota. The real acceptance_canary path deliberately
-		// retains the production platform-model concurrency limit.
-		if task.RunMode == "acceptance" && out[index].ScopeType == "platform_model" {
-			out[index].ConcurrentLimit = 0
-		}
-		if out[index].ConcurrentLimit <= 0 {
-			if task.RunMode != "acceptance" || out[index].ScopeType != "platform_model" {
-				continue
-			}
-		}
+		out[index].ScopeKey = acceptanceScopeKey(task, out[index].ScopeKey)
 		out[index].QueueLimit = acceptanceQueueLimit
 		out[index].MaxWaitSeconds = acceptanceQueueMaxWait
 	}
@@ -235,16 +224,21 @@ func acceptanceInfrastructureReservations(
 	if task.RunMode != "acceptance" {
 		return reservations
 	}
-	out := make([]store.RateLimitReservation, 0, len(reservations))
-	for _, reservation := range reservations {
-		if reservation.ScopeType == "platform_model" && reservation.Metric == "concurrent" {
-			continue
-		}
-		out = append(out, reservation)
+	out := append([]store.RateLimitReservation(nil), reservations...)
+	for index := range out {
+		out[index].ScopeKey = acceptanceScopeKey(task, out[index].ScopeKey)
 	}
 	return out
 }
 
+func acceptanceScopeKey(task store.GatewayTask, scopeKey string) string {
+	runID := strings.TrimSpace(task.AcceptanceRunID)
+	if runID == "" {
+		runID = "unbound"
+	}
+	return "acceptance:" + runID + ":" + scopeKey
+}
+
 func (s *Service) loadAsyncTaskAdmission(ctx context.Context, task store.GatewayTask) (*store.TaskAdmission, error) {
 	if !task.AsyncMode {
 		return nil, nil
@@ -265,6 +259,7 @@ func pinCandidatesToTaskAdmission(
 ) ([]store.RuntimeModelCandidate, bool) {
 	if admission == nil ||
 		(admission.Status != "waiting" && admission.Status != "admitted") ||
+		!admission.ReselectRequestedAt.IsZero() ||
 		len(candidates) < 2 {
 		return candidates, false
 	}
@@ -785,6 +780,24 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
 				return false, outcome.Err
 			}
 			if !outcome.Result.Admitted {
+				platformModelID := outcome.Result.Admission.PlatformModelID
+				if platformModelID == "" {
+					for _, input := range inputs {
+						if input.TaskID == outcome.TaskID {
+							platformModelID = input.PlatformModelID
+							break
+						}
+					}
+				}
+				if platformModelID != "" {
+					marked, markErr := s.store.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
+					if markErr != nil {
+						return false, markErr
+					}
+					if marked > 0 {
+						s.observeTaskAdmission("candidate_reselect_requested")
+					}
+				}
 				return true, nil
 			}
 		}

apps/api/internal/runner/admission_test.go

@@ -57,7 +57,7 @@ func TestDistributedAdmissionModelTypeBoundary(t *testing.T) {
 	}
 }
 
-func TestAcceptanceAdmissionScopesUseWorkerCapacityInsteadOfSupplierConcurrency(t *testing.T) {
+func TestAcceptanceAdmissionScopesIsolateRunWithoutChangingLimits(t *testing.T) {
 	input := []store.AdmissionScope{{
 		ScopeType:       "platform_model",
 		ScopeKey:        "model-1",
@@ -70,15 +70,15 @@ func TestAcceptanceAdmissionScopesUseWorkerCapacityInsteadOfSupplierConcurrency(
 		ConcurrentLimit: 48,
 	}}
 
-	got := acceptanceAdmissionScopes(store.GatewayTask{RunMode: "acceptance"}, input)
+	got := acceptanceAdmissionScopes(store.GatewayTask{RunMode: "acceptance", AcceptanceRunID: "run-1"}, input)
 
-	if got[0].ConcurrentLimit != 0 {
-		t.Fatalf("protocol-emulated acceptance must defer to worker capacity, got %+v", got[0])
+	if got[0].ConcurrentLimit != 10 || got[0].ScopeKey != "acceptance:run-1:model-1" {
+		t.Fatalf("protocol-emulated acceptance must preserve the limit in an isolated scope, got %+v", got[0])
 	}
 	if got[0].QueueLimit != acceptanceQueueLimit || got[0].MaxWaitSeconds != acceptanceQueueMaxWait {
 		t.Fatalf("acceptance must enable a bounded queue, got %+v", got[0])
 	}
-	if got[1].ConcurrentLimit != 48 {
+	if got[1].ConcurrentLimit != 48 || got[1].ScopeKey != "acceptance:run-1:global" {
 		t.Fatalf("acceptance worker capacity changed, got %+v", got[1])
 	}
 	if input[0].QueueLimit != 0 || input[0].MaxWaitSeconds != 0 {
@@ -100,15 +100,15 @@ func TestAcceptanceCanaryPreservesProductionConcurrency(t *testing.T) {
 	}
 }
 
-func TestAcceptanceInfrastructureReservationsOnlyRemoveSupplierConcurrency(t *testing.T) {
+func TestAcceptanceInfrastructureReservationsIsolateEveryMetric(t *testing.T) {
 	input := []store.RateLimitReservation{
-		{ScopeType: "platform_model", Metric: "concurrent", Limit: 10},
-		{ScopeType: "platform_model", Metric: "rpm", Limit: 600},
-		{ScopeType: "user_group", Metric: "concurrent", Limit: 20},
+		{ScopeType: "platform_model", ScopeKey: "model-1", Metric: "concurrent", Limit: 10},
+		{ScopeType: "platform_model", ScopeKey: "model-1", Metric: "rpm", Limit: 600},
+		{ScopeType: "user_group", ScopeKey: "group-1", Metric: "concurrent", Limit: 20},
 	}
 
-	got := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance"}, input)
-	if len(got) != 2 || got[0].Metric != "rpm" || got[1].ScopeType != "user_group" {
+	got := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance", AcceptanceRunID: "run-1"}, input)
+	if len(got) != 3 || got[0].ScopeKey != "acceptance:run-1:model-1" || got[1].ScopeKey != "acceptance:run-1:model-1" || got[2].ScopeKey != "acceptance:run-1:group-1" {
 		t.Fatalf("unexpected acceptance reservations: %+v", got)
 	}
 	canary := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance_canary"}, input)
@@ -192,6 +192,28 @@ func TestPinCandidatesToTaskAdmissionPreservesWaitingCandidate(t *testing.T) {
 	}
 }
 
+func TestPinCandidatesToTaskAdmissionAllowsRequestedReselection(t *testing.T) {
+	input := []store.RuntimeModelCandidate{
+		{PlatformID: "platform-a", PlatformModelID: "model-a"},
+		{PlatformID: "platform-b", PlatformModelID: "model-b"},
+	}
+	admission := &store.TaskAdmission{
+		Status:              "waiting",
+		PlatformID:          "platform-b",
+		PlatformModelID:     "model-b",
+		ReselectRequestedAt: time.Now(),
+	}
+
+	got, pinned := pinCandidatesToTaskAdmission(input, admission)
+
+	if pinned {
+		t.Fatalf("reselection request unexpectedly pinned the old candidate: %+v", got)
+	}
+	if got[0].PlatformModelID != "model-a" {
+		t.Fatalf("reselection changed the sorted candidate order: %+v", got)
+	}
+}
+
 func TestPinCandidatesToTaskAdmissionIgnoresMissingCandidate(t *testing.T) {
 	input := []store.RuntimeModelCandidate{
 		{PlatformID: "platform-a", PlatformModelID: "model-a"},

apps/api/internal/runner/queue_worker.go

@@ -10,8 +10,10 @@ import (
 
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/riverqueue/river"
 	"github.com/riverqueue/river/riverdriver/riverpgxv5"
 	"github.com/riverqueue/river/rivermigrate"
@@ -45,6 +47,10 @@ type asyncTaskWorker struct {
 	service *Service
 }
 
+type workerLoadSampler interface {
+	Sample(databaseConnections, databaseMax int32) workerload.ResourceSample
+}
+
 func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs]) error {
 	task, err := w.service.store.GetTask(ctx, job.Args.TaskID)
 	if err != nil {
@@ -53,6 +59,12 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
 	if task.Status == "succeeded" || task.Status == "failed" || task.Status == "cancelled" {
 		return nil
 	}
+	loadLease, admitted := w.service.tryStartWorkerTask()
+	if !admitted {
+		return river.JobSnooze(workerLoadRetryDelay(task.ID))
+	}
+	defer loadLease.Release()
+	ctx = context.WithValue(ctx, workerLoadLeaseContextKey{}, loadLease)
 	executionToken := uuid.NewString()
 	result, runErr := w.service.executeWithToken(ctx, task, authUserFromTask(task), nil, executionToken)
 	if runErr == nil {
@@ -323,21 +335,36 @@ func (s *Service) makeAsyncExecutionClient(capacity int) (asyncExecutionClient,
 
 func (s *Service) loadAsyncWorkerCapacity(ctx context.Context) (store.AsyncWorkerCapacitySnapshot, error) {
 	if s.asyncCapacityLoader != nil {
-		return s.asyncCapacityLoader(ctx, s.cfg.AsyncWorkerHardLimit)
+		snapshot, err := s.asyncCapacityLoader(ctx, s.cfg.AsyncWorkerHardLimit)
+		if err == nil && s.workerLoad != nil {
+			s.workerLoad.SetClaimLimit(snapshot.Capacity)
+		}
+		return snapshot, err
 	}
 	snapshot, err := s.coordinationStore.AsyncWorkerCapacity(ctx, s.cfg.AsyncWorkerHardLimit)
 	if err != nil {
 		return store.AsyncWorkerCapacitySnapshot{}, err
 	}
+	loadSnapshot := s.sampleWorkerLoad()
 	allocation, err := s.coordinationStore.RegisterWorkerInstance(ctx, store.WorkerRegistrationInput{
-		InstanceID:          s.workerInstanceID,
-		PodUID:              strings.TrimSpace(os.Getenv("POD_UID")),
-		PodName:             strings.TrimSpace(os.Getenv("POD_NAME")),
-		Site:                strings.TrimSpace(os.Getenv("EASYAI_SITE")),
-		Revision:            strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
-		DesiredCapacity:     snapshot.Capacity,
-		CapacityLimit:       s.cfg.AsyncWorkerInstanceHardLimit,
-		HeartbeatStaleAfter: time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
+		InstanceID:           s.workerInstanceID,
+		PodUID:               strings.TrimSpace(os.Getenv("POD_UID")),
+		PodName:              strings.TrimSpace(os.Getenv("POD_NAME")),
+		Site:                 strings.TrimSpace(os.Getenv("EASYAI_SITE")),
+		Revision:             strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
+		DesiredCapacity:      snapshot.Capacity,
+		CapacityLimit:        s.cfg.AsyncWorkerInstanceHardLimit,
+		LoadMode:             loadSnapshot.Mode,
+		SafeCapacity:         loadSnapshot.SafeCapacity,
+		HeavyCapacity:        loadSnapshot.HeavyLimit,
+		ActiveTasks:          loadSnapshot.ActiveTasks,
+		PreparingTasks:       loadSnapshot.PreparingTasks,
+		WaitingUpstreamTasks: loadSnapshot.WaitingUpstreamTasks,
+		FinalizingTasks:      loadSnapshot.FinalizingTasks,
+		PressureState:        string(loadSnapshot.PressureState),
+		PressureReason:       loadSnapshot.PressureReason,
+		LoadSampledAt:        loadSnapshot.SampledAt,
+		HeartbeatStaleAfter:  time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
 	})
 	if err != nil {
 		return store.AsyncWorkerCapacitySnapshot{}, err
@@ -346,9 +373,60 @@ func (s *Service) loadAsyncWorkerCapacity(ctx context.Context) (store.AsyncWorke
 	snapshot.GlobalCapacity = allocation.GlobalAllocated
 	snapshot.ActiveInstances = allocation.ActiveInstances
 	snapshot.InstanceID = allocation.InstanceID
+	loadSnapshot = s.workerLoad.SetClaimLimit(allocation.Allocated)
+	snapshot.LoadMode = loadSnapshot.Mode
+	snapshot.LocalSafeCapacity = loadSnapshot.SafeCapacity
+	snapshot.LocalHeavyCapacity = loadSnapshot.HeavyLimit
+	snapshot.LocalActiveTasks = loadSnapshot.ActiveTasks
+	snapshot.LocalPreparingTasks = loadSnapshot.PreparingTasks
+	snapshot.LocalWaitingTasks = loadSnapshot.WaitingUpstreamTasks
+	snapshot.LocalFinalizingTasks = loadSnapshot.FinalizingTasks
+	snapshot.LocalPressureState = string(loadSnapshot.PressureState)
+	snapshot.LocalPressureReason = loadSnapshot.PressureReason
 	return snapshot, nil
 }
 
+func (s *Service) sampleWorkerLoad() workerload.Snapshot {
+	if s.workerLoad == nil {
+		return workerload.Snapshot{Mode: workerload.ModeLegacy, SafeCapacity: s.cfg.AsyncWorkerInstanceHardLimit, HeavyLimit: s.cfg.AsyncWorkerInstanceHardLimit}
+	}
+	if s.workerLoadSampler == nil {
+		return s.workerLoad.Snapshot()
+	}
+	var connections int32
+	var maximum int32
+	seen := make(map[*pgxpool.Pool]struct{}, 3)
+	for _, database := range []*store.Store{s.store, s.coordinationStore, s.riverStore} {
+		if database == nil || database.Pool() == nil {
+			continue
+		}
+		pool := database.Pool()
+		if _, ok := seen[pool]; ok {
+			continue
+		}
+		seen[pool] = struct{}{}
+		statistics := pool.Stat()
+		connections += statistics.AcquiredConns()
+		maximum += statistics.MaxConns()
+	}
+	return s.workerLoad.Observe(s.workerLoadSampler.Sample(connections, maximum))
+}
+
+func (s *Service) tryStartWorkerTask() (*workerload.Lease, bool) {
+	if s.workerLoad == nil {
+		return nil, true
+	}
+	return s.workerLoad.TryStart()
+}
+
+func workerLoadRetryDelay(taskID string) time.Duration {
+	var value uint32
+	for _, character := range []byte(taskID) {
+		value = value*33 + uint32(character)
+	}
+	return 250*time.Millisecond + time.Duration(value%501)*time.Millisecond
+}
+
 func (s *Service) refreshAsyncWorkerCapacity(ctx context.Context) {
 	ticker := time.NewTicker(time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * time.Second)
 	defer ticker.Stop()
@@ -427,6 +505,12 @@ func (s *Service) resizeAsyncWorkerCapacity(ctx context.Context) {
 		"modelDesired", snapshot.ModelDesired,
 		"groupDesired", snapshot.GroupDesired,
 		"capped", snapshot.Capped,
+		"loadMode", snapshot.LoadMode,
+		"localSafeCapacity", snapshot.LocalSafeCapacity,
+		"localHeavyCapacity", snapshot.LocalHeavyCapacity,
+		"localActiveTasks", snapshot.LocalActiveTasks,
+		"pressureState", snapshot.LocalPressureState,
+		"pressureReason", snapshot.LocalPressureReason,
 	)
 	if oldClient != nil {
 		go s.drainAsyncWorkerClient(oldClient)
@@ -493,6 +577,16 @@ func (s *Service) observeAsyncWorkerCapacity(snapshot store.AsyncWorkerCapacityS
 	if ok {
 		distributedObserver.SetDistributedWorkerCapacity(snapshot.ActiveInstances, snapshot.GlobalCapacity, snapshot.Capacity)
 	}
+	loadObserver, ok := s.billingMetrics.(interface {
+		SetWorkerLoad(activeLimit, heavyLimit, active, preparing, waiting, finalizing int, pressure string)
+	})
+	if ok {
+		loadObserver.SetWorkerLoad(
+			snapshot.LocalSafeCapacity, snapshot.LocalHeavyCapacity, snapshot.LocalActiveTasks,
+			snapshot.LocalPreparingTasks, snapshot.LocalWaitingTasks, snapshot.LocalFinalizingTasks,
+			snapshot.LocalPressureState,
+		)
+	}
 }
 
 func (s *Service) observeAsyncWorkerResize(outcome string) {

apps/api/internal/runner/service.go

@@ -19,6 +19,7 @@ import (
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
 	scriptengine "github.com/easyai/easyai-ai-gateway/apps/api/internal/script"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
 	"github.com/riverqueue/river"
@@ -39,6 +40,8 @@ type Service struct {
 	riverDrainingClients   map[asyncExecutionClient]struct{}
 	riverWorkerCapacity    int
 	workerInstanceID       string
+	workerLoad             *workerload.Controller
+	workerLoadSampler      workerLoadSampler
 	asyncCapacityLoader    func(context.Context, int) (store.AsyncWorkerCapacitySnapshot, error)
 	admissionWakeMu        sync.Mutex
 	admissionWake          chan struct{}
@@ -129,6 +132,9 @@ func NewWithStores(
 	if cfg.AsyncWorkerRefreshIntervalSeconds == 0 {
 		cfg.AsyncWorkerRefreshIntervalSeconds = 5
 	}
+	if strings.TrimSpace(cfg.AsyncWorkerLoadMode) == "" {
+		cfg.AsyncWorkerLoadMode = workerload.ModeAdaptive
+	}
 	if cfg.MediaMaterializationConcurrency == 0 {
 		cfg.MediaMaterializationConcurrency = 8
 	}
@@ -184,8 +190,13 @@ func NewWithStores(
 			"universal":             clients.UniversalClient{HTTPClient: httpClients.none, ScriptExecutor: scriptExecutor},
 			"simulation":            clients.SimulationClient{},
 		},
-		httpClients:            httpClients,
-		workerInstanceID:       asyncWorkerID(),
+		httpClients:      httpClients,
+		workerInstanceID: asyncWorkerID(),
+		workerLoad: workerload.New(workerload.Config{
+			Mode: cfg.AsyncWorkerLoadMode, HardLimit: cfg.AsyncWorkerInstanceHardLimit,
+			InitialActive: 4, InitialHeavy: 1, HealthySamples: 3,
+		}),
+		workerLoadSampler:      workerload.NewSystemSampler(),
 		admissionWake:          make(chan struct{}, 4096),
 		asyncAdmissionWake:     make(chan struct{}, 1),
 		admissionTaskWaiters:   map[string]*admissionTaskWaiter{},
@@ -1266,7 +1277,7 @@ func (s *Service) runCandidate(
 	}
 	simulated := isSimulation(task, candidate)
 	baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing))
-	reservations := s.rateLimitReservations(ctx, user, candidate, body)
+	reservations := acceptanceInfrastructureReservations(task, s.rateLimitReservations(ctx, user, candidate, body))
 	if admittedPlatformModelID == candidate.PlatformModelID && len(admittedLeases) > 0 {
 		filtered := make([]store.RateLimitReservation, 0, len(reservations))
 		for _, reservation := range reservations {
@@ -1278,6 +1289,10 @@ func (s *Service) runCandidate(
 	}
 	limitResult, err := s.store.ReserveRateLimits(ctx, task.ID, "", reservations)
 	if err != nil {
+		var limitErr *store.RateLimitExceededError
+		if errors.As(err, &limitErr) {
+			s.observeProviderQuotaWait(limitErr.Metric)
+		}
 		retryable := store.RateLimitRetryable(err)
 		clientErr := &clients.ClientError{Code: "rate_limit", Message: err.Error(), Retryable: retryable}
 		return clients.Response{}, &localRateLimitError{clientErr: clientErr, cause: err, retryAfter: localRateLimitRetryAfter(err)}
@@ -1438,6 +1453,9 @@ func (s *Service) runCandidate(
 		); err != nil {
 			return clients.Response{}, fmt.Errorf("restore upstream submission status: %w", err)
 		}
+		if err := enterWorkerWaiting(ctx); err != nil {
+			return clients.Response{}, err
+		}
 	}
 	setSubmissionStatus := func(status string) error {
 		if submissionStatus == "response_received" && status != "response_received" {
@@ -1484,7 +1502,10 @@ func (s *Service) runCandidate(
 			if err := s.persistCompatibilitySubmission(context.WithoutCancel(ctx), task, candidate, remoteTaskID, checkpoint, submissionWire); err != nil {
 				return err
 			}
-			return setSubmissionStatus("response_received")
+			if err := setSubmissionStatus("response_received"); err != nil {
+				return err
+			}
+			return enterWorkerWaiting(ctx)
 		},
 		OnRemoteTaskPolled: func(remoteTaskID string, payload map[string]any) error {
 			if strings.TrimSpace(remoteTaskID) == "" {
@@ -1496,18 +1517,21 @@ func (s *Service) runCandidate(
 			}
 			task.RemoteTaskID = remoteTaskID
 			task.RemoteTaskPayload = checkpoint
-			return setSubmissionStatus("response_received")
+			if err := setSubmissionStatus("response_received"); err != nil {
+				return err
+			}
+			return enterWorkerWaiting(ctx)
 		},
 		OnUpstreamSubmissionStarted: func() error {
 			if err := setSubmissionStatus("submitting"); err != nil {
 				return err
 			}
 			markUpstreamSubmissionStarted(ctx)
-			return nil
+			return enterWorkerWaiting(ctx)
 		},
 		OnUpstreamResponseReceived: func() error {
 			submissionStatus = "response_received"
-			return nil
+			return enterWorkerFinalizing(ctx)
 		},
 		OnUpstreamWireResponse: func(wire *clients.WireResponse) error {
 			submissionWire = wire
@@ -1521,6 +1545,9 @@ func (s *Service) runCandidate(
 		UpstreamPreviousResponseID: responseExecution.UpstreamPreviousResponseID,
 		PreviousResponseTurns:      responseExecution.PreviousTurns,
 	})
+	if phaseErr := enterWorkerFinalizing(runCtx); err == nil && phaseErr != nil {
+		err = phaseErr
+	}
 	if leaseErr := stopLeaseRenewal(); leaseErr != nil {
 		err = &clients.ClientError{
 			Code:      "concurrency_lease_lost",
@@ -1717,6 +1744,15 @@ func (s *Service) runCandidate(
 	return response, nil
 }
 
+func (s *Service) observeProviderQuotaWait(metric string) {
+	observer, ok := s.billingMetrics.(interface {
+		ObserveProviderQuotaWait(string)
+	})
+	if ok {
+		observer.ObserveProviderQuotaWait(metric)
+	}
+}
+
 func minimalRemoteTaskCheckpoint(provider string, specType string, payload map[string]any) map[string]any {
 	const maxBytes = 8192
 	provider = strings.ToLower(strings.TrimSpace(provider))

apps/api/internal/runner/worker_load.go

@@ -0,0 +1,28 @@
+package runner
+
+import (
+	"context"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
+)
+
+type workerLoadLeaseContextKey struct{}
+
+func workerLoadLease(ctx context.Context) *workerload.Lease {
+	lease, _ := ctx.Value(workerLoadLeaseContextKey{}).(*workerload.Lease)
+	return lease
+}
+
+func enterWorkerWaiting(ctx context.Context) error {
+	if lease := workerLoadLease(ctx); lease != nil {
+		return lease.EnterWaiting()
+	}
+	return nil
+}
+
+func enterWorkerFinalizing(ctx context.Context) error {
+	if lease := workerLoadLease(ctx); lease != nil {
+		return lease.EnterFinalizing(ctx)
+	}
+	return nil
+}