feat(acceptance): 建立同构验收与弹性容量体系

实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。

apps/api/internal/runner/admission.go

@@ -159,7 +159,7 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
 }
 
 func (s *Service) withAsyncWorkerCapacityScope(ctx context.Context, scopes []store.AdmissionScope) ([]store.AdmissionScope, error) {
-	capacity, err := s.store.ActiveWorkerCapacity(ctx)
+	capacity, err := s.coordinationStore.ActiveWorkerCapacity(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -589,6 +589,19 @@ func (s *Service) SubmitAsyncTask(ctx context.Context, task store.GatewayTask) e
 		if s.cancelAsyncSubmissionIfDisconnected(ctx, task.ID) {
 			return ctx.Err()
 		}
+		if store.ModelCandidateRetryAfter(err) > 0 ||
+			(errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err)) {
+			if enqueueErr := s.EnqueueAsyncTask(ctx, task); enqueueErr != nil {
+				_, _ = s.store.FailQueuedTask(
+					context.WithoutCancel(ctx),
+					task.ID,
+					"enqueue_failed",
+					enqueueErr.Error(),
+				)
+				return enqueueErr
+			}
+			return nil
+		}
 		_, _ = s.store.FailQueuedTask(context.WithoutCancel(ctx), task.ID, clients.ErrorCode(err), err.Error())
 		return err
 	}

apps/api/internal/runner/execution_lease.go

@@ -21,7 +21,7 @@ func (s *Service) renewTaskExecutionLease(ctx context.Context, cancel context.Ca
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			if err := s.store.RenewTaskExecutionLease(ctx, taskID, executionToken, taskExecutionLeaseTTL); err != nil {
+			if err := s.coordinationStore.RenewTaskExecutionLease(ctx, taskID, executionToken, taskExecutionLeaseTTL); err != nil {
 				if errors.Is(err, store.ErrTaskExecutionFinished) {
 					return
 				}

apps/api/internal/runner/queue_worker.go

@@ -191,7 +191,7 @@ func retryAsyncQueueStartup(
 }
 
 func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error {
-	driver := riverpgxv5.New(s.store.Pool())
+	driver := riverpgxv5.New(s.riverStore.Pool())
 	migrator, err := rivermigrate.New(driver, nil)
 	if err != nil {
 		return err
@@ -286,7 +286,7 @@ func (s *Service) newRiverAsyncExecutionClient(capacity int) (*river.Client[pgx.
 	if err := river.AddWorkerSafely(workers, &asyncTaskWorker{service: s}); err != nil {
 		return nil, err
 	}
-	return river.NewClient(riverpgxv5.New(s.store.Pool()), &river.Config{
+	return river.NewClient(riverpgxv5.New(s.riverStore.Pool()), &river.Config{
 		ID:                          fmt.Sprintf("%s-exec-%d-%d", s.workerInstanceID, capacity, time.Now().UnixNano()),
 		JobTimeout:                  -1,
 		Logger:                      s.logger,
@@ -314,11 +314,11 @@ func (s *Service) loadAsyncWorkerCapacity(ctx context.Context) (store.AsyncWorke
 	if s.asyncCapacityLoader != nil {
 		return s.asyncCapacityLoader(ctx, s.cfg.AsyncWorkerHardLimit)
 	}
-	snapshot, err := s.store.AsyncWorkerCapacity(ctx, s.cfg.AsyncWorkerHardLimit)
+	snapshot, err := s.coordinationStore.AsyncWorkerCapacity(ctx, s.cfg.AsyncWorkerHardLimit)
 	if err != nil {
 		return store.AsyncWorkerCapacitySnapshot{}, err
 	}
-	allocation, err := s.store.RegisterWorkerInstance(ctx, store.WorkerRegistrationInput{
+	allocation, err := s.coordinationStore.RegisterWorkerInstance(ctx, store.WorkerRegistrationInput{
 		InstanceID:          s.workerInstanceID,
 		PodUID:              strings.TrimSpace(os.Getenv("POD_UID")),
 		PodName:             strings.TrimSpace(os.Getenv("POD_NAME")),
@@ -437,7 +437,7 @@ func (s *Service) stopAsyncWorkersOnShutdown(ctx context.Context) {
 	<-ctx.Done()
 	if s.store != nil && strings.TrimSpace(s.workerInstanceID) != "" {
 		drainCtx, drainCancel := context.WithTimeout(context.Background(), 5*time.Second)
-		if err := s.store.MarkWorkerDraining(drainCtx, s.workerInstanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
+		if err := s.coordinationStore.MarkWorkerDraining(drainCtx, s.workerInstanceID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
 			s.logger.Warn("mark async worker draining failed", "error", err)
 		}
 		drainCancel()
@@ -567,7 +567,7 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
 
 func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
 	recoverJobs := func() {
-		runtimeRecovery, err := s.store.RecoverInterruptedRuntimeState(ctx)
+		runtimeRecovery, err := s.coordinationStore.RecoverInterruptedRuntimeState(ctx)
 		if err != nil {
 			if ctx.Err() == nil {
 				s.logger.Warn("recover interrupted runtime state from worker failed", "error", err)
@@ -589,7 +589,7 @@ func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
 				"cleanedTaskAdmissions", runtimeRecovery.CleanedTaskAdmissions,
 			)
 		}
-		yieldedAdmissions, err := s.store.YieldStaleAsyncTaskAdmissions(
+		yieldedAdmissions, err := s.coordinationStore.YieldStaleAsyncTaskAdmissions(
 			ctx,
 			orphanedRiverJobWorkerStaleAfter,
 			orphanedRiverJobRecoveryBatchSize,
@@ -603,7 +603,7 @@ func (s *Service) recoverOrphanedAsyncRiverJobs(ctx context.Context) {
 		if yieldedAdmissions > 0 {
 			s.logger.Warn("stale async task admissions yielded", "count", yieldedAdmissions)
 		}
-		recovered, err := s.store.RecoverOrphanedAsyncRiverJobs(
+		recovered, err := s.coordinationStore.RecoverOrphanedAsyncRiverJobs(
 			ctx,
 			orphanedRiverJobWorkerStaleAfter,
 			orphanedRiverJobRecoveryBatchSize,

apps/api/internal/runner/service.go

@@ -27,6 +27,8 @@ import (
 type Service struct {
 	cfg                    config.Config
 	store                  *store.Store
+	coordinationStore      *store.Store
+	riverStore             *store.Store
 	logger                 *slog.Logger
 	clients                map[string]clients.Client
 	scriptExecutor         *scriptengine.Executor
@@ -95,6 +97,27 @@ func (e *TaskQueuedError) Is(target error) bool {
 }
 
 func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...billingMetricsObserver) *Service {
+	return NewWithStores(cfg, db, db, db, logger, observers...)
+}
+
+func NewWithCoordinationStore(
+	cfg config.Config,
+	db *store.Store,
+	coordinationDB *store.Store,
+	logger *slog.Logger,
+	observers ...billingMetricsObserver,
+) *Service {
+	return NewWithStores(cfg, db, coordinationDB, db, logger, observers...)
+}
+
+func NewWithStores(
+	cfg config.Config,
+	db *store.Store,
+	coordinationDB *store.Store,
+	riverDB *store.Store,
+	logger *slog.Logger,
+	observers ...billingMetricsObserver,
+) *Service {
 	if cfg.AsyncWorkerHardLimit == 0 {
 		cfg.AsyncWorkerHardLimit = 2048
 	}
@@ -128,10 +151,12 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
 	httpClients := newHTTPClientCache()
 	scriptExecutor := &scriptengine.Executor{Logger: logger}
 	service := &Service{
-		cfg:            cfg,
-		store:          db,
-		logger:         logger,
-		scriptExecutor: scriptExecutor,
+		cfg:               cfg,
+		store:             db,
+		coordinationStore: coordinationDB,
+		riverStore:        riverDB,
+		logger:            logger,
+		scriptExecutor:    scriptExecutor,
 		clients: map[string]clients.Client{
 			"openai":                clients.OpenAIClient{HTTPClient: httpClients.none},
 			"aliyun-bailian":        clients.AliyunBailianClient{HTTPClient: httpClients.none},
@@ -166,6 +191,9 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
 	if len(observers) > 0 {
 		service.billingMetrics = observers[0]
 	}
+	if service.coordinationStore == nil {
+		service.coordinationStore = db
+	}
 	return service
 }
 
@@ -203,9 +231,21 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 	var claimed store.GatewayTask
 	var err error
 	if distributedAdmission && !wasRunning {
-		claimed, err = s.store.ClaimTaskPreparation(ctx, task.ID, executionToken, taskExecutionLeaseTTL)
+		claimed, err = s.store.ClaimTaskPreparationForOwner(
+			ctx,
+			task.ID,
+			executionToken,
+			taskExecutionLeaseTTL,
+			s.workerInstanceID,
+		)
 	} else {
-		claimed, err = s.store.ClaimTaskExecution(ctx, task.ID, executionToken, taskExecutionLeaseTTL)
+		claimed, err = s.store.ClaimTaskExecutionForOwner(
+			ctx,
+			task.ID,
+			executionToken,
+			taskExecutionLeaseTTL,
+			s.workerInstanceID,
+		)
 	}
 	if err != nil {
 		return Result{}, err
@@ -557,6 +597,13 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 		}
 		if hasConcurrentLimit {
 			if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidates[0], body)); err != nil {
+				if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
+					queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
+					if queueErr != nil {
+						return Result{}, queueErr
+					}
+					return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
+				}
 				failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err)
 				if finishErr != nil {
 					return Result{}, finishErr
@@ -577,6 +624,9 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 				var alreadyAdmitted bool
 				admissionResult, alreadyAdmitted, admissionErr = s.activeAsyncTaskAdmission(ctx, task, plan, asyncAdmission)
 				if admissionErr == nil && !alreadyAdmitted {
+					admissionResult, admissionErr = s.tryTaskAdmission(ctx, task, plan, "")
+				}
+				if admissionErr == nil && !admissionResult.Admitted {
 					_ = s.store.ReleaseTaskPreparation(context.WithoutCancel(ctx), task.ID, task.ExecutionToken)
 					return Result{Task: task, Output: task.Result}, &TaskQueuedError{Delay: time.Second}
 				}
@@ -626,8 +676,13 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 			}
 		}
 	}
+	admissionFinalized := false
 	if distributedAdmission {
-		defer s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID)
+		defer func() {
+			if !admissionFinalized {
+				_ = s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID)
+			}
+		}()
 	}
 	firstCandidateBody := body
 	normalizedModelType := modelType
@@ -670,9 +725,6 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
 			}
 			return Result{Task: failed, Output: failed.Result}, clientErr
 		}
-		if err := s.store.MarkTaskRunning(ctx, task.ID, task.ExecutionToken, candidates[0].ModelType, s.slimTaskRequestSnapshot(task, firstCandidateBody)); err != nil {
-			return Result{}, err
-		}
 		var reserveErr error
 		walletReservations, reserveErr = s.store.ReserveTaskBilling(ctx, task, user, reservationBillings, reservationPricingSnapshot)
 		if reserveErr != nil {
@@ -872,6 +924,10 @@ candidatesLoop:
 				record.Metrics = mergeMetrics(record.Metrics, candidateCapabilityFilterMetrics(candidateFilterSummary))
 				record.Metrics = mergeMetrics(record.Metrics, parameterPreprocessingMetrics(preprocessing.Log))
 				record.Metrics = s.withAttemptHistory(ctx, task.ID, record.Metrics)
+				callbackURL, callbackErr := s.taskCallbackURL(ctx, task.ID)
+				if callbackErr != nil {
+					return Result{}, callbackErr
+				}
 				finished, finishErr := s.store.FinishTaskSuccess(ctx, store.FinishTaskSuccessInput{
 					TaskID:                task.ID,
 					ExecutionToken:        task.ExecutionToken,
@@ -891,6 +947,9 @@ candidatesLoop:
 					ResponseStartedAt:     record.ResponseStartedAt,
 					ResponseFinishedAt:    record.ResponseFinishedAt,
 					ResponseDurationMS:    record.ResponseDurationMS,
+					CompletionCallbackURL: callbackURL,
+					Simulated:             isSimulation(task, candidate),
+					FinalizeAdmission:     distributedAdmission,
 				})
 				if finishErr != nil {
 					if errors.Is(finishErr, store.ErrTaskResultBinaryNotMaterialized) {
@@ -935,23 +994,7 @@ candidatesLoop:
 					return Result{}, finishErr
 				}
 				walletReservationFinalized = true
-				if finished.BillingStatus == "pending" {
-					if err := s.emit(ctx, task.ID, "task.billing.pending", "succeeded", "billing", 0.98, "task billing queued", map[string]any{
-						"amount": finished.FinalChargeAmount, "currency": finished.BillingCurrency,
-					}, isSimulation(task, candidate)); err != nil {
-						return Result{}, err
-					}
-				}
-				if err := s.emit(ctx, task.ID, "task.completed", "succeeded", "completed", 1, "task completed", map[string]any{
-					"result":         response.Result,
-					"billings":       billings,
-					"usage":          record.Usage,
-					"metrics":        record.Metrics,
-					"billingSummary": record.BillingSummary,
-					"requestId":      record.RequestID,
-				}, isSimulation(task, candidate)); err != nil {
-					return Result{}, err
-				}
+				admissionFinalized = distributedAdmission
 				if task.AsyncMode {
 					// Async callers hydrate the canonical stored result through
 					// the task detail endpoint. The River Worker does not consume
@@ -1232,12 +1275,17 @@ func (s *Service) runCandidate(
 		limitResult.Leases = append(limitResult.Leases, admittedLeases...)
 	}
 	rateReservationsFinalized := false
+	retainAdmittedLeases := false
 	defer func() {
 		if !rateReservationsFinalized {
 			_ = s.store.ReleaseRateLimitReservations(context.WithoutCancel(ctx), limitResult.Reservations, "attempt_failed")
 		}
 	}()
-	defer s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.Leases)
+	defer func() {
+		if !retainAdmittedLeases {
+			_ = s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.Leases)
+		}
+	}()
 
 	attemptID, err := s.store.CreateTaskAttempt(ctx, store.CreateTaskAttemptInput{
 		TaskID:             task.ID,
@@ -1402,15 +1450,16 @@ func (s *Service) runCandidate(
 	var submissionWire *clients.WireResponse
 	runCtx, stopLeaseRenewal := s.startConcurrencyLeaseRenewal(ctx, task.ID, limitResult.Leases)
 	response, err := client.Run(runCtx, clients.Request{
-		Kind:              task.Kind,
-		ModelType:         candidate.ModelType,
-		Model:             task.Model,
-		Body:              providerBody,
-		OriginalBody:      preprocessing.Input,
-		Candidate:         candidate,
-		HTTPClient:        requestHTTPClient,
-		RemoteTaskID:      task.RemoteTaskID,
-		RemoteTaskPayload: task.RemoteTaskPayload,
+		Kind:                   task.Kind,
+		ModelType:              candidate.ModelType,
+		Model:                  task.Model,
+		Body:                   providerBody,
+		OriginalBody:           preprocessing.Input,
+		Candidate:              candidate,
+		HTTPClient:             requestHTTPClient,
+		RemoteTaskID:           task.RemoteTaskID,
+		RemoteTaskPayload:      task.RemoteTaskPayload,
+		UpstreamIdempotencyKey: task.ID,
 		OnRemoteTaskSubmitted: func(remoteTaskID string, payload map[string]any) error {
 			if strings.TrimSpace(remoteTaskID) == "" {
 				return nil
@@ -1446,7 +1495,8 @@ func (s *Service) runCandidate(
 			return nil
 		},
 		OnUpstreamResponseReceived: func() error {
-			return setSubmissionStatus("response_received")
+			submissionStatus = "response_received"
+			return nil
 		},
 		OnUpstreamWireResponse: func(wire *clients.WireResponse) error {
 			submissionWire = wire
@@ -1469,9 +1519,7 @@ func (s *Service) runCandidate(
 	}
 	callFinishedAt := time.Now()
 	if err == nil {
-		if markErr := setSubmissionStatus("response_received"); markErr != nil {
-			return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: markErr}
-		}
+		submissionStatus = "response_received"
 	}
 	if response.ResponseStartedAt.IsZero() {
 		response.ResponseStartedAt = callStartedAt
@@ -1501,9 +1549,7 @@ func (s *Service) runCandidate(
 	}
 	if err != nil {
 		if clients.ErrorResponseMetadata(err).StatusCode > 0 && submissionStatus != "response_received" {
-			if markErr := setSubmissionStatus("response_received"); markErr != nil {
-				return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: markErr}
-			}
+			submissionStatus = "response_received"
 		}
 		retryable := clients.IsRetryable(err)
 		requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(err, simulated)
@@ -1521,16 +1567,17 @@ func (s *Service) runCandidate(
 		}
 		metrics = mergeMetrics(baseAttemptMetrics, metrics)
 		_ = s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{
-			AttemptID:          attemptID,
-			Status:             "failed",
-			Retryable:          retryable,
-			RequestID:          requestID,
-			Metrics:            metrics,
-			ResponseStartedAt:  responseStartedAt,
-			ResponseFinishedAt: responseFinishedAt,
-			ResponseDurationMS: responseDurationMS,
-			ErrorCode:          clients.ErrorCode(err),
-			ErrorMessage:       err.Error(),
+			AttemptID:                attemptID,
+			Status:                   "failed",
+			Retryable:                retryable,
+			UpstreamSubmissionStatus: submissionStatus,
+			RequestID:                requestID,
+			Metrics:                  metrics,
+			ResponseStartedAt:        responseStartedAt,
+			ResponseFinishedAt:       responseFinishedAt,
+			ResponseDurationMS:       responseDurationMS,
+			ErrorCode:                clients.ErrorCode(err),
+			ErrorMessage:             err.Error(),
 		})
 		_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
 		if !simulated && submissionStatus == "submitting" {
@@ -1561,18 +1608,19 @@ func (s *Service) runCandidate(
 			"trace":     []any{failureTraceEntry(err, clients.IsRetryable(err))},
 		})
 		_ = s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{
-			AttemptID:          attemptID,
-			Status:             "failed",
-			Retryable:          clients.IsRetryable(err),
-			RequestID:          response.RequestID,
-			Usage:              usageToMap(response.Usage),
-			Metrics:            metrics,
-			ResponseSnapshot:   response.Result,
-			ResponseStartedAt:  response.ResponseStartedAt,
-			ResponseFinishedAt: response.ResponseFinishedAt,
-			ResponseDurationMS: response.ResponseDurationMS,
-			ErrorCode:          clients.ErrorCode(err),
-			ErrorMessage:       err.Error(),
+			AttemptID:                attemptID,
+			Status:                   "failed",
+			Retryable:                clients.IsRetryable(err),
+			UpstreamSubmissionStatus: submissionStatus,
+			RequestID:                response.RequestID,
+			Usage:                    usageToMap(response.Usage),
+			Metrics:                  metrics,
+			ResponseSnapshot:         response.Result,
+			ResponseStartedAt:        response.ResponseStartedAt,
+			ResponseFinishedAt:       response.ResponseFinishedAt,
+			ResponseDurationMS:       response.ResponseDurationMS,
+			ErrorCode:                clients.ErrorCode(err),
+			ErrorMessage:             err.Error(),
 		})
 		return clients.Response{}, err
 	}
@@ -1652,6 +1700,9 @@ func (s *Service) runCandidate(
 	}); err != nil {
 		s.logger.Warn("record cache affinity observation failed", "error", err, "clientId", candidate.ClientID)
 	}
+	if admittedPlatformModelID == candidate.PlatformModelID {
+		retainAdmittedLeases = true
+	}
 	return response, nil
 }
 
@@ -1835,17 +1886,24 @@ func (s *Service) failTask(ctx context.Context, taskID string, executionToken st
 		metrics = mergeMetrics(values...)
 	}
 	metrics = s.withAttemptHistory(ctx, taskID, metrics)
+	callbackURL, callbackErr := s.taskCallbackURL(ctx, taskID)
+	if callbackErr != nil {
+		return store.GatewayTask{}, callbackErr
+	}
 	failed, err := s.store.FinishTaskFailure(ctx, store.FinishTaskFailureInput{
-		TaskID:             taskID,
-		ExecutionToken:     executionToken,
-		Code:               code,
-		Message:            message,
-		Result:             buildFailureResult(code, message, requestID, cause),
-		RequestID:          requestID,
-		Metrics:            metrics,
-		ResponseStartedAt:  responseStartedAt,
-		ResponseFinishedAt: responseFinishedAt,
-		ResponseDurationMS: responseDurationMS,
+		TaskID:                taskID,
+		ExecutionToken:        executionToken,
+		Code:                  code,
+		Message:               message,
+		Result:                buildFailureResult(code, message, requestID, cause),
+		RequestID:             requestID,
+		Metrics:               metrics,
+		ResponseStartedAt:     responseStartedAt,
+		ResponseFinishedAt:    responseFinishedAt,
+		ResponseDurationMS:    responseDurationMS,
+		CompletionCallbackURL: callbackURL,
+		Simulated:             simulated,
+		FinalizeAdmission:     true,
 	})
 	if err != nil {
 		return store.GatewayTask{}, err
@@ -1853,9 +1911,6 @@ func (s *Service) failTask(ctx context.Context, taskID string, executionToken st
 	if failed.Status == "cancelled" {
 		return failed, nil
 	}
-	if eventErr := s.emit(ctx, taskID, "task.failed", "failed", "failed", 1, message, map[string]any{"code": code, "requestId": requestID, "metrics": metrics}, simulated); eventErr != nil {
-		return store.GatewayTask{}, eventErr
-	}
 	return failed, nil
 }
 
@@ -2056,7 +2111,7 @@ func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID strin
 				done <- nil
 				return
 			case <-ticker.C:
-				if err := s.store.RenewConcurrencyLeases(renewCtx, leases); err != nil {
+				if err := s.coordinationStore.RenewConcurrencyLeases(renewCtx, leases); err != nil {
 					if renewCtx.Err() != nil {
 						done <- nil
 						return
@@ -2113,16 +2168,10 @@ func (s *Service) withAttemptHistory(ctx context.Context, taskID string, metrics
 }
 
 func (s *Service) emit(ctx context.Context, taskID string, eventType string, status string, phase string, progress float64, message string, payload map[string]any, simulated bool) error {
-	acceptanceURL, acceptance, callbackErr := s.store.AcceptanceCallbackURL(ctx, taskID)
+	callbackURL, callbackErr := s.taskCallbackURL(ctx, taskID)
 	if callbackErr != nil {
 		return callbackErr
 	}
-	callbackURL := ""
-	if acceptance && acceptanceURL != "" {
-		callbackURL = acceptanceURL
-	} else if s.cfg.TaskProgressCallbackEnabled && s.cfg.TaskProgressCallbackURL != "" {
-		callbackURL = s.cfg.TaskProgressCallbackURL
-	}
 	event, err := s.store.AddTaskEventWithCallback(
 		ctx,
 		taskID,
@@ -2144,6 +2193,20 @@ func (s *Service) emit(ctx context.Context, taskID string, eventType string, sta
 	return nil
 }
 
+func (s *Service) taskCallbackURL(ctx context.Context, taskID string) (string, error) {
+	acceptanceURL, acceptance, err := s.store.AcceptanceCallbackURL(ctx, taskID)
+	if err != nil {
+		return "", err
+	}
+	if acceptance && acceptanceURL != "" {
+		return acceptanceURL, nil
+	}
+	if s.cfg.TaskProgressCallbackEnabled && s.cfg.TaskProgressCallbackURL != "" {
+		return s.cfg.TaskProgressCallbackURL, nil
+	}
+	return "", nil
+}
+
 func modelTypeFromKind(kind string, body map[string]any) string {
 	if requested := requestedModelTypeFromBody(body); requested != "" {
 		return requested