BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(routing): 引入多执行池智能调度

Browse Source 
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。

实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。

新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
wangbo
2026-08-05 22:25:37 +08:00
parent 03c0873649
commit 7786692d32
58 changed files with 4510 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/runner/service.go
+58 -20
View File
@@ -21,7 +21,6 @@ import (
scriptengine "github.com/easyai/easyai-ai-gateway/apps/api/internal/script"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/riverqueue/river"
)
@@ -77,24 +76,24 @@ type TaskQueuedError struct {
Delay time.Duration
}
type upstreamSubmissionUnknownError struct {
type submissionConfirmationPendingError struct {
AttemptID string
Cause error
}
func shouldClassifyUpstreamSubmissionUnknown(simulated bool, submissionStatus string, err error) bool {
func shouldEnterSubmissionConfirmationPending(simulated bool, submissionStatus string, err error) bool {
return !simulated && submissionStatus == "submitting" && clients.ErrorCode(err) != "timeout"
}
func (e *upstreamSubmissionUnknownError) Error() string {
return "upstream submission result is unknown"
func (e *submissionConfirmationPendingError) Error() string {
return "upstream submission confirmation timed out"
}
func (e *upstreamSubmissionUnknownError) ErrorCode() string {
return "upstream_submission_unknown"
func (e *submissionConfirmationPendingError) ErrorCode() string {
return "upstream_timeout"
}
func (e *upstreamSubmissionUnknownError) Unwrap() error {
func (e *submissionConfirmationPendingError) Unwrap() error {
return e.Cause
}
@@ -131,6 +130,18 @@ func NewWithStores(
if cfg.AsyncWorkerHardLimit == 0 {
cfg.AsyncWorkerHardLimit = 2048
}
if strings.TrimSpace(cfg.ExecutionPoolID) == "" {
cfg.ExecutionPoolID = "legacy-default"
}
if cfg.RouteProbeHotIntervalSeconds == 0 {
cfg.RouteProbeHotIntervalSeconds = 15
}
if cfg.RouteProbeColdIntervalSeconds == 0 {
cfg.RouteProbeColdIntervalSeconds = 60
}
if cfg.RouteProbeTimeoutMS == 0 {
cfg.RouteProbeTimeoutMS = 3000
}
if cfg.AsyncWorkerInstanceHardLimit == 0 {
cfg.AsyncWorkerInstanceHardLimit = 32
}
@@ -259,15 +270,15 @@ func (s *Service) observeResultURLSigning(outcome string) {
}
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
return s.execute(ctx, task, user, nil)
return s.executeRouted(ctx, task, user, nil)
}
func (s *Service) ExecuteStream(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
return s.execute(ctx, task, user, onDelta)
return s.executeRouted(ctx, task, user, onDelta)
}
func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
return s.executeWithToken(ctx, task, user, onDelta, uuid.NewString())
return s.executeRouted(ctx, task, user, onDelta)
}
func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta, executionToken string) (Result, error) {
@@ -511,6 +522,17 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
return Result{Task: failed, Output: failed.Result}, err
}
}
if task.RoutingPlatformModelID != "" {
candidates = pinCandidatesToRoutingAssignment(candidates, task.RoutingPlatformID, task.RoutingPlatformModelID)
if len(candidates) == 0 {
err = &clients.ClientError{Code: "routing_candidate_unavailable", Message: "the routed upstream candidate is no longer available", Retryable: true}
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err)
if finishErr != nil {
return Result{}, finishErr
}
return Result{Task: failed, Output: failed.Result}, err
}
}
for _, candidate := range candidates {
if candidate.LoadAvoided {
s.observeCandidateRouting("full_avoided")
@@ -1096,11 +1118,12 @@ candidatesLoop:
}
return Result{Task: finished, Output: output, Wire: response.Wire}, nil
}
var submissionUnknown *upstreamSubmissionUnknownError
if errors.As(err, &submissionUnknown) {
var confirmationPending *submissionConfirmationPendingError
if errors.As(err, &confirmationPending) {
_ = s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submission_confirmation_pending")
review, reviewErr := s.store.FinishTaskManualReview(context.WithoutCancel(ctx), store.FinishTaskManualReviewInput{
TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: submissionUnknown.AttemptID, TaskStatus: "failed",
Code: "upstream_submission_unknown", Message: submissionUnknown.Error(),
TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: confirmationPending.AttemptID, TaskStatus: "failed",
Code: "upstream_timeout", Message: confirmationPending.Error(),
PricingSnapshot: candidatePricing.Snapshot,
RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody),
})
@@ -1108,8 +1131,8 @@ candidatesLoop:
return Result{}, reviewErr
}
walletReservationFinalized = true
s.logger.Warn("upstream submission requires manual review", "taskID", task.ID, "attemptID", submissionUnknown.AttemptID, "error_category", "upstream_submission_unknown")
return Result{Task: review, Output: review.Result}, submissionUnknown
s.logger.Warn("upstream submission confirmation pending", "taskID", task.ID, "attemptID", confirmationPending.AttemptID, "error_category", "upstream_timeout")
return Result{Task: review, Output: review.Result}, confirmationPending
}
if isLocalRateLimitError(err) {
lastErr = err
@@ -1592,6 +1615,11 @@ func (s *Service) runCandidate(
return nil
}
var submissionWire *clients.WireResponse
var routeObserver *passiveRouteObserver
if task.RouteProfileKey != "" {
routeObserver = &passiveRouteObserver{}
requestHTTPClient = routeObserver.wrap(requestHTTPClient)
}
runCtx, stopLeaseRenewal := s.startConcurrencyLeaseRenewal(ctx, task.ID, limitResult.Leases)
response, err := client.Run(runCtx, clients.Request{
Kind: task.Kind,
@@ -1620,6 +1648,9 @@ func (s *Service) runCandidate(
if err := setSubmissionStatus("response_received"); err != nil {
return err
}
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitted"); err != nil {
return err
}
return enterWorkerWaiting(ctx)
},
OnRemoteTaskPolled: func(remoteTaskID string, payload map[string]any) error {
@@ -1641,11 +1672,17 @@ func (s *Service) runCandidate(
if err := setSubmissionStatus("submitting"); err != nil {
return err
}
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitting"); err != nil {
return err
}
markUpstreamSubmissionStarted(ctx)
return enterWorkerWaiting(ctx)
},
OnUpstreamResponseReceived: func() error {
submissionStatus = "response_received"
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitted"); err != nil {
return err
}
return enterWorkerFinalizing(ctx)
},
OnUpstreamWireResponse: func(wire *clients.WireResponse) error {
@@ -1660,6 +1697,7 @@ func (s *Service) runCandidate(
UpstreamPreviousResponseID: responseExecution.UpstreamPreviousResponseID,
PreviousResponseTurns: responseExecution.PreviousTurns,
})
s.recordPassiveRouteObservation(task, routeObserver)
if phaseErr := enterWorkerFinalizing(runCtx); err == nil && phaseErr != nil {
err = phaseErr
}
@@ -1733,8 +1771,8 @@ func (s *Service) runCandidate(
ErrorMessage: err.Error(),
})
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
if shouldClassifyUpstreamSubmissionUnknown(simulated, submissionStatus, err) {
return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: err}
if shouldEnterSubmissionConfirmationPending(simulated, submissionStatus, err) {
return clients.Response{}, &submissionConfirmationPendingError{AttemptID: attemptID, Cause: err}
}
return clients.Response{}, err
}
@@ -2352,7 +2390,7 @@ func (s *Service) observeConcurrencyLeaseRenewal(outcome string) {
}
func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) {
queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, 0, "")
queued, err := s.store.ResolveInterruptedTaskExecution(ctx, task.ID, task.ExecutionToken)
if err != nil {
return store.GatewayTask{}, err
}