feat(routing): 引入多执行池智能调度

将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口,新增 Kubernetes 和静态容量适配器,并以 shadow 模式接入生产配置。

实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌,以及提交状态不明时禁止重复分配的安全语义。

新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
2026-08-05 22:25:37 +08:00
parent 03c0873649
commit 7786692d32
58 changed files with 4510 additions and 348 deletions
+8 -1
View File
@@ -685,6 +685,10 @@ func (s *Service) SubmitAsyncTask(ctx context.Context, task store.GatewayTask) e
_, _ = s.store.FailQueuedTask(context.WithoutCancel(ctx), task.ID, clients.ErrorCode(err), err.Error())
return err
}
if err := s.routeAsyncTask(ctx, &task, user, plan.Candidate); err != nil {
_, _ = s.store.FailQueuedTask(context.WithoutCancel(ctx), task.ID, clients.ErrorCode(err), err.Error())
return err
}
if !plan.Eligible {
if err := s.EnqueueAsyncTask(ctx, task); err != nil {
if s.cancelAsyncSubmissionIfDisconnected(ctx, task.ID) {
@@ -830,7 +834,10 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
if !ok {
return fmt.Errorf("async admission batch task %s was not prepared", input.TaskID)
}
return s.enqueueAsyncTaskTx(ctx, tx, task.ID, asyncTaskInsertOpts(task))
_, publishErr := (riverExecutionBroker{service: s}).publishTx(
ctx, tx, task.AssignedPoolID, task.ID, time.Time{}, 2,
)
return publishErr
}
outcomes, batchErr := s.store.TryTaskAdmissionAtomicBatchWithAdmittedHook(
ctx,
@@ -0,0 +1,60 @@
package runner
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
)
// httpExecutionTransport is the first platform-neutral Worker transport
// adapter. The routing core sees only WorkerDescriptor and executionpool's
// versioned request/response types; Pod DNS and other orchestrator concepts do
// not cross this boundary.
type httpExecutionTransport struct {
client *http.Client
}
func (transport httpExecutionTransport) Execute(
ctx context.Context,
worker executionpool.WorkerDescriptor,
input executionpool.ExecutionRequest,
) (executionpool.ExecutionResponse, error) {
if transport.client == nil {
return executionpool.ExecutionResponse{}, errors.New("worker HTTP client is required")
}
payload, err := json.Marshal(InternalExecutionRequest{
TaskID: input.TaskID,
LeaseID: input.LeaseID,
Stream: input.Stream,
})
if err != nil {
return executionpool.ExecutionResponse{}, err
}
request, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
strings.TrimRight(worker.Endpoint, "/")+workerExecutionPath,
bytes.NewReader(payload),
)
if err != nil {
return executionpool.ExecutionResponse{}, err
}
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Authorization", "Worker "+input.AuthorizationToken)
response, err := transport.client.Do(request)
if err != nil {
return executionpool.ExecutionResponse{}, err
}
return executionpool.ExecutionResponse{
StatusCode: response.StatusCode,
Headers: response.Header,
Body: response.Body,
}, nil
}
var _ executionpool.ExecutionTransport = httpExecutionTransport{}
@@ -0,0 +1,53 @@
package runner
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
)
func TestHTTPExecutionTransportUsesAdvertisedEndpointAndProtocol(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != workerExecutionPath {
t.Fatalf("path=%q", request.URL.Path)
}
if got := request.Header.Get("Authorization"); got != "Worker signed-token" {
t.Fatalf("authorization=%q", got)
}
var payload InternalExecutionRequest
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
t.Fatal(err)
}
if payload.TaskID != "task-1" || payload.LeaseID != "lease-1" || !payload.Stream {
t.Fatalf("payload=%+v", payload)
}
response.Header().Set("X-Worker", "accepted")
response.WriteHeader(http.StatusAccepted)
_, _ = response.Write([]byte("ok"))
}))
defer server.Close()
transport := httpExecutionTransport{client: server.Client()}
result, err := transport.Execute(context.Background(), executionpool.WorkerDescriptor{
WorkerID: "worker-1", PoolID: "pool-1", Endpoint: server.URL,
}, executionpool.ExecutionRequest{
TaskID: "task-1", PoolID: "pool-1", WorkerID: "worker-1", LeaseID: "lease-1",
AuthorizationToken: "signed-token", Stream: true,
})
if err != nil {
t.Fatal(err)
}
defer result.Body.Close()
body, err := io.ReadAll(result.Body)
if err != nil {
t.Fatal(err)
}
if result.StatusCode != http.StatusAccepted || len(result.Headers["X-Worker"]) != 1 || result.Headers["X-Worker"][0] != "accepted" || string(body) != "ok" {
t.Fatalf("result=%+v body=%q", result, body)
}
}
@@ -0,0 +1,137 @@
package runner
import (
"context"
"crypto/tls"
"net/http"
"net/http/httptrace"
"sort"
"sync"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
type passiveRouteObserver struct {
mu sync.Mutex
samples int
successes int
connectTLS []time.Duration
uploadBytesRate []float64
}
type passiveRouteRoundTripper struct {
base http.RoundTripper
observer *passiveRouteObserver
}
func (observer *passiveRouteObserver) wrap(client *http.Client) *http.Client {
if client == nil {
client = http.DefaultClient
}
cloned := *client
base := cloned.Transport
if base == nil {
base = http.DefaultTransport
}
cloned.Transport = passiveRouteRoundTripper{base: base, observer: observer}
return &cloned
}
func (transport passiveRouteRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
startedAt := time.Now()
var connectStartedAt time.Time
var connectDuration time.Duration
var tlsStartedAt time.Time
var tlsDuration time.Duration
var wroteRequestAt time.Time
trace := &httptrace.ClientTrace{
ConnectStart: func(_, _ string) { connectStartedAt = time.Now() },
ConnectDone: func(_, _ string, _ error) {
if !connectStartedAt.IsZero() {
connectDuration = time.Since(connectStartedAt)
}
},
TLSHandshakeStart: func() { tlsStartedAt = time.Now() },
TLSHandshakeDone: func(tls.ConnectionState, error) {
if !tlsStartedAt.IsZero() {
tlsDuration = time.Since(tlsStartedAt)
}
},
WroteRequest: func(httptrace.WroteRequestInfo) { wroteRequestAt = time.Now() },
}
traced := request.Clone(httptrace.WithClientTrace(request.Context(), trace))
response, err := transport.base.RoundTrip(traced)
transport.observer.record(
err == nil,
connectDuration+tlsDuration,
request.ContentLength,
startedAt,
wroteRequestAt,
)
return response, err
}
func (observer *passiveRouteObserver) record(success bool, connectTLS time.Duration, contentLength int64, startedAt, wroteRequestAt time.Time) {
observer.mu.Lock()
defer observer.mu.Unlock()
observer.samples++
if success {
observer.successes++
}
if connectTLS > 0 {
observer.connectTLS = append(observer.connectTLS, connectTLS)
}
if contentLength > 0 && !wroteRequestAt.IsZero() && wroteRequestAt.After(startedAt) {
observer.uploadBytesRate = append(observer.uploadBytesRate, float64(contentLength)/wroteRequestAt.Sub(startedAt).Seconds())
}
}
func (observer *passiveRouteObserver) snapshot(poolID, routeProfileKey string) executionpool.RouteObservation {
observer.mu.Lock()
defer observer.mu.Unlock()
return executionpool.RouteObservation{
PoolID: poolID, RouteProfileKey: routeProfileKey,
SampleCount: observer.samples, SuccessCount: observer.successes,
ConnectTLSP95: durationP95(observer.connectTLS),
UploadBytesPerSecond: floatP95(observer.uploadBytesRate),
}
}
func durationP95(values []time.Duration) time.Duration {
if len(values) == 0 {
return 0
}
copyOfValues := append([]time.Duration(nil), values...)
sort.Slice(copyOfValues, func(i, j int) bool { return copyOfValues[i] < copyOfValues[j] })
return copyOfValues[(len(copyOfValues)*95+99)/100-1]
}
func floatP95(values []float64) float64 {
if len(values) == 0 {
return 0
}
copyOfValues := append([]float64(nil), values...)
sort.Float64s(copyOfValues)
return copyOfValues[(len(copyOfValues)*95+99)/100-1]
}
func (s *Service) recordPassiveRouteObservation(task store.GatewayTask, observer *passiveRouteObserver) {
if observer == nil || !s.routingEnabled() || task.RouteProfileKey == "" {
return
}
poolID := task.AssignedPoolID
if poolID == "" && s.cfg.RunsAsyncExecutionWorker() {
poolID = s.cfg.ExecutionPoolID
}
observation := observer.snapshot(poolID, task.RouteProfileKey)
if observation.PoolID == "" || observation.SampleCount == 0 {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
if err := s.coordinationStore.RecordRouteObservation(ctx, observation); err != nil && s.logger != nil {
s.logger.Warn("record passive route observation failed", "error_category", "route_observation_failed")
}
}
@@ -0,0 +1,25 @@
package runner
import (
"testing"
"time"
)
func TestPassiveRouteObserverRecordsOnlyTransportPhaseFacts(t *testing.T) {
observer := &passiveRouteObserver{}
startedAt := time.Now()
observer.record(true, 10*time.Millisecond, 1024, startedAt, startedAt.Add(time.Millisecond))
observer.record(false, 20*time.Millisecond, 2048, startedAt, startedAt.Add(2*time.Millisecond))
observer.record(true, 30*time.Millisecond, 0, startedAt, time.Time{})
observation := observer.snapshot("pool-a", "route-a")
if observation.SampleCount != 3 || observation.SuccessCount != 2 {
t.Fatalf("observation=%+v", observation)
}
if observation.ConnectTLSP95 != 30*time.Millisecond {
t.Fatalf("connect/TLS p95=%s", observation.ConnectTLSP95)
}
if observation.UploadBytesPerSecond < 1_024_000-1 || observation.UploadBytesPerSecond > 1_024_000+1 {
t.Fatalf("upload rate=%f", observation.UploadBytesPerSecond)
}
}
+73 -39
View File
@@ -2,6 +2,7 @@ package runner
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
@@ -9,6 +10,7 @@ import (
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
"github.com/google/uuid"
@@ -17,7 +19,6 @@ import (
"github.com/riverqueue/river"
"github.com/riverqueue/river/riverdriver/riverpgxv5"
"github.com/riverqueue/river/rivermigrate"
"github.com/riverqueue/river/rivertype"
)
const (
@@ -59,6 +60,11 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
if task.Status == "succeeded" || task.Status == "failed" || task.Status == "cancelled" {
return nil
}
if assignedPool := strings.TrimSpace(task.AssignedPoolID); assignedPool != "" && assignedPool != strings.TrimSpace(w.service.cfg.ExecutionPoolID) {
w.service.logger.Warn("river task claimed by a worker outside its assigned pool",
"taskID", task.ID, "assignedPool", assignedPool, "workerPool", w.service.cfg.ExecutionPoolID)
return river.JobSnooze(time.Second)
}
loadLease, admitted := w.service.tryStartWorkerTask()
if !admitted {
return river.JobSnooze(workerLoadRetryDelay(task.ID))
@@ -87,6 +93,10 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
task.ExecutionToken = executionToken
queued, queueErr := w.service.requeueInterruptedAsyncTask(context.WithoutCancel(ctx), task)
if queueErr != nil {
if errors.Is(queueErr, store.ErrTaskExecutionManualReview) {
w.service.logger.Warn("interrupted task held after upstream submission began", "taskID", task.ID, "error_category", "upstream_timeout")
return nil
}
return queueErr
}
w.service.logger.Debug("river async task interrupted and requeued", "taskID", task.ID, "status", queued.Status, "riverJobID", job.ID)
@@ -309,14 +319,19 @@ func (s *Service) newRiverAsyncExecutionClient(capacity int) (*river.Client[pgx.
if err := river.AddWorkerSafely(workers, &asyncTaskWorker{service: s}); err != nil {
return nil, err
}
queues := map[string]river.QueueConfig{
asyncTaskQueueName: {MaxWorkers: capacity},
}
poolQueue := executionpool.QueueName(s.cfg.ExecutionPoolID)
if poolQueue != asyncTaskQueueName {
queues[poolQueue] = river.QueueConfig{MaxWorkers: capacity}
}
return river.NewClient(riverpgxv5.New(s.riverStore.Pool()), &river.Config{
ID: fmt.Sprintf("%s-exec-%d-%d", s.workerInstanceID, capacity, time.Now().UnixNano()),
JobTimeout: -1,
Logger: s.logger,
CompletedJobRetentionPeriod: 24 * time.Hour,
Queues: map[string]river.QueueConfig{
asyncTaskQueueName: {MaxWorkers: capacity},
},
Queues: queues,
// Provider-backed media jobs commonly poll for 10-20 minutes. River may
// execute a still-running job again once this window elapses, so keep the
// rescue horizon above the longest configured provider poll timeout.
@@ -346,25 +361,40 @@ func (s *Service) loadAsyncWorkerCapacity(ctx context.Context) (store.AsyncWorke
return store.AsyncWorkerCapacitySnapshot{}, err
}
loadSnapshot := s.sampleWorkerLoad()
labels := map[string]string{}
if raw := strings.TrimSpace(s.cfg.ExecutionPoolLabels); raw != "" {
if err := json.Unmarshal([]byte(raw), &labels); err != nil {
return store.AsyncWorkerCapacitySnapshot{}, fmt.Errorf("decode execution pool labels: %w", err)
}
}
if endpoint := strings.TrimSpace(s.cfg.WorkerAdvertiseEndpoint); endpoint != "" {
if err := executionpool.ValidateAdvertisedEndpoint(endpoint, splitTrimmed(s.cfg.WorkerEndpointAllowedSuffixes), s.cfg.WorkerEndpointAllowPrivate); err != nil {
return store.AsyncWorkerCapacitySnapshot{}, fmt.Errorf("validate worker advertised endpoint: %w", err)
}
}
allocation, err := s.coordinationStore.RegisterWorkerInstance(ctx, store.WorkerRegistrationInput{
InstanceID: s.workerInstanceID,
PodUID: strings.TrimSpace(os.Getenv("POD_UID")),
PodName: strings.TrimSpace(os.Getenv("POD_NAME")),
Site: strings.TrimSpace(os.Getenv("EASYAI_SITE")),
Revision: strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
DesiredCapacity: snapshot.Capacity,
CapacityLimit: s.cfg.AsyncWorkerInstanceHardLimit,
LoadMode: loadSnapshot.Mode,
SafeCapacity: loadSnapshot.SafeCapacity,
HeavyCapacity: loadSnapshot.HeavyLimit,
ActiveTasks: loadSnapshot.ActiveTasks,
PreparingTasks: loadSnapshot.PreparingTasks,
WaitingUpstreamTasks: loadSnapshot.WaitingUpstreamTasks,
FinalizingTasks: loadSnapshot.FinalizingTasks,
PressureState: string(loadSnapshot.PressureState),
PressureReason: loadSnapshot.PressureReason,
LoadSampledAt: loadSnapshot.SampledAt,
HeartbeatStaleAfter: time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
InstanceID: s.workerInstanceID,
WorkerID: firstNonEmptyString(s.cfg.WorkerID, s.workerInstanceID),
PoolID: s.cfg.ExecutionPoolID,
Endpoint: s.cfg.WorkerAdvertiseEndpoint,
Labels: labels,
Capabilities: defaultWorkerCapabilities(),
ProtocolVersion: executionpool.ProtocolVersion,
OrchestratorInstanceRef: s.cfg.WorkerOrchestratorInstanceRef,
Revision: strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
DesiredCapacity: snapshot.Capacity,
CapacityLimit: s.cfg.AsyncWorkerInstanceHardLimit,
LoadMode: loadSnapshot.Mode,
SafeCapacity: loadSnapshot.SafeCapacity,
HeavyCapacity: loadSnapshot.HeavyLimit,
ActiveTasks: loadSnapshot.ActiveTasks,
PreparingTasks: loadSnapshot.PreparingTasks,
WaitingUpstreamTasks: loadSnapshot.WaitingUpstreamTasks,
FinalizingTasks: loadSnapshot.FinalizingTasks,
PressureState: string(loadSnapshot.PressureState),
PressureReason: loadSnapshot.PressureReason,
LoadSampledAt: loadSnapshot.SampledAt,
HeartbeatStaleAfter: time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
})
if err != nil {
return store.AsyncWorkerCapacitySnapshot{}, err
@@ -599,7 +629,8 @@ func (s *Service) observeAsyncWorkerResize(outcome string) {
}
func (s *Service) EnqueueAsyncTask(ctx context.Context, task store.GatewayTask) error {
return s.enqueueAsyncTaskWithOptions(ctx, task.ID, asyncTaskInsertOpts(task))
_, err := (riverExecutionBroker{service: s}).Publish(ctx, task.AssignedPoolID, task.ID, time.Time{})
return err
}
func (s *Service) enqueueAsyncTaskWithOptions(ctx context.Context, taskID string, opts *river.InsertOpts) error {
@@ -761,22 +792,25 @@ func asyncTaskInsertOpts(task store.GatewayTask) *river.InsertOpts {
if task.ID == "" {
priority = 3
}
return &river.InsertOpts{
MaxAttempts: 1000,
Priority: priority,
Queue: asyncTaskQueueName,
Tags: []string{"gateway-task"},
UniqueOpts: river.UniqueOpts{
ByArgs: true,
ByQueue: true,
ByState: []rivertype.JobState{
rivertype.JobStateAvailable,
rivertype.JobStatePending,
rivertype.JobStateRetryable,
rivertype.JobStateRunning,
rivertype.JobStateScheduled,
},
},
return riverExecutionInsertOptions(task.AssignedPoolID, time.Time{}, priority)
}
func splitTrimmed(value string) []string {
items := make([]string, 0)
for item := range strings.SplitSeq(value, ",") {
if item = strings.TrimSpace(item); item != "" {
items = append(items, item)
}
}
return items
}
func defaultWorkerCapabilities() map[string]any {
return map[string]any{
"protocolVersion": executionpool.ProtocolVersion,
"streaming": true,
"media": true,
"taskKinds": []any{"*"},
}
}
@@ -0,0 +1,264 @@
package runner
import (
"bufio"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
const (
workerExecutionAudience = "easyai-worker-execution"
workerExecutionPath = "/internal/v1/executions"
)
type InternalExecutionRequest struct {
TaskID string `json:"task_id"`
LeaseID string `json:"lease_id"`
Stream bool `json:"stream"`
}
type InternalExecutionFrame struct {
Type string `json:"type"`
Delta *clients.StreamDeltaEvent `json:"delta,omitempty"`
Output map[string]any `json:"output,omitempty"`
Wire *clients.WireResponse `json:"wire,omitempty"`
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
Status int `json:"status,omitempty"`
Retryable bool `json:"retryable,omitempty"`
Details map[string]any `json:"details,omitempty"`
}
func (s *Service) executeRouted(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
if task.AsyncMode || !s.cfg.RunsPublicHTTP() || !s.routingEnabled() || task.RunMode == "simulation" {
return s.executeLocal(ctx, task, user, onDelta)
}
candidate, err := s.routingCandidateForTask(ctx, task, user)
if err != nil {
if s.routingEnforced() {
s.observeExecutionPoolRouting("unavailable")
return Result{}, upstreamRouteUnavailable(err)
}
return s.executeLocal(ctx, task, user, onDelta)
}
decision, profile, err := s.selectExecutionPool(ctx, task, candidate)
if err != nil {
_ = s.coordinationStore.RequestRouteProbe(context.WithoutCancel(ctx), profile.Key)
if s.routingEnforced() {
s.observeExecutionPoolRouting("unavailable")
return Result{}, upstreamRouteUnavailable(err)
}
_ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion,
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
Reason: "shadow_no_eligible_pool", Snapshot: map[string]any{"mode": "shadow", "error": safeRoutingError(err)},
})
s.observeExecutionPoolRouting("shadow")
return s.executeLocal(ctx, task, user, onDelta)
}
snapshot := routingDecisionSnapshot(decision)
if !s.routingEnforced() {
snapshot["suggestedPoolId"] = decision.PoolID
_ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion,
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
Reason: decision.Reason, Snapshot: snapshot,
})
s.observeExecutionPoolRouting("shadow")
return s.executeLocal(ctx, task, user, onDelta)
}
if err := s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
TaskID: task.ID, PoolID: decision.PoolID, RouteProfileKey: profile.Key,
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
RoutingVersion: routingVersion, Reason: decision.Reason, Snapshot: snapshot,
}); err != nil {
return Result{}, err
}
s.observeExecutionPoolRouting("selected")
return s.executeThroughWorker(ctx, task, decision.PoolID, onDelta)
}
func (s *Service) executeLocal(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
return s.executeWithToken(ctx, task, user, onDelta, uuid.NewString())
}
func (s *Service) executeThroughWorker(ctx context.Context, task store.GatewayTask, poolID string, onDelta clients.StreamDelta) (Result, error) {
nonce := uuid.NewString()
lease, err := s.coordinationStore.ReserveWorkerExecution(ctx, task.ID, poolID, nonce, 30*time.Second)
if err != nil {
s.observeExecutionPoolRouting("capacity_rejected")
return Result{}, upstreamRouteUnavailable(err)
}
if err := executionpool.ValidateAdvertisedEndpoint(
lease.Endpoint, splitTrimmed(s.cfg.WorkerEndpointAllowedSuffixes), s.cfg.WorkerEndpointAllowPrivate,
); err != nil {
_ = s.coordinationStore.ReleaseWorkerExecutionLease(context.WithoutCancel(ctx), lease.LeaseID)
return Result{}, &clients.ClientError{Code: "worker_endpoint_untrusted", Message: err.Error(), StatusCode: http.StatusServiceUnavailable, Retryable: false}
}
signer := executionpool.TokenSigner{Secret: []byte(s.cfg.WorkerExecutionSecret)}
token, err := signer.Sign(executionpool.ExecutionClaims{
Audience: workerExecutionAudience, TaskID: task.ID, PoolID: poolID,
WorkerID: lease.WorkerID, Nonce: nonce, ExpiresAt: time.Now().Add(30 * time.Second).Unix(),
})
if err != nil {
_ = s.coordinationStore.ReleaseWorkerExecutionLease(context.WithoutCancel(ctx), lease.LeaseID)
return Result{}, err
}
client, err := s.workerExecutionHTTPClient()
if err != nil {
return Result{}, err
}
transport := httpExecutionTransport{client: client}
response, err := transport.Execute(ctx, executionpool.WorkerDescriptor{
WorkerID: lease.WorkerID, InstanceID: lease.InstanceID, PoolID: poolID, Endpoint: lease.Endpoint,
}, executionpool.ExecutionRequest{
TaskID: task.ID, PoolID: poolID, WorkerID: lease.WorkerID, LeaseID: lease.LeaseID,
AuthorizationToken: token, Stream: onDelta != nil,
})
if err != nil {
latest, readErr := s.store.GetTask(context.WithoutCancel(ctx), task.ID)
if readErr == nil && latest.SubmissionState != "not_started" {
return Result{Task: latest}, &clients.ClientError{
Code: "upstream_timeout", Message: "worker transport ended after upstream submission began",
StatusCode: http.StatusGatewayTimeout, Retryable: true,
}
}
return Result{}, &clients.ClientError{Code: "worker_transport_error", Message: err.Error(), StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
payload, _ := io.ReadAll(io.LimitReader(response.Body, 64*1024))
return Result{}, &clients.ClientError{Code: "worker_transport_error", Message: strings.TrimSpace(string(payload)), StatusCode: response.StatusCode, Retryable: response.StatusCode >= 500}
}
var output map[string]any
var wire *clients.WireResponse
scanner := bufio.NewScanner(response.Body)
scanner.Buffer(make([]byte, 4096), 1024*1024)
for scanner.Scan() {
var frame InternalExecutionFrame
if err := json.Unmarshal(scanner.Bytes(), &frame); err != nil {
return Result{}, &clients.ClientError{Code: "worker_protocol_error", Message: "invalid worker execution frame", StatusCode: http.StatusBadGateway, Retryable: false}
}
switch frame.Type {
case "delta":
if onDelta != nil && frame.Delta != nil {
if err := onDelta(*frame.Delta); err != nil {
return Result{}, err
}
}
case "result":
output, wire = frame.Output, frame.Wire
case "error":
return Result{}, &clients.ClientError{
Code: frame.Code, Message: frame.Message, StatusCode: frame.Status,
Retryable: frame.Retryable, Details: frame.Details,
}
}
}
if err := scanner.Err(); err != nil {
return Result{}, &clients.ClientError{Code: "worker_transport_error", Message: err.Error(), StatusCode: http.StatusBadGateway, Retryable: true}
}
finished, err := s.store.GetTask(ctx, task.ID)
if err != nil {
return Result{}, err
}
return Result{Task: finished, Output: output, Wire: wire}, nil
}
func (s *Service) ExecuteInternal(ctx context.Context, authorization string, input InternalExecutionRequest, onDelta clients.StreamDelta) (Result, error) {
if s.cfg.RunsPublicHTTP() || !s.cfg.RunsAsyncExecutionWorker() {
return Result{}, errors.New("internal execution is only available on worker processes")
}
token := strings.TrimSpace(strings.TrimPrefix(authorization, "Worker "))
if token == authorization || token == "" {
return Result{}, errors.New("worker execution authorization is required")
}
signer := executionpool.TokenSigner{Secret: []byte(s.cfg.WorkerExecutionSecret)}
claims, err := signer.Verify(token, workerExecutionAudience)
if err != nil {
return Result{}, err
}
localWorkerID := firstNonEmptyString(s.cfg.WorkerID, s.workerInstanceID)
if claims.TaskID != input.TaskID || claims.PoolID != s.cfg.ExecutionPoolID || claims.WorkerID != localWorkerID {
return Result{}, errors.New("worker execution claims do not match this worker")
}
lease, err := s.coordinationStore.ConsumeWorkerExecutionLease(ctx, input.LeaseID, claims.Nonce)
if err != nil {
return Result{}, err
}
defer func() {
_ = s.coordinationStore.ReleaseWorkerExecutionLease(context.WithoutCancel(ctx), lease.LeaseID)
}()
if lease.TaskID != input.TaskID || lease.PoolID != s.cfg.ExecutionPoolID || lease.WorkerID != localWorkerID || lease.InstanceID != s.workerInstanceID {
return Result{}, errors.New("worker execution lease does not match this worker")
}
loadLease, admitted := s.tryStartWorkerTask()
if !admitted {
return Result{}, &clients.ClientError{Code: "worker_capacity_unavailable", Message: "worker has no safe execution capacity", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
defer loadLease.Release()
ctx = context.WithValue(ctx, workerLoadLeaseContextKey{}, loadLease)
task, err := s.store.GetTask(ctx, input.TaskID)
if err != nil {
return Result{}, err
}
return s.executeLocal(ctx, task, authUserFromTask(task), onDelta)
}
func (s *Service) workerExecutionHTTPClient() (*http.Client, error) {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.Proxy = nil
if strings.TrimSpace(s.cfg.WorkerExecutionCAFile) != "" {
caBytes, err := os.ReadFile(s.cfg.WorkerExecutionCAFile)
if err != nil {
return nil, err
}
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(caBytes) {
return nil, errors.New("worker execution CA file does not contain a certificate")
}
transport.TLSClientConfig = &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}
if strings.TrimSpace(s.cfg.WorkerExecutionCertFile) != "" || strings.TrimSpace(s.cfg.WorkerExecutionKeyFile) != "" {
certificate, err := tls.LoadX509KeyPair(s.cfg.WorkerExecutionCertFile, s.cfg.WorkerExecutionKeyFile)
if err != nil {
return nil, err
}
transport.TLSClientConfig.Certificates = []tls.Certificate{certificate}
}
}
return &http.Client{Transport: transport}, nil
}
func internalExecutionStatus(err error) int {
var clientErr *clients.ClientError
if errors.As(err, &clientErr) && clientErr.StatusCode > 0 {
return clientErr.StatusCode
}
return http.StatusInternalServerError
}
func InternalExecutionFrameForError(err error) InternalExecutionFrame {
var clientErr *clients.ClientError
if errors.As(err, &clientErr) {
return InternalExecutionFrame{
Type: "error", Code: clientErr.Code, Message: clientErr.Message,
Status: internalExecutionStatus(err), Retryable: clientErr.Retryable, Details: clientErr.Details,
}
}
return InternalExecutionFrame{Type: "error", Code: "worker_execution_failed", Message: fmt.Sprint(err), Status: http.StatusInternalServerError}
}
@@ -0,0 +1,99 @@
package runner
import (
"context"
"errors"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/jackc/pgx/v5"
"github.com/riverqueue/river"
"github.com/riverqueue/river/rivertype"
)
// riverExecutionBroker is the River adapter for the platform-neutral broker
// port. Queue names and River uniqueness semantics do not escape this adapter.
type riverExecutionBroker struct {
service *Service
}
func (broker riverExecutionBroker) Publish(
ctx context.Context,
poolID string,
taskID string,
notBefore time.Time,
) (int64, error) {
tx, err := broker.service.store.Pool().Begin(ctx)
if err != nil {
return 0, err
}
defer func() {
rollbackCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = tx.Rollback(rollbackCtx)
}()
jobID, err := broker.publishTx(ctx, tx, poolID, taskID, notBefore, 2)
if err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, err
}
return jobID, nil
}
func (broker riverExecutionBroker) publishTx(
ctx context.Context,
tx pgx.Tx,
poolID string,
taskID string,
notBefore time.Time,
priority int,
) (int64, error) {
riverClient := broker.service.asyncControlClient()
if riverClient == nil {
return 0, errors.New("River execution broker is not started")
}
opts := riverExecutionInsertOptions(poolID, notBefore, priority)
result, err := riverClient.InsertTx(ctx, tx, asyncTaskArgs{TaskID: taskID}, opts)
if err != nil {
return 0, err
}
if result.Job == nil {
return 0, nil
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET river_job_id = $2,
updated_at = now()
WHERE id = $1::uuid`, taskID, result.Job.ID); err != nil {
return 0, err
}
return result.Job.ID, nil
}
func riverExecutionInsertOptions(poolID string, notBefore time.Time, priority int) *river.InsertOpts {
if priority < 1 {
priority = 2
}
opts := &river.InsertOpts{
MaxAttempts: 1000,
Priority: priority,
Queue: executionpool.QueueName(poolID),
Tags: []string{"gateway-task"},
UniqueOpts: river.UniqueOpts{
ByArgs: true, ByQueue: true,
ByState: []rivertype.JobState{
rivertype.JobStateAvailable, rivertype.JobStatePending,
rivertype.JobStateRetryable, rivertype.JobStateRunning,
rivertype.JobStateScheduled,
},
},
}
if !notBefore.IsZero() {
opts.ScheduledAt = notBefore
}
return opts
}
var _ executionpool.ExecutionBroker = riverExecutionBroker{}
+168
View File
@@ -0,0 +1,168 @@
package runner
import (
"context"
"errors"
"log/slog"
"math"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
type routeProbeState struct {
next time.Time
lastRequestObserved time.Time
}
func (s *Service) StartRouteHealthProber(ctx context.Context) {
if !s.cfg.RunsRouteProber() {
return
}
go s.runRouteHealthProber(ctx)
}
func (s *Service) runRouteHealthProber(ctx context.Context) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
states := make(map[string]routeProbeState)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
targets, err := s.coordinationStore.ListRouteProbeTargets(ctx)
if err != nil {
s.logRouteProbeFailure("list_targets", err)
continue
}
requested, requestErr := s.coordinationStore.ListRequestedRouteProbes(ctx)
if requestErr != nil {
s.logRouteProbeFailure("list_requests", requestErr)
continue
}
now := time.Now()
for _, target := range targets {
profile, profileErr := routeProfileForCandidate(target.Candidate)
if profileErr != nil {
continue
}
state := states[profile.Key]
requestedAt := requested[profile.Key]
requestDue := !requestedAt.IsZero() && requestedAt.After(state.lastRequestObserved)
if !requestDue && now.Before(state.next) {
continue
}
interval := time.Duration(s.cfg.RouteProbeColdIntervalSeconds) * time.Second
if target.Hot {
interval = time.Duration(s.cfg.RouteProbeHotIntervalSeconds) * time.Second
}
states[profile.Key] = routeProbeState{next: now.Add(interval), lastRequestObserved: requestedAt}
go s.probeRouteTarget(ctx, target, profile)
}
}
}
}
func (s *Service) probeRouteTarget(ctx context.Context, target store.RouteProbeTarget, profile executionpool.RouteProfile) {
leadership, locked, err := s.coordinationStore.TryAcquireRouteProbeLeadership(ctx, s.cfg.ExecutionPoolID, profile.Key)
if err != nil || !locked {
return
}
defer leadership.Release()
client, err := s.httpClientForCandidate(target.Candidate, false, "")
if err != nil {
s.logRouteProbeFailure("prepare_client", err)
return
}
probe := executionpool.HTTPProbe{Client: client}
result := probe.Probe(ctx, executionpool.ProbeTarget{
RouteProfile: profile,
URL: routeEndpointForCandidate(target.Candidate),
Timeout: time.Duration(s.cfg.RouteProbeTimeoutMS) * time.Millisecond,
})
previous := executionpool.RouteHealth{}
if history, historyErr := s.coordinationStore.ListRouteHealth(ctx, profile.Key, time.Now()); historyErr == nil {
for _, item := range history {
if item.PoolID == s.cfg.ExecutionPoolID {
previous = item
break
}
}
}
health := nextRouteHealth(previous, s.cfg.ExecutionPoolID, profile.Key, result)
s.observeRouteProbe(string(health.State))
if err := s.coordinationStore.RecordRouteHealth(context.WithoutCancel(ctx), health); err != nil {
s.logRouteProbeFailure("record", err)
}
}
func (s *Service) observeRouteProbe(state string) {
observer, ok := s.billingMetrics.(interface{ ObserveRouteProbe(string) })
if ok {
observer.ObserveRouteProbe(state)
}
}
func nextRouteHealth(previous executionpool.RouteHealth, poolID, profileKey string, result executionpool.ProbeResult) executionpool.RouteHealth {
health := previous
health.PoolID = poolID
health.RouteProfileKey = profileKey
health.SampleCount++
health.SampledAt = result.SampledAt
health.ExpiresAt = result.SampledAt.Add(45 * time.Second)
observation := 0.0
if result.Reachable {
observation = 1
health.ConsecutiveSuccesses++
health.ConsecutiveFailures = 0
if health.ConsecutiveSuccesses >= 2 || previous.State == executionpool.RouteHealthy {
health.State = executionpool.RouteHealthy
} else {
health.State = executionpool.RouteDegraded
}
} else {
health.ConsecutiveFailures++
health.ConsecutiveSuccesses = 0
switch {
case health.ConsecutiveFailures >= 3:
health.State = executionpool.RouteUnreachable
case previous.State == executionpool.RouteHealthy || previous.State == executionpool.RouteDegraded:
health.State = executionpool.RouteDegraded
default:
health.State = executionpool.RouteUnknown
}
}
if previous.SampleCount == 0 {
health.SuccessRate = observation
} else {
health.SuccessRate = 0.8*previous.SuccessRate + 0.2*observation
}
connectTLS := result.TCPDuration + result.TLSDuration
if connectTLS > 0 {
health.ConnectTLSP95 = ewmaDuration(previous.ConnectTLSP95, connectTLS)
}
if result.FirstByte > 0 {
if previous.FirstByteP95 > 0 {
delta := math.Abs(float64(result.FirstByte - previous.FirstByteP95))
health.JitterP95 = ewmaDuration(previous.JitterP95, time.Duration(delta))
}
health.FirstByteP95 = ewmaDuration(previous.FirstByteP95, result.FirstByte)
}
return health
}
func ewmaDuration(previous, current time.Duration) time.Duration {
if previous <= 0 {
return current
}
return time.Duration(0.8*float64(previous) + 0.2*float64(current))
}
func (s *Service) logRouteProbeFailure(stage string, err error) {
if err == nil || errors.Is(err, context.Canceled) || s.logger == nil {
return
}
s.logger.Log(context.Background(), slog.LevelWarn, "route health probe failed", "stage", stage, "error_category", "route_probe_failed")
}
@@ -0,0 +1,49 @@
package runner
import (
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestNextRouteHealthRequiresFailuresAndRecoverySamples(t *testing.T) {
now := time.Now()
health := executionpool.RouteHealth{}
for index := 0; index < 2; index++ {
health = nextRouteHealth(health, "pool", "route", executionpool.ProbeResult{SampledAt: now.Add(time.Duration(index) * time.Second), ErrorClass: "network"})
if health.State == executionpool.RouteUnreachable {
t.Fatalf("became unreachable after %d failures", index+1)
}
}
health = nextRouteHealth(health, "pool", "route", executionpool.ProbeResult{SampledAt: now.Add(2 * time.Second), ErrorClass: "network"})
if health.State != executionpool.RouteUnreachable {
t.Fatalf("state=%s, want unreachable", health.State)
}
health = nextRouteHealth(health, "pool", "route", executionpool.ProbeResult{SampledAt: now.Add(3 * time.Second), Reachable: true, TCPDuration: 5 * time.Millisecond, FirstByte: 20 * time.Millisecond})
if health.State != executionpool.RouteDegraded {
t.Fatalf("state=%s, want degraded during recovery", health.State)
}
health = nextRouteHealth(health, "pool", "route", executionpool.ProbeResult{SampledAt: now.Add(4 * time.Second), Reachable: true, TCPDuration: 5 * time.Millisecond, FirstByte: 20 * time.Millisecond})
if health.State != executionpool.RouteHealthy {
t.Fatalf("state=%s, want healthy", health.State)
}
if health.ExpiresAt.Sub(health.SampledAt) != 45*time.Second {
t.Fatalf("health TTL=%s", health.ExpiresAt.Sub(health.SampledAt))
}
}
func TestPinCandidatesToRoutingAssignment(t *testing.T) {
candidates := []store.RuntimeModelCandidate{
{PlatformID: "platform-a", PlatformModelID: "model-a"},
{PlatformID: "platform-b", PlatformModelID: "model-b"},
}
pinned := pinCandidatesToRoutingAssignment(candidates, "platform-b", "model-b")
if len(pinned) != 1 || pinned[0].PlatformModelID != "model-b" {
t.Fatalf("pinned=%+v", pinned)
}
if got := pinCandidatesToRoutingAssignment(candidates, "platform-a", "missing"); len(got) != 0 {
t.Fatalf("unexpected fallback: %+v", got)
}
}
+332
View File
@@ -0,0 +1,332 @@
package runner
import (
"context"
"errors"
"net/http"
"sort"
"strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/netproxy"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const (
routingVersion = "execution-pool-v1"
asyncRouteWaitMaximum = 10 * time.Second
)
func (s *Service) routingEnabled() bool {
mode := strings.ToLower(strings.TrimSpace(s.cfg.RoutingMode))
return mode == "shadow" || mode == "enforced"
}
func (s *Service) routingEnforced() bool {
return strings.EqualFold(strings.TrimSpace(s.cfg.RoutingMode), "enforced")
}
func (s *Service) routeAsyncTask(ctx context.Context, task *store.GatewayTask, user *auth.User, preferred store.RuntimeModelCandidate) error {
if task == nil || !task.AsyncMode || !s.routingEnabled() || task.RunMode == "simulation" {
return nil
}
startedAt := time.Now()
delays := []time.Duration{0, 2 * time.Second, 3 * time.Second, 4 * time.Second}
var lastErr error
for _, delay := range delays {
if delay > 0 {
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return ctx.Err()
case <-timer.C:
}
}
candidate := preferred
if strings.TrimSpace(candidate.PlatformID) == "" {
resolved, err := s.routingCandidateForTask(ctx, *task, user)
if err != nil {
lastErr = err
continue
}
candidate = resolved
}
decision, profile, err := s.selectExecutionPool(ctx, *task, candidate)
if err == nil {
poolID := decision.PoolID
snapshot := routingDecisionSnapshot(decision)
if !s.routingEnforced() {
s.observeExecutionPoolRouting("shadow")
snapshot["suggestedPoolId"] = poolID
poolID = ""
}
if s.routingEnforced() {
s.observeExecutionPoolRouting("selected")
}
if err := s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
TaskID: task.ID, PoolID: poolID, RouteProfileKey: profile.Key,
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
RoutingVersion: routingVersion, Reason: decision.Reason, Snapshot: snapshot,
}); err != nil {
return err
}
task.AssignedPoolID = poolID
task.RouteProfileKey = profile.Key
task.RoutingVersion = routingVersion
task.RoutingReason = decision.Reason
task.RoutingSnapshot = snapshot
return nil
}
_ = s.coordinationStore.RequestRouteProbe(context.WithoutCancel(ctx), profile.Key)
lastErr = err
if !s.routingEnforced() {
s.observeExecutionPoolRouting("shadow")
_ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion,
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
Reason: "shadow_no_eligible_pool", Snapshot: map[string]any{"mode": "shadow", "error": safeRoutingError(err)},
})
return nil
}
}
s.observeExecutionPoolRouting("unavailable")
if remaining := asyncRouteWaitMaximum - time.Since(startedAt); remaining > 0 {
timer := time.NewTimer(remaining)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return ctx.Err()
case <-timer.C:
}
}
return upstreamRouteUnavailable(lastErr)
}
func (s *Service) observeExecutionPoolRouting(outcome string) {
observer, ok := s.billingMetrics.(interface{ ObserveExecutionPoolRouting(string) })
if ok {
observer.ObserveExecutionPoolRouting(outcome)
}
}
func (s *Service) routingCandidateForTask(ctx context.Context, task store.GatewayTask, user *auth.User) (store.RuntimeModelCandidate, error) {
body := normalizeRequest(task.Kind, task.Request)
modelType := modelTypeFromKind(task.Kind, body)
candidates, err := s.store.ListModelCandidates(ctx, task.Model, modelType, user)
if err != nil {
return store.RuntimeModelCandidate{}, err
}
candidates, err = filterCandidatesByRequestedPlatform(candidates, body)
if err == nil {
candidates, _, err = filterRuntimeCandidatesByRequest(task.Kind, task.Model, modelType, body, candidates)
}
if err == nil {
candidates, _, err = filterRuntimeCandidatesByOutputTokens(task.Kind, task.Model, modelType, body, candidates)
}
if err != nil {
return store.RuntimeModelCandidate{}, err
}
if len(candidates) == 0 {
return store.RuntimeModelCandidate{}, store.ErrNoModelCandidate
}
return candidates[0], nil
}
func (s *Service) selectExecutionPool(ctx context.Context, task store.GatewayTask, candidate store.RuntimeModelCandidate) (executionpool.Decision, executionpool.RouteProfile, error) {
profile, err := routeProfileForCandidate(candidate)
if err != nil {
return executionpool.Decision{}, profile, err
}
now := time.Now()
pools, err := s.coordinationStore.ListExecutionPools(ctx)
if err != nil {
return executionpool.Decision{}, profile, err
}
workers, err := s.coordinationStore.ListWorkers(ctx, now)
if err != nil {
return executionpool.Decision{}, profile, err
}
health, err := s.coordinationStore.ListRouteHealth(ctx, profile.Key, now)
if err != nil {
return executionpool.Decision{}, profile, err
}
capacity, err := s.coordinationStore.ListCapacity(ctx, now)
if err != nil {
return executionpool.Decision{}, profile, err
}
workersByPool := make(map[string][]executionpool.WorkerDescriptor)
for _, worker := range workers {
workersByPool[worker.PoolID] = append(workersByPool[worker.PoolID], worker)
}
healthByPool := make(map[string]executionpool.RouteHealth)
for _, item := range health {
healthByPool[item.PoolID] = item
}
capacityByPool := make(map[string]executionpool.CapacitySnapshot)
for _, item := range capacity {
capacityByPool[item.PoolID] = item
}
selection := make([]executionpool.SelectionCandidate, 0, len(pools))
for _, pool := range pools {
poolWorkers := workersByPool[pool.ID]
selection = append(selection, executionpool.SelectionCandidate{
Pool: pool, Health: healthByPool[pool.ID], Capacity: capacityByPool[pool.ID],
CapabilityMatched: workerCapabilityMatched(poolWorkers, task),
})
}
selector := executionpool.NewSelector()
returnDecision, selectErr := selector.Select(executionpool.SelectionRequest{
Candidates: selection, CurrentPoolID: task.AssignedPoolID, Now: now,
})
if selectErr != nil {
return returnDecision, profile, selectErr
}
poolDecisions := make(map[string]executionpool.Decision, len(selection))
scores := make(map[string]float64, len(selection))
for _, candidate := range selection {
decision, candidateErr := selector.Select(executionpool.SelectionRequest{Candidates: []executionpool.SelectionCandidate{candidate}, Now: now})
if candidateErr == nil {
poolDecisions[decision.PoolID] = decision
scores[decision.PoolID] = decision.Score
}
}
preference, err := s.coordinationStore.ResolveRoutePreference(ctx, profile.Key, returnDecision.PoolID, scores, now)
if err != nil {
return executionpool.Decision{}, profile, err
}
if preference.CurrentPoolID != returnDecision.PoolID {
preferred, ok := poolDecisions[preference.CurrentPoolID]
if !ok {
return executionpool.Decision{}, profile, executionpool.ErrNoEligiblePool
}
preferred.Reason = "route_preference_hysteresis"
preferred.Rejected = returnDecision.Rejected
returnDecision = preferred
}
return returnDecision, profile, nil
}
func routeProfileForCandidate(candidate store.RuntimeModelCandidate) (executionpool.RouteProfile, error) {
baseURL := routeEndpointForCandidate(candidate)
host := executionpool.EndpointHost(baseURL)
if host == "" {
return executionpool.RouteProfile{}, errors.New("candidate does not expose a probeable upstream endpoint")
}
proxyMode := "none"
if proxyConfig, err := netproxy.Normalize(netproxy.FromPlatformConfig(candidate.PlatformConfig)); err == nil {
proxyMode = string(proxyConfig.Mode)
}
protocol := firstNonEmptyString(candidate.ResponseProtocol, candidate.SpecType, candidate.Provider)
profile := executionpool.RouteProfile{
Provider: candidate.Provider, Protocol: protocol, EndpointHost: host,
ProxyMode: proxyMode, ConfigRevision: candidate.PlatformID + ":" + candidate.PlatformModelID,
}
profile.Key = executionpool.RouteProfileKey(profile.Provider, profile.Protocol, profile.EndpointHost, profile.ProxyMode, profile.ConfigRevision)
return profile, nil
}
func routeEndpointForCandidate(candidate store.RuntimeModelCandidate) string {
baseURL := strings.TrimSpace(candidate.BaseURL)
if baseURL == "" {
baseURL = firstString(candidate.PlatformConfig, "endpoint", "baseURL", "base_url")
}
return baseURL
}
func workerCapabilityMatched(workers []executionpool.WorkerDescriptor, task store.GatewayTask) bool {
for _, worker := range workers {
if worker.ProtocolVersion != executionpool.ProtocolVersion || worker.AvailableCapacity() < 1 {
continue
}
if requestStreamEnabled(task.Request) && !boolValue(worker.Capabilities["streaming"]) {
continue
}
if !workerSupportsTaskKind(worker.Capabilities["taskKinds"], task.Kind) {
continue
}
return true
}
return false
}
func workerSupportsTaskKind(raw any, taskKind string) bool {
taskKind = strings.ToLower(strings.TrimSpace(taskKind))
if taskKind == "" {
return false
}
values, ok := raw.([]any)
if !ok {
if stringsList, stringsOK := raw.([]string); stringsOK {
values = make([]any, 0, len(stringsList))
for _, value := range stringsList {
values = append(values, value)
}
}
}
for _, value := range values {
capability, _ := value.(string)
capability = strings.ToLower(strings.TrimSpace(capability))
if capability == "*" || capability == taskKind ||
(strings.HasSuffix(capability, ".*") && strings.HasPrefix(taskKind, strings.TrimSuffix(capability, "*"))) {
return true
}
}
return false
}
func requestStreamEnabled(body map[string]any) bool {
value, _ := body["stream"].(bool)
return value
}
func routingDecisionSnapshot(decision executionpool.Decision) map[string]any {
rejected := make([]string, 0, len(decision.Rejected))
for poolID, reason := range decision.Rejected {
rejected = append(rejected, poolID+":"+reason)
}
sort.Strings(rejected)
return map[string]any{
"mode": "selected", "score": decision.Score, "components": decision.Components,
"rejected": rejected,
}
}
func safeRoutingError(err error) string {
if errors.Is(err, executionpool.ErrNoEligiblePool) {
return "no_eligible_pool"
}
return "routing_unavailable"
}
func upstreamRouteUnavailable(cause error) error {
details := map[string]any{"retryAfterSeconds": 10}
if cause != nil {
details["reason"] = safeRoutingError(cause)
}
return &clients.ClientError{
Code: "upstream_route_unavailable", Message: "no execution pool can currently reach the upstream route",
Retryable: true, StatusCode: http.StatusServiceUnavailable, Details: details,
}
}
func firstString(values map[string]any, keys ...string) string {
for _, key := range keys {
if value, ok := values[key].(string); ok && strings.TrimSpace(value) != "" {
return strings.TrimSpace(value)
}
}
return ""
}
func boolValue(value any) bool {
result, _ := value.(bool)
return result
}
@@ -0,0 +1,12 @@
package runner
import "github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
func pinCandidatesToRoutingAssignment(candidates []store.RuntimeModelCandidate, platformID, platformModelID string) []store.RuntimeModelCandidate {
for _, candidate := range candidates {
if candidate.PlatformModelID == platformModelID && (platformID == "" || candidate.PlatformID == platformID) {
return []store.RuntimeModelCandidate{candidate}
}
}
return nil
}
+58 -20
View File
@@ -21,7 +21,6 @@ import (
scriptengine "github.com/easyai/easyai-ai-gateway/apps/api/internal/script"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/riverqueue/river"
)
@@ -77,24 +76,24 @@ type TaskQueuedError struct {
Delay time.Duration
}
type upstreamSubmissionUnknownError struct {
type submissionConfirmationPendingError struct {
AttemptID string
Cause error
}
func shouldClassifyUpstreamSubmissionUnknown(simulated bool, submissionStatus string, err error) bool {
func shouldEnterSubmissionConfirmationPending(simulated bool, submissionStatus string, err error) bool {
return !simulated && submissionStatus == "submitting" && clients.ErrorCode(err) != "timeout"
}
func (e *upstreamSubmissionUnknownError) Error() string {
return "upstream submission result is unknown"
func (e *submissionConfirmationPendingError) Error() string {
return "upstream submission confirmation timed out"
}
func (e *upstreamSubmissionUnknownError) ErrorCode() string {
return "upstream_submission_unknown"
func (e *submissionConfirmationPendingError) ErrorCode() string {
return "upstream_timeout"
}
func (e *upstreamSubmissionUnknownError) Unwrap() error {
func (e *submissionConfirmationPendingError) Unwrap() error {
return e.Cause
}
@@ -131,6 +130,18 @@ func NewWithStores(
if cfg.AsyncWorkerHardLimit == 0 {
cfg.AsyncWorkerHardLimit = 2048
}
if strings.TrimSpace(cfg.ExecutionPoolID) == "" {
cfg.ExecutionPoolID = "legacy-default"
}
if cfg.RouteProbeHotIntervalSeconds == 0 {
cfg.RouteProbeHotIntervalSeconds = 15
}
if cfg.RouteProbeColdIntervalSeconds == 0 {
cfg.RouteProbeColdIntervalSeconds = 60
}
if cfg.RouteProbeTimeoutMS == 0 {
cfg.RouteProbeTimeoutMS = 3000
}
if cfg.AsyncWorkerInstanceHardLimit == 0 {
cfg.AsyncWorkerInstanceHardLimit = 32
}
@@ -259,15 +270,15 @@ func (s *Service) observeResultURLSigning(outcome string) {
}
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
return s.execute(ctx, task, user, nil)
return s.executeRouted(ctx, task, user, nil)
}
func (s *Service) ExecuteStream(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
return s.execute(ctx, task, user, onDelta)
return s.executeRouted(ctx, task, user, onDelta)
}
func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
return s.executeWithToken(ctx, task, user, onDelta, uuid.NewString())
return s.executeRouted(ctx, task, user, onDelta)
}
func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta, executionToken string) (Result, error) {
@@ -511,6 +522,17 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
return Result{Task: failed, Output: failed.Result}, err
}
}
if task.RoutingPlatformModelID != "" {
candidates = pinCandidatesToRoutingAssignment(candidates, task.RoutingPlatformID, task.RoutingPlatformModelID)
if len(candidates) == 0 {
err = &clients.ClientError{Code: "routing_candidate_unavailable", Message: "the routed upstream candidate is no longer available", Retryable: true}
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err)
if finishErr != nil {
return Result{}, finishErr
}
return Result{Task: failed, Output: failed.Result}, err
}
}
for _, candidate := range candidates {
if candidate.LoadAvoided {
s.observeCandidateRouting("full_avoided")
@@ -1096,11 +1118,12 @@ candidatesLoop:
}
return Result{Task: finished, Output: output, Wire: response.Wire}, nil
}
var submissionUnknown *upstreamSubmissionUnknownError
if errors.As(err, &submissionUnknown) {
var confirmationPending *submissionConfirmationPendingError
if errors.As(err, &confirmationPending) {
_ = s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submission_confirmation_pending")
review, reviewErr := s.store.FinishTaskManualReview(context.WithoutCancel(ctx), store.FinishTaskManualReviewInput{
TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: submissionUnknown.AttemptID, TaskStatus: "failed",
Code: "upstream_submission_unknown", Message: submissionUnknown.Error(),
TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: confirmationPending.AttemptID, TaskStatus: "failed",
Code: "upstream_timeout", Message: confirmationPending.Error(),
PricingSnapshot: candidatePricing.Snapshot,
RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody),
})
@@ -1108,8 +1131,8 @@ candidatesLoop:
return Result{}, reviewErr
}
walletReservationFinalized = true
s.logger.Warn("upstream submission requires manual review", "taskID", task.ID, "attemptID", submissionUnknown.AttemptID, "error_category", "upstream_submission_unknown")
return Result{Task: review, Output: review.Result}, submissionUnknown
s.logger.Warn("upstream submission confirmation pending", "taskID", task.ID, "attemptID", confirmationPending.AttemptID, "error_category", "upstream_timeout")
return Result{Task: review, Output: review.Result}, confirmationPending
}
if isLocalRateLimitError(err) {
lastErr = err
@@ -1592,6 +1615,11 @@ func (s *Service) runCandidate(
return nil
}
var submissionWire *clients.WireResponse
var routeObserver *passiveRouteObserver
if task.RouteProfileKey != "" {
routeObserver = &passiveRouteObserver{}
requestHTTPClient = routeObserver.wrap(requestHTTPClient)
}
runCtx, stopLeaseRenewal := s.startConcurrencyLeaseRenewal(ctx, task.ID, limitResult.Leases)
response, err := client.Run(runCtx, clients.Request{
Kind: task.Kind,
@@ -1620,6 +1648,9 @@ func (s *Service) runCandidate(
if err := setSubmissionStatus("response_received"); err != nil {
return err
}
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitted"); err != nil {
return err
}
return enterWorkerWaiting(ctx)
},
OnRemoteTaskPolled: func(remoteTaskID string, payload map[string]any) error {
@@ -1641,11 +1672,17 @@ func (s *Service) runCandidate(
if err := setSubmissionStatus("submitting"); err != nil {
return err
}
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitting"); err != nil {
return err
}
markUpstreamSubmissionStarted(ctx)
return enterWorkerWaiting(ctx)
},
OnUpstreamResponseReceived: func() error {
submissionStatus = "response_received"
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitted"); err != nil {
return err
}
return enterWorkerFinalizing(ctx)
},
OnUpstreamWireResponse: func(wire *clients.WireResponse) error {
@@ -1660,6 +1697,7 @@ func (s *Service) runCandidate(
UpstreamPreviousResponseID: responseExecution.UpstreamPreviousResponseID,
PreviousResponseTurns: responseExecution.PreviousTurns,
})
s.recordPassiveRouteObservation(task, routeObserver)
if phaseErr := enterWorkerFinalizing(runCtx); err == nil && phaseErr != nil {
err = phaseErr
}
@@ -1733,8 +1771,8 @@ func (s *Service) runCandidate(
ErrorMessage: err.Error(),
})
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
if shouldClassifyUpstreamSubmissionUnknown(simulated, submissionStatus, err) {
return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: err}
if shouldEnterSubmissionConfirmationPending(simulated, submissionStatus, err) {
return clients.Response{}, &submissionConfirmationPendingError{AttemptID: attemptID, Cause: err}
}
return clients.Response{}, err
}
@@ -2352,7 +2390,7 @@ func (s *Service) observeConcurrencyLeaseRenewal(outcome string) {
}
func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) {
queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, 0, "")
queued, err := s.store.ResolveInterruptedTaskExecution(ctx, task.ID, task.ExecutionToken)
if err != nil {
return store.GatewayTask{}, err
}
@@ -8,11 +8,11 @@ import (
func TestTimeoutDoesNotBecomeUpstreamSubmissionUnknown(t *testing.T) {
timeoutErr := &clients.ClientError{Code: "timeout", Message: "upstream request timed out", Retryable: false}
if shouldClassifyUpstreamSubmissionUnknown(false, "submitting", timeoutErr) {
if shouldEnterSubmissionConfirmationPending(false, "submitting", timeoutErr) {
t.Fatal("a definitive provider timeout must remain timeout instead of manual-review unknown")
}
networkErr := &clients.ClientError{Code: "network", Message: "connection reset", Retryable: true}
if !shouldClassifyUpstreamSubmissionUnknown(false, "submitting", networkErr) {
if !shouldEnterSubmissionConfirmationPending(false, "submitting", networkErr) {
t.Fatal("an ambiguous non-timeout disconnect must retain manual-review protection")
}
}
@@ -0,0 +1,23 @@
package runner
import (
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestWorkerCapabilityNegotiationRejectsLegacyDescriptor(t *testing.T) {
worker := executionpool.WorkerDescriptor{
ProtocolVersion: executionpool.ProtocolVersion, Allocated: 1, SafeCapacity: 1,
Capabilities: map[string]any{}, HeartbeatAt: time.Now(),
}
if workerCapabilityMatched([]executionpool.WorkerDescriptor{worker}, store.GatewayTask{Kind: "images.generations"}) {
t.Fatal("legacy worker without taskKinds capability was accepted")
}
worker.Capabilities = map[string]any{"taskKinds": []any{"images.*"}}
if !workerCapabilityMatched([]executionpool.WorkerDescriptor{worker}, store.GatewayTask{Kind: "images.generations"}) {
t.Fatal("compatible worker was rejected")
}
}