feat(routing): 引入多执行池智能调度
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口,新增 Kubernetes 和静态容量适配器,并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌,以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
@@ -685,6 +685,10 @@ func (s *Service) SubmitAsyncTask(ctx context.Context, task store.GatewayTask) e
|
||||
_, _ = s.store.FailQueuedTask(context.WithoutCancel(ctx), task.ID, clients.ErrorCode(err), err.Error())
|
||||
return err
|
||||
}
|
||||
if err := s.routeAsyncTask(ctx, &task, user, plan.Candidate); err != nil {
|
||||
_, _ = s.store.FailQueuedTask(context.WithoutCancel(ctx), task.ID, clients.ErrorCode(err), err.Error())
|
||||
return err
|
||||
}
|
||||
if !plan.Eligible {
|
||||
if err := s.EnqueueAsyncTask(ctx, task); err != nil {
|
||||
if s.cancelAsyncSubmissionIfDisconnected(ctx, task.ID) {
|
||||
@@ -830,7 +834,10 @@ func (s *Service) dispatchWaitingAsyncTasks(ctx context.Context, admissions []st
|
||||
if !ok {
|
||||
return fmt.Errorf("async admission batch task %s was not prepared", input.TaskID)
|
||||
}
|
||||
return s.enqueueAsyncTaskTx(ctx, tx, task.ID, asyncTaskInsertOpts(task))
|
||||
_, publishErr := (riverExecutionBroker{service: s}).publishTx(
|
||||
ctx, tx, task.AssignedPoolID, task.ID, time.Time{}, 2,
|
||||
)
|
||||
return publishErr
|
||||
}
|
||||
outcomes, batchErr := s.store.TryTaskAdmissionAtomicBatchWithAdmittedHook(
|
||||
ctx,
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
)
|
||||
|
||||
// httpExecutionTransport is the first platform-neutral Worker transport
|
||||
// adapter. The routing core sees only WorkerDescriptor and executionpool's
|
||||
// versioned request/response types; Pod DNS and other orchestrator concepts do
|
||||
// not cross this boundary.
|
||||
type httpExecutionTransport struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (transport httpExecutionTransport) Execute(
|
||||
ctx context.Context,
|
||||
worker executionpool.WorkerDescriptor,
|
||||
input executionpool.ExecutionRequest,
|
||||
) (executionpool.ExecutionResponse, error) {
|
||||
if transport.client == nil {
|
||||
return executionpool.ExecutionResponse{}, errors.New("worker HTTP client is required")
|
||||
}
|
||||
payload, err := json.Marshal(InternalExecutionRequest{
|
||||
TaskID: input.TaskID,
|
||||
LeaseID: input.LeaseID,
|
||||
Stream: input.Stream,
|
||||
})
|
||||
if err != nil {
|
||||
return executionpool.ExecutionResponse{}, err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
strings.TrimRight(worker.Endpoint, "/")+workerExecutionPath,
|
||||
bytes.NewReader(payload),
|
||||
)
|
||||
if err != nil {
|
||||
return executionpool.ExecutionResponse{}, err
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Authorization", "Worker "+input.AuthorizationToken)
|
||||
response, err := transport.client.Do(request)
|
||||
if err != nil {
|
||||
return executionpool.ExecutionResponse{}, err
|
||||
}
|
||||
return executionpool.ExecutionResponse{
|
||||
StatusCode: response.StatusCode,
|
||||
Headers: response.Header,
|
||||
Body: response.Body,
|
||||
}, nil
|
||||
}
|
||||
|
||||
var _ executionpool.ExecutionTransport = httpExecutionTransport{}
|
||||
@@ -0,0 +1,53 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
)
|
||||
|
||||
func TestHTTPExecutionTransportUsesAdvertisedEndpointAndProtocol(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != workerExecutionPath {
|
||||
t.Fatalf("path=%q", request.URL.Path)
|
||||
}
|
||||
if got := request.Header.Get("Authorization"); got != "Worker signed-token" {
|
||||
t.Fatalf("authorization=%q", got)
|
||||
}
|
||||
var payload InternalExecutionRequest
|
||||
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.TaskID != "task-1" || payload.LeaseID != "lease-1" || !payload.Stream {
|
||||
t.Fatalf("payload=%+v", payload)
|
||||
}
|
||||
response.Header().Set("X-Worker", "accepted")
|
||||
response.WriteHeader(http.StatusAccepted)
|
||||
_, _ = response.Write([]byte("ok"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
transport := httpExecutionTransport{client: server.Client()}
|
||||
result, err := transport.Execute(context.Background(), executionpool.WorkerDescriptor{
|
||||
WorkerID: "worker-1", PoolID: "pool-1", Endpoint: server.URL,
|
||||
}, executionpool.ExecutionRequest{
|
||||
TaskID: "task-1", PoolID: "pool-1", WorkerID: "worker-1", LeaseID: "lease-1",
|
||||
AuthorizationToken: "signed-token", Stream: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer result.Body.Close()
|
||||
body, err := io.ReadAll(result.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.StatusCode != http.StatusAccepted || len(result.Headers["X-Worker"]) != 1 || result.Headers["X-Worker"][0] != "accepted" || string(body) != "ok" {
|
||||
t.Fatalf("result=%+v body=%q", result, body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
type passiveRouteObserver struct {
|
||||
mu sync.Mutex
|
||||
samples int
|
||||
successes int
|
||||
connectTLS []time.Duration
|
||||
uploadBytesRate []float64
|
||||
}
|
||||
|
||||
type passiveRouteRoundTripper struct {
|
||||
base http.RoundTripper
|
||||
observer *passiveRouteObserver
|
||||
}
|
||||
|
||||
func (observer *passiveRouteObserver) wrap(client *http.Client) *http.Client {
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
cloned := *client
|
||||
base := cloned.Transport
|
||||
if base == nil {
|
||||
base = http.DefaultTransport
|
||||
}
|
||||
cloned.Transport = passiveRouteRoundTripper{base: base, observer: observer}
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func (transport passiveRouteRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
startedAt := time.Now()
|
||||
var connectStartedAt time.Time
|
||||
var connectDuration time.Duration
|
||||
var tlsStartedAt time.Time
|
||||
var tlsDuration time.Duration
|
||||
var wroteRequestAt time.Time
|
||||
trace := &httptrace.ClientTrace{
|
||||
ConnectStart: func(_, _ string) { connectStartedAt = time.Now() },
|
||||
ConnectDone: func(_, _ string, _ error) {
|
||||
if !connectStartedAt.IsZero() {
|
||||
connectDuration = time.Since(connectStartedAt)
|
||||
}
|
||||
},
|
||||
TLSHandshakeStart: func() { tlsStartedAt = time.Now() },
|
||||
TLSHandshakeDone: func(tls.ConnectionState, error) {
|
||||
if !tlsStartedAt.IsZero() {
|
||||
tlsDuration = time.Since(tlsStartedAt)
|
||||
}
|
||||
},
|
||||
WroteRequest: func(httptrace.WroteRequestInfo) { wroteRequestAt = time.Now() },
|
||||
}
|
||||
traced := request.Clone(httptrace.WithClientTrace(request.Context(), trace))
|
||||
response, err := transport.base.RoundTrip(traced)
|
||||
transport.observer.record(
|
||||
err == nil,
|
||||
connectDuration+tlsDuration,
|
||||
request.ContentLength,
|
||||
startedAt,
|
||||
wroteRequestAt,
|
||||
)
|
||||
return response, err
|
||||
}
|
||||
|
||||
func (observer *passiveRouteObserver) record(success bool, connectTLS time.Duration, contentLength int64, startedAt, wroteRequestAt time.Time) {
|
||||
observer.mu.Lock()
|
||||
defer observer.mu.Unlock()
|
||||
observer.samples++
|
||||
if success {
|
||||
observer.successes++
|
||||
}
|
||||
if connectTLS > 0 {
|
||||
observer.connectTLS = append(observer.connectTLS, connectTLS)
|
||||
}
|
||||
if contentLength > 0 && !wroteRequestAt.IsZero() && wroteRequestAt.After(startedAt) {
|
||||
observer.uploadBytesRate = append(observer.uploadBytesRate, float64(contentLength)/wroteRequestAt.Sub(startedAt).Seconds())
|
||||
}
|
||||
}
|
||||
|
||||
func (observer *passiveRouteObserver) snapshot(poolID, routeProfileKey string) executionpool.RouteObservation {
|
||||
observer.mu.Lock()
|
||||
defer observer.mu.Unlock()
|
||||
return executionpool.RouteObservation{
|
||||
PoolID: poolID, RouteProfileKey: routeProfileKey,
|
||||
SampleCount: observer.samples, SuccessCount: observer.successes,
|
||||
ConnectTLSP95: durationP95(observer.connectTLS),
|
||||
UploadBytesPerSecond: floatP95(observer.uploadBytesRate),
|
||||
}
|
||||
}
|
||||
|
||||
func durationP95(values []time.Duration) time.Duration {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
copyOfValues := append([]time.Duration(nil), values...)
|
||||
sort.Slice(copyOfValues, func(i, j int) bool { return copyOfValues[i] < copyOfValues[j] })
|
||||
return copyOfValues[(len(copyOfValues)*95+99)/100-1]
|
||||
}
|
||||
|
||||
func floatP95(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
copyOfValues := append([]float64(nil), values...)
|
||||
sort.Float64s(copyOfValues)
|
||||
return copyOfValues[(len(copyOfValues)*95+99)/100-1]
|
||||
}
|
||||
|
||||
func (s *Service) recordPassiveRouteObservation(task store.GatewayTask, observer *passiveRouteObserver) {
|
||||
if observer == nil || !s.routingEnabled() || task.RouteProfileKey == "" {
|
||||
return
|
||||
}
|
||||
poolID := task.AssignedPoolID
|
||||
if poolID == "" && s.cfg.RunsAsyncExecutionWorker() {
|
||||
poolID = s.cfg.ExecutionPoolID
|
||||
}
|
||||
observation := observer.snapshot(poolID, task.RouteProfileKey)
|
||||
if observation.PoolID == "" || observation.SampleCount == 0 {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := s.coordinationStore.RecordRouteObservation(ctx, observation); err != nil && s.logger != nil {
|
||||
s.logger.Warn("record passive route observation failed", "error_category", "route_observation_failed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPassiveRouteObserverRecordsOnlyTransportPhaseFacts(t *testing.T) {
|
||||
observer := &passiveRouteObserver{}
|
||||
startedAt := time.Now()
|
||||
observer.record(true, 10*time.Millisecond, 1024, startedAt, startedAt.Add(time.Millisecond))
|
||||
observer.record(false, 20*time.Millisecond, 2048, startedAt, startedAt.Add(2*time.Millisecond))
|
||||
observer.record(true, 30*time.Millisecond, 0, startedAt, time.Time{})
|
||||
|
||||
observation := observer.snapshot("pool-a", "route-a")
|
||||
if observation.SampleCount != 3 || observation.SuccessCount != 2 {
|
||||
t.Fatalf("observation=%+v", observation)
|
||||
}
|
||||
if observation.ConnectTLSP95 != 30*time.Millisecond {
|
||||
t.Fatalf("connect/TLS p95=%s", observation.ConnectTLSP95)
|
||||
}
|
||||
if observation.UploadBytesPerSecond < 1_024_000-1 || observation.UploadBytesPerSecond > 1_024_000+1 {
|
||||
t.Fatalf("upload rate=%f", observation.UploadBytesPerSecond)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
|
||||
"github.com/google/uuid"
|
||||
@@ -17,7 +19,6 @@ import (
|
||||
"github.com/riverqueue/river"
|
||||
"github.com/riverqueue/river/riverdriver/riverpgxv5"
|
||||
"github.com/riverqueue/river/rivermigrate"
|
||||
"github.com/riverqueue/river/rivertype"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -59,6 +60,11 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
|
||||
if task.Status == "succeeded" || task.Status == "failed" || task.Status == "cancelled" {
|
||||
return nil
|
||||
}
|
||||
if assignedPool := strings.TrimSpace(task.AssignedPoolID); assignedPool != "" && assignedPool != strings.TrimSpace(w.service.cfg.ExecutionPoolID) {
|
||||
w.service.logger.Warn("river task claimed by a worker outside its assigned pool",
|
||||
"taskID", task.ID, "assignedPool", assignedPool, "workerPool", w.service.cfg.ExecutionPoolID)
|
||||
return river.JobSnooze(time.Second)
|
||||
}
|
||||
loadLease, admitted := w.service.tryStartWorkerTask()
|
||||
if !admitted {
|
||||
return river.JobSnooze(workerLoadRetryDelay(task.ID))
|
||||
@@ -87,6 +93,10 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
|
||||
task.ExecutionToken = executionToken
|
||||
queued, queueErr := w.service.requeueInterruptedAsyncTask(context.WithoutCancel(ctx), task)
|
||||
if queueErr != nil {
|
||||
if errors.Is(queueErr, store.ErrTaskExecutionManualReview) {
|
||||
w.service.logger.Warn("interrupted task held after upstream submission began", "taskID", task.ID, "error_category", "upstream_timeout")
|
||||
return nil
|
||||
}
|
||||
return queueErr
|
||||
}
|
||||
w.service.logger.Debug("river async task interrupted and requeued", "taskID", task.ID, "status", queued.Status, "riverJobID", job.ID)
|
||||
@@ -309,14 +319,19 @@ func (s *Service) newRiverAsyncExecutionClient(capacity int) (*river.Client[pgx.
|
||||
if err := river.AddWorkerSafely(workers, &asyncTaskWorker{service: s}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
queues := map[string]river.QueueConfig{
|
||||
asyncTaskQueueName: {MaxWorkers: capacity},
|
||||
}
|
||||
poolQueue := executionpool.QueueName(s.cfg.ExecutionPoolID)
|
||||
if poolQueue != asyncTaskQueueName {
|
||||
queues[poolQueue] = river.QueueConfig{MaxWorkers: capacity}
|
||||
}
|
||||
return river.NewClient(riverpgxv5.New(s.riverStore.Pool()), &river.Config{
|
||||
ID: fmt.Sprintf("%s-exec-%d-%d", s.workerInstanceID, capacity, time.Now().UnixNano()),
|
||||
JobTimeout: -1,
|
||||
Logger: s.logger,
|
||||
CompletedJobRetentionPeriod: 24 * time.Hour,
|
||||
Queues: map[string]river.QueueConfig{
|
||||
asyncTaskQueueName: {MaxWorkers: capacity},
|
||||
},
|
||||
Queues: queues,
|
||||
// Provider-backed media jobs commonly poll for 10-20 minutes. River may
|
||||
// execute a still-running job again once this window elapses, so keep the
|
||||
// rescue horizon above the longest configured provider poll timeout.
|
||||
@@ -346,25 +361,40 @@ func (s *Service) loadAsyncWorkerCapacity(ctx context.Context) (store.AsyncWorke
|
||||
return store.AsyncWorkerCapacitySnapshot{}, err
|
||||
}
|
||||
loadSnapshot := s.sampleWorkerLoad()
|
||||
labels := map[string]string{}
|
||||
if raw := strings.TrimSpace(s.cfg.ExecutionPoolLabels); raw != "" {
|
||||
if err := json.Unmarshal([]byte(raw), &labels); err != nil {
|
||||
return store.AsyncWorkerCapacitySnapshot{}, fmt.Errorf("decode execution pool labels: %w", err)
|
||||
}
|
||||
}
|
||||
if endpoint := strings.TrimSpace(s.cfg.WorkerAdvertiseEndpoint); endpoint != "" {
|
||||
if err := executionpool.ValidateAdvertisedEndpoint(endpoint, splitTrimmed(s.cfg.WorkerEndpointAllowedSuffixes), s.cfg.WorkerEndpointAllowPrivate); err != nil {
|
||||
return store.AsyncWorkerCapacitySnapshot{}, fmt.Errorf("validate worker advertised endpoint: %w", err)
|
||||
}
|
||||
}
|
||||
allocation, err := s.coordinationStore.RegisterWorkerInstance(ctx, store.WorkerRegistrationInput{
|
||||
InstanceID: s.workerInstanceID,
|
||||
PodUID: strings.TrimSpace(os.Getenv("POD_UID")),
|
||||
PodName: strings.TrimSpace(os.Getenv("POD_NAME")),
|
||||
Site: strings.TrimSpace(os.Getenv("EASYAI_SITE")),
|
||||
Revision: strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
|
||||
DesiredCapacity: snapshot.Capacity,
|
||||
CapacityLimit: s.cfg.AsyncWorkerInstanceHardLimit,
|
||||
LoadMode: loadSnapshot.Mode,
|
||||
SafeCapacity: loadSnapshot.SafeCapacity,
|
||||
HeavyCapacity: loadSnapshot.HeavyLimit,
|
||||
ActiveTasks: loadSnapshot.ActiveTasks,
|
||||
PreparingTasks: loadSnapshot.PreparingTasks,
|
||||
WaitingUpstreamTasks: loadSnapshot.WaitingUpstreamTasks,
|
||||
FinalizingTasks: loadSnapshot.FinalizingTasks,
|
||||
PressureState: string(loadSnapshot.PressureState),
|
||||
PressureReason: loadSnapshot.PressureReason,
|
||||
LoadSampledAt: loadSnapshot.SampledAt,
|
||||
HeartbeatStaleAfter: time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
|
||||
InstanceID: s.workerInstanceID,
|
||||
WorkerID: firstNonEmptyString(s.cfg.WorkerID, s.workerInstanceID),
|
||||
PoolID: s.cfg.ExecutionPoolID,
|
||||
Endpoint: s.cfg.WorkerAdvertiseEndpoint,
|
||||
Labels: labels,
|
||||
Capabilities: defaultWorkerCapabilities(),
|
||||
ProtocolVersion: executionpool.ProtocolVersion,
|
||||
OrchestratorInstanceRef: s.cfg.WorkerOrchestratorInstanceRef,
|
||||
Revision: strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
|
||||
DesiredCapacity: snapshot.Capacity,
|
||||
CapacityLimit: s.cfg.AsyncWorkerInstanceHardLimit,
|
||||
LoadMode: loadSnapshot.Mode,
|
||||
SafeCapacity: loadSnapshot.SafeCapacity,
|
||||
HeavyCapacity: loadSnapshot.HeavyLimit,
|
||||
ActiveTasks: loadSnapshot.ActiveTasks,
|
||||
PreparingTasks: loadSnapshot.PreparingTasks,
|
||||
WaitingUpstreamTasks: loadSnapshot.WaitingUpstreamTasks,
|
||||
FinalizingTasks: loadSnapshot.FinalizingTasks,
|
||||
PressureState: string(loadSnapshot.PressureState),
|
||||
PressureReason: loadSnapshot.PressureReason,
|
||||
LoadSampledAt: loadSnapshot.SampledAt,
|
||||
HeartbeatStaleAfter: time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return store.AsyncWorkerCapacitySnapshot{}, err
|
||||
@@ -599,7 +629,8 @@ func (s *Service) observeAsyncWorkerResize(outcome string) {
|
||||
}
|
||||
|
||||
func (s *Service) EnqueueAsyncTask(ctx context.Context, task store.GatewayTask) error {
|
||||
return s.enqueueAsyncTaskWithOptions(ctx, task.ID, asyncTaskInsertOpts(task))
|
||||
_, err := (riverExecutionBroker{service: s}).Publish(ctx, task.AssignedPoolID, task.ID, time.Time{})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) enqueueAsyncTaskWithOptions(ctx context.Context, taskID string, opts *river.InsertOpts) error {
|
||||
@@ -761,22 +792,25 @@ func asyncTaskInsertOpts(task store.GatewayTask) *river.InsertOpts {
|
||||
if task.ID == "" {
|
||||
priority = 3
|
||||
}
|
||||
return &river.InsertOpts{
|
||||
MaxAttempts: 1000,
|
||||
Priority: priority,
|
||||
Queue: asyncTaskQueueName,
|
||||
Tags: []string{"gateway-task"},
|
||||
UniqueOpts: river.UniqueOpts{
|
||||
ByArgs: true,
|
||||
ByQueue: true,
|
||||
ByState: []rivertype.JobState{
|
||||
rivertype.JobStateAvailable,
|
||||
rivertype.JobStatePending,
|
||||
rivertype.JobStateRetryable,
|
||||
rivertype.JobStateRunning,
|
||||
rivertype.JobStateScheduled,
|
||||
},
|
||||
},
|
||||
return riverExecutionInsertOptions(task.AssignedPoolID, time.Time{}, priority)
|
||||
}
|
||||
|
||||
func splitTrimmed(value string) []string {
|
||||
items := make([]string, 0)
|
||||
for item := range strings.SplitSeq(value, ",") {
|
||||
if item = strings.TrimSpace(item); item != "" {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func defaultWorkerCapabilities() map[string]any {
|
||||
return map[string]any{
|
||||
"protocolVersion": executionpool.ProtocolVersion,
|
||||
"streaming": true,
|
||||
"media": true,
|
||||
"taskKinds": []any{"*"},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
workerExecutionAudience = "easyai-worker-execution"
|
||||
workerExecutionPath = "/internal/v1/executions"
|
||||
)
|
||||
|
||||
type InternalExecutionRequest struct {
|
||||
TaskID string `json:"task_id"`
|
||||
LeaseID string `json:"lease_id"`
|
||||
Stream bool `json:"stream"`
|
||||
}
|
||||
|
||||
type InternalExecutionFrame struct {
|
||||
Type string `json:"type"`
|
||||
Delta *clients.StreamDeltaEvent `json:"delta,omitempty"`
|
||||
Output map[string]any `json:"output,omitempty"`
|
||||
Wire *clients.WireResponse `json:"wire,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Status int `json:"status,omitempty"`
|
||||
Retryable bool `json:"retryable,omitempty"`
|
||||
Details map[string]any `json:"details,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) executeRouted(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
|
||||
if task.AsyncMode || !s.cfg.RunsPublicHTTP() || !s.routingEnabled() || task.RunMode == "simulation" {
|
||||
return s.executeLocal(ctx, task, user, onDelta)
|
||||
}
|
||||
candidate, err := s.routingCandidateForTask(ctx, task, user)
|
||||
if err != nil {
|
||||
if s.routingEnforced() {
|
||||
s.observeExecutionPoolRouting("unavailable")
|
||||
return Result{}, upstreamRouteUnavailable(err)
|
||||
}
|
||||
return s.executeLocal(ctx, task, user, onDelta)
|
||||
}
|
||||
decision, profile, err := s.selectExecutionPool(ctx, task, candidate)
|
||||
if err != nil {
|
||||
_ = s.coordinationStore.RequestRouteProbe(context.WithoutCancel(ctx), profile.Key)
|
||||
if s.routingEnforced() {
|
||||
s.observeExecutionPoolRouting("unavailable")
|
||||
return Result{}, upstreamRouteUnavailable(err)
|
||||
}
|
||||
_ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
||||
TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion,
|
||||
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
||||
Reason: "shadow_no_eligible_pool", Snapshot: map[string]any{"mode": "shadow", "error": safeRoutingError(err)},
|
||||
})
|
||||
s.observeExecutionPoolRouting("shadow")
|
||||
return s.executeLocal(ctx, task, user, onDelta)
|
||||
}
|
||||
snapshot := routingDecisionSnapshot(decision)
|
||||
if !s.routingEnforced() {
|
||||
snapshot["suggestedPoolId"] = decision.PoolID
|
||||
_ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
||||
TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion,
|
||||
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
||||
Reason: decision.Reason, Snapshot: snapshot,
|
||||
})
|
||||
s.observeExecutionPoolRouting("shadow")
|
||||
return s.executeLocal(ctx, task, user, onDelta)
|
||||
}
|
||||
if err := s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
||||
TaskID: task.ID, PoolID: decision.PoolID, RouteProfileKey: profile.Key,
|
||||
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
||||
RoutingVersion: routingVersion, Reason: decision.Reason, Snapshot: snapshot,
|
||||
}); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
s.observeExecutionPoolRouting("selected")
|
||||
return s.executeThroughWorker(ctx, task, decision.PoolID, onDelta)
|
||||
}
|
||||
|
||||
func (s *Service) executeLocal(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
|
||||
return s.executeWithToken(ctx, task, user, onDelta, uuid.NewString())
|
||||
}
|
||||
|
||||
func (s *Service) executeThroughWorker(ctx context.Context, task store.GatewayTask, poolID string, onDelta clients.StreamDelta) (Result, error) {
|
||||
nonce := uuid.NewString()
|
||||
lease, err := s.coordinationStore.ReserveWorkerExecution(ctx, task.ID, poolID, nonce, 30*time.Second)
|
||||
if err != nil {
|
||||
s.observeExecutionPoolRouting("capacity_rejected")
|
||||
return Result{}, upstreamRouteUnavailable(err)
|
||||
}
|
||||
if err := executionpool.ValidateAdvertisedEndpoint(
|
||||
lease.Endpoint, splitTrimmed(s.cfg.WorkerEndpointAllowedSuffixes), s.cfg.WorkerEndpointAllowPrivate,
|
||||
); err != nil {
|
||||
_ = s.coordinationStore.ReleaseWorkerExecutionLease(context.WithoutCancel(ctx), lease.LeaseID)
|
||||
return Result{}, &clients.ClientError{Code: "worker_endpoint_untrusted", Message: err.Error(), StatusCode: http.StatusServiceUnavailable, Retryable: false}
|
||||
}
|
||||
signer := executionpool.TokenSigner{Secret: []byte(s.cfg.WorkerExecutionSecret)}
|
||||
token, err := signer.Sign(executionpool.ExecutionClaims{
|
||||
Audience: workerExecutionAudience, TaskID: task.ID, PoolID: poolID,
|
||||
WorkerID: lease.WorkerID, Nonce: nonce, ExpiresAt: time.Now().Add(30 * time.Second).Unix(),
|
||||
})
|
||||
if err != nil {
|
||||
_ = s.coordinationStore.ReleaseWorkerExecutionLease(context.WithoutCancel(ctx), lease.LeaseID)
|
||||
return Result{}, err
|
||||
}
|
||||
client, err := s.workerExecutionHTTPClient()
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
transport := httpExecutionTransport{client: client}
|
||||
response, err := transport.Execute(ctx, executionpool.WorkerDescriptor{
|
||||
WorkerID: lease.WorkerID, InstanceID: lease.InstanceID, PoolID: poolID, Endpoint: lease.Endpoint,
|
||||
}, executionpool.ExecutionRequest{
|
||||
TaskID: task.ID, PoolID: poolID, WorkerID: lease.WorkerID, LeaseID: lease.LeaseID,
|
||||
AuthorizationToken: token, Stream: onDelta != nil,
|
||||
})
|
||||
if err != nil {
|
||||
latest, readErr := s.store.GetTask(context.WithoutCancel(ctx), task.ID)
|
||||
if readErr == nil && latest.SubmissionState != "not_started" {
|
||||
return Result{Task: latest}, &clients.ClientError{
|
||||
Code: "upstream_timeout", Message: "worker transport ended after upstream submission began",
|
||||
StatusCode: http.StatusGatewayTimeout, Retryable: true,
|
||||
}
|
||||
}
|
||||
return Result{}, &clients.ClientError{Code: "worker_transport_error", Message: err.Error(), StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
payload, _ := io.ReadAll(io.LimitReader(response.Body, 64*1024))
|
||||
return Result{}, &clients.ClientError{Code: "worker_transport_error", Message: strings.TrimSpace(string(payload)), StatusCode: response.StatusCode, Retryable: response.StatusCode >= 500}
|
||||
}
|
||||
var output map[string]any
|
||||
var wire *clients.WireResponse
|
||||
scanner := bufio.NewScanner(response.Body)
|
||||
scanner.Buffer(make([]byte, 4096), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
var frame InternalExecutionFrame
|
||||
if err := json.Unmarshal(scanner.Bytes(), &frame); err != nil {
|
||||
return Result{}, &clients.ClientError{Code: "worker_protocol_error", Message: "invalid worker execution frame", StatusCode: http.StatusBadGateway, Retryable: false}
|
||||
}
|
||||
switch frame.Type {
|
||||
case "delta":
|
||||
if onDelta != nil && frame.Delta != nil {
|
||||
if err := onDelta(*frame.Delta); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
case "result":
|
||||
output, wire = frame.Output, frame.Wire
|
||||
case "error":
|
||||
return Result{}, &clients.ClientError{
|
||||
Code: frame.Code, Message: frame.Message, StatusCode: frame.Status,
|
||||
Retryable: frame.Retryable, Details: frame.Details,
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return Result{}, &clients.ClientError{Code: "worker_transport_error", Message: err.Error(), StatusCode: http.StatusBadGateway, Retryable: true}
|
||||
}
|
||||
finished, err := s.store.GetTask(ctx, task.ID)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{Task: finished, Output: output, Wire: wire}, nil
|
||||
}
|
||||
|
||||
func (s *Service) ExecuteInternal(ctx context.Context, authorization string, input InternalExecutionRequest, onDelta clients.StreamDelta) (Result, error) {
|
||||
if s.cfg.RunsPublicHTTP() || !s.cfg.RunsAsyncExecutionWorker() {
|
||||
return Result{}, errors.New("internal execution is only available on worker processes")
|
||||
}
|
||||
token := strings.TrimSpace(strings.TrimPrefix(authorization, "Worker "))
|
||||
if token == authorization || token == "" {
|
||||
return Result{}, errors.New("worker execution authorization is required")
|
||||
}
|
||||
signer := executionpool.TokenSigner{Secret: []byte(s.cfg.WorkerExecutionSecret)}
|
||||
claims, err := signer.Verify(token, workerExecutionAudience)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
localWorkerID := firstNonEmptyString(s.cfg.WorkerID, s.workerInstanceID)
|
||||
if claims.TaskID != input.TaskID || claims.PoolID != s.cfg.ExecutionPoolID || claims.WorkerID != localWorkerID {
|
||||
return Result{}, errors.New("worker execution claims do not match this worker")
|
||||
}
|
||||
lease, err := s.coordinationStore.ConsumeWorkerExecutionLease(ctx, input.LeaseID, claims.Nonce)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = s.coordinationStore.ReleaseWorkerExecutionLease(context.WithoutCancel(ctx), lease.LeaseID)
|
||||
}()
|
||||
if lease.TaskID != input.TaskID || lease.PoolID != s.cfg.ExecutionPoolID || lease.WorkerID != localWorkerID || lease.InstanceID != s.workerInstanceID {
|
||||
return Result{}, errors.New("worker execution lease does not match this worker")
|
||||
}
|
||||
loadLease, admitted := s.tryStartWorkerTask()
|
||||
if !admitted {
|
||||
return Result{}, &clients.ClientError{Code: "worker_capacity_unavailable", Message: "worker has no safe execution capacity", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||
}
|
||||
defer loadLease.Release()
|
||||
ctx = context.WithValue(ctx, workerLoadLeaseContextKey{}, loadLease)
|
||||
task, err := s.store.GetTask(ctx, input.TaskID)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return s.executeLocal(ctx, task, authUserFromTask(task), onDelta)
|
||||
}
|
||||
|
||||
func (s *Service) workerExecutionHTTPClient() (*http.Client, error) {
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.Proxy = nil
|
||||
if strings.TrimSpace(s.cfg.WorkerExecutionCAFile) != "" {
|
||||
caBytes, err := os.ReadFile(s.cfg.WorkerExecutionCAFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(caBytes) {
|
||||
return nil, errors.New("worker execution CA file does not contain a certificate")
|
||||
}
|
||||
transport.TLSClientConfig = &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}
|
||||
if strings.TrimSpace(s.cfg.WorkerExecutionCertFile) != "" || strings.TrimSpace(s.cfg.WorkerExecutionKeyFile) != "" {
|
||||
certificate, err := tls.LoadX509KeyPair(s.cfg.WorkerExecutionCertFile, s.cfg.WorkerExecutionKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
transport.TLSClientConfig.Certificates = []tls.Certificate{certificate}
|
||||
}
|
||||
}
|
||||
return &http.Client{Transport: transport}, nil
|
||||
}
|
||||
|
||||
func internalExecutionStatus(err error) int {
|
||||
var clientErr *clients.ClientError
|
||||
if errors.As(err, &clientErr) && clientErr.StatusCode > 0 {
|
||||
return clientErr.StatusCode
|
||||
}
|
||||
return http.StatusInternalServerError
|
||||
}
|
||||
|
||||
func InternalExecutionFrameForError(err error) InternalExecutionFrame {
|
||||
var clientErr *clients.ClientError
|
||||
if errors.As(err, &clientErr) {
|
||||
return InternalExecutionFrame{
|
||||
Type: "error", Code: clientErr.Code, Message: clientErr.Message,
|
||||
Status: internalExecutionStatus(err), Retryable: clientErr.Retryable, Details: clientErr.Details,
|
||||
}
|
||||
}
|
||||
return InternalExecutionFrame{Type: "error", Code: "worker_execution_failed", Message: fmt.Sprint(err), Status: http.StatusInternalServerError}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/riverqueue/river"
|
||||
"github.com/riverqueue/river/rivertype"
|
||||
)
|
||||
|
||||
// riverExecutionBroker is the River adapter for the platform-neutral broker
|
||||
// port. Queue names and River uniqueness semantics do not escape this adapter.
|
||||
type riverExecutionBroker struct {
|
||||
service *Service
|
||||
}
|
||||
|
||||
func (broker riverExecutionBroker) Publish(
|
||||
ctx context.Context,
|
||||
poolID string,
|
||||
taskID string,
|
||||
notBefore time.Time,
|
||||
) (int64, error) {
|
||||
tx, err := broker.service.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
rollbackCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tx.Rollback(rollbackCtx)
|
||||
}()
|
||||
jobID, err := broker.publishTx(ctx, tx, poolID, taskID, notBefore, 2)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return jobID, nil
|
||||
}
|
||||
|
||||
func (broker riverExecutionBroker) publishTx(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
poolID string,
|
||||
taskID string,
|
||||
notBefore time.Time,
|
||||
priority int,
|
||||
) (int64, error) {
|
||||
riverClient := broker.service.asyncControlClient()
|
||||
if riverClient == nil {
|
||||
return 0, errors.New("River execution broker is not started")
|
||||
}
|
||||
opts := riverExecutionInsertOptions(poolID, notBefore, priority)
|
||||
result, err := riverClient.InsertTx(ctx, tx, asyncTaskArgs{TaskID: taskID}, opts)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if result.Job == nil {
|
||||
return 0, nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET river_job_id = $2,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid`, taskID, result.Job.ID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.Job.ID, nil
|
||||
}
|
||||
|
||||
func riverExecutionInsertOptions(poolID string, notBefore time.Time, priority int) *river.InsertOpts {
|
||||
if priority < 1 {
|
||||
priority = 2
|
||||
}
|
||||
opts := &river.InsertOpts{
|
||||
MaxAttempts: 1000,
|
||||
Priority: priority,
|
||||
Queue: executionpool.QueueName(poolID),
|
||||
Tags: []string{"gateway-task"},
|
||||
UniqueOpts: river.UniqueOpts{
|
||||
ByArgs: true, ByQueue: true,
|
||||
ByState: []rivertype.JobState{
|
||||
rivertype.JobStateAvailable, rivertype.JobStatePending,
|
||||
rivertype.JobStateRetryable, rivertype.JobStateRunning,
|
||||
rivertype.JobStateScheduled,
|
||||
},
|
||||
},
|
||||
}
|
||||
if !notBefore.IsZero() {
|
||||
opts.ScheduledAt = notBefore
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
var _ executionpool.ExecutionBroker = riverExecutionBroker{}
|
||||
@@ -0,0 +1,168 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
type routeProbeState struct {
|
||||
next time.Time
|
||||
lastRequestObserved time.Time
|
||||
}
|
||||
|
||||
func (s *Service) StartRouteHealthProber(ctx context.Context) {
|
||||
if !s.cfg.RunsRouteProber() {
|
||||
return
|
||||
}
|
||||
go s.runRouteHealthProber(ctx)
|
||||
}
|
||||
|
||||
func (s *Service) runRouteHealthProber(ctx context.Context) {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
states := make(map[string]routeProbeState)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
targets, err := s.coordinationStore.ListRouteProbeTargets(ctx)
|
||||
if err != nil {
|
||||
s.logRouteProbeFailure("list_targets", err)
|
||||
continue
|
||||
}
|
||||
requested, requestErr := s.coordinationStore.ListRequestedRouteProbes(ctx)
|
||||
if requestErr != nil {
|
||||
s.logRouteProbeFailure("list_requests", requestErr)
|
||||
continue
|
||||
}
|
||||
now := time.Now()
|
||||
for _, target := range targets {
|
||||
profile, profileErr := routeProfileForCandidate(target.Candidate)
|
||||
if profileErr != nil {
|
||||
continue
|
||||
}
|
||||
state := states[profile.Key]
|
||||
requestedAt := requested[profile.Key]
|
||||
requestDue := !requestedAt.IsZero() && requestedAt.After(state.lastRequestObserved)
|
||||
if !requestDue && now.Before(state.next) {
|
||||
continue
|
||||
}
|
||||
interval := time.Duration(s.cfg.RouteProbeColdIntervalSeconds) * time.Second
|
||||
if target.Hot {
|
||||
interval = time.Duration(s.cfg.RouteProbeHotIntervalSeconds) * time.Second
|
||||
}
|
||||
states[profile.Key] = routeProbeState{next: now.Add(interval), lastRequestObserved: requestedAt}
|
||||
go s.probeRouteTarget(ctx, target, profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) probeRouteTarget(ctx context.Context, target store.RouteProbeTarget, profile executionpool.RouteProfile) {
|
||||
leadership, locked, err := s.coordinationStore.TryAcquireRouteProbeLeadership(ctx, s.cfg.ExecutionPoolID, profile.Key)
|
||||
if err != nil || !locked {
|
||||
return
|
||||
}
|
||||
defer leadership.Release()
|
||||
client, err := s.httpClientForCandidate(target.Candidate, false, "")
|
||||
if err != nil {
|
||||
s.logRouteProbeFailure("prepare_client", err)
|
||||
return
|
||||
}
|
||||
probe := executionpool.HTTPProbe{Client: client}
|
||||
result := probe.Probe(ctx, executionpool.ProbeTarget{
|
||||
RouteProfile: profile,
|
||||
URL: routeEndpointForCandidate(target.Candidate),
|
||||
Timeout: time.Duration(s.cfg.RouteProbeTimeoutMS) * time.Millisecond,
|
||||
})
|
||||
previous := executionpool.RouteHealth{}
|
||||
if history, historyErr := s.coordinationStore.ListRouteHealth(ctx, profile.Key, time.Now()); historyErr == nil {
|
||||
for _, item := range history {
|
||||
if item.PoolID == s.cfg.ExecutionPoolID {
|
||||
previous = item
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
health := nextRouteHealth(previous, s.cfg.ExecutionPoolID, profile.Key, result)
|
||||
s.observeRouteProbe(string(health.State))
|
||||
if err := s.coordinationStore.RecordRouteHealth(context.WithoutCancel(ctx), health); err != nil {
|
||||
s.logRouteProbeFailure("record", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) observeRouteProbe(state string) {
|
||||
observer, ok := s.billingMetrics.(interface{ ObserveRouteProbe(string) })
|
||||
if ok {
|
||||
observer.ObserveRouteProbe(state)
|
||||
}
|
||||
}
|
||||
|
||||
func nextRouteHealth(previous executionpool.RouteHealth, poolID, profileKey string, result executionpool.ProbeResult) executionpool.RouteHealth {
|
||||
health := previous
|
||||
health.PoolID = poolID
|
||||
health.RouteProfileKey = profileKey
|
||||
health.SampleCount++
|
||||
health.SampledAt = result.SampledAt
|
||||
health.ExpiresAt = result.SampledAt.Add(45 * time.Second)
|
||||
observation := 0.0
|
||||
if result.Reachable {
|
||||
observation = 1
|
||||
health.ConsecutiveSuccesses++
|
||||
health.ConsecutiveFailures = 0
|
||||
if health.ConsecutiveSuccesses >= 2 || previous.State == executionpool.RouteHealthy {
|
||||
health.State = executionpool.RouteHealthy
|
||||
} else {
|
||||
health.State = executionpool.RouteDegraded
|
||||
}
|
||||
} else {
|
||||
health.ConsecutiveFailures++
|
||||
health.ConsecutiveSuccesses = 0
|
||||
switch {
|
||||
case health.ConsecutiveFailures >= 3:
|
||||
health.State = executionpool.RouteUnreachable
|
||||
case previous.State == executionpool.RouteHealthy || previous.State == executionpool.RouteDegraded:
|
||||
health.State = executionpool.RouteDegraded
|
||||
default:
|
||||
health.State = executionpool.RouteUnknown
|
||||
}
|
||||
}
|
||||
if previous.SampleCount == 0 {
|
||||
health.SuccessRate = observation
|
||||
} else {
|
||||
health.SuccessRate = 0.8*previous.SuccessRate + 0.2*observation
|
||||
}
|
||||
connectTLS := result.TCPDuration + result.TLSDuration
|
||||
if connectTLS > 0 {
|
||||
health.ConnectTLSP95 = ewmaDuration(previous.ConnectTLSP95, connectTLS)
|
||||
}
|
||||
if result.FirstByte > 0 {
|
||||
if previous.FirstByteP95 > 0 {
|
||||
delta := math.Abs(float64(result.FirstByte - previous.FirstByteP95))
|
||||
health.JitterP95 = ewmaDuration(previous.JitterP95, time.Duration(delta))
|
||||
}
|
||||
health.FirstByteP95 = ewmaDuration(previous.FirstByteP95, result.FirstByte)
|
||||
}
|
||||
return health
|
||||
}
|
||||
|
||||
func ewmaDuration(previous, current time.Duration) time.Duration {
|
||||
if previous <= 0 {
|
||||
return current
|
||||
}
|
||||
return time.Duration(0.8*float64(previous) + 0.2*float64(current))
|
||||
}
|
||||
|
||||
func (s *Service) logRouteProbeFailure(stage string, err error) {
|
||||
if err == nil || errors.Is(err, context.Canceled) || s.logger == nil {
|
||||
return
|
||||
}
|
||||
s.logger.Log(context.Background(), slog.LevelWarn, "route health probe failed", "stage", stage, "error_category", "route_probe_failed")
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestNextRouteHealthRequiresFailuresAndRecoverySamples(t *testing.T) {
|
||||
now := time.Now()
|
||||
health := executionpool.RouteHealth{}
|
||||
for index := 0; index < 2; index++ {
|
||||
health = nextRouteHealth(health, "pool", "route", executionpool.ProbeResult{SampledAt: now.Add(time.Duration(index) * time.Second), ErrorClass: "network"})
|
||||
if health.State == executionpool.RouteUnreachable {
|
||||
t.Fatalf("became unreachable after %d failures", index+1)
|
||||
}
|
||||
}
|
||||
health = nextRouteHealth(health, "pool", "route", executionpool.ProbeResult{SampledAt: now.Add(2 * time.Second), ErrorClass: "network"})
|
||||
if health.State != executionpool.RouteUnreachable {
|
||||
t.Fatalf("state=%s, want unreachable", health.State)
|
||||
}
|
||||
health = nextRouteHealth(health, "pool", "route", executionpool.ProbeResult{SampledAt: now.Add(3 * time.Second), Reachable: true, TCPDuration: 5 * time.Millisecond, FirstByte: 20 * time.Millisecond})
|
||||
if health.State != executionpool.RouteDegraded {
|
||||
t.Fatalf("state=%s, want degraded during recovery", health.State)
|
||||
}
|
||||
health = nextRouteHealth(health, "pool", "route", executionpool.ProbeResult{SampledAt: now.Add(4 * time.Second), Reachable: true, TCPDuration: 5 * time.Millisecond, FirstByte: 20 * time.Millisecond})
|
||||
if health.State != executionpool.RouteHealthy {
|
||||
t.Fatalf("state=%s, want healthy", health.State)
|
||||
}
|
||||
if health.ExpiresAt.Sub(health.SampledAt) != 45*time.Second {
|
||||
t.Fatalf("health TTL=%s", health.ExpiresAt.Sub(health.SampledAt))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPinCandidatesToRoutingAssignment(t *testing.T) {
|
||||
candidates := []store.RuntimeModelCandidate{
|
||||
{PlatformID: "platform-a", PlatformModelID: "model-a"},
|
||||
{PlatformID: "platform-b", PlatformModelID: "model-b"},
|
||||
}
|
||||
pinned := pinCandidatesToRoutingAssignment(candidates, "platform-b", "model-b")
|
||||
if len(pinned) != 1 || pinned[0].PlatformModelID != "model-b" {
|
||||
t.Fatalf("pinned=%+v", pinned)
|
||||
}
|
||||
if got := pinCandidatesToRoutingAssignment(candidates, "platform-a", "missing"); len(got) != 0 {
|
||||
t.Fatalf("unexpected fallback: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/netproxy"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
routingVersion = "execution-pool-v1"
|
||||
asyncRouteWaitMaximum = 10 * time.Second
|
||||
)
|
||||
|
||||
func (s *Service) routingEnabled() bool {
|
||||
mode := strings.ToLower(strings.TrimSpace(s.cfg.RoutingMode))
|
||||
return mode == "shadow" || mode == "enforced"
|
||||
}
|
||||
|
||||
func (s *Service) routingEnforced() bool {
|
||||
return strings.EqualFold(strings.TrimSpace(s.cfg.RoutingMode), "enforced")
|
||||
}
|
||||
|
||||
func (s *Service) routeAsyncTask(ctx context.Context, task *store.GatewayTask, user *auth.User, preferred store.RuntimeModelCandidate) error {
|
||||
if task == nil || !task.AsyncMode || !s.routingEnabled() || task.RunMode == "simulation" {
|
||||
return nil
|
||||
}
|
||||
startedAt := time.Now()
|
||||
delays := []time.Duration{0, 2 * time.Second, 3 * time.Second, 4 * time.Second}
|
||||
var lastErr error
|
||||
for _, delay := range delays {
|
||||
if delay > 0 {
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
candidate := preferred
|
||||
if strings.TrimSpace(candidate.PlatformID) == "" {
|
||||
resolved, err := s.routingCandidateForTask(ctx, *task, user)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
candidate = resolved
|
||||
}
|
||||
decision, profile, err := s.selectExecutionPool(ctx, *task, candidate)
|
||||
if err == nil {
|
||||
poolID := decision.PoolID
|
||||
snapshot := routingDecisionSnapshot(decision)
|
||||
if !s.routingEnforced() {
|
||||
s.observeExecutionPoolRouting("shadow")
|
||||
snapshot["suggestedPoolId"] = poolID
|
||||
poolID = ""
|
||||
}
|
||||
if s.routingEnforced() {
|
||||
s.observeExecutionPoolRouting("selected")
|
||||
}
|
||||
if err := s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
||||
TaskID: task.ID, PoolID: poolID, RouteProfileKey: profile.Key,
|
||||
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
||||
RoutingVersion: routingVersion, Reason: decision.Reason, Snapshot: snapshot,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
task.AssignedPoolID = poolID
|
||||
task.RouteProfileKey = profile.Key
|
||||
task.RoutingVersion = routingVersion
|
||||
task.RoutingReason = decision.Reason
|
||||
task.RoutingSnapshot = snapshot
|
||||
return nil
|
||||
}
|
||||
_ = s.coordinationStore.RequestRouteProbe(context.WithoutCancel(ctx), profile.Key)
|
||||
lastErr = err
|
||||
if !s.routingEnforced() {
|
||||
s.observeExecutionPoolRouting("shadow")
|
||||
_ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
||||
TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion,
|
||||
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
||||
Reason: "shadow_no_eligible_pool", Snapshot: map[string]any{"mode": "shadow", "error": safeRoutingError(err)},
|
||||
})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
s.observeExecutionPoolRouting("unavailable")
|
||||
if remaining := asyncRouteWaitMaximum - time.Since(startedAt); remaining > 0 {
|
||||
timer := time.NewTimer(remaining)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return upstreamRouteUnavailable(lastErr)
|
||||
}
|
||||
|
||||
func (s *Service) observeExecutionPoolRouting(outcome string) {
|
||||
observer, ok := s.billingMetrics.(interface{ ObserveExecutionPoolRouting(string) })
|
||||
if ok {
|
||||
observer.ObserveExecutionPoolRouting(outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) routingCandidateForTask(ctx context.Context, task store.GatewayTask, user *auth.User) (store.RuntimeModelCandidate, error) {
|
||||
body := normalizeRequest(task.Kind, task.Request)
|
||||
modelType := modelTypeFromKind(task.Kind, body)
|
||||
candidates, err := s.store.ListModelCandidates(ctx, task.Model, modelType, user)
|
||||
if err != nil {
|
||||
return store.RuntimeModelCandidate{}, err
|
||||
}
|
||||
candidates, err = filterCandidatesByRequestedPlatform(candidates, body)
|
||||
if err == nil {
|
||||
candidates, _, err = filterRuntimeCandidatesByRequest(task.Kind, task.Model, modelType, body, candidates)
|
||||
}
|
||||
if err == nil {
|
||||
candidates, _, err = filterRuntimeCandidatesByOutputTokens(task.Kind, task.Model, modelType, body, candidates)
|
||||
}
|
||||
if err != nil {
|
||||
return store.RuntimeModelCandidate{}, err
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return store.RuntimeModelCandidate{}, store.ErrNoModelCandidate
|
||||
}
|
||||
return candidates[0], nil
|
||||
}
|
||||
|
||||
func (s *Service) selectExecutionPool(ctx context.Context, task store.GatewayTask, candidate store.RuntimeModelCandidate) (executionpool.Decision, executionpool.RouteProfile, error) {
|
||||
profile, err := routeProfileForCandidate(candidate)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
now := time.Now()
|
||||
pools, err := s.coordinationStore.ListExecutionPools(ctx)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
workers, err := s.coordinationStore.ListWorkers(ctx, now)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
health, err := s.coordinationStore.ListRouteHealth(ctx, profile.Key, now)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
capacity, err := s.coordinationStore.ListCapacity(ctx, now)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
workersByPool := make(map[string][]executionpool.WorkerDescriptor)
|
||||
for _, worker := range workers {
|
||||
workersByPool[worker.PoolID] = append(workersByPool[worker.PoolID], worker)
|
||||
}
|
||||
healthByPool := make(map[string]executionpool.RouteHealth)
|
||||
for _, item := range health {
|
||||
healthByPool[item.PoolID] = item
|
||||
}
|
||||
capacityByPool := make(map[string]executionpool.CapacitySnapshot)
|
||||
for _, item := range capacity {
|
||||
capacityByPool[item.PoolID] = item
|
||||
}
|
||||
selection := make([]executionpool.SelectionCandidate, 0, len(pools))
|
||||
for _, pool := range pools {
|
||||
poolWorkers := workersByPool[pool.ID]
|
||||
selection = append(selection, executionpool.SelectionCandidate{
|
||||
Pool: pool, Health: healthByPool[pool.ID], Capacity: capacityByPool[pool.ID],
|
||||
CapabilityMatched: workerCapabilityMatched(poolWorkers, task),
|
||||
})
|
||||
}
|
||||
selector := executionpool.NewSelector()
|
||||
returnDecision, selectErr := selector.Select(executionpool.SelectionRequest{
|
||||
Candidates: selection, CurrentPoolID: task.AssignedPoolID, Now: now,
|
||||
})
|
||||
if selectErr != nil {
|
||||
return returnDecision, profile, selectErr
|
||||
}
|
||||
poolDecisions := make(map[string]executionpool.Decision, len(selection))
|
||||
scores := make(map[string]float64, len(selection))
|
||||
for _, candidate := range selection {
|
||||
decision, candidateErr := selector.Select(executionpool.SelectionRequest{Candidates: []executionpool.SelectionCandidate{candidate}, Now: now})
|
||||
if candidateErr == nil {
|
||||
poolDecisions[decision.PoolID] = decision
|
||||
scores[decision.PoolID] = decision.Score
|
||||
}
|
||||
}
|
||||
preference, err := s.coordinationStore.ResolveRoutePreference(ctx, profile.Key, returnDecision.PoolID, scores, now)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
if preference.CurrentPoolID != returnDecision.PoolID {
|
||||
preferred, ok := poolDecisions[preference.CurrentPoolID]
|
||||
if !ok {
|
||||
return executionpool.Decision{}, profile, executionpool.ErrNoEligiblePool
|
||||
}
|
||||
preferred.Reason = "route_preference_hysteresis"
|
||||
preferred.Rejected = returnDecision.Rejected
|
||||
returnDecision = preferred
|
||||
}
|
||||
return returnDecision, profile, nil
|
||||
}
|
||||
|
||||
func routeProfileForCandidate(candidate store.RuntimeModelCandidate) (executionpool.RouteProfile, error) {
|
||||
baseURL := routeEndpointForCandidate(candidate)
|
||||
host := executionpool.EndpointHost(baseURL)
|
||||
if host == "" {
|
||||
return executionpool.RouteProfile{}, errors.New("candidate does not expose a probeable upstream endpoint")
|
||||
}
|
||||
proxyMode := "none"
|
||||
if proxyConfig, err := netproxy.Normalize(netproxy.FromPlatformConfig(candidate.PlatformConfig)); err == nil {
|
||||
proxyMode = string(proxyConfig.Mode)
|
||||
}
|
||||
protocol := firstNonEmptyString(candidate.ResponseProtocol, candidate.SpecType, candidate.Provider)
|
||||
profile := executionpool.RouteProfile{
|
||||
Provider: candidate.Provider, Protocol: protocol, EndpointHost: host,
|
||||
ProxyMode: proxyMode, ConfigRevision: candidate.PlatformID + ":" + candidate.PlatformModelID,
|
||||
}
|
||||
profile.Key = executionpool.RouteProfileKey(profile.Provider, profile.Protocol, profile.EndpointHost, profile.ProxyMode, profile.ConfigRevision)
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func routeEndpointForCandidate(candidate store.RuntimeModelCandidate) string {
|
||||
baseURL := strings.TrimSpace(candidate.BaseURL)
|
||||
if baseURL == "" {
|
||||
baseURL = firstString(candidate.PlatformConfig, "endpoint", "baseURL", "base_url")
|
||||
}
|
||||
return baseURL
|
||||
}
|
||||
|
||||
func workerCapabilityMatched(workers []executionpool.WorkerDescriptor, task store.GatewayTask) bool {
|
||||
for _, worker := range workers {
|
||||
if worker.ProtocolVersion != executionpool.ProtocolVersion || worker.AvailableCapacity() < 1 {
|
||||
continue
|
||||
}
|
||||
if requestStreamEnabled(task.Request) && !boolValue(worker.Capabilities["streaming"]) {
|
||||
continue
|
||||
}
|
||||
if !workerSupportsTaskKind(worker.Capabilities["taskKinds"], task.Kind) {
|
||||
continue
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func workerSupportsTaskKind(raw any, taskKind string) bool {
|
||||
taskKind = strings.ToLower(strings.TrimSpace(taskKind))
|
||||
if taskKind == "" {
|
||||
return false
|
||||
}
|
||||
values, ok := raw.([]any)
|
||||
if !ok {
|
||||
if stringsList, stringsOK := raw.([]string); stringsOK {
|
||||
values = make([]any, 0, len(stringsList))
|
||||
for _, value := range stringsList {
|
||||
values = append(values, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, value := range values {
|
||||
capability, _ := value.(string)
|
||||
capability = strings.ToLower(strings.TrimSpace(capability))
|
||||
if capability == "*" || capability == taskKind ||
|
||||
(strings.HasSuffix(capability, ".*") && strings.HasPrefix(taskKind, strings.TrimSuffix(capability, "*"))) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func requestStreamEnabled(body map[string]any) bool {
|
||||
value, _ := body["stream"].(bool)
|
||||
return value
|
||||
}
|
||||
|
||||
func routingDecisionSnapshot(decision executionpool.Decision) map[string]any {
|
||||
rejected := make([]string, 0, len(decision.Rejected))
|
||||
for poolID, reason := range decision.Rejected {
|
||||
rejected = append(rejected, poolID+":"+reason)
|
||||
}
|
||||
sort.Strings(rejected)
|
||||
return map[string]any{
|
||||
"mode": "selected", "score": decision.Score, "components": decision.Components,
|
||||
"rejected": rejected,
|
||||
}
|
||||
}
|
||||
|
||||
func safeRoutingError(err error) string {
|
||||
if errors.Is(err, executionpool.ErrNoEligiblePool) {
|
||||
return "no_eligible_pool"
|
||||
}
|
||||
return "routing_unavailable"
|
||||
}
|
||||
|
||||
func upstreamRouteUnavailable(cause error) error {
|
||||
details := map[string]any{"retryAfterSeconds": 10}
|
||||
if cause != nil {
|
||||
details["reason"] = safeRoutingError(cause)
|
||||
}
|
||||
return &clients.ClientError{
|
||||
Code: "upstream_route_unavailable", Message: "no execution pool can currently reach the upstream route",
|
||||
Retryable: true, StatusCode: http.StatusServiceUnavailable, Details: details,
|
||||
}
|
||||
}
|
||||
|
||||
func firstString(values map[string]any, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value, ok := values[key].(string); ok && strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func boolValue(value any) bool {
|
||||
result, _ := value.(bool)
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package runner
|
||||
|
||||
import "github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
|
||||
func pinCandidatesToRoutingAssignment(candidates []store.RuntimeModelCandidate, platformID, platformModelID string) []store.RuntimeModelCandidate {
|
||||
for _, candidate := range candidates {
|
||||
if candidate.PlatformModelID == platformModelID && (platformID == "" || candidate.PlatformID == platformID) {
|
||||
return []store.RuntimeModelCandidate{candidate}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
scriptengine "github.com/easyai/easyai-ai-gateway/apps/api/internal/script"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/workerload"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/riverqueue/river"
|
||||
)
|
||||
@@ -77,24 +76,24 @@ type TaskQueuedError struct {
|
||||
Delay time.Duration
|
||||
}
|
||||
|
||||
type upstreamSubmissionUnknownError struct {
|
||||
type submissionConfirmationPendingError struct {
|
||||
AttemptID string
|
||||
Cause error
|
||||
}
|
||||
|
||||
func shouldClassifyUpstreamSubmissionUnknown(simulated bool, submissionStatus string, err error) bool {
|
||||
func shouldEnterSubmissionConfirmationPending(simulated bool, submissionStatus string, err error) bool {
|
||||
return !simulated && submissionStatus == "submitting" && clients.ErrorCode(err) != "timeout"
|
||||
}
|
||||
|
||||
func (e *upstreamSubmissionUnknownError) Error() string {
|
||||
return "upstream submission result is unknown"
|
||||
func (e *submissionConfirmationPendingError) Error() string {
|
||||
return "upstream submission confirmation timed out"
|
||||
}
|
||||
|
||||
func (e *upstreamSubmissionUnknownError) ErrorCode() string {
|
||||
return "upstream_submission_unknown"
|
||||
func (e *submissionConfirmationPendingError) ErrorCode() string {
|
||||
return "upstream_timeout"
|
||||
}
|
||||
|
||||
func (e *upstreamSubmissionUnknownError) Unwrap() error {
|
||||
func (e *submissionConfirmationPendingError) Unwrap() error {
|
||||
return e.Cause
|
||||
}
|
||||
|
||||
@@ -131,6 +130,18 @@ func NewWithStores(
|
||||
if cfg.AsyncWorkerHardLimit == 0 {
|
||||
cfg.AsyncWorkerHardLimit = 2048
|
||||
}
|
||||
if strings.TrimSpace(cfg.ExecutionPoolID) == "" {
|
||||
cfg.ExecutionPoolID = "legacy-default"
|
||||
}
|
||||
if cfg.RouteProbeHotIntervalSeconds == 0 {
|
||||
cfg.RouteProbeHotIntervalSeconds = 15
|
||||
}
|
||||
if cfg.RouteProbeColdIntervalSeconds == 0 {
|
||||
cfg.RouteProbeColdIntervalSeconds = 60
|
||||
}
|
||||
if cfg.RouteProbeTimeoutMS == 0 {
|
||||
cfg.RouteProbeTimeoutMS = 3000
|
||||
}
|
||||
if cfg.AsyncWorkerInstanceHardLimit == 0 {
|
||||
cfg.AsyncWorkerInstanceHardLimit = 32
|
||||
}
|
||||
@@ -259,15 +270,15 @@ func (s *Service) observeResultURLSigning(outcome string) {
|
||||
}
|
||||
|
||||
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
|
||||
return s.execute(ctx, task, user, nil)
|
||||
return s.executeRouted(ctx, task, user, nil)
|
||||
}
|
||||
|
||||
func (s *Service) ExecuteStream(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
|
||||
return s.execute(ctx, task, user, onDelta)
|
||||
return s.executeRouted(ctx, task, user, onDelta)
|
||||
}
|
||||
|
||||
func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
|
||||
return s.executeWithToken(ctx, task, user, onDelta, uuid.NewString())
|
||||
return s.executeRouted(ctx, task, user, onDelta)
|
||||
}
|
||||
|
||||
func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta, executionToken string) (Result, error) {
|
||||
@@ -511,6 +522,17 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
|
||||
return Result{Task: failed, Output: failed.Result}, err
|
||||
}
|
||||
}
|
||||
if task.RoutingPlatformModelID != "" {
|
||||
candidates = pinCandidatesToRoutingAssignment(candidates, task.RoutingPlatformID, task.RoutingPlatformModelID)
|
||||
if len(candidates) == 0 {
|
||||
err = &clients.ClientError{Code: "routing_candidate_unavailable", Message: "the routed upstream candidate is no longer available", Retryable: true}
|
||||
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err)
|
||||
if finishErr != nil {
|
||||
return Result{}, finishErr
|
||||
}
|
||||
return Result{Task: failed, Output: failed.Result}, err
|
||||
}
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if candidate.LoadAvoided {
|
||||
s.observeCandidateRouting("full_avoided")
|
||||
@@ -1096,11 +1118,12 @@ candidatesLoop:
|
||||
}
|
||||
return Result{Task: finished, Output: output, Wire: response.Wire}, nil
|
||||
}
|
||||
var submissionUnknown *upstreamSubmissionUnknownError
|
||||
if errors.As(err, &submissionUnknown) {
|
||||
var confirmationPending *submissionConfirmationPendingError
|
||||
if errors.As(err, &confirmationPending) {
|
||||
_ = s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submission_confirmation_pending")
|
||||
review, reviewErr := s.store.FinishTaskManualReview(context.WithoutCancel(ctx), store.FinishTaskManualReviewInput{
|
||||
TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: submissionUnknown.AttemptID, TaskStatus: "failed",
|
||||
Code: "upstream_submission_unknown", Message: submissionUnknown.Error(),
|
||||
TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: confirmationPending.AttemptID, TaskStatus: "failed",
|
||||
Code: "upstream_timeout", Message: confirmationPending.Error(),
|
||||
PricingSnapshot: candidatePricing.Snapshot,
|
||||
RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody),
|
||||
})
|
||||
@@ -1108,8 +1131,8 @@ candidatesLoop:
|
||||
return Result{}, reviewErr
|
||||
}
|
||||
walletReservationFinalized = true
|
||||
s.logger.Warn("upstream submission requires manual review", "taskID", task.ID, "attemptID", submissionUnknown.AttemptID, "error_category", "upstream_submission_unknown")
|
||||
return Result{Task: review, Output: review.Result}, submissionUnknown
|
||||
s.logger.Warn("upstream submission confirmation pending", "taskID", task.ID, "attemptID", confirmationPending.AttemptID, "error_category", "upstream_timeout")
|
||||
return Result{Task: review, Output: review.Result}, confirmationPending
|
||||
}
|
||||
if isLocalRateLimitError(err) {
|
||||
lastErr = err
|
||||
@@ -1592,6 +1615,11 @@ func (s *Service) runCandidate(
|
||||
return nil
|
||||
}
|
||||
var submissionWire *clients.WireResponse
|
||||
var routeObserver *passiveRouteObserver
|
||||
if task.RouteProfileKey != "" {
|
||||
routeObserver = &passiveRouteObserver{}
|
||||
requestHTTPClient = routeObserver.wrap(requestHTTPClient)
|
||||
}
|
||||
runCtx, stopLeaseRenewal := s.startConcurrencyLeaseRenewal(ctx, task.ID, limitResult.Leases)
|
||||
response, err := client.Run(runCtx, clients.Request{
|
||||
Kind: task.Kind,
|
||||
@@ -1620,6 +1648,9 @@ func (s *Service) runCandidate(
|
||||
if err := setSubmissionStatus("response_received"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitted"); err != nil {
|
||||
return err
|
||||
}
|
||||
return enterWorkerWaiting(ctx)
|
||||
},
|
||||
OnRemoteTaskPolled: func(remoteTaskID string, payload map[string]any) error {
|
||||
@@ -1641,11 +1672,17 @@ func (s *Service) runCandidate(
|
||||
if err := setSubmissionStatus("submitting"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitting"); err != nil {
|
||||
return err
|
||||
}
|
||||
markUpstreamSubmissionStarted(ctx)
|
||||
return enterWorkerWaiting(ctx)
|
||||
},
|
||||
OnUpstreamResponseReceived: func() error {
|
||||
submissionStatus = "response_received"
|
||||
if err := s.store.SetTaskSubmissionState(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, "submitted"); err != nil {
|
||||
return err
|
||||
}
|
||||
return enterWorkerFinalizing(ctx)
|
||||
},
|
||||
OnUpstreamWireResponse: func(wire *clients.WireResponse) error {
|
||||
@@ -1660,6 +1697,7 @@ func (s *Service) runCandidate(
|
||||
UpstreamPreviousResponseID: responseExecution.UpstreamPreviousResponseID,
|
||||
PreviousResponseTurns: responseExecution.PreviousTurns,
|
||||
})
|
||||
s.recordPassiveRouteObservation(task, routeObserver)
|
||||
if phaseErr := enterWorkerFinalizing(runCtx); err == nil && phaseErr != nil {
|
||||
err = phaseErr
|
||||
}
|
||||
@@ -1733,8 +1771,8 @@ func (s *Service) runCandidate(
|
||||
ErrorMessage: err.Error(),
|
||||
})
|
||||
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
|
||||
if shouldClassifyUpstreamSubmissionUnknown(simulated, submissionStatus, err) {
|
||||
return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: err}
|
||||
if shouldEnterSubmissionConfirmationPending(simulated, submissionStatus, err) {
|
||||
return clients.Response{}, &submissionConfirmationPendingError{AttemptID: attemptID, Cause: err}
|
||||
}
|
||||
return clients.Response{}, err
|
||||
}
|
||||
@@ -2352,7 +2390,7 @@ func (s *Service) observeConcurrencyLeaseRenewal(outcome string) {
|
||||
}
|
||||
|
||||
func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) {
|
||||
queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, 0, "")
|
||||
queued, err := s.store.ResolveInterruptedTaskExecution(ctx, task.ID, task.ExecutionToken)
|
||||
if err != nil {
|
||||
return store.GatewayTask{}, err
|
||||
}
|
||||
|
||||
@@ -8,11 +8,11 @@ import (
|
||||
|
||||
func TestTimeoutDoesNotBecomeUpstreamSubmissionUnknown(t *testing.T) {
|
||||
timeoutErr := &clients.ClientError{Code: "timeout", Message: "upstream request timed out", Retryable: false}
|
||||
if shouldClassifyUpstreamSubmissionUnknown(false, "submitting", timeoutErr) {
|
||||
if shouldEnterSubmissionConfirmationPending(false, "submitting", timeoutErr) {
|
||||
t.Fatal("a definitive provider timeout must remain timeout instead of manual-review unknown")
|
||||
}
|
||||
networkErr := &clients.ClientError{Code: "network", Message: "connection reset", Retryable: true}
|
||||
if !shouldClassifyUpstreamSubmissionUnknown(false, "submitting", networkErr) {
|
||||
if !shouldEnterSubmissionConfirmationPending(false, "submitting", networkErr) {
|
||||
t.Fatal("an ambiguous non-timeout disconnect must retain manual-review protection")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestWorkerCapabilityNegotiationRejectsLegacyDescriptor(t *testing.T) {
|
||||
worker := executionpool.WorkerDescriptor{
|
||||
ProtocolVersion: executionpool.ProtocolVersion, Allocated: 1, SafeCapacity: 1,
|
||||
Capabilities: map[string]any{}, HeartbeatAt: time.Now(),
|
||||
}
|
||||
if workerCapabilityMatched([]executionpool.WorkerDescriptor{worker}, store.GatewayTask{Kind: "images.generations"}) {
|
||||
t.Fatal("legacy worker without taskKinds capability was accepted")
|
||||
}
|
||||
worker.Capabilities = map[string]any{"taskKinds": []any{"images.*"}}
|
||||
if !workerCapabilityMatched([]executionpool.WorkerDescriptor{worker}, store.GatewayTask{Kind: "images.generations"}) {
|
||||
t.Fatal("compatible worker was rejected")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user