package runner import ( "context" "errors" "net/http" "sort" "strings" "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" "github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool" "github.com/easyai/easyai-ai-gateway/apps/api/internal/netproxy" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) const ( routingVersion = "execution-pool-v1" asyncRouteWaitMaximum = 10 * time.Second ) func (s *Service) routingEnabled() bool { mode := strings.ToLower(strings.TrimSpace(s.cfg.RoutingMode)) return mode == "shadow" || mode == "enforced" } func (s *Service) routingEnforced() bool { return strings.EqualFold(strings.TrimSpace(s.cfg.RoutingMode), "enforced") } func (s *Service) routeAsyncTask(ctx context.Context, task *store.GatewayTask, user *auth.User, preferred store.RuntimeModelCandidate) error { if task == nil || !task.AsyncMode || !s.routingEnabled() || task.RunMode == "simulation" { return nil } startedAt := time.Now() delays := []time.Duration{0, 2 * time.Second, 3 * time.Second, 4 * time.Second} var lastErr error for _, delay := range delays { if delay > 0 { timer := time.NewTimer(delay) select { case <-ctx.Done(): if !timer.Stop() { <-timer.C } return ctx.Err() case <-timer.C: } } candidate := preferred if strings.TrimSpace(candidate.PlatformID) == "" { resolved, err := s.routingCandidateForTask(ctx, *task, user) if err != nil { lastErr = err continue } candidate = resolved } decision, profile, err := s.selectExecutionPool(ctx, *task, candidate) if err == nil { poolID := decision.PoolID snapshot := routingDecisionSnapshot(decision) if !s.routingEnforced() { s.observeExecutionPoolRouting("shadow") snapshot["suggestedPoolId"] = poolID poolID = "" } if s.routingEnforced() { s.observeExecutionPoolRouting("selected") } if err := s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{ TaskID: task.ID, PoolID: poolID, RouteProfileKey: profile.Key, PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID, RoutingVersion: routingVersion, Reason: decision.Reason, Snapshot: snapshot, }); err != nil { return err } task.AssignedPoolID = poolID task.RouteProfileKey = profile.Key task.RoutingVersion = routingVersion task.RoutingReason = decision.Reason task.RoutingSnapshot = snapshot return nil } _ = s.coordinationStore.RequestRouteProbe(context.WithoutCancel(ctx), profile.Key) lastErr = err if !s.routingEnforced() { s.observeExecutionPoolRouting("shadow") _ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{ TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion, PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID, Reason: "shadow_no_eligible_pool", Snapshot: map[string]any{"mode": "shadow", "error": safeRoutingError(err)}, }) return nil } } s.observeExecutionPoolRouting("unavailable") if remaining := asyncRouteWaitMaximum - time.Since(startedAt); remaining > 0 { timer := time.NewTimer(remaining) select { case <-ctx.Done(): if !timer.Stop() { <-timer.C } return ctx.Err() case <-timer.C: } } return upstreamRouteUnavailable(lastErr) } func (s *Service) observeExecutionPoolRouting(outcome string) { observer, ok := s.billingMetrics.(interface{ ObserveExecutionPoolRouting(string) }) if ok { observer.ObserveExecutionPoolRouting(outcome) } } func (s *Service) routingCandidateForTask(ctx context.Context, task store.GatewayTask, user *auth.User) (store.RuntimeModelCandidate, error) { body := normalizeRequest(task.Kind, task.Request) modelType := modelTypeFromKind(task.Kind, body) candidates, err := s.store.ListModelCandidates(ctx, task.Model, modelType, user) if err != nil { return store.RuntimeModelCandidate{}, err } candidates, err = filterCandidatesByRequestedPlatform(candidates, body) if err == nil { candidates, _, err = filterRuntimeCandidatesByRequest(task.Kind, task.Model, modelType, body, candidates) } if err == nil { candidates, _, err = filterRuntimeCandidatesByOutputTokens(task.Kind, task.Model, modelType, body, candidates) } if err != nil { return store.RuntimeModelCandidate{}, err } if len(candidates) == 0 { return store.RuntimeModelCandidate{}, store.ErrNoModelCandidate } return candidates[0], nil } func (s *Service) selectExecutionPool(ctx context.Context, task store.GatewayTask, candidate store.RuntimeModelCandidate) (executionpool.Decision, executionpool.RouteProfile, error) { profile, err := routeProfileForCandidate(candidate) if err != nil { return executionpool.Decision{}, profile, err } now := time.Now() pools, err := s.coordinationStore.ListExecutionPools(ctx) if err != nil { return executionpool.Decision{}, profile, err } workers, err := s.coordinationStore.ListWorkers(ctx, now) if err != nil { return executionpool.Decision{}, profile, err } health, err := s.coordinationStore.ListRouteHealth(ctx, profile.Key, now) if err != nil { return executionpool.Decision{}, profile, err } capacity, err := s.coordinationStore.ListCapacity(ctx, now) if err != nil { return executionpool.Decision{}, profile, err } workersByPool := make(map[string][]executionpool.WorkerDescriptor) for _, worker := range workers { workersByPool[worker.PoolID] = append(workersByPool[worker.PoolID], worker) } healthByPool := make(map[string]executionpool.RouteHealth) for _, item := range health { healthByPool[item.PoolID] = item } capacityByPool := make(map[string]executionpool.CapacitySnapshot) for _, item := range capacity { capacityByPool[item.PoolID] = item } selection := make([]executionpool.SelectionCandidate, 0, len(pools)) for _, pool := range pools { poolWorkers := workersByPool[pool.ID] selection = append(selection, executionpool.SelectionCandidate{ Pool: pool, Health: healthByPool[pool.ID], Capacity: capacityByPool[pool.ID], CapabilityMatched: workerCapabilityMatched(poolWorkers, task), }) } selector := executionpool.NewSelector() returnDecision, selectErr := selector.Select(executionpool.SelectionRequest{ Candidates: selection, CurrentPoolID: task.AssignedPoolID, Now: now, }) if selectErr != nil { return returnDecision, profile, selectErr } poolDecisions := make(map[string]executionpool.Decision, len(selection)) scores := make(map[string]float64, len(selection)) for _, candidate := range selection { decision, candidateErr := selector.Select(executionpool.SelectionRequest{Candidates: []executionpool.SelectionCandidate{candidate}, Now: now}) if candidateErr == nil { poolDecisions[decision.PoolID] = decision scores[decision.PoolID] = decision.Score } } preference, err := s.coordinationStore.ResolveRoutePreference(ctx, profile.Key, returnDecision.PoolID, scores, now) if err != nil { return executionpool.Decision{}, profile, err } if preference.CurrentPoolID != returnDecision.PoolID { preferred, ok := poolDecisions[preference.CurrentPoolID] if !ok { return executionpool.Decision{}, profile, executionpool.ErrNoEligiblePool } preferred.Reason = "route_preference_hysteresis" preferred.Rejected = returnDecision.Rejected returnDecision = preferred } return returnDecision, profile, nil } func routeProfileForCandidate(candidate store.RuntimeModelCandidate) (executionpool.RouteProfile, error) { baseURL := routeEndpointForCandidate(candidate) host := executionpool.EndpointHost(baseURL) if host == "" { return executionpool.RouteProfile{}, errors.New("candidate does not expose a probeable upstream endpoint") } proxyMode := "none" if proxyConfig, err := netproxy.Normalize(netproxy.FromPlatformConfig(candidate.PlatformConfig)); err == nil { proxyMode = string(proxyConfig.Mode) } protocol := firstNonEmptyString(candidate.ResponseProtocol, candidate.SpecType, candidate.Provider) profile := executionpool.RouteProfile{ Provider: candidate.Provider, Protocol: protocol, EndpointHost: host, ProxyMode: proxyMode, ConfigRevision: candidate.PlatformID + ":" + candidate.PlatformModelID, } profile.Key = executionpool.RouteProfileKey(profile.Provider, profile.Protocol, profile.EndpointHost, profile.ProxyMode, profile.ConfigRevision) return profile, nil } func routeEndpointForCandidate(candidate store.RuntimeModelCandidate) string { baseURL := strings.TrimSpace(candidate.BaseURL) if baseURL == "" { baseURL = firstString(candidate.PlatformConfig, "endpoint", "baseURL", "base_url") } return baseURL } func workerCapabilityMatched(workers []executionpool.WorkerDescriptor, task store.GatewayTask) bool { for _, worker := range workers { if worker.ProtocolVersion != executionpool.ProtocolVersion || worker.AvailableCapacity() < 1 { continue } if requestStreamEnabled(task.Request) && !boolValue(worker.Capabilities["streaming"]) { continue } if !workerSupportsTaskKind(worker.Capabilities["taskKinds"], task.Kind) { continue } return true } return false } func workerSupportsTaskKind(raw any, taskKind string) bool { taskKind = strings.ToLower(strings.TrimSpace(taskKind)) if taskKind == "" { return false } values, ok := raw.([]any) if !ok { if stringsList, stringsOK := raw.([]string); stringsOK { values = make([]any, 0, len(stringsList)) for _, value := range stringsList { values = append(values, value) } } } for _, value := range values { capability, _ := value.(string) capability = strings.ToLower(strings.TrimSpace(capability)) if capability == "*" || capability == taskKind || (strings.HasSuffix(capability, ".*") && strings.HasPrefix(taskKind, strings.TrimSuffix(capability, "*"))) { return true } } return false } func requestStreamEnabled(body map[string]any) bool { value, _ := body["stream"].(bool) return value } func routingDecisionSnapshot(decision executionpool.Decision) map[string]any { rejected := make([]string, 0, len(decision.Rejected)) for poolID, reason := range decision.Rejected { rejected = append(rejected, poolID+":"+reason) } sort.Strings(rejected) return map[string]any{ "mode": "selected", "score": decision.Score, "components": decision.Components, "rejected": rejected, } } func safeRoutingError(err error) string { if errors.Is(err, executionpool.ErrNoEligiblePool) { return "no_eligible_pool" } return "routing_unavailable" } func upstreamRouteUnavailable(cause error) error { details := map[string]any{"retryAfterSeconds": 10} if cause != nil { details["reason"] = safeRoutingError(cause) } return &clients.ClientError{ Code: "upstream_route_unavailable", Message: "no execution pool can currently reach the upstream route", Retryable: true, StatusCode: http.StatusServiceUnavailable, Details: details, } } func firstString(values map[string]any, keys ...string) string { for _, key := range keys { if value, ok := values[key].(string); ok && strings.TrimSpace(value) != "" { return strings.TrimSpace(value) } } return "" } func boolValue(value any) bool { result, _ := value.(bool) return result }