feat(routing): 引入多执行池智能调度
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
@@ -0,0 +1,332 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/netproxy"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
routingVersion = "execution-pool-v1"
|
||||
asyncRouteWaitMaximum = 10 * time.Second
|
||||
)
|
||||
|
||||
func (s *Service) routingEnabled() bool {
|
||||
mode := strings.ToLower(strings.TrimSpace(s.cfg.RoutingMode))
|
||||
return mode == "shadow" || mode == "enforced"
|
||||
}
|
||||
|
||||
func (s *Service) routingEnforced() bool {
|
||||
return strings.EqualFold(strings.TrimSpace(s.cfg.RoutingMode), "enforced")
|
||||
}
|
||||
|
||||
func (s *Service) routeAsyncTask(ctx context.Context, task *store.GatewayTask, user *auth.User, preferred store.RuntimeModelCandidate) error {
|
||||
if task == nil || !task.AsyncMode || !s.routingEnabled() || task.RunMode == "simulation" {
|
||||
return nil
|
||||
}
|
||||
startedAt := time.Now()
|
||||
delays := []time.Duration{0, 2 * time.Second, 3 * time.Second, 4 * time.Second}
|
||||
var lastErr error
|
||||
for _, delay := range delays {
|
||||
if delay > 0 {
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
candidate := preferred
|
||||
if strings.TrimSpace(candidate.PlatformID) == "" {
|
||||
resolved, err := s.routingCandidateForTask(ctx, *task, user)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
candidate = resolved
|
||||
}
|
||||
decision, profile, err := s.selectExecutionPool(ctx, *task, candidate)
|
||||
if err == nil {
|
||||
poolID := decision.PoolID
|
||||
snapshot := routingDecisionSnapshot(decision)
|
||||
if !s.routingEnforced() {
|
||||
s.observeExecutionPoolRouting("shadow")
|
||||
snapshot["suggestedPoolId"] = poolID
|
||||
poolID = ""
|
||||
}
|
||||
if s.routingEnforced() {
|
||||
s.observeExecutionPoolRouting("selected")
|
||||
}
|
||||
if err := s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
||||
TaskID: task.ID, PoolID: poolID, RouteProfileKey: profile.Key,
|
||||
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
||||
RoutingVersion: routingVersion, Reason: decision.Reason, Snapshot: snapshot,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
task.AssignedPoolID = poolID
|
||||
task.RouteProfileKey = profile.Key
|
||||
task.RoutingVersion = routingVersion
|
||||
task.RoutingReason = decision.Reason
|
||||
task.RoutingSnapshot = snapshot
|
||||
return nil
|
||||
}
|
||||
_ = s.coordinationStore.RequestRouteProbe(context.WithoutCancel(ctx), profile.Key)
|
||||
lastErr = err
|
||||
if !s.routingEnforced() {
|
||||
s.observeExecutionPoolRouting("shadow")
|
||||
_ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
||||
TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion,
|
||||
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
||||
Reason: "shadow_no_eligible_pool", Snapshot: map[string]any{"mode": "shadow", "error": safeRoutingError(err)},
|
||||
})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
s.observeExecutionPoolRouting("unavailable")
|
||||
if remaining := asyncRouteWaitMaximum - time.Since(startedAt); remaining > 0 {
|
||||
timer := time.NewTimer(remaining)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return upstreamRouteUnavailable(lastErr)
|
||||
}
|
||||
|
||||
func (s *Service) observeExecutionPoolRouting(outcome string) {
|
||||
observer, ok := s.billingMetrics.(interface{ ObserveExecutionPoolRouting(string) })
|
||||
if ok {
|
||||
observer.ObserveExecutionPoolRouting(outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) routingCandidateForTask(ctx context.Context, task store.GatewayTask, user *auth.User) (store.RuntimeModelCandidate, error) {
|
||||
body := normalizeRequest(task.Kind, task.Request)
|
||||
modelType := modelTypeFromKind(task.Kind, body)
|
||||
candidates, err := s.store.ListModelCandidates(ctx, task.Model, modelType, user)
|
||||
if err != nil {
|
||||
return store.RuntimeModelCandidate{}, err
|
||||
}
|
||||
candidates, err = filterCandidatesByRequestedPlatform(candidates, body)
|
||||
if err == nil {
|
||||
candidates, _, err = filterRuntimeCandidatesByRequest(task.Kind, task.Model, modelType, body, candidates)
|
||||
}
|
||||
if err == nil {
|
||||
candidates, _, err = filterRuntimeCandidatesByOutputTokens(task.Kind, task.Model, modelType, body, candidates)
|
||||
}
|
||||
if err != nil {
|
||||
return store.RuntimeModelCandidate{}, err
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return store.RuntimeModelCandidate{}, store.ErrNoModelCandidate
|
||||
}
|
||||
return candidates[0], nil
|
||||
}
|
||||
|
||||
func (s *Service) selectExecutionPool(ctx context.Context, task store.GatewayTask, candidate store.RuntimeModelCandidate) (executionpool.Decision, executionpool.RouteProfile, error) {
|
||||
profile, err := routeProfileForCandidate(candidate)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
now := time.Now()
|
||||
pools, err := s.coordinationStore.ListExecutionPools(ctx)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
workers, err := s.coordinationStore.ListWorkers(ctx, now)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
health, err := s.coordinationStore.ListRouteHealth(ctx, profile.Key, now)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
capacity, err := s.coordinationStore.ListCapacity(ctx, now)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
workersByPool := make(map[string][]executionpool.WorkerDescriptor)
|
||||
for _, worker := range workers {
|
||||
workersByPool[worker.PoolID] = append(workersByPool[worker.PoolID], worker)
|
||||
}
|
||||
healthByPool := make(map[string]executionpool.RouteHealth)
|
||||
for _, item := range health {
|
||||
healthByPool[item.PoolID] = item
|
||||
}
|
||||
capacityByPool := make(map[string]executionpool.CapacitySnapshot)
|
||||
for _, item := range capacity {
|
||||
capacityByPool[item.PoolID] = item
|
||||
}
|
||||
selection := make([]executionpool.SelectionCandidate, 0, len(pools))
|
||||
for _, pool := range pools {
|
||||
poolWorkers := workersByPool[pool.ID]
|
||||
selection = append(selection, executionpool.SelectionCandidate{
|
||||
Pool: pool, Health: healthByPool[pool.ID], Capacity: capacityByPool[pool.ID],
|
||||
CapabilityMatched: workerCapabilityMatched(poolWorkers, task),
|
||||
})
|
||||
}
|
||||
selector := executionpool.NewSelector()
|
||||
returnDecision, selectErr := selector.Select(executionpool.SelectionRequest{
|
||||
Candidates: selection, CurrentPoolID: task.AssignedPoolID, Now: now,
|
||||
})
|
||||
if selectErr != nil {
|
||||
return returnDecision, profile, selectErr
|
||||
}
|
||||
poolDecisions := make(map[string]executionpool.Decision, len(selection))
|
||||
scores := make(map[string]float64, len(selection))
|
||||
for _, candidate := range selection {
|
||||
decision, candidateErr := selector.Select(executionpool.SelectionRequest{Candidates: []executionpool.SelectionCandidate{candidate}, Now: now})
|
||||
if candidateErr == nil {
|
||||
poolDecisions[decision.PoolID] = decision
|
||||
scores[decision.PoolID] = decision.Score
|
||||
}
|
||||
}
|
||||
preference, err := s.coordinationStore.ResolveRoutePreference(ctx, profile.Key, returnDecision.PoolID, scores, now)
|
||||
if err != nil {
|
||||
return executionpool.Decision{}, profile, err
|
||||
}
|
||||
if preference.CurrentPoolID != returnDecision.PoolID {
|
||||
preferred, ok := poolDecisions[preference.CurrentPoolID]
|
||||
if !ok {
|
||||
return executionpool.Decision{}, profile, executionpool.ErrNoEligiblePool
|
||||
}
|
||||
preferred.Reason = "route_preference_hysteresis"
|
||||
preferred.Rejected = returnDecision.Rejected
|
||||
returnDecision = preferred
|
||||
}
|
||||
return returnDecision, profile, nil
|
||||
}
|
||||
|
||||
func routeProfileForCandidate(candidate store.RuntimeModelCandidate) (executionpool.RouteProfile, error) {
|
||||
baseURL := routeEndpointForCandidate(candidate)
|
||||
host := executionpool.EndpointHost(baseURL)
|
||||
if host == "" {
|
||||
return executionpool.RouteProfile{}, errors.New("candidate does not expose a probeable upstream endpoint")
|
||||
}
|
||||
proxyMode := "none"
|
||||
if proxyConfig, err := netproxy.Normalize(netproxy.FromPlatformConfig(candidate.PlatformConfig)); err == nil {
|
||||
proxyMode = string(proxyConfig.Mode)
|
||||
}
|
||||
protocol := firstNonEmptyString(candidate.ResponseProtocol, candidate.SpecType, candidate.Provider)
|
||||
profile := executionpool.RouteProfile{
|
||||
Provider: candidate.Provider, Protocol: protocol, EndpointHost: host,
|
||||
ProxyMode: proxyMode, ConfigRevision: candidate.PlatformID + ":" + candidate.PlatformModelID,
|
||||
}
|
||||
profile.Key = executionpool.RouteProfileKey(profile.Provider, profile.Protocol, profile.EndpointHost, profile.ProxyMode, profile.ConfigRevision)
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func routeEndpointForCandidate(candidate store.RuntimeModelCandidate) string {
|
||||
baseURL := strings.TrimSpace(candidate.BaseURL)
|
||||
if baseURL == "" {
|
||||
baseURL = firstString(candidate.PlatformConfig, "endpoint", "baseURL", "base_url")
|
||||
}
|
||||
return baseURL
|
||||
}
|
||||
|
||||
func workerCapabilityMatched(workers []executionpool.WorkerDescriptor, task store.GatewayTask) bool {
|
||||
for _, worker := range workers {
|
||||
if worker.ProtocolVersion != executionpool.ProtocolVersion || worker.AvailableCapacity() < 1 {
|
||||
continue
|
||||
}
|
||||
if requestStreamEnabled(task.Request) && !boolValue(worker.Capabilities["streaming"]) {
|
||||
continue
|
||||
}
|
||||
if !workerSupportsTaskKind(worker.Capabilities["taskKinds"], task.Kind) {
|
||||
continue
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func workerSupportsTaskKind(raw any, taskKind string) bool {
|
||||
taskKind = strings.ToLower(strings.TrimSpace(taskKind))
|
||||
if taskKind == "" {
|
||||
return false
|
||||
}
|
||||
values, ok := raw.([]any)
|
||||
if !ok {
|
||||
if stringsList, stringsOK := raw.([]string); stringsOK {
|
||||
values = make([]any, 0, len(stringsList))
|
||||
for _, value := range stringsList {
|
||||
values = append(values, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, value := range values {
|
||||
capability, _ := value.(string)
|
||||
capability = strings.ToLower(strings.TrimSpace(capability))
|
||||
if capability == "*" || capability == taskKind ||
|
||||
(strings.HasSuffix(capability, ".*") && strings.HasPrefix(taskKind, strings.TrimSuffix(capability, "*"))) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func requestStreamEnabled(body map[string]any) bool {
|
||||
value, _ := body["stream"].(bool)
|
||||
return value
|
||||
}
|
||||
|
||||
func routingDecisionSnapshot(decision executionpool.Decision) map[string]any {
|
||||
rejected := make([]string, 0, len(decision.Rejected))
|
||||
for poolID, reason := range decision.Rejected {
|
||||
rejected = append(rejected, poolID+":"+reason)
|
||||
}
|
||||
sort.Strings(rejected)
|
||||
return map[string]any{
|
||||
"mode": "selected", "score": decision.Score, "components": decision.Components,
|
||||
"rejected": rejected,
|
||||
}
|
||||
}
|
||||
|
||||
func safeRoutingError(err error) string {
|
||||
if errors.Is(err, executionpool.ErrNoEligiblePool) {
|
||||
return "no_eligible_pool"
|
||||
}
|
||||
return "routing_unavailable"
|
||||
}
|
||||
|
||||
func upstreamRouteUnavailable(cause error) error {
|
||||
details := map[string]any{"retryAfterSeconds": 10}
|
||||
if cause != nil {
|
||||
details["reason"] = safeRoutingError(cause)
|
||||
}
|
||||
return &clients.ClientError{
|
||||
Code: "upstream_route_unavailable", Message: "no execution pool can currently reach the upstream route",
|
||||
Retryable: true, StatusCode: http.StatusServiceUnavailable, Details: details,
|
||||
}
|
||||
}
|
||||
|
||||
func firstString(values map[string]any, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value, ok := values[key].(string); ok && strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func boolValue(value any) bool {
|
||||
result, _ := value.(bool)
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user