wangbo
7786692d32
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
333 lines
11 KiB
Go
333 lines
11 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/netproxy"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
const (
|
|
routingVersion = "execution-pool-v1"
|
|
asyncRouteWaitMaximum = 10 * time.Second
|
|
)
|
|
|
|
func (s *Service) routingEnabled() bool {
|
|
mode := strings.ToLower(strings.TrimSpace(s.cfg.RoutingMode))
|
|
return mode == "shadow" || mode == "enforced"
|
|
}
|
|
|
|
func (s *Service) routingEnforced() bool {
|
|
return strings.EqualFold(strings.TrimSpace(s.cfg.RoutingMode), "enforced")
|
|
}
|
|
|
|
func (s *Service) routeAsyncTask(ctx context.Context, task *store.GatewayTask, user *auth.User, preferred store.RuntimeModelCandidate) error {
|
|
if task == nil || !task.AsyncMode || !s.routingEnabled() || task.RunMode == "simulation" {
|
|
return nil
|
|
}
|
|
startedAt := time.Now()
|
|
delays := []time.Duration{0, 2 * time.Second, 3 * time.Second, 4 * time.Second}
|
|
var lastErr error
|
|
for _, delay := range delays {
|
|
if delay > 0 {
|
|
timer := time.NewTimer(delay)
|
|
select {
|
|
case <-ctx.Done():
|
|
if !timer.Stop() {
|
|
<-timer.C
|
|
}
|
|
return ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
candidate := preferred
|
|
if strings.TrimSpace(candidate.PlatformID) == "" {
|
|
resolved, err := s.routingCandidateForTask(ctx, *task, user)
|
|
if err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
candidate = resolved
|
|
}
|
|
decision, profile, err := s.selectExecutionPool(ctx, *task, candidate)
|
|
if err == nil {
|
|
poolID := decision.PoolID
|
|
snapshot := routingDecisionSnapshot(decision)
|
|
if !s.routingEnforced() {
|
|
s.observeExecutionPoolRouting("shadow")
|
|
snapshot["suggestedPoolId"] = poolID
|
|
poolID = ""
|
|
}
|
|
if s.routingEnforced() {
|
|
s.observeExecutionPoolRouting("selected")
|
|
}
|
|
if err := s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
|
TaskID: task.ID, PoolID: poolID, RouteProfileKey: profile.Key,
|
|
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
|
RoutingVersion: routingVersion, Reason: decision.Reason, Snapshot: snapshot,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
task.AssignedPoolID = poolID
|
|
task.RouteProfileKey = profile.Key
|
|
task.RoutingVersion = routingVersion
|
|
task.RoutingReason = decision.Reason
|
|
task.RoutingSnapshot = snapshot
|
|
return nil
|
|
}
|
|
_ = s.coordinationStore.RequestRouteProbe(context.WithoutCancel(ctx), profile.Key)
|
|
lastErr = err
|
|
if !s.routingEnforced() {
|
|
s.observeExecutionPoolRouting("shadow")
|
|
_ = s.store.AssignTaskRouting(ctx, store.TaskRoutingDecision{
|
|
TaskID: task.ID, RouteProfileKey: profile.Key, RoutingVersion: routingVersion,
|
|
PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID,
|
|
Reason: "shadow_no_eligible_pool", Snapshot: map[string]any{"mode": "shadow", "error": safeRoutingError(err)},
|
|
})
|
|
return nil
|
|
}
|
|
}
|
|
s.observeExecutionPoolRouting("unavailable")
|
|
if remaining := asyncRouteWaitMaximum - time.Since(startedAt); remaining > 0 {
|
|
timer := time.NewTimer(remaining)
|
|
select {
|
|
case <-ctx.Done():
|
|
if !timer.Stop() {
|
|
<-timer.C
|
|
}
|
|
return ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
return upstreamRouteUnavailable(lastErr)
|
|
}
|
|
|
|
func (s *Service) observeExecutionPoolRouting(outcome string) {
|
|
observer, ok := s.billingMetrics.(interface{ ObserveExecutionPoolRouting(string) })
|
|
if ok {
|
|
observer.ObserveExecutionPoolRouting(outcome)
|
|
}
|
|
}
|
|
|
|
func (s *Service) routingCandidateForTask(ctx context.Context, task store.GatewayTask, user *auth.User) (store.RuntimeModelCandidate, error) {
|
|
body := normalizeRequest(task.Kind, task.Request)
|
|
modelType := modelTypeFromKind(task.Kind, body)
|
|
candidates, err := s.store.ListModelCandidates(ctx, task.Model, modelType, user)
|
|
if err != nil {
|
|
return store.RuntimeModelCandidate{}, err
|
|
}
|
|
candidates, err = filterCandidatesByRequestedPlatform(candidates, body)
|
|
if err == nil {
|
|
candidates, _, err = filterRuntimeCandidatesByRequest(task.Kind, task.Model, modelType, body, candidates)
|
|
}
|
|
if err == nil {
|
|
candidates, _, err = filterRuntimeCandidatesByOutputTokens(task.Kind, task.Model, modelType, body, candidates)
|
|
}
|
|
if err != nil {
|
|
return store.RuntimeModelCandidate{}, err
|
|
}
|
|
if len(candidates) == 0 {
|
|
return store.RuntimeModelCandidate{}, store.ErrNoModelCandidate
|
|
}
|
|
return candidates[0], nil
|
|
}
|
|
|
|
func (s *Service) selectExecutionPool(ctx context.Context, task store.GatewayTask, candidate store.RuntimeModelCandidate) (executionpool.Decision, executionpool.RouteProfile, error) {
|
|
profile, err := routeProfileForCandidate(candidate)
|
|
if err != nil {
|
|
return executionpool.Decision{}, profile, err
|
|
}
|
|
now := time.Now()
|
|
pools, err := s.coordinationStore.ListExecutionPools(ctx)
|
|
if err != nil {
|
|
return executionpool.Decision{}, profile, err
|
|
}
|
|
workers, err := s.coordinationStore.ListWorkers(ctx, now)
|
|
if err != nil {
|
|
return executionpool.Decision{}, profile, err
|
|
}
|
|
health, err := s.coordinationStore.ListRouteHealth(ctx, profile.Key, now)
|
|
if err != nil {
|
|
return executionpool.Decision{}, profile, err
|
|
}
|
|
capacity, err := s.coordinationStore.ListCapacity(ctx, now)
|
|
if err != nil {
|
|
return executionpool.Decision{}, profile, err
|
|
}
|
|
workersByPool := make(map[string][]executionpool.WorkerDescriptor)
|
|
for _, worker := range workers {
|
|
workersByPool[worker.PoolID] = append(workersByPool[worker.PoolID], worker)
|
|
}
|
|
healthByPool := make(map[string]executionpool.RouteHealth)
|
|
for _, item := range health {
|
|
healthByPool[item.PoolID] = item
|
|
}
|
|
capacityByPool := make(map[string]executionpool.CapacitySnapshot)
|
|
for _, item := range capacity {
|
|
capacityByPool[item.PoolID] = item
|
|
}
|
|
selection := make([]executionpool.SelectionCandidate, 0, len(pools))
|
|
for _, pool := range pools {
|
|
poolWorkers := workersByPool[pool.ID]
|
|
selection = append(selection, executionpool.SelectionCandidate{
|
|
Pool: pool, Health: healthByPool[pool.ID], Capacity: capacityByPool[pool.ID],
|
|
CapabilityMatched: workerCapabilityMatched(poolWorkers, task),
|
|
})
|
|
}
|
|
selector := executionpool.NewSelector()
|
|
returnDecision, selectErr := selector.Select(executionpool.SelectionRequest{
|
|
Candidates: selection, CurrentPoolID: task.AssignedPoolID, Now: now,
|
|
})
|
|
if selectErr != nil {
|
|
return returnDecision, profile, selectErr
|
|
}
|
|
poolDecisions := make(map[string]executionpool.Decision, len(selection))
|
|
scores := make(map[string]float64, len(selection))
|
|
for _, candidate := range selection {
|
|
decision, candidateErr := selector.Select(executionpool.SelectionRequest{Candidates: []executionpool.SelectionCandidate{candidate}, Now: now})
|
|
if candidateErr == nil {
|
|
poolDecisions[decision.PoolID] = decision
|
|
scores[decision.PoolID] = decision.Score
|
|
}
|
|
}
|
|
preference, err := s.coordinationStore.ResolveRoutePreference(ctx, profile.Key, returnDecision.PoolID, scores, now)
|
|
if err != nil {
|
|
return executionpool.Decision{}, profile, err
|
|
}
|
|
if preference.CurrentPoolID != returnDecision.PoolID {
|
|
preferred, ok := poolDecisions[preference.CurrentPoolID]
|
|
if !ok {
|
|
return executionpool.Decision{}, profile, executionpool.ErrNoEligiblePool
|
|
}
|
|
preferred.Reason = "route_preference_hysteresis"
|
|
preferred.Rejected = returnDecision.Rejected
|
|
returnDecision = preferred
|
|
}
|
|
return returnDecision, profile, nil
|
|
}
|
|
|
|
func routeProfileForCandidate(candidate store.RuntimeModelCandidate) (executionpool.RouteProfile, error) {
|
|
baseURL := routeEndpointForCandidate(candidate)
|
|
host := executionpool.EndpointHost(baseURL)
|
|
if host == "" {
|
|
return executionpool.RouteProfile{}, errors.New("candidate does not expose a probeable upstream endpoint")
|
|
}
|
|
proxyMode := "none"
|
|
if proxyConfig, err := netproxy.Normalize(netproxy.FromPlatformConfig(candidate.PlatformConfig)); err == nil {
|
|
proxyMode = string(proxyConfig.Mode)
|
|
}
|
|
protocol := firstNonEmptyString(candidate.ResponseProtocol, candidate.SpecType, candidate.Provider)
|
|
profile := executionpool.RouteProfile{
|
|
Provider: candidate.Provider, Protocol: protocol, EndpointHost: host,
|
|
ProxyMode: proxyMode, ConfigRevision: candidate.PlatformID + ":" + candidate.PlatformModelID,
|
|
}
|
|
profile.Key = executionpool.RouteProfileKey(profile.Provider, profile.Protocol, profile.EndpointHost, profile.ProxyMode, profile.ConfigRevision)
|
|
return profile, nil
|
|
}
|
|
|
|
func routeEndpointForCandidate(candidate store.RuntimeModelCandidate) string {
|
|
baseURL := strings.TrimSpace(candidate.BaseURL)
|
|
if baseURL == "" {
|
|
baseURL = firstString(candidate.PlatformConfig, "endpoint", "baseURL", "base_url")
|
|
}
|
|
return baseURL
|
|
}
|
|
|
|
func workerCapabilityMatched(workers []executionpool.WorkerDescriptor, task store.GatewayTask) bool {
|
|
for _, worker := range workers {
|
|
if worker.ProtocolVersion != executionpool.ProtocolVersion || worker.AvailableCapacity() < 1 {
|
|
continue
|
|
}
|
|
if requestStreamEnabled(task.Request) && !boolValue(worker.Capabilities["streaming"]) {
|
|
continue
|
|
}
|
|
if !workerSupportsTaskKind(worker.Capabilities["taskKinds"], task.Kind) {
|
|
continue
|
|
}
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func workerSupportsTaskKind(raw any, taskKind string) bool {
|
|
taskKind = strings.ToLower(strings.TrimSpace(taskKind))
|
|
if taskKind == "" {
|
|
return false
|
|
}
|
|
values, ok := raw.([]any)
|
|
if !ok {
|
|
if stringsList, stringsOK := raw.([]string); stringsOK {
|
|
values = make([]any, 0, len(stringsList))
|
|
for _, value := range stringsList {
|
|
values = append(values, value)
|
|
}
|
|
}
|
|
}
|
|
for _, value := range values {
|
|
capability, _ := value.(string)
|
|
capability = strings.ToLower(strings.TrimSpace(capability))
|
|
if capability == "*" || capability == taskKind ||
|
|
(strings.HasSuffix(capability, ".*") && strings.HasPrefix(taskKind, strings.TrimSuffix(capability, "*"))) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func requestStreamEnabled(body map[string]any) bool {
|
|
value, _ := body["stream"].(bool)
|
|
return value
|
|
}
|
|
|
|
func routingDecisionSnapshot(decision executionpool.Decision) map[string]any {
|
|
rejected := make([]string, 0, len(decision.Rejected))
|
|
for poolID, reason := range decision.Rejected {
|
|
rejected = append(rejected, poolID+":"+reason)
|
|
}
|
|
sort.Strings(rejected)
|
|
return map[string]any{
|
|
"mode": "selected", "score": decision.Score, "components": decision.Components,
|
|
"rejected": rejected,
|
|
}
|
|
}
|
|
|
|
func safeRoutingError(err error) string {
|
|
if errors.Is(err, executionpool.ErrNoEligiblePool) {
|
|
return "no_eligible_pool"
|
|
}
|
|
return "routing_unavailable"
|
|
}
|
|
|
|
func upstreamRouteUnavailable(cause error) error {
|
|
details := map[string]any{"retryAfterSeconds": 10}
|
|
if cause != nil {
|
|
details["reason"] = safeRoutingError(cause)
|
|
}
|
|
return &clients.ClientError{
|
|
Code: "upstream_route_unavailable", Message: "no execution pool can currently reach the upstream route",
|
|
Retryable: true, StatusCode: http.StatusServiceUnavailable, Details: details,
|
|
}
|
|
}
|
|
|
|
func firstString(values map[string]any, keys ...string) string {
|
|
for _, key := range keys {
|
|
if value, ok := values[key].(string); ok && strings.TrimSpace(value) != "" {
|
|
return strings.TrimSpace(value)
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func boolValue(value any) bool {
|
|
result, _ := value.(bool)
|
|
return result
|
|
}
|