feat(routing): 引入多执行池智能调度
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
type routeProbeState struct {
|
||||
next time.Time
|
||||
lastRequestObserved time.Time
|
||||
}
|
||||
|
||||
func (s *Service) StartRouteHealthProber(ctx context.Context) {
|
||||
if !s.cfg.RunsRouteProber() {
|
||||
return
|
||||
}
|
||||
go s.runRouteHealthProber(ctx)
|
||||
}
|
||||
|
||||
func (s *Service) runRouteHealthProber(ctx context.Context) {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
states := make(map[string]routeProbeState)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
targets, err := s.coordinationStore.ListRouteProbeTargets(ctx)
|
||||
if err != nil {
|
||||
s.logRouteProbeFailure("list_targets", err)
|
||||
continue
|
||||
}
|
||||
requested, requestErr := s.coordinationStore.ListRequestedRouteProbes(ctx)
|
||||
if requestErr != nil {
|
||||
s.logRouteProbeFailure("list_requests", requestErr)
|
||||
continue
|
||||
}
|
||||
now := time.Now()
|
||||
for _, target := range targets {
|
||||
profile, profileErr := routeProfileForCandidate(target.Candidate)
|
||||
if profileErr != nil {
|
||||
continue
|
||||
}
|
||||
state := states[profile.Key]
|
||||
requestedAt := requested[profile.Key]
|
||||
requestDue := !requestedAt.IsZero() && requestedAt.After(state.lastRequestObserved)
|
||||
if !requestDue && now.Before(state.next) {
|
||||
continue
|
||||
}
|
||||
interval := time.Duration(s.cfg.RouteProbeColdIntervalSeconds) * time.Second
|
||||
if target.Hot {
|
||||
interval = time.Duration(s.cfg.RouteProbeHotIntervalSeconds) * time.Second
|
||||
}
|
||||
states[profile.Key] = routeProbeState{next: now.Add(interval), lastRequestObserved: requestedAt}
|
||||
go s.probeRouteTarget(ctx, target, profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) probeRouteTarget(ctx context.Context, target store.RouteProbeTarget, profile executionpool.RouteProfile) {
|
||||
leadership, locked, err := s.coordinationStore.TryAcquireRouteProbeLeadership(ctx, s.cfg.ExecutionPoolID, profile.Key)
|
||||
if err != nil || !locked {
|
||||
return
|
||||
}
|
||||
defer leadership.Release()
|
||||
client, err := s.httpClientForCandidate(target.Candidate, false, "")
|
||||
if err != nil {
|
||||
s.logRouteProbeFailure("prepare_client", err)
|
||||
return
|
||||
}
|
||||
probe := executionpool.HTTPProbe{Client: client}
|
||||
result := probe.Probe(ctx, executionpool.ProbeTarget{
|
||||
RouteProfile: profile,
|
||||
URL: routeEndpointForCandidate(target.Candidate),
|
||||
Timeout: time.Duration(s.cfg.RouteProbeTimeoutMS) * time.Millisecond,
|
||||
})
|
||||
previous := executionpool.RouteHealth{}
|
||||
if history, historyErr := s.coordinationStore.ListRouteHealth(ctx, profile.Key, time.Now()); historyErr == nil {
|
||||
for _, item := range history {
|
||||
if item.PoolID == s.cfg.ExecutionPoolID {
|
||||
previous = item
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
health := nextRouteHealth(previous, s.cfg.ExecutionPoolID, profile.Key, result)
|
||||
s.observeRouteProbe(string(health.State))
|
||||
if err := s.coordinationStore.RecordRouteHealth(context.WithoutCancel(ctx), health); err != nil {
|
||||
s.logRouteProbeFailure("record", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) observeRouteProbe(state string) {
|
||||
observer, ok := s.billingMetrics.(interface{ ObserveRouteProbe(string) })
|
||||
if ok {
|
||||
observer.ObserveRouteProbe(state)
|
||||
}
|
||||
}
|
||||
|
||||
func nextRouteHealth(previous executionpool.RouteHealth, poolID, profileKey string, result executionpool.ProbeResult) executionpool.RouteHealth {
|
||||
health := previous
|
||||
health.PoolID = poolID
|
||||
health.RouteProfileKey = profileKey
|
||||
health.SampleCount++
|
||||
health.SampledAt = result.SampledAt
|
||||
health.ExpiresAt = result.SampledAt.Add(45 * time.Second)
|
||||
observation := 0.0
|
||||
if result.Reachable {
|
||||
observation = 1
|
||||
health.ConsecutiveSuccesses++
|
||||
health.ConsecutiveFailures = 0
|
||||
if health.ConsecutiveSuccesses >= 2 || previous.State == executionpool.RouteHealthy {
|
||||
health.State = executionpool.RouteHealthy
|
||||
} else {
|
||||
health.State = executionpool.RouteDegraded
|
||||
}
|
||||
} else {
|
||||
health.ConsecutiveFailures++
|
||||
health.ConsecutiveSuccesses = 0
|
||||
switch {
|
||||
case health.ConsecutiveFailures >= 3:
|
||||
health.State = executionpool.RouteUnreachable
|
||||
case previous.State == executionpool.RouteHealthy || previous.State == executionpool.RouteDegraded:
|
||||
health.State = executionpool.RouteDegraded
|
||||
default:
|
||||
health.State = executionpool.RouteUnknown
|
||||
}
|
||||
}
|
||||
if previous.SampleCount == 0 {
|
||||
health.SuccessRate = observation
|
||||
} else {
|
||||
health.SuccessRate = 0.8*previous.SuccessRate + 0.2*observation
|
||||
}
|
||||
connectTLS := result.TCPDuration + result.TLSDuration
|
||||
if connectTLS > 0 {
|
||||
health.ConnectTLSP95 = ewmaDuration(previous.ConnectTLSP95, connectTLS)
|
||||
}
|
||||
if result.FirstByte > 0 {
|
||||
if previous.FirstByteP95 > 0 {
|
||||
delta := math.Abs(float64(result.FirstByte - previous.FirstByteP95))
|
||||
health.JitterP95 = ewmaDuration(previous.JitterP95, time.Duration(delta))
|
||||
}
|
||||
health.FirstByteP95 = ewmaDuration(previous.FirstByteP95, result.FirstByte)
|
||||
}
|
||||
return health
|
||||
}
|
||||
|
||||
func ewmaDuration(previous, current time.Duration) time.Duration {
|
||||
if previous <= 0 {
|
||||
return current
|
||||
}
|
||||
return time.Duration(0.8*float64(previous) + 0.2*float64(current))
|
||||
}
|
||||
|
||||
func (s *Service) logRouteProbeFailure(stage string, err error) {
|
||||
if err == nil || errors.Is(err, context.Canceled) || s.logger == nil {
|
||||
return
|
||||
}
|
||||
s.logger.Log(context.Background(), slog.LevelWarn, "route health probe failed", "stage", stage, "error_category", "route_probe_failed")
|
||||
}
|
||||
Reference in New Issue
Block a user