feat(routing): 引入多执行池智能调度
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
@@ -0,0 +1,137 @@
|
||||
package executionpool
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func QueueName(poolID string) string {
|
||||
poolID = strings.TrimSpace(poolID)
|
||||
if poolID == "" {
|
||||
return "gateway_tasks"
|
||||
}
|
||||
digest := sha256.Sum256([]byte(poolID))
|
||||
return "gateway_pool_" + fmt.Sprintf("%x", digest[:8])
|
||||
}
|
||||
|
||||
func RouteProfileKey(provider, protocol, endpointHost, proxyMode, configRevision string) string {
|
||||
value := strings.Join([]string{
|
||||
strings.ToLower(strings.TrimSpace(provider)),
|
||||
strings.ToLower(strings.TrimSpace(protocol)),
|
||||
strings.ToLower(strings.TrimSpace(endpointHost)),
|
||||
strings.ToLower(strings.TrimSpace(proxyMode)),
|
||||
strings.TrimSpace(configRevision),
|
||||
}, "\x00")
|
||||
digest := sha256.Sum256([]byte(value))
|
||||
return fmt.Sprintf("route_%x", digest[:16])
|
||||
}
|
||||
|
||||
func EndpointHost(rawURL string) string {
|
||||
parsed, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(parsed.Hostname())
|
||||
}
|
||||
|
||||
func ValidateAdvertisedEndpoint(rawURL string, allowedSuffixes []string, allowPrivate bool) error {
|
||||
parsed, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil || parsed.Host == "" || parsed.User != nil {
|
||||
return errors.New("worker endpoint must be an absolute URL without user info")
|
||||
}
|
||||
if parsed.Scheme != "https" && parsed.Scheme != "http" {
|
||||
return errors.New("worker endpoint scheme must be http or https")
|
||||
}
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
if host == "" {
|
||||
return errors.New("worker endpoint host is required")
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if allowPrivate && (ip.IsPrivate() || ip.IsLoopback()) {
|
||||
return nil
|
||||
}
|
||||
return errors.New("worker endpoint IP is outside the configured trust boundary")
|
||||
}
|
||||
for _, suffix := range allowedSuffixes {
|
||||
suffix = strings.ToLower(strings.TrimSpace(suffix))
|
||||
if suffix != "" && (host == strings.TrimPrefix(suffix, ".") || strings.HasSuffix(host, "."+strings.TrimPrefix(suffix, "."))) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errors.New("worker endpoint host is outside the configured trust boundary")
|
||||
}
|
||||
|
||||
type ExecutionClaims struct {
|
||||
Audience string `json:"aud"`
|
||||
TaskID string `json:"task_id"`
|
||||
PoolID string `json:"pool_id"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
Nonce string `json:"nonce"`
|
||||
ExpiresAt int64 `json:"exp"`
|
||||
}
|
||||
|
||||
type TokenSigner struct {
|
||||
Secret []byte
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (s TokenSigner) Sign(claims ExecutionClaims) (string, error) {
|
||||
if len(s.Secret) < 32 {
|
||||
return "", errors.New("execution token secret must be at least 32 bytes")
|
||||
}
|
||||
if claims.Audience == "" || claims.TaskID == "" || claims.PoolID == "" || claims.WorkerID == "" || claims.Nonce == "" || claims.ExpiresAt <= 0 {
|
||||
return "", errors.New("execution token claims are incomplete")
|
||||
}
|
||||
payload, err := json.Marshal(claims)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
encoded := base64.RawURLEncoding.EncodeToString(payload)
|
||||
mac := hmac.New(sha256.New, s.Secret)
|
||||
_, _ = mac.Write([]byte(encoded))
|
||||
signature := base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
|
||||
return encoded + "." + signature, nil
|
||||
}
|
||||
|
||||
func (s TokenSigner) Verify(token, audience string) (ExecutionClaims, error) {
|
||||
var claims ExecutionClaims
|
||||
if len(s.Secret) < 32 {
|
||||
return claims, errors.New("execution token secret must be at least 32 bytes")
|
||||
}
|
||||
parts := strings.Split(token, ".")
|
||||
if len(parts) != 2 {
|
||||
return claims, errors.New("invalid execution token")
|
||||
}
|
||||
signature, err := base64.RawURLEncoding.DecodeString(parts[1])
|
||||
if err != nil {
|
||||
return claims, errors.New("invalid execution token signature")
|
||||
}
|
||||
mac := hmac.New(sha256.New, s.Secret)
|
||||
_, _ = mac.Write([]byte(parts[0]))
|
||||
if !hmac.Equal(signature, mac.Sum(nil)) {
|
||||
return claims, errors.New("invalid execution token signature")
|
||||
}
|
||||
payload, err := base64.RawURLEncoding.DecodeString(parts[0])
|
||||
if err != nil || json.Unmarshal(payload, &claims) != nil {
|
||||
return ExecutionClaims{}, errors.New("invalid execution token payload")
|
||||
}
|
||||
now := time.Now()
|
||||
if s.Now != nil {
|
||||
now = s.Now()
|
||||
}
|
||||
if claims.ExpiresAt <= now.Unix() {
|
||||
return ExecutionClaims{}, errors.New("execution token expired")
|
||||
}
|
||||
if claims.Audience != audience {
|
||||
return ExecutionClaims{}, errors.New("execution token audience mismatch")
|
||||
}
|
||||
return claims, nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package executionpool
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueueNameDoesNotExposePoolID(t *testing.T) {
|
||||
name := QueueName("region-sensitive-name")
|
||||
if strings.Contains(name, "sensitive") || name == QueueName("other") {
|
||||
t.Fatalf("unexpected queue name %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenSigner(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
signer := TokenSigner{Secret: []byte("01234567890123456789012345678901"), Now: func() time.Time { return now }}
|
||||
token, err := signer.Sign(ExecutionClaims{
|
||||
Audience: "worker", TaskID: "task", PoolID: "pool", WorkerID: "worker", Nonce: "nonce", ExpiresAt: now.Add(30 * time.Second).Unix(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claims, err := signer.Verify(token, "worker")
|
||||
if err != nil || claims.TaskID != "task" {
|
||||
t.Fatalf("claims=%+v err=%v", claims, err)
|
||||
}
|
||||
if _, err := signer.Verify(token, "other"); err == nil {
|
||||
t.Fatal("expected audience mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAdvertisedEndpoint(t *testing.T) {
|
||||
if err := ValidateAdvertisedEndpoint("http://10.0.0.2:8088", nil, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateAdvertisedEndpoint("https://worker.internal.example", []string{"internal.example"}, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateAdvertisedEndpoint("https://public.example", []string{"internal.example"}, false); err == nil {
|
||||
t.Fatal("expected untrusted endpoint rejection")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package executionpool
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RoutePreference struct {
|
||||
RouteProfileKey string
|
||||
CurrentPoolID string
|
||||
CurrentSince time.Time
|
||||
ChallengerPoolID string
|
||||
ChallengerWins int
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func AdvanceRoutePreference(
|
||||
previous RoutePreference,
|
||||
proposedPoolID string,
|
||||
scores map[string]float64,
|
||||
now time.Time,
|
||||
) RoutePreference {
|
||||
if now.IsZero() {
|
||||
now = time.Now()
|
||||
}
|
||||
proposedPoolID = strings.TrimSpace(proposedPoolID)
|
||||
next := previous
|
||||
next.UpdatedAt = now
|
||||
if proposedPoolID == "" {
|
||||
return next
|
||||
}
|
||||
if _, eligible := scores[next.CurrentPoolID]; next.CurrentPoolID == "" || !eligible {
|
||||
next.CurrentPoolID = proposedPoolID
|
||||
next.CurrentSince = now
|
||||
next.ChallengerPoolID = ""
|
||||
next.ChallengerWins = 0
|
||||
return next
|
||||
}
|
||||
if proposedPoolID == next.CurrentPoolID {
|
||||
next.ChallengerPoolID = ""
|
||||
next.ChallengerWins = 0
|
||||
return next
|
||||
}
|
||||
if relativeImprovement(scores[proposedPoolID], scores[next.CurrentPoolID]) < defaultSwitchImprovement {
|
||||
next.ChallengerPoolID = ""
|
||||
next.ChallengerWins = 0
|
||||
return next
|
||||
}
|
||||
if next.ChallengerPoolID == proposedPoolID {
|
||||
next.ChallengerWins++
|
||||
} else {
|
||||
next.ChallengerPoolID = proposedPoolID
|
||||
next.ChallengerWins = 1
|
||||
}
|
||||
if next.ChallengerWins >= defaultBetterWindowCount &&
|
||||
!next.CurrentSince.IsZero() && now.Sub(next.CurrentSince) >= defaultMinimumDwell {
|
||||
next.CurrentPoolID = proposedPoolID
|
||||
next.CurrentSince = now
|
||||
next.ChallengerPoolID = ""
|
||||
next.ChallengerWins = 0
|
||||
}
|
||||
return next
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package executionpool
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAdvanceRoutePreferenceRequiresThreeWinsAndMinimumDwell(t *testing.T) {
|
||||
now := time.Date(2026, 8, 5, 12, 0, 0, 0, time.UTC)
|
||||
state := RoutePreference{CurrentPoolID: "slow", CurrentSince: now}
|
||||
scores := map[string]float64{"slow": 0.5, "fast": 0.7}
|
||||
for window := 1; window <= 3; window++ {
|
||||
state = AdvanceRoutePreference(state, "fast", scores, now.Add(time.Duration(window)*10*time.Second))
|
||||
if state.CurrentPoolID != "slow" {
|
||||
t.Fatalf("switched before minimum dwell in window %d", window)
|
||||
}
|
||||
}
|
||||
state = AdvanceRoutePreference(state, "fast", scores, now.Add(2*time.Minute))
|
||||
if state.CurrentPoolID != "fast" {
|
||||
t.Fatalf("current pool=%q, want fast", state.CurrentPoolID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvanceRoutePreferenceImmediatelyLeavesIneligibleCurrentPool(t *testing.T) {
|
||||
now := time.Now()
|
||||
state := AdvanceRoutePreference(RoutePreference{
|
||||
CurrentPoolID: "offline", CurrentSince: now.Add(-time.Minute),
|
||||
}, "healthy", map[string]float64{"healthy": 0.6}, now)
|
||||
if state.CurrentPoolID != "healthy" || state.ChallengerWins != 0 {
|
||||
t.Fatalf("unexpected preference: %#v", state)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package executionpool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ProbeTarget struct {
|
||||
RouteProfile RouteProfile
|
||||
URL string
|
||||
Method string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type ProbeResult struct {
|
||||
Reachable bool
|
||||
StatusCode int
|
||||
DNSDuration time.Duration
|
||||
TCPDuration time.Duration
|
||||
TLSDuration time.Duration
|
||||
FirstByte time.Duration
|
||||
SampledAt time.Time
|
||||
ErrorClass string
|
||||
}
|
||||
|
||||
type NetworkProbe interface {
|
||||
Probe(context.Context, ProbeTarget) ProbeResult
|
||||
}
|
||||
|
||||
type HTTPProbe struct {
|
||||
Client *http.Client
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (p HTTPProbe) Probe(ctx context.Context, target ProbeTarget) ProbeResult {
|
||||
result := ProbeResult{}
|
||||
now := time.Now
|
||||
if p.Now != nil {
|
||||
now = p.Now
|
||||
}
|
||||
result.SampledAt = now()
|
||||
method := target.Method
|
||||
if method == "" {
|
||||
method = http.MethodHead
|
||||
}
|
||||
timeout := target.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 3 * time.Second
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
var dnsStart, tcpStart, tlsStart time.Time
|
||||
requestStart := now()
|
||||
trace := &httptrace.ClientTrace{
|
||||
DNSStart: func(httptrace.DNSStartInfo) { dnsStart = now() },
|
||||
DNSDone: func(httptrace.DNSDoneInfo) {
|
||||
if !dnsStart.IsZero() {
|
||||
result.DNSDuration = now().Sub(dnsStart)
|
||||
}
|
||||
},
|
||||
ConnectStart: func(_, _ string) { tcpStart = now() },
|
||||
ConnectDone: func(_, _ string, _ error) {
|
||||
if !tcpStart.IsZero() {
|
||||
result.TCPDuration = now().Sub(tcpStart)
|
||||
}
|
||||
},
|
||||
TLSHandshakeStart: func() { tlsStart = now() },
|
||||
TLSHandshakeDone: func(tls.ConnectionState, error) {
|
||||
if !tlsStart.IsZero() {
|
||||
result.TLSDuration = now().Sub(tlsStart)
|
||||
}
|
||||
},
|
||||
GotFirstResponseByte: func() { result.FirstByte = now().Sub(requestStart) },
|
||||
}
|
||||
request, err := http.NewRequestWithContext(httptrace.WithClientTrace(probeCtx, trace), method, target.URL, nil)
|
||||
if err != nil {
|
||||
result.ErrorClass = "invalid_target"
|
||||
return result
|
||||
}
|
||||
client := p.Client
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
if probeCtx.Err() != nil {
|
||||
result.ErrorClass = "timeout"
|
||||
} else {
|
||||
result.ErrorClass = "network"
|
||||
}
|
||||
return result
|
||||
}
|
||||
defer response.Body.Close()
|
||||
result.StatusCode = response.StatusCode
|
||||
// Any HTTP response, including 401/403/404/405, proves transport reachability.
|
||||
result.Reachable = true
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package executionpool
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHealthMaxAge = 45 * time.Second
|
||||
defaultMinimumDwell = 2 * time.Minute
|
||||
defaultSwitchImprovement = 0.20
|
||||
defaultBetterWindowCount = 3
|
||||
)
|
||||
|
||||
var ErrNoEligiblePool = errors.New("no eligible execution pool")
|
||||
|
||||
type SelectionCandidate struct {
|
||||
Pool ExecutionPool
|
||||
Health RouteHealth
|
||||
Capacity CapacitySnapshot
|
||||
CapabilityMatched bool
|
||||
BetterWindows int
|
||||
}
|
||||
|
||||
type SelectionRequest struct {
|
||||
Candidates []SelectionCandidate
|
||||
CurrentPoolID string
|
||||
CurrentPoolSince time.Time
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type Decision struct {
|
||||
PoolID string
|
||||
Score float64
|
||||
Reason string
|
||||
Components map[string]float64
|
||||
Rejected map[string]string
|
||||
}
|
||||
|
||||
type Selector struct {
|
||||
HealthMaxAge time.Duration
|
||||
MinimumDwell time.Duration
|
||||
SwitchImprovement float64
|
||||
BetterWindows int
|
||||
}
|
||||
|
||||
func NewSelector() Selector {
|
||||
return Selector{
|
||||
HealthMaxAge: defaultHealthMaxAge, MinimumDwell: defaultMinimumDwell,
|
||||
SwitchImprovement: defaultSwitchImprovement, BetterWindows: defaultBetterWindowCount,
|
||||
}
|
||||
}
|
||||
|
||||
type scoredCandidate struct {
|
||||
candidate SelectionCandidate
|
||||
score float64
|
||||
components map[string]float64
|
||||
}
|
||||
|
||||
func (s Selector) Select(request SelectionRequest) (Decision, error) {
|
||||
now := request.Now
|
||||
if now.IsZero() {
|
||||
now = time.Now()
|
||||
}
|
||||
if s.HealthMaxAge <= 0 {
|
||||
s.HealthMaxAge = defaultHealthMaxAge
|
||||
}
|
||||
if s.MinimumDwell <= 0 {
|
||||
s.MinimumDwell = defaultMinimumDwell
|
||||
}
|
||||
if s.SwitchImprovement <= 0 {
|
||||
s.SwitchImprovement = defaultSwitchImprovement
|
||||
}
|
||||
if s.BetterWindows <= 0 {
|
||||
s.BetterWindows = defaultBetterWindowCount
|
||||
}
|
||||
rejected := make(map[string]string)
|
||||
scored := make([]scoredCandidate, 0, len(request.Candidates))
|
||||
for _, candidate := range request.Candidates {
|
||||
if reason := rejectionReason(candidate, now, s.HealthMaxAge); reason != "" {
|
||||
rejected[candidate.Pool.ID] = reason
|
||||
continue
|
||||
}
|
||||
network := routeQuality(candidate.Health)
|
||||
queue := queueQuality(candidate.Capacity)
|
||||
resources := clamp01(candidate.Capacity.ResourceHeadroom)
|
||||
stability := clamp01(candidate.Capacity.Stability)
|
||||
components := map[string]float64{
|
||||
"network": network, "queue": queue, "resources": resources, "stability": stability,
|
||||
}
|
||||
scored = append(scored, scoredCandidate{
|
||||
candidate: candidate,
|
||||
score: 0.55*network + 0.25*queue + 0.15*resources + 0.05*stability,
|
||||
components: components,
|
||||
})
|
||||
}
|
||||
if len(scored) == 0 {
|
||||
return Decision{Rejected: rejected}, ErrNoEligiblePool
|
||||
}
|
||||
sort.SliceStable(scored, func(i, j int) bool {
|
||||
if math.Abs(scored[i].score-scored[j].score) > 0.000001 {
|
||||
return scored[i].score > scored[j].score
|
||||
}
|
||||
return scored[i].candidate.Pool.ID < scored[j].candidate.Pool.ID
|
||||
})
|
||||
selected := scored[0]
|
||||
if currentIndex := indexOfPool(scored, strings.TrimSpace(request.CurrentPoolID)); currentIndex >= 0 && currentIndex != 0 {
|
||||
current := scored[currentIndex]
|
||||
withinDwell := !request.CurrentPoolSince.IsZero() && now.Sub(request.CurrentPoolSince) < s.MinimumDwell
|
||||
improvement := relativeImprovement(selected.score, current.score)
|
||||
if withinDwell || improvement < s.SwitchImprovement || selected.candidate.BetterWindows < s.BetterWindows {
|
||||
selected = current
|
||||
}
|
||||
}
|
||||
return Decision{
|
||||
PoolID: selected.candidate.Pool.ID,
|
||||
Score: selected.score,
|
||||
Reason: "network_quality_then_safe_capacity",
|
||||
Components: selected.components,
|
||||
Rejected: rejected,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func rejectionReason(candidate SelectionCandidate, now time.Time, maxAge time.Duration) string {
|
||||
if strings.TrimSpace(candidate.Pool.ID) == "" {
|
||||
return "invalid_pool"
|
||||
}
|
||||
if candidate.Pool.State != PoolActive {
|
||||
return "pool_not_active"
|
||||
}
|
||||
if !candidate.CapabilityMatched {
|
||||
return "capability_mismatch"
|
||||
}
|
||||
if candidate.Health.State == RouteUnreachable {
|
||||
return "route_unreachable"
|
||||
}
|
||||
if candidate.Health.State == RouteUnknown || candidate.Health.State == "" {
|
||||
return "route_unknown"
|
||||
}
|
||||
if candidate.Health.SampledAt.IsZero() || now.Sub(candidate.Health.SampledAt) > maxAge || (!candidate.Health.ExpiresAt.IsZero() && !now.Before(candidate.Health.ExpiresAt)) {
|
||||
return "route_stale"
|
||||
}
|
||||
if candidate.Capacity.Critical {
|
||||
return "resource_critical"
|
||||
}
|
||||
if candidate.Capacity.WorkerCount < 1 {
|
||||
return "no_worker"
|
||||
}
|
||||
if candidate.Capacity.SafeCapacity-candidate.Capacity.ActiveTasks < 1 {
|
||||
return "capacity_exhausted"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func routeQuality(health RouteHealth) float64 {
|
||||
success := clamp01(health.SuccessRate)
|
||||
connect := latencyQuality(health.ConnectTLSP95, 100*time.Millisecond)
|
||||
firstByte := latencyQuality(health.FirstByteP95, 250*time.Millisecond)
|
||||
throughput := clamp01(health.UploadBytesPerSecond / (10 * 1024 * 1024))
|
||||
jitter := latencyQuality(health.JitterP95, 100*time.Millisecond)
|
||||
return 0.35*success + 0.25*connect + 0.25*firstByte + 0.10*throughput + 0.05*jitter
|
||||
}
|
||||
|
||||
func queueQuality(capacity CapacitySnapshot) float64 {
|
||||
available := capacity.SafeCapacity - capacity.ActiveTasks
|
||||
if available <= 0 {
|
||||
return 0
|
||||
}
|
||||
headroom := float64(available) / float64(max(capacity.SafeCapacity, 1))
|
||||
waitPenalty := 1 / (1 + capacity.EstimatedQueueWait.Seconds())
|
||||
return clamp01(0.6*headroom + 0.4*waitPenalty)
|
||||
}
|
||||
|
||||
func latencyQuality(value, target time.Duration) float64 {
|
||||
if value <= 0 {
|
||||
return 0
|
||||
}
|
||||
return clamp01(1 / (1 + float64(value)/float64(target)))
|
||||
}
|
||||
|
||||
func relativeImprovement(candidate, current float64) float64 {
|
||||
if current <= 0 {
|
||||
return 1
|
||||
}
|
||||
return (candidate - current) / current
|
||||
}
|
||||
|
||||
func indexOfPool(candidates []scoredCandidate, poolID string) int {
|
||||
for index := range candidates {
|
||||
if candidates[index].candidate.Pool.ID == poolID {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func clamp01(value float64) float64 {
|
||||
return math.Max(0, math.Min(1, value))
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package executionpool
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSelectorRejectsUnreachableAndPrefersNetwork(t *testing.T) {
|
||||
now := time.Now()
|
||||
selector := NewSelector()
|
||||
decision, err := selector.Select(SelectionRequest{Now: now, Candidates: []SelectionCandidate{
|
||||
candidate("slow", RouteHealthy, now, 120*time.Millisecond, 8, 1),
|
||||
candidate("fast", RouteHealthy, now, 20*time.Millisecond, 8, 1),
|
||||
candidate("down", RouteUnreachable, now, time.Millisecond, 32, 0),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision.PoolID != "fast" {
|
||||
t.Fatalf("pool=%s, want fast", decision.PoolID)
|
||||
}
|
||||
if decision.Rejected["down"] != "route_unreachable" {
|
||||
t.Fatalf("down rejection=%q", decision.Rejected["down"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectorUsesCapacityAsHardGate(t *testing.T) {
|
||||
now := time.Now()
|
||||
selector := NewSelector()
|
||||
decision, err := selector.Select(SelectionRequest{Now: now, Candidates: []SelectionCandidate{
|
||||
candidate("fast-full", RouteHealthy, now, 10*time.Millisecond, 2, 2),
|
||||
candidate("slower-free", RouteHealthy, now, 80*time.Millisecond, 8, 1),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision.PoolID != "slower-free" {
|
||||
t.Fatalf("pool=%s, want slower-free", decision.PoolID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectorRejectsUnknownAndStale(t *testing.T) {
|
||||
now := time.Now()
|
||||
selector := NewSelector()
|
||||
_, err := selector.Select(SelectionRequest{Now: now, Candidates: []SelectionCandidate{
|
||||
candidate("unknown", RouteUnknown, now, time.Millisecond, 8, 0),
|
||||
candidate("stale", RouteHealthy, now.Add(-time.Minute), time.Millisecond, 8, 0),
|
||||
}})
|
||||
if !errors.Is(err, ErrNoEligiblePool) {
|
||||
t.Fatalf("err=%v, want ErrNoEligiblePool", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectorHonorsHysteresis(t *testing.T) {
|
||||
now := time.Now()
|
||||
selector := NewSelector()
|
||||
fast := candidate("fast", RouteHealthy, now, 20*time.Millisecond, 8, 1)
|
||||
fast.BetterWindows = 3
|
||||
current := candidate("current", RouteHealthy, now, 80*time.Millisecond, 8, 1)
|
||||
decision, err := selector.Select(SelectionRequest{
|
||||
Now: now, Candidates: []SelectionCandidate{fast, current},
|
||||
CurrentPoolID: "current", CurrentPoolSince: now.Add(-time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision.PoolID != "current" {
|
||||
t.Fatalf("pool=%s, want current during dwell", decision.PoolID)
|
||||
}
|
||||
}
|
||||
|
||||
func candidate(id string, state RouteState, sampled time.Time, latency time.Duration, capacity, active int) SelectionCandidate {
|
||||
return SelectionCandidate{
|
||||
Pool: ExecutionPool{ID: id, State: PoolActive}, CapabilityMatched: true,
|
||||
Health: RouteHealth{
|
||||
PoolID: id, State: state, SuccessRate: 1, ConnectTLSP95: latency,
|
||||
FirstByteP95: latency, UploadBytesPerSecond: 10 * 1024 * 1024,
|
||||
JitterP95: latency / 10, SampledAt: sampled, ExpiresAt: sampled.Add(45 * time.Second),
|
||||
},
|
||||
Capacity: CapacitySnapshot{
|
||||
PoolID: id, WorkerCount: 1, SafeCapacity: capacity, ActiveTasks: active,
|
||||
ResourceHeadroom: 0.8, Stability: 1,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package executionpool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
const ProtocolVersion = "v1"
|
||||
|
||||
type PoolState string
|
||||
|
||||
const (
|
||||
PoolActive PoolState = "active"
|
||||
PoolDraining PoolState = "draining"
|
||||
PoolDisabled PoolState = "disabled"
|
||||
)
|
||||
|
||||
type RouteState string
|
||||
|
||||
const (
|
||||
RouteHealthy RouteState = "healthy"
|
||||
RouteDegraded RouteState = "degraded"
|
||||
RouteUnreachable RouteState = "unreachable"
|
||||
RouteUnknown RouteState = "unknown"
|
||||
)
|
||||
|
||||
type ExecutionPool struct {
|
||||
ID string
|
||||
Labels map[string]string
|
||||
Capabilities map[string]any
|
||||
State PoolState
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type WorkerDescriptor struct {
|
||||
WorkerID string
|
||||
InstanceID string
|
||||
PoolID string
|
||||
Endpoint string
|
||||
ProtocolVersion string
|
||||
Revision string
|
||||
Capabilities map[string]any
|
||||
Allocated int
|
||||
SafeCapacity int
|
||||
HeavyCapacity int
|
||||
ActiveTasks int
|
||||
PressureState string
|
||||
HeartbeatAt time.Time
|
||||
LoadSampledAt time.Time
|
||||
}
|
||||
|
||||
func (w WorkerDescriptor) AvailableCapacity() int {
|
||||
limit := w.Allocated
|
||||
if w.SafeCapacity < limit {
|
||||
limit = w.SafeCapacity
|
||||
}
|
||||
if limit < 0 {
|
||||
return 0
|
||||
}
|
||||
available := limit - w.ActiveTasks
|
||||
if available < 0 {
|
||||
return 0
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
type RouteProfile struct {
|
||||
Key string
|
||||
Provider string
|
||||
Protocol string
|
||||
EndpointHost string
|
||||
ProxyMode string
|
||||
ConfigRevision string
|
||||
}
|
||||
|
||||
type RouteHealth struct {
|
||||
PoolID string
|
||||
RouteProfileKey string
|
||||
State RouteState
|
||||
SuccessRate float64
|
||||
ConnectTLSP95 time.Duration
|
||||
FirstByteP95 time.Duration
|
||||
UploadBytesPerSecond float64
|
||||
JitterP95 time.Duration
|
||||
ConsecutiveFailures int
|
||||
ConsecutiveSuccesses int
|
||||
SampleCount int
|
||||
SampledAt time.Time
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// RouteObservation contains only transport-phase facts. Model generation time
|
||||
// is deliberately excluded from this structure and therefore from routing
|
||||
// quality updates.
|
||||
type RouteObservation struct {
|
||||
PoolID string
|
||||
RouteProfileKey string
|
||||
SampleCount int
|
||||
SuccessCount int
|
||||
ConnectTLSP95 time.Duration
|
||||
UploadBytesPerSecond float64
|
||||
}
|
||||
|
||||
type CapacitySnapshot struct {
|
||||
PoolID string
|
||||
WorkerCount int
|
||||
Allocated int
|
||||
SafeCapacity int
|
||||
ActiveTasks int
|
||||
HeavyCapacity int
|
||||
HeavyTasks int
|
||||
QueueDepth int
|
||||
EstimatedQueueWait time.Duration
|
||||
ResourceHeadroom float64
|
||||
Stability float64
|
||||
Critical bool
|
||||
SampledAt time.Time
|
||||
}
|
||||
|
||||
type DesiredCapacity struct {
|
||||
PoolID string
|
||||
Desired int
|
||||
Reason string
|
||||
ValidUntil time.Time
|
||||
}
|
||||
|
||||
// WorkerDirectory is the platform-neutral discovery boundary. Implementations
|
||||
// may use PostgreSQL, Consul, etcd, or another registry.
|
||||
type WorkerDirectory interface {
|
||||
ListWorkers(context.Context, time.Time) ([]WorkerDescriptor, error)
|
||||
}
|
||||
|
||||
// ExecutionBroker hides the durable queue implementation from routing code.
|
||||
type ExecutionBroker interface {
|
||||
Publish(context.Context, string, string, time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// ExecutionTransport hides HTTP, gRPC, or service-mesh transport details.
|
||||
type ExecutionTransport interface {
|
||||
Execute(context.Context, WorkerDescriptor, ExecutionRequest) (ExecutionResponse, error)
|
||||
}
|
||||
|
||||
type ExecutionRequest struct {
|
||||
TaskID string
|
||||
PoolID string
|
||||
WorkerID string
|
||||
LeaseID string
|
||||
AuthorizationToken string
|
||||
Deadline time.Time
|
||||
Stream bool
|
||||
}
|
||||
|
||||
type ExecutionResponse struct {
|
||||
StatusCode int
|
||||
Headers map[string][]string
|
||||
Body io.ReadCloser
|
||||
}
|
||||
|
||||
type RouteHealthRepository interface {
|
||||
ListRouteHealth(context.Context, string, time.Time) ([]RouteHealth, error)
|
||||
RecordRouteHealth(context.Context, RouteHealth) error
|
||||
RecordRouteObservation(context.Context, RouteObservation) error
|
||||
}
|
||||
|
||||
type CapacityProvider interface {
|
||||
ListCapacity(context.Context, time.Time) ([]CapacitySnapshot, error)
|
||||
PublishDesiredCapacity(context.Context, DesiredCapacity) error
|
||||
}
|
||||
Reference in New Issue
Block a user