BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(routing): 完善平台满载避让与故障轮转

Browse Source 
为未配置并发上限的平台增加基于运行和等待任务数的软负载，并按非满载、有效优先级、缓存亲和与负载稳定排序。\n\n平台模型 RPM、TPM 和并发额度竞争失败时只轮转候选，不触发冷却、禁用或降级；用户组额度保持不可绕过。补齐异步冷却排队恢复、满载原因、选择原因与低基数指标。\n\n验证：go test ./...；go vet ./...；PostgreSQL 原子额度、准入队列、Worker 容量回收及故障策略 HTTP acceptance。
This commit is contained in:
wangbo
2026-08-03 17:42:34 +08:00
parent b125599354
commit 8362d6d27a
15 changed files with 468 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/store/candidates.go
+60 -8
View File
@@ -699,7 +699,14 @@ func applyRuntimeCandidateLoad(candidate *RuntimeModelCandidate, input runtimeCa
concurrentLimit := rateLimitForMetric(input.Policy, "concurrent")
rpmCurrent := input.RPMUsed + input.RPMReserved
tpmCurrent := input.TPMUsed + input.TPMReserved
concurrentCurrent := input.ConcurrentActive + input.QueuedWaiting
activeCurrent := maxFloat(input.ConcurrentActive, input.StateRunningCount)
waitingCurrent := maxFloat(input.QueuedWaiting, input.StateWaitingCount)
concurrentCurrent := activeCurrent + waitingCurrent
softRatio := unboundedLoadRatio(concurrentCurrent)
concurrentRatio := softRatio
if concurrentLimit > 0 {
concurrentRatio = ratioIfLimited(concurrentCurrent, concurrentLimit)
}
metrics := RuntimeCandidateLoadMetrics{
RPMCurrent: rpmCurrent,
RPMLimit: rpmLimit,
@@ -709,15 +716,20 @@ func applyRuntimeCandidateLoad(candidate *RuntimeModelCandidate, input runtimeCa
TPMRatio: ratioIfLimited(tpmCurrent, tpmLimitValue),
ConcurrentCurrent: concurrentCurrent,
ConcurrentLimit: concurrentLimit,
ConcurrentRatio: ratioIfLimited(concurrentCurrent, concurrentLimit),
QueuedCount: input.QueuedWaiting,
ConcurrentRatio: concurrentRatio,
SoftCurrent: concurrentCurrent,
SoftRatio: softRatio,
QueuedCount: waitingCurrent,
StateRunningCount: input.StateRunningCount,
StateWaitingCount: input.StateWaitingCount,
StateLimiterRatio: input.StateLimiterRatio,
}
candidate.RunningCount = activeCurrent
candidate.WaitingCount = waitingCurrent
candidate.LoadMetrics = metrics
candidate.LoadLimited = rpmLimit > 0 || tpmLimitValue > 0 || concurrentLimit > 0
candidate.LoadRatio = maxFloat(metrics.RPMRatio, metrics.TPMRatio, metrics.ConcurrentRatio)
candidate.FullReasons = runtimeCandidateFullReasons(*candidate)
}
func ratioIfLimited(current float64, limit float64) float64 {
@@ -727,11 +739,20 @@ func ratioIfLimited(current float64, limit float64) float64 {
return current / limit
}
func unboundedLoadRatio(current float64) float64 {
if current <= 0 {
return 0
}
return current / (current + 1)
}
func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
hasFull := false
hasNonFull := false
for index := range items {
items[index].LoadAvoided = false
items[index].SelectionReason = "normal_rotation"
items[index].FullReasons = runtimeCandidateFullReasons(items[index])
if !items[index].CacheAffinity.Applied && items[index].CacheAffinity.AdjustedPriority == 0 {
items[index].CacheAffinity.AdjustedPriority = float64(items[index].PlatformPriority)
}
@@ -744,6 +765,13 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
if hasFull && hasNonFull {
for index := range items {
items[index].LoadAvoided = runtimeCandidateFull(items[index])
if items[index].LoadAvoided {
items[index].SelectionReason = "full_avoided"
}
}
} else if hasFull {
for index := range items {
items[index].SelectionReason = "all_full_fallback"
}
}
sort.SliceStable(items, func(i, j int) bool {
@@ -778,16 +806,16 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
if items[i].LoadRatio != items[j].LoadRatio {
return items[i].LoadRatio < items[j].LoadRatio
}
if items[i].RunningCount != items[j].RunningCount {
return items[i].RunningCount < items[j].RunningCount
}
if items[i].WaitingCount != items[j].WaitingCount {
return items[i].WaitingCount < items[j].WaitingCount
}
if items[i].RunningCount != items[j].RunningCount {
return items[i].RunningCount < items[j].RunningCount
}
if items[i].LastAssignedUnix != items[j].LastAssignedUnix {
return items[i].LastAssignedUnix < items[j].LastAssignedUnix
}
return false
return items[i].PlatformModelID < items[j].PlatformModelID
})
appliedCount := 0
for index := range items {
@@ -818,7 +846,31 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
}
func runtimeCandidateFull(candidate RuntimeModelCandidate) bool {
return candidate.LoadLimited && candidate.LoadRatio >= 1
return len(runtimeCandidateFullReasons(candidate)) > 0
}
func runtimeCandidateFullReasons(candidate RuntimeModelCandidate) []string {
if len(candidate.FullReasons) > 0 {
return append([]string(nil), candidate.FullReasons...)
}
metrics := candidate.LoadMetrics
reasons := make([]string, 0, 4)
if metrics.ConcurrentLimit > 0 && metrics.ConcurrentRatio >= 1 {
reasons = append(reasons, "concurrent")
}
if metrics.RPMLimit > 0 && metrics.RPMRatio >= 1 {
reasons = append(reasons, "rpm")
}
if metrics.TPMLimit > 0 && metrics.TPMRatio >= 1 {
reasons = append(reasons, "tpm")
}
if metrics.QueuedCount > 0 {
reasons = append(reasons, "waiting")
}
if len(reasons) == 0 && candidate.LoadLimited && candidate.LoadRatio >= 1 {
reasons = append(reasons, "limited")
}
return reasons
}
func (s *Store) modelCandidateCooldownError(ctx context.Context, model string, modelType string) (error, error) {
apps/api/internal/store/candidates_test.go
+138
View File
@@ -86,6 +86,144 @@ func TestRuntimeCandidateLoadUsesMaxLimitedMetric(t *testing.T) {
}
}
func TestRuntimeCandidateLoadUsesSoftConcurrencyWithoutLimit(t *testing.T) {
tests := []struct {
name string
active float64
waiting float64
stateRun float64
stateWait float64
wantCurrent float64
wantRatio float64
}{
{name: "idle", wantCurrent: 0, wantRatio: 0},
{name: "one active", active: 1, wantCurrent: 1, wantRatio: 0.5},
{name: "two from runtime state", stateRun: 2, wantCurrent: 2, wantRatio: 2.0 / 3.0},
{name: "uses maximum per phase without double counting", active: 1, waiting: 2, stateRun: 3, stateWait: 1, wantCurrent: 5, wantRatio: 5.0 / 6.0},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
candidate := RuntimeModelCandidate{}
applyRuntimeCandidateLoad(&candidate, runtimeCandidateLoadInput{
ConcurrentActive: test.active,
QueuedWaiting: test.waiting,
StateRunningCount: test.stateRun,
StateWaitingCount: test.stateWait,
})
if candidate.LoadLimited {
t.Fatal("soft concurrency must not become a hard limit")
}
if candidate.LoadMetrics.SoftCurrent != test.wantCurrent {
t.Fatalf("soft current=%v, want %v", candidate.LoadMetrics.SoftCurrent, test.wantCurrent)
}
if diff := candidate.LoadRatio - test.wantRatio; diff < -1e-9 || diff > 1e-9 {
t.Fatalf("load ratio=%v, want %v", candidate.LoadRatio, test.wantRatio)
}
})
}
}
func TestRuntimeCandidateLoadUsesSoftConcurrencyAlongsideRPM(t *testing.T) {
candidate := RuntimeModelCandidate{}
applyRuntimeCandidateLoad(&candidate, runtimeCandidateLoadInput{
Policy: map[string]any{"rules": []any{map[string]any{"metric": "rpm", "limit": 100}}},
RPMUsed: 20,
StateRunningCount: 4,
})
if !candidate.LoadLimited {
t.Fatal("rpm policy should remain a hard limit")
}
if candidate.LoadMetrics.ConcurrentLimit != 0 || candidate.LoadMetrics.ConcurrentRatio != 0.8 {
t.Fatalf("unexpected soft concurrent metric: %+v", candidate.LoadMetrics)
}
if candidate.LoadRatio != 0.8 {
t.Fatalf("load ratio=%v, want 0.8", candidate.LoadRatio)
}
}
func TestRuntimeCandidateWaitingMarksCandidateFull(t *testing.T) {
candidates := []RuntimeModelCandidate{
{
PlatformID: "higher-priority-waiting",
PlatformModelID: "model-a",
PlatformPriority: 10,
LoadRatio: 0.5,
LoadMetrics: RuntimeCandidateLoadMetrics{QueuedCount: 1},
},
{
PlatformID: "lower-priority-idle",
PlatformModelID: "model-b",
PlatformPriority: 20,
},
}
sortRuntimeModelCandidates(candidates)
if candidates[0].PlatformID != "lower-priority-idle" {
t.Fatalf("non-full lower-priority candidate should receive overflow: %+v", candidates)
}
if !candidates[1].LoadAvoided || !containsString(candidates[1].FullReasons, "waiting") {
t.Fatalf("waiting candidate should be marked full and avoided: %+v", candidates[1])
}
}
func TestRuntimeCandidateSortingUsesStableIDAfterLoadTies(t *testing.T) {
candidates := []RuntimeModelCandidate{
{PlatformID: "second", PlatformModelID: "model-b", PlatformPriority: 10},
{PlatformID: "first", PlatformModelID: "model-a", PlatformPriority: 10},
}
sortRuntimeModelCandidates(candidates)
if candidates[0].PlatformModelID != "model-a" {
t.Fatalf("stable candidate order=%+v", candidates)
}
}
func TestRuntimeCandidateSortingKeepsDeterministicFallbackWhenAllFull(t *testing.T) {
candidates := []RuntimeModelCandidate{
{
PlatformID: "second",
PlatformModelID: "model-b",
PlatformPriority: 10,
LoadMetrics: RuntimeCandidateLoadMetrics{
ConcurrentLimit: 1,
ConcurrentRatio: 1,
},
},
{
PlatformID: "first",
PlatformModelID: "model-a",
PlatformPriority: 10,
LoadMetrics: RuntimeCandidateLoadMetrics{
ConcurrentLimit: 1,
ConcurrentRatio: 1,
},
},
}
sortRuntimeModelCandidates(candidates)
if candidates[0].PlatformModelID != "model-a" || candidates[0].LoadAvoided || candidates[1].LoadAvoided {
t.Fatalf("all-full fallback should be stable without pretending a candidate was avoidable: %+v", candidates)
}
for _, candidate := range candidates {
if candidate.SelectionReason != "all_full_fallback" {
t.Fatalf("all-full candidate reason=%q, want all_full_fallback", candidate.SelectionReason)
}
}
}
func containsString(values []string, expected string) bool {
for _, value := range values {
if value == expected {
return true
}
}
return false
}
func TestRuntimeCandidateSortingAvoidsFullCandidatesButKeepsFallback(t *testing.T) {
candidates := []RuntimeModelCandidate{
{
apps/api/internal/store/runtime_policies.go
+22 -16
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"encoding/json"
"errors"
"strings"
"time"
@@ -200,25 +201,30 @@ WHERE id = $1::uuid`, platformModelID, cooldownSeconds)
return err
}
func (s *Store) RuntimeCandidateAvailable(ctx context.Context, platformID string, platformModelID string) (bool, error) {
func (s *Store) RuntimeCandidateAvailability(ctx context.Context, platformID string, platformModelID string) (bool, string, error) {
if strings.TrimSpace(platformID) == "" || strings.TrimSpace(platformModelID) == "" {
return false, nil
return false, "disabled_skipped", nil
}
var available bool
var reason string
err := s.pool.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1
FROM platform_models model
JOIN integration_platforms platform ON platform.id = model.platform_id
WHERE platform.id = $1::uuid
AND model.id = $2::uuid
AND platform.status = 'enabled'
AND platform.deleted_at IS NULL
AND (platform.cooldown_until IS NULL OR platform.cooldown_until <= now())
AND model.enabled = true
AND (model.cooldown_until IS NULL OR model.cooldown_until <= now())
)`, platformID, platformModelID).Scan(&available)
return available, err
SELECT CASE
WHEN platform.deleted_at IS NOT NULL OR platform.status <> 'enabled' OR model.enabled = false
THEN 'disabled_skipped'
WHEN platform.cooldown_until > now() OR model.cooldown_until > now()
THEN 'cooldown_skipped'
ELSE ''
END
FROM platform_models model
JOIN integration_platforms platform ON platform.id = model.platform_id
WHERE platform.id = $1::uuid
AND model.id = $2::uuid`, platformID, platformModelID).Scan(&reason)
if errors.Is(err, pgx.ErrNoRows) {
return false, "disabled_skipped", nil
}
if err != nil {
return false, "", err
}
return reason == "", reason, nil
}
func (s *Store) ApplyCandidateFailureEffect(ctx context.Context, input CandidateFailureEffectInput) (CandidateFailureEffectResult, error) {
apps/api/internal/store/runtime_types.go
+4
View File
@@ -194,6 +194,8 @@ type RuntimeModelCandidate struct {
LoadRatio float64
LoadLimited bool
LoadAvoided bool
FullReasons []string
SelectionReason string
LoadMetrics RuntimeCandidateLoadMetrics
CacheAffinity RuntimeCandidateCacheAffinity
RunningCount float64
@@ -230,6 +232,8 @@ type RuntimeCandidateLoadMetrics struct {
ConcurrentCurrent float64
ConcurrentLimit float64
ConcurrentRatio float64
SoftCurrent float64
SoftRatio float64
QueuedCount float64
StateRunningCount float64
StateWaitingCount float64