feat(routing): 完善平台满载避让与故障轮转
为未配置并发上限的平台增加基于运行和等待任务数的软负载,并按非满载、有效优先级、缓存亲和与负载稳定排序。\n\n平台模型 RPM、TPM 和并发额度竞争失败时只轮转候选,不触发冷却、禁用或降级;用户组额度保持不可绕过。补齐异步冷却排队恢复、满载原因、选择原因与低基数指标。\n\n验证:go test ./...;go vet ./...;PostgreSQL 原子额度、准入队列、Worker 容量回收及故障策略 HTTP acceptance。
This commit is contained in:
@@ -699,7 +699,14 @@ func applyRuntimeCandidateLoad(candidate *RuntimeModelCandidate, input runtimeCa
|
||||
concurrentLimit := rateLimitForMetric(input.Policy, "concurrent")
|
||||
rpmCurrent := input.RPMUsed + input.RPMReserved
|
||||
tpmCurrent := input.TPMUsed + input.TPMReserved
|
||||
concurrentCurrent := input.ConcurrentActive + input.QueuedWaiting
|
||||
activeCurrent := maxFloat(input.ConcurrentActive, input.StateRunningCount)
|
||||
waitingCurrent := maxFloat(input.QueuedWaiting, input.StateWaitingCount)
|
||||
concurrentCurrent := activeCurrent + waitingCurrent
|
||||
softRatio := unboundedLoadRatio(concurrentCurrent)
|
||||
concurrentRatio := softRatio
|
||||
if concurrentLimit > 0 {
|
||||
concurrentRatio = ratioIfLimited(concurrentCurrent, concurrentLimit)
|
||||
}
|
||||
metrics := RuntimeCandidateLoadMetrics{
|
||||
RPMCurrent: rpmCurrent,
|
||||
RPMLimit: rpmLimit,
|
||||
@@ -709,15 +716,20 @@ func applyRuntimeCandidateLoad(candidate *RuntimeModelCandidate, input runtimeCa
|
||||
TPMRatio: ratioIfLimited(tpmCurrent, tpmLimitValue),
|
||||
ConcurrentCurrent: concurrentCurrent,
|
||||
ConcurrentLimit: concurrentLimit,
|
||||
ConcurrentRatio: ratioIfLimited(concurrentCurrent, concurrentLimit),
|
||||
QueuedCount: input.QueuedWaiting,
|
||||
ConcurrentRatio: concurrentRatio,
|
||||
SoftCurrent: concurrentCurrent,
|
||||
SoftRatio: softRatio,
|
||||
QueuedCount: waitingCurrent,
|
||||
StateRunningCount: input.StateRunningCount,
|
||||
StateWaitingCount: input.StateWaitingCount,
|
||||
StateLimiterRatio: input.StateLimiterRatio,
|
||||
}
|
||||
candidate.RunningCount = activeCurrent
|
||||
candidate.WaitingCount = waitingCurrent
|
||||
candidate.LoadMetrics = metrics
|
||||
candidate.LoadLimited = rpmLimit > 0 || tpmLimitValue > 0 || concurrentLimit > 0
|
||||
candidate.LoadRatio = maxFloat(metrics.RPMRatio, metrics.TPMRatio, metrics.ConcurrentRatio)
|
||||
candidate.FullReasons = runtimeCandidateFullReasons(*candidate)
|
||||
}
|
||||
|
||||
func ratioIfLimited(current float64, limit float64) float64 {
|
||||
@@ -727,11 +739,20 @@ func ratioIfLimited(current float64, limit float64) float64 {
|
||||
return current / limit
|
||||
}
|
||||
|
||||
func unboundedLoadRatio(current float64) float64 {
|
||||
if current <= 0 {
|
||||
return 0
|
||||
}
|
||||
return current / (current + 1)
|
||||
}
|
||||
|
||||
func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
|
||||
hasFull := false
|
||||
hasNonFull := false
|
||||
for index := range items {
|
||||
items[index].LoadAvoided = false
|
||||
items[index].SelectionReason = "normal_rotation"
|
||||
items[index].FullReasons = runtimeCandidateFullReasons(items[index])
|
||||
if !items[index].CacheAffinity.Applied && items[index].CacheAffinity.AdjustedPriority == 0 {
|
||||
items[index].CacheAffinity.AdjustedPriority = float64(items[index].PlatformPriority)
|
||||
}
|
||||
@@ -744,6 +765,13 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
|
||||
if hasFull && hasNonFull {
|
||||
for index := range items {
|
||||
items[index].LoadAvoided = runtimeCandidateFull(items[index])
|
||||
if items[index].LoadAvoided {
|
||||
items[index].SelectionReason = "full_avoided"
|
||||
}
|
||||
}
|
||||
} else if hasFull {
|
||||
for index := range items {
|
||||
items[index].SelectionReason = "all_full_fallback"
|
||||
}
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
@@ -778,16 +806,16 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
|
||||
if items[i].LoadRatio != items[j].LoadRatio {
|
||||
return items[i].LoadRatio < items[j].LoadRatio
|
||||
}
|
||||
if items[i].RunningCount != items[j].RunningCount {
|
||||
return items[i].RunningCount < items[j].RunningCount
|
||||
}
|
||||
if items[i].WaitingCount != items[j].WaitingCount {
|
||||
return items[i].WaitingCount < items[j].WaitingCount
|
||||
}
|
||||
if items[i].RunningCount != items[j].RunningCount {
|
||||
return items[i].RunningCount < items[j].RunningCount
|
||||
}
|
||||
if items[i].LastAssignedUnix != items[j].LastAssignedUnix {
|
||||
return items[i].LastAssignedUnix < items[j].LastAssignedUnix
|
||||
}
|
||||
return false
|
||||
return items[i].PlatformModelID < items[j].PlatformModelID
|
||||
})
|
||||
appliedCount := 0
|
||||
for index := range items {
|
||||
@@ -818,7 +846,31 @@ func sortRuntimeModelCandidates(items []RuntimeModelCandidate) {
|
||||
}
|
||||
|
||||
func runtimeCandidateFull(candidate RuntimeModelCandidate) bool {
|
||||
return candidate.LoadLimited && candidate.LoadRatio >= 1
|
||||
return len(runtimeCandidateFullReasons(candidate)) > 0
|
||||
}
|
||||
|
||||
func runtimeCandidateFullReasons(candidate RuntimeModelCandidate) []string {
|
||||
if len(candidate.FullReasons) > 0 {
|
||||
return append([]string(nil), candidate.FullReasons...)
|
||||
}
|
||||
metrics := candidate.LoadMetrics
|
||||
reasons := make([]string, 0, 4)
|
||||
if metrics.ConcurrentLimit > 0 && metrics.ConcurrentRatio >= 1 {
|
||||
reasons = append(reasons, "concurrent")
|
||||
}
|
||||
if metrics.RPMLimit > 0 && metrics.RPMRatio >= 1 {
|
||||
reasons = append(reasons, "rpm")
|
||||
}
|
||||
if metrics.TPMLimit > 0 && metrics.TPMRatio >= 1 {
|
||||
reasons = append(reasons, "tpm")
|
||||
}
|
||||
if metrics.QueuedCount > 0 {
|
||||
reasons = append(reasons, "waiting")
|
||||
}
|
||||
if len(reasons) == 0 && candidate.LoadLimited && candidate.LoadRatio >= 1 {
|
||||
reasons = append(reasons, "limited")
|
||||
}
|
||||
return reasons
|
||||
}
|
||||
|
||||
func (s *Store) modelCandidateCooldownError(ctx context.Context, model string, modelType string) (error, error) {
|
||||
|
||||
Reference in New Issue
Block a user