feat(routing): 完善平台满载避让与故障轮转
为未配置并发上限的平台增加基于运行和等待任务数的软负载,并按非满载、有效优先级、缓存亲和与负载稳定排序。\n\n平台模型 RPM、TPM 和并发额度竞争失败时只轮转候选,不触发冷却、禁用或降级;用户组额度保持不可绕过。补齐异步冷却排队恢复、满载原因、选择原因与低基数指标。\n\n验证:go test ./...;go vet ./...;PostgreSQL 原子额度、准入队列、Worker 容量回收及故障策略 HTTP acceptance。
This commit is contained in:
@@ -79,6 +79,13 @@ type Metrics struct {
|
||||
taskAdmissionCancelled atomic.Uint64
|
||||
taskAdmissionExpired atomic.Uint64
|
||||
taskAdmissionMigrated atomic.Uint64
|
||||
candidateNormalRotation atomic.Uint64
|
||||
candidateFullAvoided atomic.Uint64
|
||||
candidateQuotaRaceRotated atomic.Uint64
|
||||
candidateCooldownSkipped atomic.Uint64
|
||||
candidateDisabledSkipped atomic.Uint64
|
||||
candidateAllFullQueued atomic.Uint64
|
||||
candidateRoutingOther atomic.Uint64
|
||||
taskAdmissionWaitBuckets [11]atomic.Uint64
|
||||
taskAdmissionWaitMicros atomic.Uint64
|
||||
}
|
||||
@@ -239,6 +246,25 @@ func (m *Metrics) ObserveTaskAdmission(event string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveCandidateRouting(reason string) {
|
||||
switch reason {
|
||||
case "normal_rotation":
|
||||
m.candidateNormalRotation.Add(1)
|
||||
case "full_avoided":
|
||||
m.candidateFullAvoided.Add(1)
|
||||
case "quota_race_rotated":
|
||||
m.candidateQuotaRaceRotated.Add(1)
|
||||
case "cooldown_skipped":
|
||||
m.candidateCooldownSkipped.Add(1)
|
||||
case "disabled_skipped":
|
||||
m.candidateDisabledSkipped.Add(1)
|
||||
case "all_full_queued":
|
||||
m.candidateAllFullQueued.Add(1)
|
||||
default:
|
||||
m.candidateRoutingOther.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveTaskAdmissionWait(wait time.Duration) {
|
||||
if wait < 0 {
|
||||
wait = 0
|
||||
@@ -436,6 +462,15 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
{"concurrent", m.providerQuotaWaitConcurrent.Load()},
|
||||
{"other", m.providerQuotaWaitOther.Load()},
|
||||
})
|
||||
outcomeCounters(w, "easyai_gateway_candidate_routing_total", "Candidate routing decisions by bounded reason.", []outcomeValue{
|
||||
{"normal_rotation", m.candidateNormalRotation.Load()},
|
||||
{"full_avoided", m.candidateFullAvoided.Load()},
|
||||
{"quota_race_rotated", m.candidateQuotaRaceRotated.Load()},
|
||||
{"cooldown_skipped", m.candidateCooldownSkipped.Load()},
|
||||
{"disabled_skipped", m.candidateDisabledSkipped.Load()},
|
||||
{"all_full_queued", m.candidateAllFullQueued.Load()},
|
||||
{"other", m.candidateRoutingOther.Load()},
|
||||
})
|
||||
platformModelRateLimitUtilizationGauges(w, modelRateLimits)
|
||||
plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections))
|
||||
plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections))
|
||||
|
||||
Reference in New Issue
Block a user