feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -25,6 +25,15 @@ func (m metricsSnapshot) PostgresPoolMetrics() store.PostgresPoolMetricsSnapshot
|
||||
return m.pool
|
||||
}
|
||||
|
||||
func (m metricsSnapshot) ListModelRateLimitStatuses(context.Context) ([]store.ModelRateLimitStatus, error) {
|
||||
return []store.ModelRateLimitStatus{{
|
||||
PlatformModelID: "platform-model-1",
|
||||
RPM: store.RateLimitMetricStatus{Ratio: .25},
|
||||
TPM: store.RateLimitMetricStatus{Ratio: .5},
|
||||
Concurrent: store.RateLimitMetricStatus{Ratio: .75},
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
||||
metrics := &Metrics{}
|
||||
metrics.ObserveReceipt("accepted", 20*time.Millisecond, 2)
|
||||
@@ -34,6 +43,9 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
||||
metrics.ObserveIntrospection("failed")
|
||||
metrics.ObserveJWKSRefreshFailure("ssf")
|
||||
metrics.SetAsyncWorkerCapacity(96, 128, 96, true)
|
||||
metrics.SetWorkerLoad(12, 3, 8, 2, 5, 1, "busy")
|
||||
metrics.ObserveProviderQuotaWait("rpm")
|
||||
metrics.ObserveProviderQuotaWait("tpm_total")
|
||||
metrics.ObserveAsyncWorkerResize("success")
|
||||
metrics.ObserveConcurrencyLeaseRenewal("success")
|
||||
metrics.ObserveConcurrencyLeaseRenewal("lost")
|
||||
@@ -65,6 +77,16 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
||||
`easyai_gateway_async_worker_capacity 96`,
|
||||
`easyai_gateway_async_worker_desired_capacity 128`,
|
||||
`easyai_gateway_async_worker_capacity_capped 1`,
|
||||
`easyai_gateway_worker_safe_capacity 12`,
|
||||
`easyai_gateway_worker_heavy_capacity 3`,
|
||||
`easyai_gateway_worker_active_tasks 8`,
|
||||
`easyai_gateway_worker_preparing_tasks 2`,
|
||||
`easyai_gateway_worker_waiting_upstream_tasks 5`,
|
||||
`easyai_gateway_worker_finalizing_tasks 1`,
|
||||
`easyai_gateway_worker_pressure_state 1`,
|
||||
`easyai_gateway_provider_quota_waits_total{outcome="rpm"} 1`,
|
||||
`easyai_gateway_provider_quota_waits_total{outcome="tpm"} 1`,
|
||||
`easyai_gateway_platform_model_rate_limit_utilization{platform_model_id="platform-model-1",metric="concurrent"} 0.750000`,
|
||||
`easyai_gateway_async_worker_resizes_total{outcome="success"} 1`,
|
||||
`easyai_gateway_concurrency_lease_renewals_total{outcome="success"} 1`,
|
||||
`easyai_gateway_concurrency_lease_renewals_total{outcome="lost"} 1`,
|
||||
|
||||
Reference in New Issue
Block a user