feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -22,53 +23,64 @@ type DynamicMetricsSnapshotProvider interface {
|
||||
}
|
||||
|
||||
type Metrics struct {
|
||||
accepted atomic.Uint64
|
||||
rejected atomic.Uint64
|
||||
duplicate atomic.Uint64
|
||||
sessionsDeleted atomic.Uint64
|
||||
watermarkRejected atomic.Uint64
|
||||
verificationAccepted atomic.Uint64
|
||||
heartbeatAccepted atomic.Uint64
|
||||
heartbeatFailed atomic.Uint64
|
||||
introspectionActive atomic.Uint64
|
||||
introspectionInactive atomic.Uint64
|
||||
introspectionFailed atomic.Uint64
|
||||
jwksSSFFailed atomic.Uint64
|
||||
jwksOIDCFailed atomic.Uint64
|
||||
processingCount atomic.Uint64
|
||||
processingNanos atomic.Uint64
|
||||
processingBuckets [6]atomic.Uint64
|
||||
billingSettlementCompleted atomic.Uint64
|
||||
billingSettlementRetry atomic.Uint64
|
||||
billingManualReview atomic.Uint64
|
||||
billingEstimateFailed atomic.Uint64
|
||||
billingIdempotentReplay atomic.Uint64
|
||||
billingPricingUnavailable atomic.Uint64
|
||||
asyncWorkerCapacity atomic.Int64
|
||||
asyncWorkerDesiredCapacity atomic.Int64
|
||||
asyncWorkerHardLimit atomic.Int64
|
||||
asyncWorkerCapacityCapped atomic.Int64
|
||||
asyncWorkerActiveInstances atomic.Int64
|
||||
asyncWorkerGlobalCapacity atomic.Int64
|
||||
asyncWorkerAllocated atomic.Int64
|
||||
asyncWorkerResizeSuccess atomic.Uint64
|
||||
asyncWorkerRefreshFailed atomic.Uint64
|
||||
asyncWorkerCreateFailed atomic.Uint64
|
||||
asyncWorkerStartFailed atomic.Uint64
|
||||
leaseRenewalSuccess atomic.Uint64
|
||||
leaseRenewalFailure atomic.Uint64
|
||||
leaseRenewalLost atomic.Uint64
|
||||
taskEventDuplicate atomic.Uint64
|
||||
taskEventUnknownType atomic.Uint64
|
||||
taskEventBudgetExceeded atomic.Uint64
|
||||
taskAdmissionAdmitted atomic.Uint64
|
||||
taskAdmissionQueueFull atomic.Uint64
|
||||
taskAdmissionTimeout atomic.Uint64
|
||||
taskAdmissionCancelled atomic.Uint64
|
||||
taskAdmissionExpired atomic.Uint64
|
||||
taskAdmissionMigrated atomic.Uint64
|
||||
taskAdmissionWaitBuckets [11]atomic.Uint64
|
||||
taskAdmissionWaitMicros atomic.Uint64
|
||||
accepted atomic.Uint64
|
||||
rejected atomic.Uint64
|
||||
duplicate atomic.Uint64
|
||||
sessionsDeleted atomic.Uint64
|
||||
watermarkRejected atomic.Uint64
|
||||
verificationAccepted atomic.Uint64
|
||||
heartbeatAccepted atomic.Uint64
|
||||
heartbeatFailed atomic.Uint64
|
||||
introspectionActive atomic.Uint64
|
||||
introspectionInactive atomic.Uint64
|
||||
introspectionFailed atomic.Uint64
|
||||
jwksSSFFailed atomic.Uint64
|
||||
jwksOIDCFailed atomic.Uint64
|
||||
processingCount atomic.Uint64
|
||||
processingNanos atomic.Uint64
|
||||
processingBuckets [6]atomic.Uint64
|
||||
billingSettlementCompleted atomic.Uint64
|
||||
billingSettlementRetry atomic.Uint64
|
||||
billingManualReview atomic.Uint64
|
||||
billingEstimateFailed atomic.Uint64
|
||||
billingIdempotentReplay atomic.Uint64
|
||||
billingPricingUnavailable atomic.Uint64
|
||||
asyncWorkerCapacity atomic.Int64
|
||||
asyncWorkerDesiredCapacity atomic.Int64
|
||||
asyncWorkerHardLimit atomic.Int64
|
||||
asyncWorkerCapacityCapped atomic.Int64
|
||||
asyncWorkerActiveInstances atomic.Int64
|
||||
asyncWorkerGlobalCapacity atomic.Int64
|
||||
asyncWorkerAllocated atomic.Int64
|
||||
workerSafeCapacity atomic.Int64
|
||||
workerHeavyCapacity atomic.Int64
|
||||
workerActiveTasks atomic.Int64
|
||||
workerPreparingTasks atomic.Int64
|
||||
workerWaitingTasks atomic.Int64
|
||||
workerFinalizingTasks atomic.Int64
|
||||
workerPressureState atomic.Int64
|
||||
providerQuotaWaitRPM atomic.Uint64
|
||||
providerQuotaWaitTPM atomic.Uint64
|
||||
providerQuotaWaitConcurrent atomic.Uint64
|
||||
providerQuotaWaitOther atomic.Uint64
|
||||
asyncWorkerResizeSuccess atomic.Uint64
|
||||
asyncWorkerRefreshFailed atomic.Uint64
|
||||
asyncWorkerCreateFailed atomic.Uint64
|
||||
asyncWorkerStartFailed atomic.Uint64
|
||||
leaseRenewalSuccess atomic.Uint64
|
||||
leaseRenewalFailure atomic.Uint64
|
||||
leaseRenewalLost atomic.Uint64
|
||||
taskEventDuplicate atomic.Uint64
|
||||
taskEventUnknownType atomic.Uint64
|
||||
taskEventBudgetExceeded atomic.Uint64
|
||||
taskAdmissionAdmitted atomic.Uint64
|
||||
taskAdmissionQueueFull atomic.Uint64
|
||||
taskAdmissionTimeout atomic.Uint64
|
||||
taskAdmissionCancelled atomic.Uint64
|
||||
taskAdmissionExpired atomic.Uint64
|
||||
taskAdmissionMigrated atomic.Uint64
|
||||
taskAdmissionWaitBuckets [11]atomic.Uint64
|
||||
taskAdmissionWaitMicros atomic.Uint64
|
||||
}
|
||||
|
||||
var processingDurationBounds = [...]time.Duration{
|
||||
@@ -178,6 +190,38 @@ func (m *Metrics) SetDistributedWorkerCapacity(activeInstances, globalCapacity,
|
||||
m.asyncWorkerAllocated.Store(int64(allocatedCapacity))
|
||||
}
|
||||
|
||||
func (m *Metrics) SetWorkerLoad(activeLimit, heavyLimit, active, preparing, waiting, finalizing int, pressure string) {
|
||||
m.workerSafeCapacity.Store(int64(activeLimit))
|
||||
m.workerHeavyCapacity.Store(int64(heavyLimit))
|
||||
m.workerActiveTasks.Store(int64(active))
|
||||
m.workerPreparingTasks.Store(int64(preparing))
|
||||
m.workerWaitingTasks.Store(int64(waiting))
|
||||
m.workerFinalizingTasks.Store(int64(finalizing))
|
||||
state := int64(-1)
|
||||
switch pressure {
|
||||
case "normal":
|
||||
state = 0
|
||||
case "busy":
|
||||
state = 1
|
||||
case "critical":
|
||||
state = 2
|
||||
}
|
||||
m.workerPressureState.Store(state)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveProviderQuotaWait(metric string) {
|
||||
switch metric {
|
||||
case "rpm":
|
||||
m.providerQuotaWaitRPM.Add(1)
|
||||
case "tpm", "tpm_total", "tpm_input", "tpm_output":
|
||||
m.providerQuotaWaitTPM.Add(1)
|
||||
case "concurrent":
|
||||
m.providerQuotaWaitConcurrent.Add(1)
|
||||
default:
|
||||
m.providerQuotaWaitOther.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveTaskAdmission(event string) {
|
||||
switch event {
|
||||
case "admitted":
|
||||
@@ -297,6 +341,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
}); ok {
|
||||
riverPostgresPool = poolProvider.RiverPostgresPoolMetrics()
|
||||
}
|
||||
modelRateLimits := []store.ModelRateLimitStatus{}
|
||||
if rateLimitProvider, ok := provider.(interface {
|
||||
ListModelRateLimitStatuses(context.Context) ([]store.ModelRateLimitStatus, error)
|
||||
}); ok {
|
||||
var err error
|
||||
modelRateLimits, err = rateLimitProvider.ListModelRateLimitStatuses(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "metrics unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
outcomeCounters(w, "easyai_gateway_ssf_receipts_total", "Received SETs by bounded outcome.", []outcomeValue{
|
||||
{"accepted", m.accepted.Load()}, {"rejected", m.rejected.Load()}, {"duplicate", m.duplicate.Load()},
|
||||
@@ -368,6 +423,20 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
plainGauge(w, "easyai_gateway_worker_active_instances", "Active distributed worker instances with a fresh heartbeat.", m.asyncWorkerActiveInstances.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_global_capacity", "Global asynchronous execution capacity before instance allocation.", m.asyncWorkerGlobalCapacity.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_allocated_capacity", "Asynchronous execution capacity allocated to this worker instance.", m.asyncWorkerAllocated.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_safe_capacity", "Locally resource-safe asynchronous task capacity.", m.workerSafeCapacity.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_heavy_capacity", "Locally resource-safe preparing and finalizing capacity.", m.workerHeavyCapacity.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_active_tasks", "Tasks currently owned by this Worker process.", m.workerActiveTasks.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_preparing_tasks", "Tasks in the local preparing phase.", m.workerPreparingTasks.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_waiting_upstream_tasks", "Tasks waiting for an upstream result.", m.workerWaitingTasks.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_finalizing_tasks", "Tasks in the local finalizing phase.", m.workerFinalizingTasks.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_pressure_state", "Local Worker pressure state: -1 unknown, 0 normal, 1 busy, 2 critical.", m.workerPressureState.Load())
|
||||
outcomeCounters(w, "easyai_gateway_provider_quota_waits_total", "Tasks delayed before an upstream call by a cluster-wide provider quota.", []outcomeValue{
|
||||
{"rpm", m.providerQuotaWaitRPM.Load()},
|
||||
{"tpm", m.providerQuotaWaitTPM.Load()},
|
||||
{"concurrent", m.providerQuotaWaitConcurrent.Load()},
|
||||
{"other", m.providerQuotaWaitOther.Load()},
|
||||
})
|
||||
platformModelRateLimitUtilizationGauges(w, modelRateLimits)
|
||||
plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections))
|
||||
plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections))
|
||||
plainGauge(w, "easyai_gateway_postgres_pool_acquired_connections", "Currently acquired PostgreSQL connections in this process pool.", int64(postgresPool.AcquiredConnections))
|
||||
@@ -463,6 +532,28 @@ func plainFloatGauge(w http.ResponseWriter, name, help string, value float64) {
|
||||
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %.6f\n", name, help, name, name, value)
|
||||
}
|
||||
|
||||
func platformModelRateLimitUtilizationGauges(w http.ResponseWriter, statuses []store.ModelRateLimitStatus) {
|
||||
const name = "easyai_gateway_platform_model_rate_limit_utilization"
|
||||
fmt.Fprintf(w, "# HELP %s Cluster-wide platform model quota utilization.\n# TYPE %s gauge\n", name, name)
|
||||
for _, status := range statuses {
|
||||
platformModelID := escapePrometheusLabel(status.PlatformModelID)
|
||||
for _, metric := range []struct {
|
||||
name string
|
||||
ratio float64
|
||||
}{
|
||||
{name: "rpm", ratio: status.RPM.Ratio},
|
||||
{name: "tpm", ratio: status.TPM.Ratio},
|
||||
{name: "concurrent", ratio: status.Concurrent.Ratio},
|
||||
} {
|
||||
fmt.Fprintf(w, "%s{platform_model_id=\"%s\",metric=\"%s\"} %.6f\n", name, platformModelID, metric.name, metric.ratio)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func escapePrometheusLabel(value string) string {
|
||||
return strings.NewReplacer("\\", "\\\\", "\n", "\\n", "\"", "\\\"").Replace(value)
|
||||
}
|
||||
|
||||
func taskAdmissionWaitHistogram(w http.ResponseWriter, metrics *Metrics) {
|
||||
const name = "easyai_gateway_task_admission_wait_seconds"
|
||||
fmt.Fprintf(w, "# HELP %s Time spent waiting for persistent task admission.\n# TYPE %s histogram\n", name, name)
|
||||
|
||||
Reference in New Issue
Block a user