wangbo
c28bf74230
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
107 lines
4.2 KiB
Go
107 lines
4.2 KiB
Go
package securityevents
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
type metricsSnapshot struct {
|
|
mode string
|
|
last time.Time
|
|
pool store.PostgresPoolMetricsSnapshot
|
|
}
|
|
|
|
func (m metricsSnapshot) SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error) {
|
|
return m.mode, &m.last, nil
|
|
}
|
|
|
|
func (m metricsSnapshot) PostgresPoolMetrics() store.PostgresPoolMetricsSnapshot {
|
|
return m.pool
|
|
}
|
|
|
|
func (m metricsSnapshot) ListModelRateLimitStatuses(context.Context) ([]store.ModelRateLimitStatus, error) {
|
|
return []store.ModelRateLimitStatus{{
|
|
PlatformModelID: "platform-model-1",
|
|
RPM: store.RateLimitMetricStatus{Ratio: .25},
|
|
TPM: store.RateLimitMetricStatus{Ratio: .5},
|
|
Concurrent: store.RateLimitMetricStatus{Ratio: .75},
|
|
}}, nil
|
|
}
|
|
|
|
func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
|
metrics := &Metrics{}
|
|
metrics.ObserveReceipt("accepted", 20*time.Millisecond, 2)
|
|
metrics.ObserveReceipt("duplicate", 10*time.Millisecond, 0)
|
|
metrics.ObserveWatermarkRejection()
|
|
metrics.ObserveHeartbeat("accepted")
|
|
metrics.ObserveIntrospection("failed")
|
|
metrics.ObserveJWKSRefreshFailure("ssf")
|
|
metrics.SetAsyncWorkerCapacity(96, 128, 96, true)
|
|
metrics.SetWorkerLoad(12, 3, 8, 2, 5, 1, "busy")
|
|
metrics.ObserveProviderQuotaWait("rpm")
|
|
metrics.ObserveProviderQuotaWait("tpm_total")
|
|
metrics.ObserveAsyncWorkerResize("success")
|
|
metrics.ObserveConcurrencyLeaseRenewal("success")
|
|
metrics.ObserveConcurrencyLeaseRenewal("lost")
|
|
metrics.ObserveTaskEventSkip("duplicate")
|
|
metrics.ObserveTaskEventSkip("budget_exceeded")
|
|
|
|
recorder := httptest.NewRecorder()
|
|
metrics.Handler(metricsSnapshot{
|
|
mode: "push_healthy",
|
|
last: time.Now().Add(-time.Minute),
|
|
pool: store.PostgresPoolMetricsSnapshot{
|
|
MaxConnections: 32, TotalConnections: 18, AcquiredConnections: 12,
|
|
IdleConnections: 6, EmptyAcquireCount: 3, CanceledAcquireCount: 1,
|
|
},
|
|
}, "issuer", "audience", true).
|
|
ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil))
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("metrics status=%d", recorder.Code)
|
|
}
|
|
for _, expected := range []string{
|
|
`easyai_gateway_ssf_receipts_total{outcome="accepted"} 1`,
|
|
`easyai_gateway_ssf_receipts_total{outcome="duplicate"} 1`,
|
|
`easyai_gateway_ssf_sessions_deleted_total 2`,
|
|
`easyai_gateway_ssf_watermark_rejections_total 1`,
|
|
`easyai_gateway_ssf_processing_duration_seconds_bucket{le="0.05"} 2`,
|
|
`easyai_gateway_ssf_mode{mode="push_healthy"} 1`,
|
|
`easyai_gateway_oidc_introspection_total{outcome="failed"} 1`,
|
|
`easyai_gateway_jwks_refresh_failures_total{outcome="ssf"} 1`,
|
|
`easyai_gateway_async_worker_capacity 96`,
|
|
`easyai_gateway_async_worker_desired_capacity 128`,
|
|
`easyai_gateway_async_worker_capacity_capped 1`,
|
|
`easyai_gateway_worker_safe_capacity 12`,
|
|
`easyai_gateway_worker_heavy_capacity 3`,
|
|
`easyai_gateway_worker_active_tasks 8`,
|
|
`easyai_gateway_worker_preparing_tasks 2`,
|
|
`easyai_gateway_worker_waiting_upstream_tasks 5`,
|
|
`easyai_gateway_worker_finalizing_tasks 1`,
|
|
`easyai_gateway_worker_pressure_state 1`,
|
|
`easyai_gateway_provider_quota_waits_total{outcome="rpm"} 1`,
|
|
`easyai_gateway_provider_quota_waits_total{outcome="tpm"} 1`,
|
|
`easyai_gateway_platform_model_rate_limit_utilization{platform_model_id="platform-model-1",metric="concurrent"} 0.750000`,
|
|
`easyai_gateway_async_worker_resizes_total{outcome="success"} 1`,
|
|
`easyai_gateway_concurrency_lease_renewals_total{outcome="success"} 1`,
|
|
`easyai_gateway_concurrency_lease_renewals_total{outcome="lost"} 1`,
|
|
`easyai_gateway_task_events_skipped_total{outcome="duplicate"} 1`,
|
|
`easyai_gateway_task_events_skipped_total{outcome="budget_exceeded"} 1`,
|
|
`easyai_gateway_postgres_pool_max_connections 32`,
|
|
`easyai_gateway_postgres_pool_total_connections 18`,
|
|
`easyai_gateway_postgres_pool_acquired_connections 12`,
|
|
`easyai_gateway_postgres_pool_idle_connections 6`,
|
|
`easyai_gateway_postgres_pool_empty_acquire_total 3`,
|
|
`easyai_gateway_postgres_pool_canceled_acquire_total 1`,
|
|
} {
|
|
if !strings.Contains(recorder.Body.String(), expected) {
|
|
t.Fatalf("missing metric %q in:\n%s", expected, recorder.Body.String())
|
|
}
|
|
}
|
|
}
|