package securityevents import ( "context" "errors" "fmt" "net/http" "sync/atomic" "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) type MetricsSnapshotProvider interface { SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error) } type DynamicMetricsSnapshotProvider interface { MetricsSnapshotProvider SecurityEventConnection(context.Context) (store.SecurityEventConnection, error) BillingMetrics(context.Context) (store.BillingMetricsSnapshot, error) } type Metrics struct { accepted atomic.Uint64 rejected atomic.Uint64 duplicate atomic.Uint64 sessionsDeleted atomic.Uint64 watermarkRejected atomic.Uint64 verificationAccepted atomic.Uint64 heartbeatAccepted atomic.Uint64 heartbeatFailed atomic.Uint64 introspectionActive atomic.Uint64 introspectionInactive atomic.Uint64 introspectionFailed atomic.Uint64 jwksSSFFailed atomic.Uint64 jwksOIDCFailed atomic.Uint64 processingCount atomic.Uint64 processingNanos atomic.Uint64 processingBuckets [6]atomic.Uint64 billingSettlementCompleted atomic.Uint64 billingSettlementRetry atomic.Uint64 billingManualReview atomic.Uint64 billingEstimateFailed atomic.Uint64 billingIdempotentReplay atomic.Uint64 billingPricingUnavailable atomic.Uint64 asyncWorkerCapacity atomic.Int64 asyncWorkerDesiredCapacity atomic.Int64 asyncWorkerHardLimit atomic.Int64 asyncWorkerCapacityCapped atomic.Int64 asyncWorkerActiveInstances atomic.Int64 asyncWorkerGlobalCapacity atomic.Int64 asyncWorkerAllocated atomic.Int64 asyncWorkerResizeSuccess atomic.Uint64 asyncWorkerRefreshFailed atomic.Uint64 asyncWorkerCreateFailed atomic.Uint64 asyncWorkerStartFailed atomic.Uint64 leaseRenewalSuccess atomic.Uint64 leaseRenewalFailure atomic.Uint64 leaseRenewalLost atomic.Uint64 taskEventDuplicate atomic.Uint64 taskEventUnknownType atomic.Uint64 taskEventBudgetExceeded atomic.Uint64 taskAdmissionAdmitted atomic.Uint64 taskAdmissionQueueFull atomic.Uint64 taskAdmissionTimeout atomic.Uint64 taskAdmissionCancelled atomic.Uint64 taskAdmissionExpired atomic.Uint64 taskAdmissionMigrated atomic.Uint64 taskAdmissionWaitBuckets [11]atomic.Uint64 taskAdmissionWaitMicros atomic.Uint64 } var processingDurationBounds = [...]time.Duration{ 10 * time.Millisecond, 50 * time.Millisecond, 100 * time.Millisecond, 500 * time.Millisecond, time.Second, 3 * time.Second, } var taskAdmissionWaitBounds = [...]time.Duration{ time.Second, 5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute, 5 * time.Minute, 10 * time.Minute, 30 * time.Minute, time.Hour, } func (m *Metrics) ObserveReceipt(outcome string, duration time.Duration, sessionsDeleted int64) { switch outcome { case "accepted": m.accepted.Add(1) case "duplicate": m.duplicate.Add(1) default: m.rejected.Add(1) } if sessionsDeleted > 0 { m.sessionsDeleted.Add(uint64(sessionsDeleted)) } if duration < 0 { duration = 0 } m.processingCount.Add(1) m.processingNanos.Add(uint64(duration)) for index, bound := range processingDurationBounds { if duration <= bound { m.processingBuckets[index].Add(1) } } } func (m *Metrics) ObserveWatermarkRejection() { m.watermarkRejected.Add(1) } func (m *Metrics) ObserveVerificationAccepted() { m.verificationAccepted.Add(1) } func (m *Metrics) ObserveHeartbeat(outcome string) { if outcome == "accepted" { m.heartbeatAccepted.Add(1) return } m.heartbeatFailed.Add(1) } func (m *Metrics) ObserveIntrospection(outcome string) { switch outcome { case "active": m.introspectionActive.Add(1) case "inactive": m.introspectionInactive.Add(1) default: m.introspectionFailed.Add(1) } } func (m *Metrics) ObserveJWKSRefreshFailure(source string) { if source == "ssf" { m.jwksSSFFailed.Add(1) return } m.jwksOIDCFailed.Add(1) } func (m *Metrics) ObserveBillingEvent(event string) { switch event { case "settlement_completed": m.billingSettlementCompleted.Add(1) case "settlement_retry": m.billingSettlementRetry.Add(1) case "manual_review": m.billingManualReview.Add(1) case "estimate_failed": m.billingEstimateFailed.Add(1) case "idempotent_replay": m.billingIdempotentReplay.Add(1) case "pricing_unavailable": m.billingPricingUnavailable.Add(1) } } func (m *Metrics) SetAsyncWorkerCapacity(current, desired, hardLimit int, capped bool) { m.asyncWorkerCapacity.Store(int64(current)) m.asyncWorkerDesiredCapacity.Store(int64(desired)) m.asyncWorkerHardLimit.Store(int64(hardLimit)) cappedValue := int64(0) if capped { cappedValue = 1 } m.asyncWorkerCapacityCapped.Store(cappedValue) } func (m *Metrics) SetDistributedWorkerCapacity(activeInstances, globalCapacity, allocatedCapacity int) { m.asyncWorkerActiveInstances.Store(int64(activeInstances)) m.asyncWorkerGlobalCapacity.Store(int64(globalCapacity)) m.asyncWorkerAllocated.Store(int64(allocatedCapacity)) } func (m *Metrics) ObserveTaskAdmission(event string) { switch event { case "admitted": m.taskAdmissionAdmitted.Add(1) case "queue_full": m.taskAdmissionQueueFull.Add(1) case "timeout": m.taskAdmissionTimeout.Add(1) case "cancelled": m.taskAdmissionCancelled.Add(1) case "expired": m.taskAdmissionExpired.Add(1) case "candidate_migrated": m.taskAdmissionMigrated.Add(1) } } func (m *Metrics) ObserveTaskAdmissionWait(wait time.Duration) { if wait < 0 { wait = 0 } for index, bound := range taskAdmissionWaitBounds { if wait <= bound { m.taskAdmissionWaitBuckets[index].Add(1) } } m.taskAdmissionWaitBuckets[len(m.taskAdmissionWaitBuckets)-1].Add(1) m.taskAdmissionWaitMicros.Add(uint64(wait / time.Microsecond)) } func (m *Metrics) ObserveAsyncWorkerResize(outcome string) { switch outcome { case "success": m.asyncWorkerResizeSuccess.Add(1) case "refresh_failed": m.asyncWorkerRefreshFailed.Add(1) case "create_failed": m.asyncWorkerCreateFailed.Add(1) case "start_failed": m.asyncWorkerStartFailed.Add(1) } } func (m *Metrics) ObserveConcurrencyLeaseRenewal(outcome string) { switch outcome { case "success": m.leaseRenewalSuccess.Add(1) case "lost": m.leaseRenewalLost.Add(1) default: m.leaseRenewalFailure.Add(1) } } func (m *Metrics) ObserveTaskEventSkip(reason string) { switch reason { case "duplicate": m.taskEventDuplicate.Add(1) case "budget_exceeded": m.taskEventBudgetExceeded.Add(1) default: m.taskEventUnknownType.Add(1) } } func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience string, enabled bool) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mode := "disabled" var lastVerificationAt *time.Time if enabled { ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) defer cancel() var err error mode, lastVerificationAt, err = provider.SecurityEventMetrics(ctx, issuer, audience) if err != nil { http.Error(w, "metrics unavailable", http.StatusServiceUnavailable) return } } billing := store.BillingMetricsSnapshot{} if billingProvider, ok := provider.(interface { BillingMetrics(context.Context) (store.BillingMetricsSnapshot, error) }); ok { var err error billing, err = billingProvider.BillingMetrics(r.Context()) if err != nil { http.Error(w, "metrics unavailable", http.StatusServiceUnavailable) return } } admission := store.TaskAdmissionMetricsSnapshot{} if admissionProvider, ok := provider.(interface { TaskAdmissionMetrics(context.Context) (store.TaskAdmissionMetricsSnapshot, error) }); ok { var err error admission, err = admissionProvider.TaskAdmissionMetrics(r.Context()) if err != nil { http.Error(w, "metrics unavailable", http.StatusServiceUnavailable) return } } w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") outcomeCounters(w, "easyai_gateway_ssf_receipts_total", "Received SETs by bounded outcome.", []outcomeValue{ {"accepted", m.accepted.Load()}, {"rejected", m.rejected.Load()}, {"duplicate", m.duplicate.Load()}, }) plainCounter(w, "easyai_gateway_ssf_sessions_deleted_total", "OIDC browser sessions deleted by accepted session-revoked events.", m.sessionsDeleted.Load()) plainCounter(w, "easyai_gateway_ssf_watermark_rejections_total", "OIDC access tokens rejected by a local revocation watermark.", m.watermarkRejected.Load()) plainCounter(w, "easyai_gateway_ssf_verifications_total", "Valid verification SETs accepted by the receiver.", m.verificationAccepted.Load()) outcomeCounters(w, "easyai_gateway_ssf_heartbeat_requests_total", "Verification requests by bounded outcome.", []outcomeValue{ {"accepted", m.heartbeatAccepted.Load()}, {"failed", m.heartbeatFailed.Load()}, }) outcomeCounters(w, "easyai_gateway_oidc_introspection_total", "OIDC introspection requests by bounded outcome.", []outcomeValue{ {"active", m.introspectionActive.Load()}, {"inactive", m.introspectionInactive.Load()}, {"failed", m.introspectionFailed.Load()}, }) outcomeCounters(w, "easyai_gateway_jwks_refresh_failures_total", "JWKS refresh failures by bounded source.", []outcomeValue{ {"ssf", m.jwksSSFFailed.Load()}, {"oidc", m.jwksOIDCFailed.Load()}, }) fmt.Fprintln(w, "# HELP easyai_gateway_ssf_processing_duration_seconds SSF receiver transaction processing time.") fmt.Fprintln(w, "# TYPE easyai_gateway_ssf_processing_duration_seconds histogram") for index, bound := range processingDurationBounds { fmt.Fprintf(w, "easyai_gateway_ssf_processing_duration_seconds_bucket{le=\"%.2f\"} %d\n", bound.Seconds(), m.processingBuckets[index].Load()) } fmt.Fprintf(w, "easyai_gateway_ssf_processing_duration_seconds_bucket{le=\"+Inf\"} %d\n", m.processingCount.Load()) fmt.Fprintf(w, "easyai_gateway_ssf_processing_duration_seconds_sum %.6f\n", float64(m.processingNanos.Load())/float64(time.Second)) fmt.Fprintf(w, "easyai_gateway_ssf_processing_duration_seconds_count %d\n", m.processingCount.Load()) verificationAge := float64(0) if lastVerificationAt != nil { verificationAge = time.Since(*lastVerificationAt).Seconds() if verificationAge < 0 { verificationAge = 0 } } fmt.Fprintln(w, "# HELP easyai_gateway_ssf_verification_age_seconds Seconds since the last matched verification SET.") fmt.Fprintln(w, "# TYPE easyai_gateway_ssf_verification_age_seconds gauge") fmt.Fprintf(w, "easyai_gateway_ssf_verification_age_seconds %.6f\n", verificationAge) fmt.Fprintln(w, "# HELP easyai_gateway_ssf_mode Current SSF receiver mode as a one-hot gauge.") fmt.Fprintln(w, "# TYPE easyai_gateway_ssf_mode gauge") for _, modeName := range []string{"disabled", "bootstrap", "push_healthy", "introspection_fallback"} { value := 0 if mode == modeName { value = 1 } fmt.Fprintf(w, "easyai_gateway_ssf_mode{mode=\"%s\"} %d\n", modeName, value) } fmt.Fprintln(w, "# HELP easyai_gateway_billing_settlement_backlog Current unsettled Outbox records.") fmt.Fprintln(w, "# TYPE easyai_gateway_billing_settlement_backlog gauge") fmt.Fprintf(w, "easyai_gateway_billing_settlement_backlog %d\n", billing.SettlementBacklog) fmt.Fprintln(w, "# HELP easyai_gateway_billing_settlement_delay_seconds Age of the oldest unsettled Outbox record.") fmt.Fprintln(w, "# TYPE easyai_gateway_billing_settlement_delay_seconds gauge") fmt.Fprintf(w, "easyai_gateway_billing_settlement_delay_seconds %.6f\n", billing.SettlementDelaySecs) fmt.Fprintln(w, "# HELP easyai_gateway_billing_manual_review Current billing records requiring manual review.") fmt.Fprintln(w, "# TYPE easyai_gateway_billing_manual_review gauge") fmt.Fprintf(w, "easyai_gateway_billing_manual_review %d\n", billing.ManualReview) fmt.Fprintln(w, "# HELP easyai_gateway_billing_orphan_frozen Wallets whose frozen amount differs from active reservations.") fmt.Fprintln(w, "# TYPE easyai_gateway_billing_orphan_frozen gauge") fmt.Fprintf(w, "easyai_gateway_billing_orphan_frozen %d\n", billing.OrphanFrozen) fmt.Fprintln(w, "# HELP easyai_gateway_billing_pricing_unavailable_tasks Current tasks rejected because no effective price was available.") fmt.Fprintln(w, "# TYPE easyai_gateway_billing_pricing_unavailable_tasks gauge") fmt.Fprintf(w, "easyai_gateway_billing_pricing_unavailable_tasks %d\n", billing.PricingUnavailable) plainCounter(w, "easyai_gateway_billing_settlements_completed_total", "Billing settlements completed by this process.", m.billingSettlementCompleted.Load()) plainCounter(w, "easyai_gateway_billing_settlement_retries_total", "Billing settlement retries scheduled by this process.", m.billingSettlementRetry.Load()) plainCounter(w, "easyai_gateway_billing_manual_review_transitions_total", "Billing records transitioned to manual review by this process.", m.billingManualReview.Load()) plainCounter(w, "easyai_gateway_billing_estimate_failures_total", "Pricing estimate requests that failed.", m.billingEstimateFailed.Load()) plainCounter(w, "easyai_gateway_billing_idempotent_replays_total", "Generation requests replayed idempotently.", m.billingIdempotentReplay.Load()) plainCounter(w, "easyai_gateway_billing_pricing_unavailable_total", "Pricing requests rejected because no effective price was available.", m.billingPricingUnavailable.Load()) plainGauge(w, "easyai_gateway_async_worker_capacity", "Current River asynchronous worker capacity.", m.asyncWorkerCapacity.Load()) plainGauge(w, "easyai_gateway_async_worker_desired_capacity", "Uncapped asynchronous worker capacity derived from model and user-group policies.", m.asyncWorkerDesiredCapacity.Load()) plainGauge(w, "easyai_gateway_async_worker_hard_limit", "Per-process asynchronous worker safety limit.", m.asyncWorkerHardLimit.Load()) plainGauge(w, "easyai_gateway_async_worker_capacity_capped", "Whether desired asynchronous worker capacity is capped by the hard limit.", m.asyncWorkerCapacityCapped.Load()) plainGauge(w, "easyai_gateway_worker_active_instances", "Active distributed worker instances with a fresh heartbeat.", m.asyncWorkerActiveInstances.Load()) plainGauge(w, "easyai_gateway_worker_global_capacity", "Global asynchronous execution capacity before instance allocation.", m.asyncWorkerGlobalCapacity.Load()) plainGauge(w, "easyai_gateway_worker_allocated_capacity", "Asynchronous execution capacity allocated to this worker instance.", m.asyncWorkerAllocated.Load()) outcomeCounters(w, "easyai_gateway_async_worker_resizes_total", "Asynchronous worker resize attempts by bounded outcome.", []outcomeValue{ {"success", m.asyncWorkerResizeSuccess.Load()}, {"refresh_failed", m.asyncWorkerRefreshFailed.Load()}, {"create_failed", m.asyncWorkerCreateFailed.Load()}, {"start_failed", m.asyncWorkerStartFailed.Load()}, }) outcomeCounters(w, "easyai_gateway_concurrency_lease_renewals_total", "Concurrency lease renewals by bounded outcome.", []outcomeValue{ {"success", m.leaseRenewalSuccess.Load()}, {"failure", m.leaseRenewalFailure.Load()}, {"lost", m.leaseRenewalLost.Load()}, }) outcomeCounters(w, "easyai_gateway_task_events_skipped_total", "Task events skipped before persistence by bounded reason.", []outcomeValue{ {"duplicate", m.taskEventDuplicate.Load()}, {"unknown_type", m.taskEventUnknownType.Load()}, {"budget_exceeded", m.taskEventBudgetExceeded.Load()}, }) plainGauge(w, "easyai_gateway_task_admission_queue_depth", "Current persistent non-text task admission queue depth.", int64(admission.QueueDepth)) plainGauge(w, "easyai_gateway_task_admission_waiting_sync", "Current synchronous admission waiters.", int64(admission.WaitingSync)) plainGauge(w, "easyai_gateway_task_admission_waiting_async", "Current asynchronous admission waiters.", int64(admission.WaitingAsync)) plainFloatGauge(w, "easyai_gateway_task_admission_oldest_wait_seconds", "Age of the oldest persistent admission waiter.", admission.OldestWaitSeconds) plainGauge(w, "easyai_gateway_task_admission_expired_waiter_backlog", "Expired synchronous waiter leases awaiting reclamation.", int64(admission.ExpiredWaiterBacklog)) plainGauge(w, "easyai_gateway_task_admission_expired_deadline_backlog", "Expired queue deadlines awaiting reclamation.", int64(admission.ExpiredDeadlineBacklog)) outcomeCounters(w, "easyai_gateway_task_admissions_total", "Task admission transitions by bounded outcome.", []outcomeValue{ {"admitted", m.taskAdmissionAdmitted.Load()}, {"queue_full", m.taskAdmissionQueueFull.Load()}, {"timeout", m.taskAdmissionTimeout.Load()}, {"cancelled", m.taskAdmissionCancelled.Load()}, {"expired", m.taskAdmissionExpired.Load()}, {"candidate_migrated", m.taskAdmissionMigrated.Load()}, }) taskAdmissionWaitHistogram(w, m) }) } func (m *Metrics) DynamicHandler(provider DynamicMetricsSnapshotProvider) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { connection, err := provider.SecurityEventConnection(r.Context()) if errors.Is(err, store.ErrSecurityEventConnectionNotFound) { m.Handler(provider, "", "", false).ServeHTTP(w, r) return } if err != nil { http.Error(w, "metrics unavailable", http.StatusServiceUnavailable) return } if connection.Audience == nil { m.Handler(provider, "", "", false).ServeHTTP(w, r) return } m.Handler(provider, connection.TransmitterIssuer, *connection.Audience, true).ServeHTTP(w, r) }) } type outcomeValue struct { name string value uint64 } func outcomeCounters(w http.ResponseWriter, name, help string, values []outcomeValue) { fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n", name, help, name) for _, value := range values { fmt.Fprintf(w, "%s{outcome=\"%s\"} %d\n", name, value.name, value.value) } } func plainCounter(w http.ResponseWriter, name, help string, value uint64) { fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n%s %d\n", name, help, name, name, value) } func plainGauge(w http.ResponseWriter, name, help string, value int64) { fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %d\n", name, help, name, name, value) } func plainFloatGauge(w http.ResponseWriter, name, help string, value float64) { fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %.6f\n", name, help, name, name, value) } func taskAdmissionWaitHistogram(w http.ResponseWriter, metrics *Metrics) { const name = "easyai_gateway_task_admission_wait_seconds" fmt.Fprintf(w, "# HELP %s Time spent waiting for persistent task admission.\n# TYPE %s histogram\n", name, name) for index, bound := range taskAdmissionWaitBounds { fmt.Fprintf( w, "%s_bucket{le=\"%g\"} %d\n", name, bound.Seconds(), metrics.taskAdmissionWaitBuckets[index].Load(), ) } count := metrics.taskAdmissionWaitBuckets[len(metrics.taskAdmissionWaitBuckets)-1].Load() fmt.Fprintf(w, "%s_bucket{le=\"+Inf\"} %d\n", name, count) fmt.Fprintf(w, "%s_sum %.6f\n", name, float64(metrics.taskAdmissionWaitMicros.Load())/1_000_000) fmt.Fprintf(w, "%s_count %d\n", name, count) }