fix(queue): 隔离异步准入故障并持久化退避
将 X-Async 调度改为逐任务阻塞协议，区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障，避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈，并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证：API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过；pnpm audit 命中未改动的 Nx 工具链既有漏洞。
This commit is contained in:
@@ -80,6 +80,15 @@ type Metrics struct {
|
||||
taskAdmissionCancelled atomic.Uint64
|
||||
taskAdmissionExpired atomic.Uint64
|
||||
taskAdmissionMigrated atomic.Uint64
|
||||
asyncAdmissionAdmitted atomic.Uint64
|
||||
asyncAdmissionGlobalWait atomic.Uint64
|
||||
asyncAdmissionPlatformWait atomic.Uint64
|
||||
asyncAdmissionGroupWait atomic.Uint64
|
||||
asyncAdmissionReselect atomic.Uint64
|
||||
asyncAdmissionDeferred atomic.Uint64
|
||||
asyncAdmissionTerminal atomic.Uint64
|
||||
asyncAdmissionTaskError atomic.Uint64
|
||||
asyncAdmissionSystemError atomic.Uint64
|
||||
candidateNormalRotation atomic.Uint64
|
||||
candidateFullAvoided atomic.Uint64
|
||||
candidateQuotaRaceRotated atomic.Uint64
|
||||
@@ -260,6 +269,29 @@ func (m *Metrics) ObserveTaskAdmission(event string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveAsyncAdmissionDispatch(outcome string) {
|
||||
switch outcome {
|
||||
case "admitted":
|
||||
m.asyncAdmissionAdmitted.Add(1)
|
||||
case "global_wait":
|
||||
m.asyncAdmissionGlobalWait.Add(1)
|
||||
case "platform_wait":
|
||||
m.asyncAdmissionPlatformWait.Add(1)
|
||||
case "group_wait":
|
||||
m.asyncAdmissionGroupWait.Add(1)
|
||||
case "reselect":
|
||||
m.asyncAdmissionReselect.Add(1)
|
||||
case "deferred":
|
||||
m.asyncAdmissionDeferred.Add(1)
|
||||
case "terminal":
|
||||
m.asyncAdmissionTerminal.Add(1)
|
||||
case "task_error":
|
||||
m.asyncAdmissionTaskError.Add(1)
|
||||
case "system_error":
|
||||
m.asyncAdmissionSystemError.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveCandidateRouting(reason string) {
|
||||
switch reason {
|
||||
case "normal_rotation":
|
||||
@@ -568,7 +600,10 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
plainGauge(w, "easyai_gateway_task_admission_queue_depth", "Current persistent non-text task admission queue depth.", int64(admission.QueueDepth))
|
||||
plainGauge(w, "easyai_gateway_task_admission_waiting_sync", "Current synchronous admission waiters.", int64(admission.WaitingSync))
|
||||
plainGauge(w, "easyai_gateway_task_admission_waiting_async", "Current asynchronous admission waiters.", int64(admission.WaitingAsync))
|
||||
plainGauge(w, "easyai_gateway_task_admission_deferred_async", "Asynchronous admissions quarantined until a future dispatch attempt.", int64(admission.DeferredAsync))
|
||||
plainGauge(w, "easyai_gateway_task_admission_pending_reselect", "Waiting asynchronous admissions marked for candidate reselection.", int64(admission.PendingReselect))
|
||||
plainFloatGauge(w, "easyai_gateway_task_admission_oldest_wait_seconds", "Age of the oldest persistent admission waiter.", admission.OldestWaitSeconds)
|
||||
plainFloatGauge(w, "easyai_gateway_task_admission_oldest_dispatchable_wait_seconds", "Age of the oldest asynchronous admission currently eligible for dispatch.", admission.OldestDispatchableWaitSeconds)
|
||||
plainGauge(w, "easyai_gateway_task_admission_expired_waiter_backlog", "Expired synchronous waiter leases awaiting reclamation.", int64(admission.ExpiredWaiterBacklog))
|
||||
plainGauge(w, "easyai_gateway_task_admission_expired_deadline_backlog", "Expired queue deadlines awaiting reclamation.", int64(admission.ExpiredDeadlineBacklog))
|
||||
outcomeCounters(w, "easyai_gateway_task_admissions_total", "Task admission transitions by bounded outcome.", []outcomeValue{
|
||||
@@ -579,6 +614,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
{"expired", m.taskAdmissionExpired.Load()},
|
||||
{"candidate_migrated", m.taskAdmissionMigrated.Load()},
|
||||
})
|
||||
outcomeCounters(w, "easyai_gateway_async_admission_dispatch_total", "Asynchronous admission dispatcher decisions by bounded outcome.", []outcomeValue{
|
||||
{"admitted", m.asyncAdmissionAdmitted.Load()},
|
||||
{"global_wait", m.asyncAdmissionGlobalWait.Load()},
|
||||
{"platform_wait", m.asyncAdmissionPlatformWait.Load()},
|
||||
{"group_wait", m.asyncAdmissionGroupWait.Load()},
|
||||
{"reselect", m.asyncAdmissionReselect.Load()},
|
||||
{"deferred", m.asyncAdmissionDeferred.Load()},
|
||||
{"terminal", m.asyncAdmissionTerminal.Load()},
|
||||
{"task_error", m.asyncAdmissionTaskError.Load()},
|
||||
{"system_error", m.asyncAdmissionSystemError.Load()},
|
||||
})
|
||||
taskAdmissionWaitHistogram(w, m)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -62,6 +62,13 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
||||
metrics.ObserveConcurrencyLeaseRenewal("lost")
|
||||
metrics.ObserveTaskEventSkip("duplicate")
|
||||
metrics.ObserveTaskEventSkip("budget_exceeded")
|
||||
metrics.ObserveAsyncAdmissionDispatch("admitted")
|
||||
metrics.ObserveAsyncAdmissionDispatch("global_wait")
|
||||
metrics.ObserveAsyncAdmissionDispatch("reselect")
|
||||
metrics.ObserveAsyncAdmissionDispatch("deferred")
|
||||
metrics.ObserveAsyncAdmissionDispatch("terminal")
|
||||
metrics.ObserveAsyncAdmissionDispatch("task_error")
|
||||
metrics.ObserveAsyncAdmissionDispatch("system_error")
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
metrics.Handler(metricsSnapshot{
|
||||
@@ -116,6 +123,13 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
||||
`easyai_gateway_concurrency_lease_renewals_total{outcome="lost"} 1`,
|
||||
`easyai_gateway_task_events_skipped_total{outcome="duplicate"} 1`,
|
||||
`easyai_gateway_task_events_skipped_total{outcome="budget_exceeded"} 1`,
|
||||
`easyai_gateway_async_admission_dispatch_total{outcome="admitted"} 1`,
|
||||
`easyai_gateway_async_admission_dispatch_total{outcome="global_wait"} 1`,
|
||||
`easyai_gateway_async_admission_dispatch_total{outcome="reselect"} 1`,
|
||||
`easyai_gateway_async_admission_dispatch_total{outcome="deferred"} 1`,
|
||||
`easyai_gateway_async_admission_dispatch_total{outcome="terminal"} 1`,
|
||||
`easyai_gateway_async_admission_dispatch_total{outcome="task_error"} 1`,
|
||||
`easyai_gateway_async_admission_dispatch_total{outcome="system_error"} 1`,
|
||||
`easyai_gateway_postgres_pool_max_connections 32`,
|
||||
`easyai_gateway_postgres_pool_total_connections 18`,
|
||||
`easyai_gateway_postgres_pool_acquired_connections 12`,
|
||||
|
||||
Reference in New Issue
Block a user