fix(queue): 隔离异步准入故障并持久化退避

将 X-Async 调度改为逐任务阻塞协议,区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障,避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈,并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证:API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过;pnpm audit 命中未改动的 Nx 工具链既有漏洞。
This commit is contained in:
2026-08-04 18:23:35 +08:00
parent 44d5cf2b9d
commit f4214dd489
12 changed files with 1654 additions and 133 deletions
@@ -80,6 +80,15 @@ type Metrics struct {
taskAdmissionCancelled atomic.Uint64
taskAdmissionExpired atomic.Uint64
taskAdmissionMigrated atomic.Uint64
asyncAdmissionAdmitted atomic.Uint64
asyncAdmissionGlobalWait atomic.Uint64
asyncAdmissionPlatformWait atomic.Uint64
asyncAdmissionGroupWait atomic.Uint64
asyncAdmissionReselect atomic.Uint64
asyncAdmissionDeferred atomic.Uint64
asyncAdmissionTerminal atomic.Uint64
asyncAdmissionTaskError atomic.Uint64
asyncAdmissionSystemError atomic.Uint64
candidateNormalRotation atomic.Uint64
candidateFullAvoided atomic.Uint64
candidateQuotaRaceRotated atomic.Uint64
@@ -260,6 +269,29 @@ func (m *Metrics) ObserveTaskAdmission(event string) {
}
}
func (m *Metrics) ObserveAsyncAdmissionDispatch(outcome string) {
switch outcome {
case "admitted":
m.asyncAdmissionAdmitted.Add(1)
case "global_wait":
m.asyncAdmissionGlobalWait.Add(1)
case "platform_wait":
m.asyncAdmissionPlatformWait.Add(1)
case "group_wait":
m.asyncAdmissionGroupWait.Add(1)
case "reselect":
m.asyncAdmissionReselect.Add(1)
case "deferred":
m.asyncAdmissionDeferred.Add(1)
case "terminal":
m.asyncAdmissionTerminal.Add(1)
case "task_error":
m.asyncAdmissionTaskError.Add(1)
case "system_error":
m.asyncAdmissionSystemError.Add(1)
}
}
func (m *Metrics) ObserveCandidateRouting(reason string) {
switch reason {
case "normal_rotation":
@@ -568,7 +600,10 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
plainGauge(w, "easyai_gateway_task_admission_queue_depth", "Current persistent non-text task admission queue depth.", int64(admission.QueueDepth))
plainGauge(w, "easyai_gateway_task_admission_waiting_sync", "Current synchronous admission waiters.", int64(admission.WaitingSync))
plainGauge(w, "easyai_gateway_task_admission_waiting_async", "Current asynchronous admission waiters.", int64(admission.WaitingAsync))
plainGauge(w, "easyai_gateway_task_admission_deferred_async", "Asynchronous admissions quarantined until a future dispatch attempt.", int64(admission.DeferredAsync))
plainGauge(w, "easyai_gateway_task_admission_pending_reselect", "Waiting asynchronous admissions marked for candidate reselection.", int64(admission.PendingReselect))
plainFloatGauge(w, "easyai_gateway_task_admission_oldest_wait_seconds", "Age of the oldest persistent admission waiter.", admission.OldestWaitSeconds)
plainFloatGauge(w, "easyai_gateway_task_admission_oldest_dispatchable_wait_seconds", "Age of the oldest asynchronous admission currently eligible for dispatch.", admission.OldestDispatchableWaitSeconds)
plainGauge(w, "easyai_gateway_task_admission_expired_waiter_backlog", "Expired synchronous waiter leases awaiting reclamation.", int64(admission.ExpiredWaiterBacklog))
plainGauge(w, "easyai_gateway_task_admission_expired_deadline_backlog", "Expired queue deadlines awaiting reclamation.", int64(admission.ExpiredDeadlineBacklog))
outcomeCounters(w, "easyai_gateway_task_admissions_total", "Task admission transitions by bounded outcome.", []outcomeValue{
@@ -579,6 +614,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
{"expired", m.taskAdmissionExpired.Load()},
{"candidate_migrated", m.taskAdmissionMigrated.Load()},
})
outcomeCounters(w, "easyai_gateway_async_admission_dispatch_total", "Asynchronous admission dispatcher decisions by bounded outcome.", []outcomeValue{
{"admitted", m.asyncAdmissionAdmitted.Load()},
{"global_wait", m.asyncAdmissionGlobalWait.Load()},
{"platform_wait", m.asyncAdmissionPlatformWait.Load()},
{"group_wait", m.asyncAdmissionGroupWait.Load()},
{"reselect", m.asyncAdmissionReselect.Load()},
{"deferred", m.asyncAdmissionDeferred.Load()},
{"terminal", m.asyncAdmissionTerminal.Load()},
{"task_error", m.asyncAdmissionTaskError.Load()},
{"system_error", m.asyncAdmissionSystemError.Load()},
})
taskAdmissionWaitHistogram(w, m)
})
}