feat(queue): 增加非文本模型分布式准入队列
使用 PostgreSQL 统一同步与异步非文本任务的并发准入、持久化等待和 Worker 容量分配，并将生产 API 与独立 Worker 角色拆分。 补充策略管理、共享契约、OpenAPI、Kubernetes 双节点 Worker 清单及跨节点验收脚本；未默认启用任何生产 queue_size 策略。 已在原基线完成 Go、前端、迁移、Shell、Kustomize 与长任务容量验收；合入最新主干后将重新执行发布门禁。
This commit is contained in:
@@ -48,6 +48,9 @@ type Metrics struct {
|
||||
asyncWorkerDesiredCapacity atomic.Int64
|
||||
asyncWorkerHardLimit atomic.Int64
|
||||
asyncWorkerCapacityCapped atomic.Int64
|
||||
asyncWorkerActiveInstances atomic.Int64
|
||||
asyncWorkerGlobalCapacity atomic.Int64
|
||||
asyncWorkerAllocated atomic.Int64
|
||||
asyncWorkerResizeSuccess atomic.Uint64
|
||||
asyncWorkerRefreshFailed atomic.Uint64
|
||||
asyncWorkerCreateFailed atomic.Uint64
|
||||
@@ -58,6 +61,14 @@ type Metrics struct {
|
||||
taskEventDuplicate atomic.Uint64
|
||||
taskEventUnknownType atomic.Uint64
|
||||
taskEventBudgetExceeded atomic.Uint64
|
||||
taskAdmissionAdmitted atomic.Uint64
|
||||
taskAdmissionQueueFull atomic.Uint64
|
||||
taskAdmissionTimeout atomic.Uint64
|
||||
taskAdmissionCancelled atomic.Uint64
|
||||
taskAdmissionExpired atomic.Uint64
|
||||
taskAdmissionMigrated atomic.Uint64
|
||||
taskAdmissionWaitBuckets [11]atomic.Uint64
|
||||
taskAdmissionWaitMicros atomic.Uint64
|
||||
}
|
||||
|
||||
var processingDurationBounds = [...]time.Duration{
|
||||
@@ -65,6 +76,19 @@ var processingDurationBounds = [...]time.Duration{
|
||||
500 * time.Millisecond, time.Second, 3 * time.Second,
|
||||
}
|
||||
|
||||
var taskAdmissionWaitBounds = [...]time.Duration{
|
||||
time.Second,
|
||||
5 * time.Second,
|
||||
15 * time.Second,
|
||||
30 * time.Second,
|
||||
time.Minute,
|
||||
2 * time.Minute,
|
||||
5 * time.Minute,
|
||||
10 * time.Minute,
|
||||
30 * time.Minute,
|
||||
time.Hour,
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveReceipt(outcome string, duration time.Duration, sessionsDeleted int64) {
|
||||
switch outcome {
|
||||
case "accepted":
|
||||
@@ -148,6 +172,42 @@ func (m *Metrics) SetAsyncWorkerCapacity(current, desired, hardLimit int, capped
|
||||
m.asyncWorkerCapacityCapped.Store(cappedValue)
|
||||
}
|
||||
|
||||
func (m *Metrics) SetDistributedWorkerCapacity(activeInstances, globalCapacity, allocatedCapacity int) {
|
||||
m.asyncWorkerActiveInstances.Store(int64(activeInstances))
|
||||
m.asyncWorkerGlobalCapacity.Store(int64(globalCapacity))
|
||||
m.asyncWorkerAllocated.Store(int64(allocatedCapacity))
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveTaskAdmission(event string) {
|
||||
switch event {
|
||||
case "admitted":
|
||||
m.taskAdmissionAdmitted.Add(1)
|
||||
case "queue_full":
|
||||
m.taskAdmissionQueueFull.Add(1)
|
||||
case "timeout":
|
||||
m.taskAdmissionTimeout.Add(1)
|
||||
case "cancelled":
|
||||
m.taskAdmissionCancelled.Add(1)
|
||||
case "expired":
|
||||
m.taskAdmissionExpired.Add(1)
|
||||
case "candidate_migrated":
|
||||
m.taskAdmissionMigrated.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveTaskAdmissionWait(wait time.Duration) {
|
||||
if wait < 0 {
|
||||
wait = 0
|
||||
}
|
||||
for index, bound := range taskAdmissionWaitBounds {
|
||||
if wait <= bound {
|
||||
m.taskAdmissionWaitBuckets[index].Add(1)
|
||||
}
|
||||
}
|
||||
m.taskAdmissionWaitBuckets[len(m.taskAdmissionWaitBuckets)-1].Add(1)
|
||||
m.taskAdmissionWaitMicros.Add(uint64(wait / time.Microsecond))
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveAsyncWorkerResize(outcome string) {
|
||||
switch outcome {
|
||||
case "success":
|
||||
@@ -208,6 +268,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
return
|
||||
}
|
||||
}
|
||||
admission := store.TaskAdmissionMetricsSnapshot{}
|
||||
if admissionProvider, ok := provider.(interface {
|
||||
TaskAdmissionMetrics(context.Context) (store.TaskAdmissionMetricsSnapshot, error)
|
||||
}); ok {
|
||||
var err error
|
||||
admission, err = admissionProvider.TaskAdmissionMetrics(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "metrics unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
outcomeCounters(w, "easyai_gateway_ssf_receipts_total", "Received SETs by bounded outcome.", []outcomeValue{
|
||||
{"accepted", m.accepted.Load()}, {"rejected", m.rejected.Load()}, {"duplicate", m.duplicate.Load()},
|
||||
@@ -276,6 +347,9 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
plainGauge(w, "easyai_gateway_async_worker_desired_capacity", "Uncapped asynchronous worker capacity derived from model and user-group policies.", m.asyncWorkerDesiredCapacity.Load())
|
||||
plainGauge(w, "easyai_gateway_async_worker_hard_limit", "Per-process asynchronous worker safety limit.", m.asyncWorkerHardLimit.Load())
|
||||
plainGauge(w, "easyai_gateway_async_worker_capacity_capped", "Whether desired asynchronous worker capacity is capped by the hard limit.", m.asyncWorkerCapacityCapped.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_active_instances", "Active distributed worker instances with a fresh heartbeat.", m.asyncWorkerActiveInstances.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_global_capacity", "Global asynchronous execution capacity before instance allocation.", m.asyncWorkerGlobalCapacity.Load())
|
||||
plainGauge(w, "easyai_gateway_worker_allocated_capacity", "Asynchronous execution capacity allocated to this worker instance.", m.asyncWorkerAllocated.Load())
|
||||
outcomeCounters(w, "easyai_gateway_async_worker_resizes_total", "Asynchronous worker resize attempts by bounded outcome.", []outcomeValue{
|
||||
{"success", m.asyncWorkerResizeSuccess.Load()},
|
||||
{"refresh_failed", m.asyncWorkerRefreshFailed.Load()},
|
||||
@@ -292,6 +366,21 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
{"unknown_type", m.taskEventUnknownType.Load()},
|
||||
{"budget_exceeded", m.taskEventBudgetExceeded.Load()},
|
||||
})
|
||||
plainGauge(w, "easyai_gateway_task_admission_queue_depth", "Current persistent non-text task admission queue depth.", int64(admission.QueueDepth))
|
||||
plainGauge(w, "easyai_gateway_task_admission_waiting_sync", "Current synchronous admission waiters.", int64(admission.WaitingSync))
|
||||
plainGauge(w, "easyai_gateway_task_admission_waiting_async", "Current asynchronous admission waiters.", int64(admission.WaitingAsync))
|
||||
plainFloatGauge(w, "easyai_gateway_task_admission_oldest_wait_seconds", "Age of the oldest persistent admission waiter.", admission.OldestWaitSeconds)
|
||||
plainGauge(w, "easyai_gateway_task_admission_expired_waiter_backlog", "Expired synchronous waiter leases awaiting reclamation.", int64(admission.ExpiredWaiterBacklog))
|
||||
plainGauge(w, "easyai_gateway_task_admission_expired_deadline_backlog", "Expired queue deadlines awaiting reclamation.", int64(admission.ExpiredDeadlineBacklog))
|
||||
outcomeCounters(w, "easyai_gateway_task_admissions_total", "Task admission transitions by bounded outcome.", []outcomeValue{
|
||||
{"admitted", m.taskAdmissionAdmitted.Load()},
|
||||
{"queue_full", m.taskAdmissionQueueFull.Load()},
|
||||
{"timeout", m.taskAdmissionTimeout.Load()},
|
||||
{"cancelled", m.taskAdmissionCancelled.Load()},
|
||||
{"expired", m.taskAdmissionExpired.Load()},
|
||||
{"candidate_migrated", m.taskAdmissionMigrated.Load()},
|
||||
})
|
||||
taskAdmissionWaitHistogram(w, m)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -333,3 +422,25 @@ func plainCounter(w http.ResponseWriter, name, help string, value uint64) {
|
||||
func plainGauge(w http.ResponseWriter, name, help string, value int64) {
|
||||
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %d\n", name, help, name, name, value)
|
||||
}
|
||||
|
||||
func plainFloatGauge(w http.ResponseWriter, name, help string, value float64) {
|
||||
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %.6f\n", name, help, name, name, value)
|
||||
}
|
||||
|
||||
func taskAdmissionWaitHistogram(w http.ResponseWriter, metrics *Metrics) {
|
||||
const name = "easyai_gateway_task_admission_wait_seconds"
|
||||
fmt.Fprintf(w, "# HELP %s Time spent waiting for persistent task admission.\n# TYPE %s histogram\n", name, name)
|
||||
for index, bound := range taskAdmissionWaitBounds {
|
||||
fmt.Fprintf(
|
||||
w,
|
||||
"%s_bucket{le=\"%g\"} %d\n",
|
||||
name,
|
||||
bound.Seconds(),
|
||||
metrics.taskAdmissionWaitBuckets[index].Load(),
|
||||
)
|
||||
}
|
||||
count := metrics.taskAdmissionWaitBuckets[len(metrics.taskAdmissionWaitBuckets)-1].Load()
|
||||
fmt.Fprintf(w, "%s_bucket{le=\"+Inf\"} %d\n", name, count)
|
||||
fmt.Fprintf(w, "%s_sum %.6f\n", name, float64(metrics.taskAdmissionWaitMicros.Load())/1_000_000)
|
||||
fmt.Fprintf(w, "%s_count %d\n", name, count)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user