fix(queue): 重新调度终态 River 任务
等待异步准入的任务只跳过仍处于活跃状态的 River Job;missing 或 terminal Job 交由准入调度器重新创建,避免非空 river_job_id 永久阻断旧任务。\n\n通用 River 恢复排除等待准入任务,消除双恢复循环与误导日志;事务回滚继续使用独立有界上下文。\n\n验证:真实 PostgreSQL 下异步 Worker 准入验收和跨 Store 队列集成测试均通过。
This commit is contained in:
@@ -596,7 +596,7 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if task.Status != "queued" || task.RiverJobID > 0 {
|
||||
if task.Status != "queued" {
|
||||
continue
|
||||
}
|
||||
if err := s.dispatchWaitingAsyncTask(ctx, task); err != nil {
|
||||
|
||||
@@ -502,7 +502,11 @@ func (s *Service) enqueueAsyncTaskWithOptions(ctx context.Context, taskID string
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
defer func() {
|
||||
rollbackCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tx.Rollback(rollbackCtx)
|
||||
}()
|
||||
if err := s.enqueueAsyncTaskTx(ctx, tx, taskID, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -548,18 +552,15 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recovered := 0
|
||||
for _, item := range items {
|
||||
if admission, admissionErr := s.store.GetTaskAdmission(ctx, item.ID); admissionErr == nil && admission.Status == "waiting" {
|
||||
continue
|
||||
} else if admissionErr != nil && !errors.Is(admissionErr, pgx.ErrNoRows) {
|
||||
return admissionErr
|
||||
}
|
||||
if err := s.enqueueAsyncTaskWithOptions(ctx, item.ID, asyncTaskRecoveryInsertOpts(item, time.Now())); err != nil {
|
||||
return err
|
||||
}
|
||||
recovered++
|
||||
}
|
||||
if len(items) > 0 {
|
||||
s.logger.Info("river async queue recovered persisted tasks", "count", len(items))
|
||||
if recovered > 0 {
|
||||
s.logger.Info("river async queue recovered persisted tasks", "count", recovered)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -684,11 +684,19 @@ func (s *Store) ListWaitingAsyncAdmissionTaskIDs(ctx context.Context, limit int)
|
||||
limit = 100
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT task_id::text
|
||||
FROM gateway_task_admissions
|
||||
WHERE status = 'waiting'
|
||||
AND mode = 'async'
|
||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
||||
SELECT admission.task_id::text
|
||||
FROM gateway_task_admissions admission
|
||||
JOIN gateway_tasks task ON task.id = admission.task_id
|
||||
LEFT JOIN river_job job ON job.id = task.river_job_id
|
||||
WHERE admission.status = 'waiting'
|
||||
AND admission.mode = 'async'
|
||||
AND task.status = 'queued'
|
||||
AND (
|
||||
task.river_job_id IS NULL
|
||||
OR job.id IS NULL
|
||||
OR job.state NOT IN ('available', 'pending', 'retryable', 'running', 'scheduled')
|
||||
)
|
||||
ORDER BY admission.priority ASC, admission.enqueued_at ASC, admission.task_id ASC
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -447,6 +447,29 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID).Scan(&riverJobID, &queuedLeases); err
|
||||
if riverJobID != queuedSyntheticRiverJobID || queuedLeases != 0 {
|
||||
t.Fatalf("queued River marker=%d leases=%d, want %d/0", riverJobID, queuedLeases, queuedSyntheticRiverJobID)
|
||||
}
|
||||
waitingAsyncTaskIDs, err := first.ListWaitingAsyncAdmissionTaskIDs(ctx, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("list waiting async tasks with missing River job: %v", err)
|
||||
}
|
||||
waitingAsyncFound := false
|
||||
for _, taskID := range waitingAsyncTaskIDs {
|
||||
if taskID == queuedAtomicTask.ID {
|
||||
waitingAsyncFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !waitingAsyncFound {
|
||||
t.Fatal("waiting async task with missing River job was not dispatchable")
|
||||
}
|
||||
recoverableTasks, err := first.ListRecoverableAsyncTasks(ctx, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("list recoverable tasks while admission is waiting: %v", err)
|
||||
}
|
||||
for _, item := range recoverableTasks {
|
||||
if item.ID == queuedAtomicTask.ID {
|
||||
t.Fatal("generic River recovery claimed a task owned by the async admission dispatcher")
|
||||
}
|
||||
}
|
||||
result, err = second.TryTaskAdmission(ctx, inputFor(queuedAtomicTask, 100, ""))
|
||||
if err != nil || !result.Admitted || len(result.Leases) != 1 {
|
||||
t.Fatalf("worker-time queued admission result=%+v err=%v", result, err)
|
||||
|
||||
@@ -947,6 +947,12 @@ WHERE task.async_mode = true
|
||||
OR job.id IS NULL
|
||||
OR job.state NOT IN ('available', 'pending', 'retryable', 'running', 'scheduled')
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_task_admissions admission
|
||||
WHERE admission.task_id = task.id
|
||||
AND admission.status = 'waiting'
|
||||
)
|
||||
ORDER BY task.priority ASC, task.created_at ASC
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user