fix(queue): 重新调度终态 River 任务

等待异步准入的任务只跳过仍处于活跃状态的 River Job;missing 或 terminal Job 交由准入调度器重新创建,避免非空 river_job_id 永久阻断旧任务。\n\n通用 River 恢复排除等待准入任务,消除双恢复循环与误导日志;事务回滚继续使用独立有界上下文。\n\n验证:真实 PostgreSQL 下异步 Worker 准入验收和跨 Store 队列集成测试均通过。
This commit is contained in:
2026-07-30 17:42:33 +08:00
parent e8893e2b8f
commit fa818e9ebf
5 changed files with 52 additions and 14 deletions
+1 -1
View File
@@ -596,7 +596,7 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
}
continue
}
if task.Status != "queued" || task.RiverJobID > 0 {
if task.Status != "queued" {
continue
}
if err := s.dispatchWaitingAsyncTask(ctx, task); err != nil {
+9 -8
View File
@@ -502,7 +502,11 @@ func (s *Service) enqueueAsyncTaskWithOptions(ctx context.Context, taskID string
if err != nil {
return err
}
defer tx.Rollback(ctx)
defer func() {
rollbackCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = tx.Rollback(rollbackCtx)
}()
if err := s.enqueueAsyncTaskTx(ctx, tx, taskID, opts); err != nil {
return err
}
@@ -548,18 +552,15 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
if err != nil {
return err
}
recovered := 0
for _, item := range items {
if admission, admissionErr := s.store.GetTaskAdmission(ctx, item.ID); admissionErr == nil && admission.Status == "waiting" {
continue
} else if admissionErr != nil && !errors.Is(admissionErr, pgx.ErrNoRows) {
return admissionErr
}
if err := s.enqueueAsyncTaskWithOptions(ctx, item.ID, asyncTaskRecoveryInsertOpts(item, time.Now())); err != nil {
return err
}
recovered++
}
if len(items) > 0 {
s.logger.Info("river async queue recovered persisted tasks", "count", len(items))
if recovered > 0 {
s.logger.Info("river async queue recovered persisted tasks", "count", recovered)
}
return nil
}
+13 -5
View File
@@ -684,11 +684,19 @@ func (s *Store) ListWaitingAsyncAdmissionTaskIDs(ctx context.Context, limit int)
limit = 100
}
rows, err := s.pool.Query(ctx, `
SELECT task_id::text
FROM gateway_task_admissions
WHERE status = 'waiting'
AND mode = 'async'
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
SELECT admission.task_id::text
FROM gateway_task_admissions admission
JOIN gateway_tasks task ON task.id = admission.task_id
LEFT JOIN river_job job ON job.id = task.river_job_id
WHERE admission.status = 'waiting'
AND admission.mode = 'async'
AND task.status = 'queued'
AND (
task.river_job_id IS NULL
OR job.id IS NULL
OR job.state NOT IN ('available', 'pending', 'retryable', 'running', 'scheduled')
)
ORDER BY admission.priority ASC, admission.enqueued_at ASC, admission.task_id ASC
LIMIT $1`, limit)
if err != nil {
return nil, err
@@ -447,6 +447,29 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID).Scan(&riverJobID, &queuedLeases); err
if riverJobID != queuedSyntheticRiverJobID || queuedLeases != 0 {
t.Fatalf("queued River marker=%d leases=%d, want %d/0", riverJobID, queuedLeases, queuedSyntheticRiverJobID)
}
waitingAsyncTaskIDs, err := first.ListWaitingAsyncAdmissionTaskIDs(ctx, 1000)
if err != nil {
t.Fatalf("list waiting async tasks with missing River job: %v", err)
}
waitingAsyncFound := false
for _, taskID := range waitingAsyncTaskIDs {
if taskID == queuedAtomicTask.ID {
waitingAsyncFound = true
break
}
}
if !waitingAsyncFound {
t.Fatal("waiting async task with missing River job was not dispatchable")
}
recoverableTasks, err := first.ListRecoverableAsyncTasks(ctx, 1000)
if err != nil {
t.Fatalf("list recoverable tasks while admission is waiting: %v", err)
}
for _, item := range recoverableTasks {
if item.ID == queuedAtomicTask.ID {
t.Fatal("generic River recovery claimed a task owned by the async admission dispatcher")
}
}
result, err = second.TryTaskAdmission(ctx, inputFor(queuedAtomicTask, 100, ""))
if err != nil || !result.Admitted || len(result.Leases) != 1 {
t.Fatalf("worker-time queued admission result=%+v err=%v", result, err)
+6
View File
@@ -947,6 +947,12 @@ WHERE task.async_mode = true
OR job.id IS NULL
OR job.state NOT IN ('available', 'pending', 'retryable', 'running', 'scheduled')
)
AND NOT EXISTS (
SELECT 1
FROM gateway_task_admissions admission
WHERE admission.task_id = task.id
AND admission.status = 'waiting'
)
ORDER BY task.priority ASC, task.created_at ASC
LIMIT $1`, limit)
if err != nil {