fix(queue): 重新调度终态 River 任务
等待异步准入的任务只跳过仍处于活跃状态的 River Job;missing 或 terminal Job 交由准入调度器重新创建,避免非空 river_job_id 永久阻断旧任务。\n\n通用 River 恢复排除等待准入任务,消除双恢复循环与误导日志;事务回滚继续使用独立有界上下文。\n\n验证:真实 PostgreSQL 下异步 Worker 准入验收和跨 Store 队列集成测试均通过。
This commit is contained in:
@@ -596,7 +596,7 @@ func (s *Service) dispatchWaitingAsyncAdmissions(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if task.Status != "queued" || task.RiverJobID > 0 {
|
if task.Status != "queued" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := s.dispatchWaitingAsyncTask(ctx, task); err != nil {
|
if err := s.dispatchWaitingAsyncTask(ctx, task); err != nil {
|
||||||
|
|||||||
@@ -502,7 +502,11 @@ func (s *Service) enqueueAsyncTaskWithOptions(ctx context.Context, taskID string
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer tx.Rollback(ctx)
|
defer func() {
|
||||||
|
rollbackCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = tx.Rollback(rollbackCtx)
|
||||||
|
}()
|
||||||
if err := s.enqueueAsyncTaskTx(ctx, tx, taskID, opts); err != nil {
|
if err := s.enqueueAsyncTaskTx(ctx, tx, taskID, opts); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -548,18 +552,15 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
recovered := 0
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
if admission, admissionErr := s.store.GetTaskAdmission(ctx, item.ID); admissionErr == nil && admission.Status == "waiting" {
|
|
||||||
continue
|
|
||||||
} else if admissionErr != nil && !errors.Is(admissionErr, pgx.ErrNoRows) {
|
|
||||||
return admissionErr
|
|
||||||
}
|
|
||||||
if err := s.enqueueAsyncTaskWithOptions(ctx, item.ID, asyncTaskRecoveryInsertOpts(item, time.Now())); err != nil {
|
if err := s.enqueueAsyncTaskWithOptions(ctx, item.ID, asyncTaskRecoveryInsertOpts(item, time.Now())); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
recovered++
|
||||||
}
|
}
|
||||||
if len(items) > 0 {
|
if recovered > 0 {
|
||||||
s.logger.Info("river async queue recovered persisted tasks", "count", len(items))
|
s.logger.Info("river async queue recovered persisted tasks", "count", recovered)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -684,11 +684,19 @@ func (s *Store) ListWaitingAsyncAdmissionTaskIDs(ctx context.Context, limit int)
|
|||||||
limit = 100
|
limit = 100
|
||||||
}
|
}
|
||||||
rows, err := s.pool.Query(ctx, `
|
rows, err := s.pool.Query(ctx, `
|
||||||
SELECT task_id::text
|
SELECT admission.task_id::text
|
||||||
FROM gateway_task_admissions
|
FROM gateway_task_admissions admission
|
||||||
WHERE status = 'waiting'
|
JOIN gateway_tasks task ON task.id = admission.task_id
|
||||||
AND mode = 'async'
|
LEFT JOIN river_job job ON job.id = task.river_job_id
|
||||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
WHERE admission.status = 'waiting'
|
||||||
|
AND admission.mode = 'async'
|
||||||
|
AND task.status = 'queued'
|
||||||
|
AND (
|
||||||
|
task.river_job_id IS NULL
|
||||||
|
OR job.id IS NULL
|
||||||
|
OR job.state NOT IN ('available', 'pending', 'retryable', 'running', 'scheduled')
|
||||||
|
)
|
||||||
|
ORDER BY admission.priority ASC, admission.enqueued_at ASC, admission.task_id ASC
|
||||||
LIMIT $1`, limit)
|
LIMIT $1`, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -447,6 +447,29 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID).Scan(&riverJobID, &queuedLeases); err
|
|||||||
if riverJobID != queuedSyntheticRiverJobID || queuedLeases != 0 {
|
if riverJobID != queuedSyntheticRiverJobID || queuedLeases != 0 {
|
||||||
t.Fatalf("queued River marker=%d leases=%d, want %d/0", riverJobID, queuedLeases, queuedSyntheticRiverJobID)
|
t.Fatalf("queued River marker=%d leases=%d, want %d/0", riverJobID, queuedLeases, queuedSyntheticRiverJobID)
|
||||||
}
|
}
|
||||||
|
waitingAsyncTaskIDs, err := first.ListWaitingAsyncAdmissionTaskIDs(ctx, 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list waiting async tasks with missing River job: %v", err)
|
||||||
|
}
|
||||||
|
waitingAsyncFound := false
|
||||||
|
for _, taskID := range waitingAsyncTaskIDs {
|
||||||
|
if taskID == queuedAtomicTask.ID {
|
||||||
|
waitingAsyncFound = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !waitingAsyncFound {
|
||||||
|
t.Fatal("waiting async task with missing River job was not dispatchable")
|
||||||
|
}
|
||||||
|
recoverableTasks, err := first.ListRecoverableAsyncTasks(ctx, 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list recoverable tasks while admission is waiting: %v", err)
|
||||||
|
}
|
||||||
|
for _, item := range recoverableTasks {
|
||||||
|
if item.ID == queuedAtomicTask.ID {
|
||||||
|
t.Fatal("generic River recovery claimed a task owned by the async admission dispatcher")
|
||||||
|
}
|
||||||
|
}
|
||||||
result, err = second.TryTaskAdmission(ctx, inputFor(queuedAtomicTask, 100, ""))
|
result, err = second.TryTaskAdmission(ctx, inputFor(queuedAtomicTask, 100, ""))
|
||||||
if err != nil || !result.Admitted || len(result.Leases) != 1 {
|
if err != nil || !result.Admitted || len(result.Leases) != 1 {
|
||||||
t.Fatalf("worker-time queued admission result=%+v err=%v", result, err)
|
t.Fatalf("worker-time queued admission result=%+v err=%v", result, err)
|
||||||
|
|||||||
@@ -947,6 +947,12 @@ WHERE task.async_mode = true
|
|||||||
OR job.id IS NULL
|
OR job.id IS NULL
|
||||||
OR job.state NOT IN ('available', 'pending', 'retryable', 'running', 'scheduled')
|
OR job.state NOT IN ('available', 'pending', 'retryable', 'running', 'scheduled')
|
||||||
)
|
)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM gateway_task_admissions admission
|
||||||
|
WHERE admission.task_id = task.id
|
||||||
|
AND admission.status = 'waiting'
|
||||||
|
)
|
||||||
ORDER BY task.priority ASC, task.created_at ASC
|
ORDER BY task.priority ASC, task.created_at ASC
|
||||||
LIMIT $1`, limit)
|
LIMIT $1`, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user