fix(queue): 隔离异步准入故障并持久化退避

将 X-Async 调度改为逐任务阻塞协议，区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障，避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈，并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证：API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过；pnpm audit 命中未改动的 Nx 工具链既有漏洞。
2026-08-04 18:23:35 +08:00
parent 44d5cf2b9d
commit f4214dd489
12 changed files with 1654 additions and 133 deletions
@@ -131,6 +131,9 @@ RETURNING id::text`, platform.ID, "queue-model-"+suffix).Scan(&platformModelID);
 	if err != nil || result.Admitted {
 		t.Fatalf("second task should wait: result=%+v err=%v", result, err)
 	}
+	if len(result.Blockers) != 1 || result.Blockers[0].Reason != "saturated" || result.Blockers[0].ScopeType != "platform_model" {
+		t.Fatalf("second task blockers=%+v, want saturated platform_model", result.Blockers)
+	}
 	higherPriorityAsync := createTask(true)
 	result, err = first.TryTaskAdmission(ctx, inputFor(higherPriorityAsync, 10, ""))
 	if err != nil || result.Admitted {
@@ -176,6 +179,9 @@ WHERE task_id = ANY($1::uuid[])`, taskIDs).Scan(&attempts); err != nil {
 	if result.Admitted {
 		t.Fatal("lower-priority task bypassed higher-priority asynchronous waiter")
 	}
+	if len(result.Blockers) != 1 || result.Blockers[0].Reason != "not_queue_head" || result.Blockers[0].ScopeType != "platform_model" {
+		t.Fatalf("lower-priority blockers=%+v, want platform FIFO head blocker", result.Blockers)
+	}
 	result, err = first.TryTaskAdmission(ctx, inputFor(higherPriorityAsync, 10, ""))
 	if err != nil || !result.Admitted {
 		t.Fatalf("higher-priority async task was not admitted first: result=%+v err=%v", result, err)
@@ -451,12 +457,23 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID, queuedSyntheticRiverJobID)
 	if listedSnapshot == nil || len(listedSnapshot.Scopes) != len(queuedAdmission.Scopes) {
 		t.Fatalf("listed admission snapshot=%+v, want %d scopes", listedSnapshot, len(queuedAdmission.Scopes))
 	}
-	markedForReselect, err := first.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
+	_, markedForReselect, err := first.RequestTaskAdmissionReselect(ctx, queuedAtomicTask.ID, platformModelID)
 	if err != nil {
 		t.Fatalf("request waiting admission reselection: %v", err)
 	}
-	if markedForReselect < 1 {
-		t.Fatalf("marked admissions=%d, want at least the queued task", markedForReselect)
+	if !markedForReselect {
+		t.Fatal("queued task was not marked for reselection")
+	}
+	var platformReselectCount int
+	if err := first.pool.QueryRow(ctx, `
+SELECT count(*)
+FROM gateway_task_admissions
+WHERE platform_model_id = $1::uuid
+  AND reselect_requested_at IS NOT NULL`, platformModelID).Scan(&platformReselectCount); err != nil {
+		t.Fatalf("count task-scoped reselections: %v", err)
+	}
+	if platformReselectCount != 1 {
+		t.Fatalf("platform reselection fanout=%d, want exactly one task", platformReselectCount)
 	}
 	reselectAdmission, err := first.GetTaskAdmission(ctx, queuedAtomicTask.ID)
 	if err != nil {
@@ -1070,6 +1087,276 @@ WHERE id = $1::uuid`, ambiguousTask.ID).Scan(&ambiguousStatus); err != nil {
 		t.Fatalf("ambiguous submission task status = %s, want running", ambiguousStatus)
 	}

+	globalHolder := createTask(true)
+	globalWaiter := createTask(true)
+	globalScope := AdmissionScope{
+		ScopeType:       "worker_capacity",
+		ScopeKey:        "global-blocker-" + suffix,
+		ScopeName:       "global blocker",
+		ConcurrentLimit: 1,
+		Amount:          1,
+		LeaseTTLSeconds: 120,
+		QueueLimit:      10,
+		MaxWaitSeconds:  600,
+	}
+	globalHolderInput := inputFor(globalHolder, 100, "")
+	globalHolderInput.Scopes = []AdmissionScope{globalScope}
+	globalWaiterInput := inputFor(globalWaiter, 100, "")
+	globalWaiterInput.Scopes = []AdmissionScope{globalScope}
+	if result, err := first.TryTaskAdmission(ctx, globalHolderInput); err != nil || !result.Admitted {
+		t.Fatalf("admit global capacity holder: result=%+v err=%v", result, err)
+	}
+	if _, err := first.QueueTaskAdmissionWithHook(ctx, globalWaiterInput, nil); err != nil {
+		t.Fatalf("queue global capacity waiter: %v", err)
+	}
+	globalWaitResult, err := second.TryTaskAdmission(ctx, globalWaiterInput)
+	if err != nil || globalWaitResult.Admitted || len(globalWaitResult.Blockers) != 1 ||
+		globalWaitResult.Blockers[0].Reason != "saturated" ||
+		globalWaitResult.Blockers[0].ScopeType != "worker_capacity" {
+		t.Fatalf("global capacity waiter result=%+v err=%v", globalWaitResult, err)
+	}
+	if !globalWaitResult.Admission.ReselectRequestedAt.IsZero() {
+		t.Fatal("global capacity wait unexpectedly requested candidate reselection")
+	}
+	if err := first.DeleteTaskAdmission(ctx, globalHolder.ID); err != nil {
+		t.Fatalf("release global capacity holder: %v", err)
+	}
+	if err := first.DeleteTaskAdmission(ctx, globalWaiter.ID); err != nil {
+		t.Fatalf("delete global capacity waiter: %v", err)
+	}
+
+	orphanedAdmittedTask := createTask(true)
+	orphanedAdmittedInput := inputFor(orphanedAdmittedTask, 100, "")
+	orphanedAdmittedInput.Scopes = []AdmissionScope{{
+		ScopeType:       "worker_capacity",
+		ScopeKey:        "orphaned-admitted-" + suffix,
+		ConcurrentLimit: 1,
+		Amount:          1,
+		LeaseTTLSeconds: 120,
+		QueueLimit:      10,
+		MaxWaitSeconds:  600,
+	}}
+	if result, err := first.TryTaskAdmission(ctx, orphanedAdmittedInput); err != nil || !result.Admitted {
+		t.Fatalf("admit orphan candidate: result=%+v err=%v", result, err)
+	}
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_concurrency_leases
+SET released_at = now()
+WHERE task_id = $1::uuid
+  AND released_at IS NULL`, orphanedAdmittedTask.ID); err != nil {
+		t.Fatalf("orphan admitted task leases: %v", err)
+	}
+	if result, err := second.TryTaskAdmission(ctx, orphanedAdmittedInput); err != nil || !result.Admitted || !result.NewlyAdmitted {
+		t.Fatalf("recover admitted row without lease: result=%+v err=%v", result, err)
+	}
+	if err := first.DeleteTaskAdmission(ctx, orphanedAdmittedTask.ID); err != nil {
+		t.Fatalf("delete recovered admitted row: %v", err)
+	}
+
+	resilienceTask := createTask(true)
+	resilienceInput := inputFor(resilienceTask, 100, "")
+	resilienceInput.Scopes = []AdmissionScope{{
+		ScopeType:       "worker_capacity",
+		ScopeKey:        "resilience-" + suffix,
+		ScopeName:       "resilience capacity",
+		ConcurrentLimit: 5,
+		Amount:          1,
+		LeaseTTLSeconds: 120,
+		QueueLimit:      10,
+		MaxWaitSeconds:  600,
+	}}
+	if _, err := first.QueueTaskAdmissionWithHook(ctx, resilienceInput, nil); err != nil {
+		t.Fatalf("queue resilience task: %v", err)
+	}
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_task_admissions
+SET scope_snapshot = '[{"ScopeType":123}]'::jsonb
+WHERE task_id = $1::uuid`, resilienceTask.ID); err != nil {
+		t.Fatalf("corrupt resilience scope snapshot: %v", err)
+	}
+	listedAdmissions, err := first.ListWaitingAsyncAdmissions(ctx, 1000)
+	if err != nil {
+		t.Fatalf("list admissions with one corrupt snapshot: %v", err)
+	}
+	corruptFound := false
+	for _, listed := range listedAdmissions {
+		if listed.TaskID == resilienceTask.ID {
+			corruptFound = listed.SnapshotInvalid
+			break
+		}
+	}
+	if !corruptFound {
+		t.Fatal("corrupt admission snapshot was not isolated for rebuilding")
+	}
+	repairedAdmission, err := first.RebindWaitingTaskAdmission(ctx, resilienceInput)
+	if err != nil || repairedAdmission.SnapshotInvalid || len(repairedAdmission.Scopes) != 1 {
+		t.Fatalf("repair corrupt admission snapshot=%+v err=%v", repairedAdmission, err)
+	}
+	deferred, changed, err := first.DeferWaitingTaskAdmission(
+		ctx,
+		resilienceTask.ID,
+		platformModelID,
+		time.Now().Add(time.Minute),
+		"deferred",
+		"snapshot_invalid",
+	)
+	if err != nil || !changed || deferred.DispatchFailureCount != 1 || deferred.DispatchLastErrorCode != "snapshot_invalid" {
+		t.Fatalf("defer corrupt admission=%+v changed=%v err=%v", deferred, changed, err)
+	}
+	listedAdmissions, err = first.ListWaitingAsyncAdmissions(ctx, 1000)
+	if err != nil {
+		t.Fatalf("list admissions after defer: %v", err)
+	}
+	for _, listed := range listedAdmissions {
+		if listed.TaskID == resilienceTask.ID {
+			t.Fatal("deferred admission remained dispatchable")
+		}
+	}
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_task_admissions
+SET dispatch_next_at = now(), scope_snapshot = '[]'::jsonb
+WHERE task_id = $1::uuid`, resilienceTask.ID); err != nil {
+		t.Fatalf("make resilience admission dispatchable: %v", err)
+	}
+	reselected, changed, err := first.RequestTaskAdmissionReselect(ctx, resilienceTask.ID, platformModelID)
+	if err != nil || !changed || reselected.ReselectRequestedAt.IsZero() {
+		t.Fatalf("task-scoped reselection=%+v changed=%v err=%v", reselected, changed, err)
+	}
+	if _, changed, err := second.RequestTaskAdmissionReselect(ctx, resilienceTask.ID, platformModelID); err != nil || changed {
+		t.Fatalf("duplicate task-scoped reselection changed=%v err=%v", changed, err)
+	}
+	if _, err := first.FailQueuedTaskWithCallback(
+		ctx,
+		resilienceTask.ID,
+		"no_model_candidate",
+		"no enabled platform model matches request",
+		"https://callback.invalid/admission-resilience",
+		false,
+	); err != nil {
+		t.Fatalf("terminally fail resilience task: %v", err)
+	}
+	if _, err := second.FailQueuedTaskWithCallback(
+		ctx,
+		resilienceTask.ID,
+		"no_model_candidate",
+		"duplicate terminalization",
+		"https://callback.invalid/admission-resilience",
+		false,
+	); !errors.Is(err, ErrTaskExecutionFinished) {
+		t.Fatalf("duplicate terminalization error=%v, want task finished", err)
+	}
+	var resilienceStatus, resilienceCode string
+	var resilienceAdmissions, resilienceLeases, resilienceFailedEvents, resilienceCallbacks int
+	if err := first.pool.QueryRow(ctx, `
+SELECT task.status, COALESCE(task.error_code, ''),
+       (SELECT count(*) FROM gateway_task_admissions admission WHERE admission.task_id = task.id),
+       (SELECT count(*) FROM gateway_concurrency_leases lease WHERE lease.task_id = task.id AND lease.released_at IS NULL),
+       (SELECT count(*) FROM gateway_task_events event WHERE event.task_id = task.id AND event.event_type = 'task.failed'),
+       (SELECT count(*) FROM gateway_task_callback_outbox callback WHERE callback.task_id = task.id)
+FROM gateway_tasks task
+WHERE task.id = $1::uuid`, resilienceTask.ID).Scan(
+		&resilienceStatus,
+		&resilienceCode,
+		&resilienceAdmissions,
+		&resilienceLeases,
+		&resilienceFailedEvents,
+		&resilienceCallbacks,
+	); err != nil {
+		t.Fatalf("read resilience terminal state: %v", err)
+	}
+	if resilienceStatus != "failed" || resilienceCode != "no_model_candidate" ||
+		resilienceAdmissions != 0 || resilienceLeases != 0 ||
+		resilienceFailedEvents != 1 || resilienceCallbacks != 1 {
+		t.Fatalf(
+			"resilience terminal state=%s/%s admissions=%d leases=%d events=%d callbacks=%d",
+			resilienceStatus,
+			resilienceCode,
+			resilienceAdmissions,
+			resilienceLeases,
+			resilienceFailedEvents,
+			resilienceCallbacks,
+		)
+	}
+	retryExpiredTask := createTask(true)
+	retryExpiredInput := resilienceInput
+	retryExpiredInput.TaskID = retryExpiredTask.ID
+	if _, err := first.QueueTaskAdmissionWithHook(ctx, retryExpiredInput, nil); err != nil {
+		t.Fatalf("queue retry-expired task: %v", err)
+	}
+	if _, err := first.pool.Exec(ctx, `
+UPDATE gateway_task_admissions
+SET enqueued_at = now() - interval '2 seconds',
+    wait_deadline_at = now() - interval '1 second',
+    dispatch_failure_count = 3,
+    dispatch_last_outcome = 'deferred',
+    dispatch_last_error_code = 'client_error',
+    dispatch_last_error_at = now() - interval '1 minute'
+WHERE task_id = $1::uuid`, retryExpiredTask.ID); err != nil {
+		t.Fatalf("expire deferred task: %v", err)
+	}
+	reaped, err = first.ReapExpiredTaskAdmissions(ctx, 10, "https://callback.invalid/admission-timeout")
+	if err != nil || reaped.ExpiredDeadlines < 1 {
+		t.Fatalf("reap retry-expired task=%+v err=%v", reaped, err)
+	}
+	var retryExpiredCode string
+	var retryExpiredEvents, retryExpiredCallbacks int
+	if err := first.pool.QueryRow(ctx, `
+SELECT COALESCE(task.error_code, ''),
+       (SELECT count(*) FROM gateway_task_events event WHERE event.task_id = task.id AND event.event_type = 'task.failed'),
+       (SELECT count(*) FROM gateway_task_callback_outbox callback WHERE callback.task_id = task.id)
+FROM gateway_tasks task
+WHERE task.id = $1::uuid`, retryExpiredTask.ID).Scan(
+		&retryExpiredCode,
+		&retryExpiredEvents,
+		&retryExpiredCallbacks,
+	); err != nil {
+		t.Fatalf("read retry-expired task: %v", err)
+	}
+	if retryExpiredCode != "admission_dispatch_failed" || retryExpiredEvents != 1 || retryExpiredCallbacks != 1 {
+		t.Fatalf(
+			"retry-expired task code=%s events=%d callbacks=%d, want admission_dispatch_failed/1/1",
+			retryExpiredCode,
+			retryExpiredEvents,
+			retryExpiredCallbacks,
+		)
+	}
+
+	disconnectedTask := createTask(true)
+	disconnectedInput := resilienceInput
+	disconnectedInput.TaskID = disconnectedTask.ID
+	if _, err := first.QueueTaskAdmissionWithHook(ctx, disconnectedInput, nil); err != nil {
+		t.Fatalf("queue disconnected task: %v", err)
+	}
+	if _, changed, err := first.CancelQueuedTaskWithCallback(
+		ctx,
+		disconnectedTask.ID,
+		"client disconnected before upstream submission",
+		"https://callback.invalid/admission-disconnect",
+	); err != nil || !changed {
+		t.Fatalf("cancel disconnected task changed=%v err=%v", changed, err)
+	}
+	var disconnectedEvents, disconnectedCallbacks, disconnectedAdmissions int
+	if err := first.pool.QueryRow(ctx, `
+SELECT (SELECT count(*) FROM gateway_task_events event WHERE event.task_id = task.id AND event.event_type = 'task.cancelled'),
+       (SELECT count(*) FROM gateway_task_callback_outbox callback WHERE callback.task_id = task.id),
+       (SELECT count(*) FROM gateway_task_admissions admission WHERE admission.task_id = task.id)
+FROM gateway_tasks task
+WHERE task.id = $1::uuid`, disconnectedTask.ID).Scan(
+		&disconnectedEvents,
+		&disconnectedCallbacks,
+		&disconnectedAdmissions,
+	); err != nil {
+		t.Fatalf("read disconnected task cleanup: %v", err)
+	}
+	if disconnectedEvents != 1 || disconnectedCallbacks != 1 || disconnectedAdmissions != 0 {
+		t.Fatalf(
+			"disconnected task events=%d callbacks=%d admissions=%d, want 1/1/0",
+			disconnectedEvents,
+			disconnectedCallbacks,
+			disconnectedAdmissions,
+		)
+	}
+
 	terminalResidue := createTask(true)
 	result, err = first.TryTaskAdmission(ctx, inputFor(terminalResidue, 100, ""))
 	if err != nil || !result.Admitted {
@@ -1247,6 +1534,150 @@ WHERE task_id = ANY($1::uuid[])`, taskIDs); err != nil {
 	if !leaderSet[tasks[0].ID] || leaderSet[tasks[2].ID] {
 		t.Fatalf("grouped candidate leaders = %v, want only group head %s", leaders, tasks[0].ID)
 	}
+	if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = ANY($1::uuid[])`, taskIDs); err != nil {
+		t.Fatalf("clear grouped admission leaders: %v", err)
+	}
+
+	createAsyncTask := func(modelIndex int) GatewayTask {
+		t.Helper()
+		task, createErr := db.CreateTask(ctx, CreateTaskInput{
+			Kind:    "images.generate",
+			Model:   fmt.Sprintf("leader-model-%d-%s", modelIndex, suffix),
+			RunMode: "production",
+			Async:   true,
+			Request: map[string]any{"prompt": "async admission independence test"},
+		}, &auth.User{ID: "leader-test-user-" + suffix, Source: "gateway"})
+		if createErr != nil {
+			t.Fatalf("create async task: %v", createErr)
+		}
+		taskIDs = append(taskIDs, task.ID)
+		return task
+	}
+	platformScope := func(modelID string) AdmissionScope {
+		return AdmissionScope{
+			ScopeType:       "platform_model",
+			ScopeKey:        modelID,
+			ConcurrentLimit: 1,
+			Amount:          1,
+			LeaseTTLSeconds: 120,
+			QueueLimit:      10,
+			MaxWaitSeconds:  600,
+		}
+	}
+	groupScope := AdmissionScope{
+		ScopeType:       "user_group",
+		ScopeKey:        group.ID,
+		ConcurrentLimit: 1,
+		Amount:          1,
+		LeaseTTLSeconds: 120,
+		QueueLimit:      10,
+		MaxWaitSeconds:  600,
+	}
+	holder := createAsyncTask(0)
+	blocked := createAsyncTask(0)
+	independent := createAsyncTask(1)
+	holderInput := TaskAdmissionInput{
+		TaskID: holder.ID, PlatformID: platform.ID, PlatformModelID: modelIDs[0],
+		UserGroupID: group.ID, QueueKey: "leader-test:" + modelIDs[0], Mode: "async", Priority: 100,
+		Scopes: []AdmissionScope{platformScope(modelIDs[0]), groupScope},
+	}
+	blockedInput := holderInput
+	blockedInput.TaskID = blocked.ID
+	independentInput := TaskAdmissionInput{
+		TaskID: independent.ID, PlatformID: platform.ID, PlatformModelID: modelIDs[1],
+		QueueKey: "leader-test:" + modelIDs[1], Mode: "async", Priority: 100,
+		Scopes: []AdmissionScope{platformScope(modelIDs[1])},
+	}
+	if result, err := db.TryTaskAdmission(ctx, holderInput); err != nil || !result.Admitted {
+		t.Fatalf("admit local blocker holder: result=%+v err=%v", result, err)
+	}
+	if _, err := db.QueueTaskAdmissionWithHook(ctx, blockedInput, nil); err != nil {
+		t.Fatalf("queue locally blocked task: %v", err)
+	}
+	if _, err := db.QueueTaskAdmissionWithHook(ctx, independentInput, nil); err != nil {
+		t.Fatalf("queue independent task: %v", err)
+	}
+	outcomes, err := db.TryTaskAdmissionAtomicBatchWithAdmittedHook(
+		ctx,
+		[]TaskAdmissionInput{blockedInput, independentInput},
+		nil,
+	)
+	if err != nil || len(outcomes) != 2 {
+		t.Fatalf("independent atomic outcomes=%+v err=%v", outcomes, err)
+	}
+	if outcomes[0].Result.Admitted || len(outcomes[0].Result.Blockers) == 0 {
+		t.Fatalf("locally blocked outcome=%+v, want explicit blockers", outcomes[0])
+	}
+	if !outcomes[1].Result.Admitted {
+		t.Fatalf("independent candidate was cross-blocked: %+v", outcomes[1])
+	}
+	for _, taskID := range []string{holder.ID, blocked.ID, independent.ID} {
+		if err := db.DeleteTaskAdmission(ctx, taskID); err != nil {
+			t.Fatalf("clear independent atomic admission %s: %v", taskID, err)
+		}
+	}
+
+	hookTasks := []GatewayTask{createAsyncTask(0), createAsyncTask(0), createAsyncTask(1)}
+	hookInputs := make([]TaskAdmissionInput, 0, len(hookTasks))
+	for index, task := range hookTasks {
+		input := TaskAdmissionInput{
+			TaskID: task.ID, PlatformID: platform.ID, PlatformModelID: modelIDs[index%len(modelIDs)],
+			QueueKey: "hook-isolation:" + task.ID, Mode: "async", Priority: 100,
+			Scopes: []AdmissionScope{{
+				ScopeType:       "worker_capacity",
+				ScopeKey:        "hook-isolation-" + task.ID,
+				ConcurrentLimit: 1,
+				Amount:          1,
+				LeaseTTLSeconds: 120,
+				QueueLimit:      10,
+				MaxWaitSeconds:  600,
+			}},
+		}
+		if _, err := db.QueueTaskAdmissionWithHook(ctx, input, nil); err != nil {
+			t.Fatalf("queue hook isolation task %d: %v", index, err)
+		}
+		hookInputs = append(hookInputs, input)
+	}
+	hookFailure := errors.New("synthetic per-task hook failure")
+	outcomes, err = db.TryTaskAdmissionAtomicBatchWithAdmittedHook(
+		ctx,
+		hookInputs,
+		func(_ pgx.Tx, input TaskAdmissionInput) error {
+			if input.TaskID == hookTasks[1].ID {
+				return hookFailure
+			}
+			return nil
+		},
+	)
+	if !errors.Is(err, hookFailure) || len(outcomes) != 2 || outcomes[1].TaskID != hookTasks[1].ID {
+		t.Fatalf("hook failure outcomes=%+v err=%v", outcomes, err)
+	}
+	var hookAdmitted, hookLeases int
+	if err := db.pool.QueryRow(ctx, `
+SELECT (SELECT count(*) FROM gateway_task_admissions WHERE task_id = ANY($1::uuid[]) AND status = 'admitted'),
+       (SELECT count(*) FROM gateway_concurrency_leases WHERE task_id = ANY($1::uuid[]) AND released_at IS NULL)`,
+		[]string{hookTasks[0].ID, hookTasks[1].ID, hookTasks[2].ID},
+	).Scan(&hookAdmitted, &hookLeases); err != nil {
+		t.Fatalf("read rolled back hook batch: %v", err)
+	}
+	if hookAdmitted != 0 || hookLeases != 0 {
+		t.Fatalf("failed hook batch left admitted=%d leases=%d", hookAdmitted, hookLeases)
+	}
+	if _, changed, err := db.DeferWaitingTaskAdmission(
+		ctx,
+		hookTasks[1].ID,
+		hookInputs[1].PlatformModelID,
+		time.Now().Add(time.Minute),
+		"deferred",
+		"synthetic_hook_failure",
+	); err != nil || !changed {
+		t.Fatalf("isolate hook failure changed=%v err=%v", changed, err)
+	}
+	for _, index := range []int{0, 2} {
+		if result, err := db.TryTaskAdmission(ctx, hookInputs[index]); err != nil || !result.Admitted {
+			t.Fatalf("admit hook survivor %d result=%+v err=%v", index, result, err)
+		}
+	}
 }

 func TestWorkerCapacityAllocationAndFailover(t *testing.T) {