fix(queue): 隔离异步准入故障并持久化退避

将 X-Async 调度改为逐任务阻塞协议，区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障，避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈，并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证：API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过；pnpm audit 命中未改动的 Nx 工具链既有漏洞。
2026-08-04 18:23:35 +08:00
parent 44d5cf2b9d
commit f4214dd489
12 changed files with 1654 additions and 133 deletions
@@ -96,6 +96,141 @@ SELECT
 	assertAcceptanceAttempts(t, ctx, pool, taskIDs, len(taskIDs))
 }

+func TestAsyncAdmissionPoisonTaskDoesNotBlockHealthyFollower(t *testing.T) {
+	databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
+	if databaseURL == "" {
+		t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run async admission resilience tests")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+	applyMigration(t, ctx, databaseURL)
+
+	db, err := store.Connect(ctx, databaseURL)
+	if err != nil {
+		t.Fatalf("connect store: %v", err)
+	}
+	defer db.Close()
+	pool, err := pgxpool.New(ctx, databaseURL)
+	if err != nil {
+		t.Fatalf("connect acceptance pool: %v", err)
+	}
+	defer pool.Close()
+	restoreAcceptanceState := isolateHTTPAcceptanceState(t, ctx, pool, false)
+	defer restoreAcceptanceState()
+
+	serverCtx, cancelServer := context.WithCancel(ctx)
+	defer cancelServer()
+	server := httptest.NewServer(NewServerWithContext(serverCtx, config.Config{
+		AppEnv:                             "test",
+		HTTPAddr:                           ":0",
+		DatabaseURL:                        databaseURL,
+		IdentityMode:                       "hybrid",
+		JWTSecret:                          "test-secret",
+		BillingEngineMode:                  "observe",
+		CORSAllowedOrigin:                  "*",
+		AsyncQueueWorkerEnabled:            true,
+		AsyncWorkerHardLimit:               1,
+		AsyncWorkerInstanceHardLimit:       1,
+		AsyncWorkerRefreshIntervalSeconds:  1,
+		AsyncAdmissionMicrobatchSize:       3,
+		AsyncAdmissionDispatcherEnabled:    true,
+		AsyncAdmissionDispatcherConfigured: true,
+	}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
+	defer server.Close()
+
+	adminToken := createAsyncAcceptanceAdmin(t, ctx, pool, server.URL)
+	if _, err := pool.Exec(ctx, `
+UPDATE gateway_user_groups
+SET rate_limit_policy = '{"rules":[{"metric":"concurrent","limit":256,"leaseTtlSeconds":120}]}'::jsonb
+WHERE status = 'active'`); err != nil {
+		t.Fatalf("raise resilience user-group concurrency: %v", err)
+	}
+	suffix := strconv.FormatInt(time.Now().UnixNano(), 10)
+	model := "admission-poison-" + suffix
+	platform := createAsyncAcceptancePlatform(t, server.URL, adminToken, "admission-poison-"+suffix, "Admission Poison Simulation", 3)
+	defer updateAsyncAcceptancePlatform(t, server.URL, adminToken, platform, 3, "disabled")
+	createAsyncAcceptanceModel(t, server.URL, adminToken, platform.ID, model, "video_generate", "inherit", 0, 120)
+
+	waitForAsyncWorkerMetric(t, server.URL, "easyai_gateway_async_worker_capacity", 1, 15*time.Second)
+	taskIDs := submitAsyncSimulationTasks(t, server.URL, adminToken, "/api/v1/videos/generations", model, 3, 10*time.Second)
+	waitForTaskCount(t, ctx, pool, taskIDs, "running", 1, 10*time.Second)
+
+	var poisonTaskID, followerTaskID string
+	if err := pool.QueryRow(ctx, `
+SELECT (array_agg(admission.task_id::text ORDER BY admission.priority, admission.enqueued_at, admission.task_id))[1],
+       (array_agg(admission.task_id::text ORDER BY admission.priority, admission.enqueued_at, admission.task_id))[2]
+FROM gateway_task_admissions admission
+JOIN gateway_tasks task ON task.id = admission.task_id
+WHERE task.id = ANY($1::uuid[])
+  AND task.status = 'queued'
+  AND admission.status = 'waiting'`, taskIDs).Scan(&poisonTaskID, &followerTaskID); err != nil {
+		t.Fatalf("select waiting poison and follower tasks: %v", err)
+	}
+	var prematureReselect int
+	if err := pool.QueryRow(ctx, `
+SELECT count(*)
+FROM gateway_task_admissions
+WHERE task_id = ANY($1::uuid[])
+  AND status = 'waiting'
+  AND reselect_requested_at IS NOT NULL`, taskIDs).Scan(&prematureReselect); err != nil {
+		t.Fatalf("count global-capacity reselections: %v", err)
+	}
+	if prematureReselect != 0 {
+		t.Fatalf("global worker saturation marked %d tasks for reselection", prematureReselect)
+	}
+	if _, err := pool.Exec(ctx, `
+UPDATE gateway_tasks
+SET model = $2,
+    requested_model = $2,
+    updated_at = now()
+WHERE id = $1::uuid`, poisonTaskID, "missing-model-"+suffix); err != nil {
+		t.Fatalf("poison queued task model: %v", err)
+	}
+	if _, err := pool.Exec(ctx, `
+UPDATE gateway_task_admissions
+SET reselect_requested_at = now(),
+    dispatch_next_at = now(),
+    updated_at = now()
+WHERE task_id = $1::uuid`, poisonTaskID); err != nil {
+		t.Fatalf("poison queued admission binding: %v", err)
+	}
+
+	waitForTaskCount(t, ctx, pool, []string{poisonTaskID}, "failed", 1, 20*time.Second)
+	waitForTaskCount(t, ctx, pool, []string{followerTaskID}, "succeeded", 1, 30*time.Second)
+	var poisonCode string
+	var poisonAttempts, poisonAdmissions, poisonLeases, poisonFailedEvents int
+	var poisonRiverJobID int64
+	if err := pool.QueryRow(ctx, `
+SELECT COALESCE(task.error_code, ''), COALESCE(task.river_job_id, 0),
+       (SELECT count(*) FROM gateway_task_attempts attempt WHERE attempt.task_id = task.id),
+       (SELECT count(*) FROM gateway_task_admissions admission WHERE admission.task_id = task.id),
+       (SELECT count(*) FROM gateway_concurrency_leases lease WHERE lease.task_id = task.id AND lease.released_at IS NULL),
+       (SELECT count(*) FROM gateway_task_events event WHERE event.task_id = task.id AND event.event_type = 'task.failed')
+FROM gateway_tasks task
+WHERE task.id = $1::uuid`, poisonTaskID).Scan(
+		&poisonCode,
+		&poisonRiverJobID,
+		&poisonAttempts,
+		&poisonAdmissions,
+		&poisonLeases,
+		&poisonFailedEvents,
+	); err != nil {
+		t.Fatalf("read poison task terminal state: %v", err)
+	}
+	if poisonCode != "no_model_candidate" || poisonRiverJobID != 0 || poisonAttempts != 0 ||
+		poisonAdmissions != 0 || poisonLeases != 0 || poisonFailedEvents != 1 {
+		t.Fatalf(
+			"poison state code=%s river=%v attempts=%d admissions=%d leases=%d events=%d",
+			poisonCode,
+			poisonRiverJobID,
+			poisonAttempts,
+			poisonAdmissions,
+			poisonLeases,
+			poisonFailedEvents,
+		)
+	}
+}
+
 func TestAsyncWorkerThousandConcurrentSchedulingAcceptance(t *testing.T) {
 	databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
 	if databaseURL == "" {
@@ -537,7 +672,24 @@ func waitForTaskCount(t *testing.T, ctx context.Context, pool *pgxpool.Pool, tas
 		}
 		time.Sleep(100 * time.Millisecond)
 	}
-	t.Fatalf("task status %s count did not reach %d within %s", status, count, timeout)
+	rows, err := pool.Query(ctx, `
+SELECT task.status, COALESCE(task.error_code, ''), count(*)
+FROM gateway_tasks task
+WHERE task.id = ANY($1::uuid[])
+GROUP BY task.status, COALESCE(task.error_code, '')
+ORDER BY task.status, COALESCE(task.error_code, '')`, taskIDs)
+	diagnostics := make([]string, 0)
+	if err == nil {
+		defer rows.Close()
+		for rows.Next() {
+			var taskStatus, code string
+			var taskCount int
+			if scanErr := rows.Scan(&taskStatus, &code, &taskCount); scanErr == nil {
+				diagnostics = append(diagnostics, fmt.Sprintf("%s/%s=%d", taskStatus, code, taskCount))
+			}
+		}
+	}
+	t.Fatalf("task status %s count did not reach %d within %s; observed=%v", status, count, timeout, diagnostics)
 }

 func activeLeaseExpiry(t *testing.T, ctx context.Context, pool *pgxpool.Pool, modelID string, wantCount int) time.Time {