fix(queue): 隔离异步准入故障并持久化退避

将 X-Async 调度改为逐任务阻塞协议，区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障，避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈，并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证：API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过；pnpm audit 命中未改动的 Nx 工具链既有漏洞。
2026-08-04 18:23:35 +08:00
parent 44d5cf2b9d
commit f4214dd489
12 changed files with 1654 additions and 133 deletions
@@ -1,12 +1,84 @@
 package runner

 import (
+	"errors"
 	"testing"
 	"time"

+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+	"github.com/jackc/pgx/v5/pgconn"
 )

+func TestAsyncAdmissionTaskErrorDisposition(t *testing.T) {
+	for _, testCase := range []struct {
+		name     string
+		err      error
+		terminal bool
+		code     string
+	}{
+		{name: "bare no candidate", err: store.ErrNoModelCandidate, terminal: true, code: "no_model_candidate"},
+		{name: "cooldown", err: &store.ModelCandidateUnavailableError{Code: "model_cooling_down", Message: "cooling", RetryAfter: time.Minute}, code: "model_cooling_down"},
+		{name: "retryable rate limit", err: &store.RateLimitExceededError{Retryable: true}, code: "gateway_rate_limited"},
+		{name: "invalid parameter", err: &clients.ClientError{Code: "invalid_parameter", Retryable: false}, terminal: true, code: "invalid_parameter"},
+		{name: "retryable client", err: &clients.ClientError{Code: "network", Retryable: true}, code: "network"},
+		{name: "unknown", err: errors.New("unknown dispatcher failure"), code: "client_error"},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			if got := asyncAdmissionTaskErrorTerminal(testCase.err); got != testCase.terminal {
+				t.Fatalf("terminal=%v, want %v", got, testCase.terminal)
+			}
+			if got := asyncAdmissionTaskErrorCode(testCase.err); got != testCase.code {
+				t.Fatalf("code=%q, want %q", got, testCase.code)
+			}
+		})
+	}
+}
+
+func TestAsyncAdmissionSystemErrorClassification(t *testing.T) {
+	for _, testCase := range []struct {
+		name   string
+		code   string
+		system bool
+	}{
+		{name: "connection", code: "08006", system: true},
+		{name: "deadlock", code: "40P01", system: true},
+		{name: "resource exhaustion", code: "53300", system: true},
+		{name: "task data violation", code: "22023", system: false},
+		{name: "task unique violation", code: "23505", system: false},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			err := &pgconn.PgError{Code: testCase.code}
+			if got := isAsyncAdmissionSystemError(err); got != testCase.system {
+				t.Fatalf("SQLSTATE %s system=%v, want %v", testCase.code, got, testCase.system)
+			}
+		})
+	}
+}
+
+func TestAsyncAdmissionRetryAtUsesRecoveryAndBoundedBackoff(t *testing.T) {
+	now := time.Unix(1000, 0)
+	recoveryAt := now.Add(7 * time.Minute)
+	if got := asyncAdmissionRetryAt(now, "task-recovery", 9, &store.ModelCandidateUnavailableError{
+		Message:    "cooling",
+		RetryAfter: time.Minute,
+		RecoveryAt: recoveryAt,
+	}); !got.Equal(recoveryAt) {
+		t.Fatalf("recovery retry=%s, want %s", got, recoveryAt)
+	}
+	previous := time.Duration(0)
+	for failureCount := 0; failureCount < 10; failureCount++ {
+		delay := asyncAdmissionRetryAt(now, "stable-task", failureCount, errors.New("boom")).Sub(now)
+		if delay < time.Second || delay > time.Minute {
+			t.Fatalf("failure %d delay=%s outside jittered 1s..60s bounds", failureCount, delay)
+		}
+		if delay < previous {
+			t.Fatalf("failure %d delay=%s decreased from %s", failureCount, delay, previous)
+		}
+		previous = delay
+	}
+}
+
 func TestAdmissionInputFromSnapshotRefreshesWorkerCapacity(t *testing.T) {
 	admission := store.TaskAdmission{
 		TaskID:          "task-1",