refactor(runner): 收敛上游失败决策与轮转策略

将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。

apps/api/internal/store/candidates.go

@@ -2,12 +2,15 @@ package store
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
+	"github.com/jackc/pgx/v5"
 )
 
 type ListModelCandidatesOptions struct {
@@ -810,13 +813,20 @@ func runtimeCandidateFull(candidate RuntimeModelCandidate) bool {
 
 func (s *Store) modelCandidateCooldownError(ctx context.Context, model string, modelType string) (error, error) {
 	exactModel := strings.TrimSpace(model)
-	rows, err := s.pool.Query(ctx, `
-SELECT p.name,
-       COALESCE(NULLIF(m.display_name, ''), NULLIF(m.model_alias, ''), m.model_name),
-       COALESCE(to_char(p.cooldown_until AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), ''),
-       GREATEST(COALESCE(EXTRACT(EPOCH FROM p.cooldown_until - now()), 0), 0)::float8,
-       COALESCE(to_char(m.cooldown_until AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), ''),
-       GREATEST(COALESCE(EXTRACT(EPOCH FROM m.cooldown_until - now()), 0), 0)::float8
+	var code string
+	var recoveryAt time.Time
+	var remainingSeconds float64
+	err := s.pool.QueryRow(ctx, `
+SELECT CASE
+         WHEN COALESCE(m.cooldown_until, to_timestamp(0)) >= COALESCE(p.cooldown_until, to_timestamp(0))
+         THEN 'model_cooling_down'
+         ELSE 'platform_cooling_down'
+       END,
+       GREATEST(COALESCE(p.cooldown_until, to_timestamp(0)), COALESCE(m.cooldown_until, to_timestamp(0))) AS recovery_at,
+       GREATEST(
+         EXTRACT(EPOCH FROM GREATEST(COALESCE(p.cooldown_until, to_timestamp(0)), COALESCE(m.cooldown_until, to_timestamp(0))) - now()),
+         0
+       )::float8
 FROM platform_models m
 JOIN integration_platforms p ON p.id = m.platform_id
 LEFT JOIN base_model_catalog b ON b.id = m.base_model_id
@@ -836,64 +846,39 @@ WHERE p.status = 'enabled'
 	      AND compatibility_alias.active = true
 	      AND (compatibility_alias.expires_at IS NULL OR compatibility_alias.expires_at > now())
 	      AND (compatibility_alias.base_model_id = b.id OR alias_base.invocation_name = b.invocation_name)
-	  )
-	)
-ORDER BY GREATEST(COALESCE(p.cooldown_until, to_timestamp(0)), COALESCE(m.cooldown_until, to_timestamp(0))) DESC,
-         p.priority ASC,
-         m.created_at ASC`, exactModel, modelType)
+		  )
+		)
+  AND (
+    p.cooldown_until > now()
+    OR m.cooldown_until > now()
+  )
+ORDER BY GREATEST(COALESCE(p.cooldown_until, to_timestamp(0)), COALESCE(m.cooldown_until, to_timestamp(0))) ASC,
+	         p.priority ASC,
+	         m.created_at ASC
+LIMIT 1`, exactModel, modelType).Scan(&code, &recoveryAt, &remainingSeconds)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return nil, nil
+	}
 	if err != nil {
 		return nil, err
 	}
-	defer rows.Close()
-
-	for rows.Next() {
-		var platformName string
-		var displayName string
-		var platformCooldownUntil string
-		var platformRemainingSeconds float64
-		var modelCooldownUntil string
-		var modelRemainingSeconds float64
-		if err := rows.Scan(
-			&platformName,
-			&displayName,
-			&platformCooldownUntil,
-			&platformRemainingSeconds,
-			&modelCooldownUntil,
-			&modelRemainingSeconds,
-		); err != nil {
-			return nil, err
-		}
-		if modelRemainingSeconds > 0 {
-			return &ModelCandidateUnavailableError{
-				Code:    "model_cooling_down",
-				Message: cooldownErrorMessage("模型", displayName, modelRemainingSeconds, modelCooldownUntil),
-			}, nil
-		}
-		if platformRemainingSeconds > 0 {
-			return &ModelCandidateUnavailableError{
-				Code:    "platform_cooling_down",
-				Message: cooldownErrorMessage("平台", platformName, platformRemainingSeconds, platformCooldownUntil),
-			}, nil
-		}
+	retryAfterSeconds := int(math.Ceil(remainingSeconds))
+	if retryAfterSeconds < 1 {
+		retryAfterSeconds = 1
 	}
-	if err := rows.Err(); err != nil {
-		return nil, err
+	message := "请求的模型暂时不可用，请稍后重试"
+	if code == "platform_cooling_down" {
+		message = "可用平台暂时不可用，请稍后重试"
 	}
-	return nil, nil
-}
-
-func cooldownErrorMessage(scope string, name string, remainingSeconds float64, cooldownUntil string) string {
-	name = strings.TrimSpace(name)
-	if name == "" {
-		name = "候选"
-	}
-	remainingMinutes := remainingSeconds / 60
-	if remainingMinutes < 0.1 {
-		remainingMinutes = 0.1
-	}
-	message := fmt.Sprintf("%s %s 冷却中，剩余 %.1f 分钟", scope, name, remainingMinutes)
-	if strings.TrimSpace(cooldownUntil) != "" {
-		message += "，预计恢复时间 " + cooldownUntil
-	}
-	return message
+	recoveryAt = recoveryAt.UTC()
+	return &ModelCandidateUnavailableError{
+		Code:       code,
+		Message:    message,
+		RetryAfter: time.Duration(retryAfterSeconds) * time.Second,
+		RecoveryAt: recoveryAt,
+		Details: map[string]any{
+			"retryAfterSeconds": retryAfterSeconds,
+			"recoveryAt":        recoveryAt.Format(time.RFC3339),
+		},
+	}, nil
 }

apps/api/internal/store/runtime_policies.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
 )
 
 const runtimePolicyColumns = `
@@ -38,6 +39,24 @@ type PlatformDynamicPriorityState struct {
 	UpdatedAt         time.Time `json:"updatedAt"`
 }
 
+type CandidateFailureEffectInput struct {
+	Effect                 string
+	Target                 string
+	PlatformID             string
+	PlatformModelID        string
+	RequestedModel         string
+	ModelType              string
+	CooldownSeconds        int
+	DemoteSteps            int
+	SingleSourceProtection bool
+}
+
+type CandidateFailureEffectResult struct {
+	Applied         bool
+	GuardReason     string
+	DynamicPriority int
+}
+
 func (s *Store) ListRuntimePolicySets(ctx context.Context) ([]RuntimePolicySet, error) {
 	rows, err := s.pool.Query(ctx, `SELECT `+runtimePolicyColumns+` FROM model_runtime_policy_sets ORDER BY policy_key ASC`)
 	if err != nil {
@@ -171,6 +190,164 @@ WHERE id = $1::uuid`, platformModelID, cooldownSeconds)
 	return err
 }
 
+func (s *Store) RuntimeCandidateAvailable(ctx context.Context, platformID string, platformModelID string) (bool, error) {
+	if strings.TrimSpace(platformID) == "" || strings.TrimSpace(platformModelID) == "" {
+		return false, nil
+	}
+	var available bool
+	err := s.pool.QueryRow(ctx, `
+SELECT EXISTS (
+  SELECT 1
+  FROM platform_models model
+  JOIN integration_platforms platform ON platform.id = model.platform_id
+  WHERE platform.id = $1::uuid
+    AND model.id = $2::uuid
+    AND platform.status = 'enabled'
+    AND platform.deleted_at IS NULL
+    AND (platform.cooldown_until IS NULL OR platform.cooldown_until <= now())
+    AND model.enabled = true
+    AND (model.cooldown_until IS NULL OR model.cooldown_until <= now())
+)`, platformID, platformModelID).Scan(&available)
+	return available, err
+}
+
+func (s *Store) ApplyCandidateFailureEffect(ctx context.Context, input CandidateFailureEffectInput) (CandidateFailureEffectResult, error) {
+	if input.Effect == "" || input.Effect == "none" {
+		return CandidateFailureEffectResult{}, nil
+	}
+	if input.Effect == "demote" {
+		steps := input.DemoteSteps
+		if steps <= 0 {
+			steps = 1
+		}
+		priority, err := s.DemoteCandidatePlatformPriorityBySteps(
+			ctx,
+			input.PlatformID,
+			input.PlatformModelID,
+			input.RequestedModel,
+			input.ModelType,
+			steps,
+		)
+		return CandidateFailureEffectResult{Applied: err == nil, DynamicPriority: priority}, err
+	}
+	if input.Effect != "cooldown" && input.Effect != "disable" {
+		return CandidateFailureEffectResult{}, nil
+	}
+
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return CandidateFailureEffectResult{}, err
+	}
+	defer func() {
+		_ = tx.Rollback(ctx)
+	}()
+
+	lockKey := strings.TrimSpace(input.RequestedModel) + "\x1f" + strings.TrimSpace(input.ModelType)
+	if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))`, lockKey); err != nil {
+		return CandidateFailureEffectResult{}, err
+	}
+	if input.SingleSourceProtection {
+		alternativeAvailable, err := candidateAlternativeAvailable(ctx, tx, input)
+		if err != nil {
+			return CandidateFailureEffectResult{}, err
+		}
+		if !alternativeAvailable {
+			if err := tx.Commit(ctx); err != nil {
+				return CandidateFailureEffectResult{}, err
+			}
+			return CandidateFailureEffectResult{GuardReason: "single_source"}, nil
+		}
+	}
+
+	var commandTag pgconn.CommandTag
+	switch {
+	case input.Effect == "disable" && input.Target == "model":
+		commandTag, err = tx.Exec(ctx, `
+UPDATE platform_models
+SET enabled = false,
+    updated_at = now()
+WHERE id = $1::uuid
+  AND enabled = true`, input.PlatformModelID)
+	case input.Effect == "disable":
+		commandTag, err = tx.Exec(ctx, `
+UPDATE integration_platforms
+SET status = 'disabled',
+    updated_at = now()
+WHERE id = $1::uuid
+  AND status = 'enabled'
+  AND deleted_at IS NULL`, input.PlatformID)
+	case input.Effect == "cooldown" && input.Target == "platform":
+		commandTag, err = tx.Exec(ctx, `
+UPDATE integration_platforms
+SET cooldown_until = GREATEST(
+      COALESCE(cooldown_until, to_timestamp(0)),
+      now() + ($2::int * interval '1 second')
+    ),
+    updated_at = now()
+WHERE id = $1::uuid
+  AND status = 'enabled'
+  AND deleted_at IS NULL`, input.PlatformID, positiveCooldownSeconds(input.CooldownSeconds))
+	default:
+		commandTag, err = tx.Exec(ctx, `
+UPDATE platform_models
+SET cooldown_until = GREATEST(
+      COALESCE(cooldown_until, to_timestamp(0)),
+      now() + ($2::int * interval '1 second')
+    ),
+    updated_at = now()
+WHERE id = $1::uuid
+  AND enabled = true`, input.PlatformModelID, positiveCooldownSeconds(input.CooldownSeconds))
+	}
+	if err != nil {
+		return CandidateFailureEffectResult{}, err
+	}
+	applied := commandTag.RowsAffected() > 0
+	if err := tx.Commit(ctx); err != nil {
+		return CandidateFailureEffectResult{}, err
+	}
+	return CandidateFailureEffectResult{Applied: applied}, nil
+}
+
+func candidateAlternativeAvailable(ctx context.Context, tx pgx.Tx, input CandidateFailureEffectInput) (bool, error) {
+	var available bool
+	err := tx.QueryRow(ctx, `
+SELECT EXISTS (
+  SELECT 1
+  FROM platform_models model
+  JOIN integration_platforms platform ON platform.id = model.platform_id
+  LEFT JOIN base_model_catalog base ON base.id = model.base_model_id
+  WHERE platform.id <> $1::uuid
+    AND platform.status = 'enabled'
+    AND platform.deleted_at IS NULL
+    AND (platform.cooldown_until IS NULL OR platform.cooldown_until <= now())
+    AND model.enabled = true
+    AND (model.cooldown_until IS NULL OR model.cooldown_until <= now())
+    AND (NULLIF($3::text, '') IS NULL OR model.model_type @> jsonb_build_array($3::text))
+    AND (
+      base.invocation_name = $2::text
+      OR (base.id IS NULL AND model.model_name = $2::text)
+      OR EXISTS (
+        SELECT 1
+        FROM model_compatibility_aliases compatibility_alias
+        JOIN base_model_catalog alias_base ON alias_base.id = compatibility_alias.base_model_id
+        WHERE compatibility_alias.alias = $2::text
+          AND compatibility_alias.model_type = $3::text
+          AND compatibility_alias.active = true
+          AND (compatibility_alias.expires_at IS NULL OR compatibility_alias.expires_at > now())
+          AND (compatibility_alias.base_model_id = base.id OR alias_base.invocation_name = base.invocation_name)
+      )
+    )
+)`, input.PlatformID, strings.TrimSpace(input.RequestedModel), strings.TrimSpace(input.ModelType)).Scan(&available)
+	return available, err
+}
+
+func positiveCooldownSeconds(value int) int {
+	if value <= 0 {
+		return 300
+	}
+	return value
+}
+
 func (s *Store) DemoteCandidatePlatformPriority(ctx context.Context, platformID string, platformModelID string, requestedModel string, modelType string) (int, error) {
 	return s.DemoteCandidatePlatformPriorityBySteps(ctx, platformID, platformModelID, requestedModel, modelType, 99)
 }

apps/api/internal/store/runtime_types.go

@@ -13,9 +13,11 @@ var (
 )
 
 type ModelCandidateUnavailableError struct {
-	Code    string
-	Message string
-	Details map[string]any
+	Code       string
+	Message    string
+	Details    map[string]any
+	RetryAfter time.Duration
+	RecoveryAt time.Time
 }
 
 func (e *ModelCandidateUnavailableError) Error() string {
@@ -42,6 +44,22 @@ func ModelCandidateErrorDetails(err error) map[string]any {
 	return nil
 }
 
+func ModelCandidateRetryAfter(err error) time.Duration {
+	var candidateErr *ModelCandidateUnavailableError
+	if errors.As(err, &candidateErr) && candidateErr.RetryAfter > 0 {
+		return candidateErr.RetryAfter
+	}
+	return 0
+}
+
+func ModelCandidateRecoveryAt(err error) time.Time {
+	var candidateErr *ModelCandidateUnavailableError
+	if errors.As(err, &candidateErr) {
+		return candidateErr.RecoveryAt
+	}
+	return time.Time{}
+}
+
 type RateLimitExceededError struct {
 	ScopeType     string
 	ScopeKey      string

apps/api/internal/store/tasks_runtime.go

@@ -470,6 +470,13 @@ func (s *Store) RequeueTask(ctx context.Context, taskID string, executionToken s
 		delay = 10 * time.Minute
 	}
 	nextRunAt := time.Now().Add(delay)
+	return s.RequeueTaskUntil(ctx, taskID, executionToken, nextRunAt, queueKey)
+}
+
+func (s *Store) RequeueTaskUntil(ctx context.Context, taskID string, executionToken string, nextRunAt time.Time, queueKey string) (GatewayTask, error) {
+	if nextRunAt.Before(time.Now().Add(time.Second)) {
+		nextRunAt = time.Now().Add(time.Second)
+	}
 	return scanGatewayTask(s.pool.QueryRow(ctx, `
 UPDATE gateway_tasks
 SET status = 'queued',
@@ -820,6 +827,29 @@ ORDER BY COALESCE(attempt_no, 0), created_at`, taskID)
 }
 
 func (s *Store) AppendTaskAttemptTrace(ctx context.Context, taskID string, attemptNo int, entry map[string]any) error {
+	if strings.TrimSpace(taskID) == "" || attemptNo <= 0 || len(entry) == 0 {
+		return nil
+	}
+	encoded, err := json.Marshal(entry)
+	if err != nil {
+		return err
+	}
+	result, err := s.pool.Exec(ctx, `
+UPDATE gateway_task_attempts
+SET metrics = jsonb_set(
+      COALESCE(metrics, '{}'::jsonb),
+      '{trace}',
+      COALESCE(metrics->'trace', '[]'::jsonb) || jsonb_build_array($3::jsonb),
+      true
+    )
+WHERE task_id = $1::uuid
+  AND attempt_no = $2::int`, taskID, attemptNo, encoded)
+	if err != nil {
+		return err
+	}
+	if result.RowsAffected() == 0 {
+		return pgx.ErrNoRows
+	}
 	return nil
 }