refactor(runner): 收敛上游失败决策与轮转策略
将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
This commit is contained in:
@@ -2,12 +2,15 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type ListModelCandidatesOptions struct {
|
||||
@@ -810,13 +813,20 @@ func runtimeCandidateFull(candidate RuntimeModelCandidate) bool {
|
||||
|
||||
func (s *Store) modelCandidateCooldownError(ctx context.Context, model string, modelType string) (error, error) {
|
||||
exactModel := strings.TrimSpace(model)
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT p.name,
|
||||
COALESCE(NULLIF(m.display_name, ''), NULLIF(m.model_alias, ''), m.model_name),
|
||||
COALESCE(to_char(p.cooldown_until AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), ''),
|
||||
GREATEST(COALESCE(EXTRACT(EPOCH FROM p.cooldown_until - now()), 0), 0)::float8,
|
||||
COALESCE(to_char(m.cooldown_until AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), ''),
|
||||
GREATEST(COALESCE(EXTRACT(EPOCH FROM m.cooldown_until - now()), 0), 0)::float8
|
||||
var code string
|
||||
var recoveryAt time.Time
|
||||
var remainingSeconds float64
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT CASE
|
||||
WHEN COALESCE(m.cooldown_until, to_timestamp(0)) >= COALESCE(p.cooldown_until, to_timestamp(0))
|
||||
THEN 'model_cooling_down'
|
||||
ELSE 'platform_cooling_down'
|
||||
END,
|
||||
GREATEST(COALESCE(p.cooldown_until, to_timestamp(0)), COALESCE(m.cooldown_until, to_timestamp(0))) AS recovery_at,
|
||||
GREATEST(
|
||||
EXTRACT(EPOCH FROM GREATEST(COALESCE(p.cooldown_until, to_timestamp(0)), COALESCE(m.cooldown_until, to_timestamp(0))) - now()),
|
||||
0
|
||||
)::float8
|
||||
FROM platform_models m
|
||||
JOIN integration_platforms p ON p.id = m.platform_id
|
||||
LEFT JOIN base_model_catalog b ON b.id = m.base_model_id
|
||||
@@ -836,64 +846,39 @@ WHERE p.status = 'enabled'
|
||||
AND compatibility_alias.active = true
|
||||
AND (compatibility_alias.expires_at IS NULL OR compatibility_alias.expires_at > now())
|
||||
AND (compatibility_alias.base_model_id = b.id OR alias_base.invocation_name = b.invocation_name)
|
||||
)
|
||||
)
|
||||
ORDER BY GREATEST(COALESCE(p.cooldown_until, to_timestamp(0)), COALESCE(m.cooldown_until, to_timestamp(0))) DESC,
|
||||
p.priority ASC,
|
||||
m.created_at ASC`, exactModel, modelType)
|
||||
)
|
||||
)
|
||||
AND (
|
||||
p.cooldown_until > now()
|
||||
OR m.cooldown_until > now()
|
||||
)
|
||||
ORDER BY GREATEST(COALESCE(p.cooldown_until, to_timestamp(0)), COALESCE(m.cooldown_until, to_timestamp(0))) ASC,
|
||||
p.priority ASC,
|
||||
m.created_at ASC
|
||||
LIMIT 1`, exactModel, modelType).Scan(&code, &recoveryAt, &remainingSeconds)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var platformName string
|
||||
var displayName string
|
||||
var platformCooldownUntil string
|
||||
var platformRemainingSeconds float64
|
||||
var modelCooldownUntil string
|
||||
var modelRemainingSeconds float64
|
||||
if err := rows.Scan(
|
||||
&platformName,
|
||||
&displayName,
|
||||
&platformCooldownUntil,
|
||||
&platformRemainingSeconds,
|
||||
&modelCooldownUntil,
|
||||
&modelRemainingSeconds,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if modelRemainingSeconds > 0 {
|
||||
return &ModelCandidateUnavailableError{
|
||||
Code: "model_cooling_down",
|
||||
Message: cooldownErrorMessage("模型", displayName, modelRemainingSeconds, modelCooldownUntil),
|
||||
}, nil
|
||||
}
|
||||
if platformRemainingSeconds > 0 {
|
||||
return &ModelCandidateUnavailableError{
|
||||
Code: "platform_cooling_down",
|
||||
Message: cooldownErrorMessage("平台", platformName, platformRemainingSeconds, platformCooldownUntil),
|
||||
}, nil
|
||||
}
|
||||
retryAfterSeconds := int(math.Ceil(remainingSeconds))
|
||||
if retryAfterSeconds < 1 {
|
||||
retryAfterSeconds = 1
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
message := "请求的模型暂时不可用，请稍后重试"
|
||||
if code == "platform_cooling_down" {
|
||||
message = "可用平台暂时不可用，请稍后重试"
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func cooldownErrorMessage(scope string, name string, remainingSeconds float64, cooldownUntil string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = "候选"
|
||||
}
|
||||
remainingMinutes := remainingSeconds / 60
|
||||
if remainingMinutes < 0.1 {
|
||||
remainingMinutes = 0.1
|
||||
}
|
||||
message := fmt.Sprintf("%s %s 冷却中，剩余 %.1f 分钟", scope, name, remainingMinutes)
|
||||
if strings.TrimSpace(cooldownUntil) != "" {
|
||||
message += "，预计恢复时间 " + cooldownUntil
|
||||
}
|
||||
return message
|
||||
recoveryAt = recoveryAt.UTC()
|
||||
return &ModelCandidateUnavailableError{
|
||||
Code: code,
|
||||
Message: message,
|
||||
RetryAfter: time.Duration(retryAfterSeconds) * time.Second,
|
||||
RecoveryAt: recoveryAt,
|
||||
Details: map[string]any{
|
||||
"retryAfterSeconds": retryAfterSeconds,
|
||||
"recoveryAt": recoveryAt.Format(time.RFC3339),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user