refactor(runner): 收敛上游失败决策与轮转策略
将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
const runtimePolicyColumns = `
|
||||
@@ -38,6 +39,24 @@ type PlatformDynamicPriorityState struct {
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type CandidateFailureEffectInput struct {
|
||||
Effect string
|
||||
Target string
|
||||
PlatformID string
|
||||
PlatformModelID string
|
||||
RequestedModel string
|
||||
ModelType string
|
||||
CooldownSeconds int
|
||||
DemoteSteps int
|
||||
SingleSourceProtection bool
|
||||
}
|
||||
|
||||
type CandidateFailureEffectResult struct {
|
||||
Applied bool
|
||||
GuardReason string
|
||||
DynamicPriority int
|
||||
}
|
||||
|
||||
func (s *Store) ListRuntimePolicySets(ctx context.Context) ([]RuntimePolicySet, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT `+runtimePolicyColumns+` FROM model_runtime_policy_sets ORDER BY policy_key ASC`)
|
||||
if err != nil {
|
||||
@@ -171,6 +190,164 @@ WHERE id = $1::uuid`, platformModelID, cooldownSeconds)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) RuntimeCandidateAvailable(ctx context.Context, platformID string, platformModelID string) (bool, error) {
|
||||
if strings.TrimSpace(platformID) == "" || strings.TrimSpace(platformModelID) == "" {
|
||||
return false, nil
|
||||
}
|
||||
var available bool
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM platform_models model
|
||||
JOIN integration_platforms platform ON platform.id = model.platform_id
|
||||
WHERE platform.id = $1::uuid
|
||||
AND model.id = $2::uuid
|
||||
AND platform.status = 'enabled'
|
||||
AND platform.deleted_at IS NULL
|
||||
AND (platform.cooldown_until IS NULL OR platform.cooldown_until <= now())
|
||||
AND model.enabled = true
|
||||
AND (model.cooldown_until IS NULL OR model.cooldown_until <= now())
|
||||
)`, platformID, platformModelID).Scan(&available)
|
||||
return available, err
|
||||
}
|
||||
|
||||
func (s *Store) ApplyCandidateFailureEffect(ctx context.Context, input CandidateFailureEffectInput) (CandidateFailureEffectResult, error) {
|
||||
if input.Effect == "" || input.Effect == "none" {
|
||||
return CandidateFailureEffectResult{}, nil
|
||||
}
|
||||
if input.Effect == "demote" {
|
||||
steps := input.DemoteSteps
|
||||
if steps <= 0 {
|
||||
steps = 1
|
||||
}
|
||||
priority, err := s.DemoteCandidatePlatformPriorityBySteps(
|
||||
ctx,
|
||||
input.PlatformID,
|
||||
input.PlatformModelID,
|
||||
input.RequestedModel,
|
||||
input.ModelType,
|
||||
steps,
|
||||
)
|
||||
return CandidateFailureEffectResult{Applied: err == nil, DynamicPriority: priority}, err
|
||||
}
|
||||
if input.Effect != "cooldown" && input.Effect != "disable" {
|
||||
return CandidateFailureEffectResult{}, nil
|
||||
}
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return CandidateFailureEffectResult{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback(ctx)
|
||||
}()
|
||||
|
||||
lockKey := strings.TrimSpace(input.RequestedModel) + "\x1f" + strings.TrimSpace(input.ModelType)
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))`, lockKey); err != nil {
|
||||
return CandidateFailureEffectResult{}, err
|
||||
}
|
||||
if input.SingleSourceProtection {
|
||||
alternativeAvailable, err := candidateAlternativeAvailable(ctx, tx, input)
|
||||
if err != nil {
|
||||
return CandidateFailureEffectResult{}, err
|
||||
}
|
||||
if !alternativeAvailable {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return CandidateFailureEffectResult{}, err
|
||||
}
|
||||
return CandidateFailureEffectResult{GuardReason: "single_source"}, nil
|
||||
}
|
||||
}
|
||||
|
||||
var commandTag pgconn.CommandTag
|
||||
switch {
|
||||
case input.Effect == "disable" && input.Target == "model":
|
||||
commandTag, err = tx.Exec(ctx, `
|
||||
UPDATE platform_models
|
||||
SET enabled = false,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND enabled = true`, input.PlatformModelID)
|
||||
case input.Effect == "disable":
|
||||
commandTag, err = tx.Exec(ctx, `
|
||||
UPDATE integration_platforms
|
||||
SET status = 'disabled',
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'enabled'
|
||||
AND deleted_at IS NULL`, input.PlatformID)
|
||||
case input.Effect == "cooldown" && input.Target == "platform":
|
||||
commandTag, err = tx.Exec(ctx, `
|
||||
UPDATE integration_platforms
|
||||
SET cooldown_until = GREATEST(
|
||||
COALESCE(cooldown_until, to_timestamp(0)),
|
||||
now() + ($2::int * interval '1 second')
|
||||
),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'enabled'
|
||||
AND deleted_at IS NULL`, input.PlatformID, positiveCooldownSeconds(input.CooldownSeconds))
|
||||
default:
|
||||
commandTag, err = tx.Exec(ctx, `
|
||||
UPDATE platform_models
|
||||
SET cooldown_until = GREATEST(
|
||||
COALESCE(cooldown_until, to_timestamp(0)),
|
||||
now() + ($2::int * interval '1 second')
|
||||
),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND enabled = true`, input.PlatformModelID, positiveCooldownSeconds(input.CooldownSeconds))
|
||||
}
|
||||
if err != nil {
|
||||
return CandidateFailureEffectResult{}, err
|
||||
}
|
||||
applied := commandTag.RowsAffected() > 0
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return CandidateFailureEffectResult{}, err
|
||||
}
|
||||
return CandidateFailureEffectResult{Applied: applied}, nil
|
||||
}
|
||||
|
||||
func candidateAlternativeAvailable(ctx context.Context, tx pgx.Tx, input CandidateFailureEffectInput) (bool, error) {
|
||||
var available bool
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM platform_models model
|
||||
JOIN integration_platforms platform ON platform.id = model.platform_id
|
||||
LEFT JOIN base_model_catalog base ON base.id = model.base_model_id
|
||||
WHERE platform.id <> $1::uuid
|
||||
AND platform.status = 'enabled'
|
||||
AND platform.deleted_at IS NULL
|
||||
AND (platform.cooldown_until IS NULL OR platform.cooldown_until <= now())
|
||||
AND model.enabled = true
|
||||
AND (model.cooldown_until IS NULL OR model.cooldown_until <= now())
|
||||
AND (NULLIF($3::text, '') IS NULL OR model.model_type @> jsonb_build_array($3::text))
|
||||
AND (
|
||||
base.invocation_name = $2::text
|
||||
OR (base.id IS NULL AND model.model_name = $2::text)
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM model_compatibility_aliases compatibility_alias
|
||||
JOIN base_model_catalog alias_base ON alias_base.id = compatibility_alias.base_model_id
|
||||
WHERE compatibility_alias.alias = $2::text
|
||||
AND compatibility_alias.model_type = $3::text
|
||||
AND compatibility_alias.active = true
|
||||
AND (compatibility_alias.expires_at IS NULL OR compatibility_alias.expires_at > now())
|
||||
AND (compatibility_alias.base_model_id = base.id OR alias_base.invocation_name = base.invocation_name)
|
||||
)
|
||||
)
|
||||
)`, input.PlatformID, strings.TrimSpace(input.RequestedModel), strings.TrimSpace(input.ModelType)).Scan(&available)
|
||||
return available, err
|
||||
}
|
||||
|
||||
func positiveCooldownSeconds(value int) int {
|
||||
if value <= 0 {
|
||||
return 300
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func (s *Store) DemoteCandidatePlatformPriority(ctx context.Context, platformID string, platformModelID string, requestedModel string, modelType string) (int, error) {
|
||||
return s.DemoteCandidatePlatformPriorityBySteps(ctx, platformID, platformModelID, requestedModel, modelType, 99)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user