refactor(runner): 收敛上游失败决策与轮转策略

将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策,避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证:真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过;go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
This commit is contained in:
2026-07-27 23:50:17 +08:00
parent 046c16fc69
commit 31565af07a
19 changed files with 1988 additions and 307 deletions
+43 -137
View File
@@ -2,153 +2,59 @@ package runner
import (
"context"
"errors"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, cause error, simulated bool, singleSourceProtected bool) {
code := clients.ErrorCode(cause)
message := ""
if cause != nil {
message = cause.Error()
func (s *Service) applyFailureDecision(ctx context.Context, taskID string, requestedModel string, candidate store.RuntimeModelCandidate, decision failureDecision, singleSourceProtection bool, simulated bool) (store.CandidateFailureEffectResult, error) {
result, err := s.store.ApplyCandidateFailureEffect(ctx, store.CandidateFailureEffectInput{
Effect: decision.Effect,
Target: decision.Target,
PlatformID: candidate.PlatformID,
PlatformModelID: candidate.PlatformModelID,
RequestedModel: requestedModel,
ModelType: candidate.ModelType,
CooldownSeconds: decision.CooldownSeconds,
DemoteSteps: decision.DemoteSteps,
SingleSourceProtection: singleSourceProtection,
})
if err != nil {
return result, err
}
autoDisablePolicy := effectiveRuntimePolicy(candidate.AutoDisablePolicy, candidate.RuntimePolicyOverride, "autoDisablePolicy")
if failurePolicyMatches(autoDisablePolicy, code, message) && intFromPolicy(autoDisablePolicy, "threshold") <= 1 {
if singleSourceProtected {
s.emitSingleSourceProtected(ctx, taskID, candidate, "auto_disable", code, simulated)
} else if err := s.store.DisableCandidatePlatform(ctx, candidate.PlatformID); err == nil {
_ = s.emit(ctx, taskID, "task.policy.auto_disabled", "running", "auto_disable", 0.48, "candidate platform disabled by failure policy", map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"code": code,
}, simulated)
}
}
degradePolicy := effectiveRuntimePolicy(candidate.DegradePolicy, candidate.RuntimePolicyOverride, "degradePolicy")
if failurePolicyMatches(degradePolicy, code, message) {
cooldownSeconds := intFromPolicy(degradePolicy, "cooldownSeconds")
if singleSourceProtected {
s.emitSingleSourceProtected(ctx, taskID, candidate, "degrade_cooldown", code, simulated)
} else if err := s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, cooldownSeconds); err == nil {
_ = s.emit(ctx, taskID, "task.policy.degraded", "running", "degrade", 0.5, "candidate model cooled down by failure policy", map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"cooldownSeconds": cooldownSeconds,
"code": code,
}, simulated)
}
}
}
func (s *Service) applyFailoverAction(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, requestedModel string, decision failoverDecision, simulated bool, singleSourceProtected bool) {
switch decision.Action {
case "disable_and_next":
if singleSourceProtected {
s.emitSingleSourceProtected(ctx, taskID, candidate, decision.Action, decision.Info.Code, simulated)
return
}
target := decision.Target
if target == "" {
target = "platform"
}
var err error
if target == "model" {
err = s.store.DisableCandidatePlatformModel(ctx, candidate.PlatformModelID)
} else {
err = s.store.DisableCandidatePlatform(ctx, candidate.PlatformID)
}
if err == nil {
_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate platform disabled by runner failover policy", addPolicyTracePayload(map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"action": decision.Action,
"target": target,
"reason": decision.Reason,
}, decision.Match, decision.Info), simulated)
}
case "cooldown_and_next":
if singleSourceProtected {
s.emitSingleSourceProtected(ctx, taskID, candidate, decision.Action, decision.Info.Code, simulated)
return
}
target := decision.Target
if target == "" {
target = "model"
}
var err error
if target == "platform" {
err = s.store.CooldownCandidatePlatform(ctx, candidate.PlatformID, decision.CooldownSeconds)
} else {
err = s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, decision.CooldownSeconds)
}
if err == nil {
_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate model cooled down by runner failover policy", addPolicyTracePayload(map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"cooldownSeconds": decision.CooldownSeconds,
"action": decision.Action,
"target": target,
"reason": decision.Reason,
}, decision.Match, decision.Info), simulated)
}
case "demote_and_next":
demoteSteps := decision.DemoteSteps
if demoteSteps <= 0 {
demoteSteps = 1
}
if dynamicPriority, err := s.store.DemoteCandidatePlatformPriorityBySteps(ctx, candidate.PlatformID, candidate.PlatformModelID, requestedModel, candidate.ModelType, demoteSteps); err == nil {
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner failover policy", addPolicyTracePayload(map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"dynamicPriority": dynamicPriority,
"demoteSteps": demoteSteps,
"action": decision.Action,
"target": "platform",
"reason": decision.Reason,
"errorMessage": decision.Info.Message,
}, decision.Match, decision.Info), simulated)
}
}
}
func (s *Service) emitSingleSourceProtected(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, action string, code string, simulated bool) {
_ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"action": action,
"code": code,
}, simulated)
}
func (s *Service) applyPriorityDemotePolicy(ctx context.Context, taskID string, attemptNo int, runnerPolicy store.RunnerPolicy, candidate store.RuntimeModelCandidate, requestedModel string, cause error, simulated bool) {
if errors.Is(cause, store.ErrRateLimited) {
return
}
decision := priorityDemoteDecisionForCandidate(runnerPolicy, cause)
if !decision.Demote {
return
}
demoteSteps := decision.DemoteSteps
if demoteSteps <= 0 {
demoteSteps = 99
}
if dynamicPriority, err := s.store.DemoteCandidatePlatformPriorityBySteps(ctx, candidate.PlatformID, candidate.PlatformModelID, requestedModel, candidate.ModelType, demoteSteps); err == nil {
s.recordAttemptTrace(ctx, taskID, attemptNo, priorityDemoteTraceEntry(decision, candidate.PlatformID, candidate.PlatformModelID, dynamicPriority))
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner policy", addPolicyTracePayload(map[string]any{
if result.GuardReason == "single_source" {
_ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", addPolicyTracePayload(map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"dynamicPriority": dynamicPriority,
"demoteSteps": demoteSteps,
"code": clients.ErrorCode(cause),
"reason": decision.Reason,
"errorMessage": messageFromError(cause),
"effect": decision.Effect,
"target": decision.Target,
"guardReason": result.GuardReason,
}, decision.Match, decision.Info), simulated)
return result, nil
}
if !result.Applied {
return result, nil
}
payload := addPolicyTracePayload(map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"effect": decision.Effect,
"target": decision.Target,
"reason": decision.Reason,
}, decision.Match, decision.Info)
switch decision.Effect {
case "disable":
_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate disabled by runner failover policy", payload, simulated)
case "cooldown":
payload["cooldownSeconds"] = decision.CooldownSeconds
_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate cooled down by runner failover policy", payload, simulated)
case "demote":
payload["demoteSteps"] = decision.DemoteSteps
payload["dynamicPriority"] = result.DynamicPriority
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner failover policy", payload, simulated)
}
return result, nil
}
func messageFromError(err error) string {