refactor(runner): 收敛上游失败决策与轮转策略
将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策,避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证:真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过;go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
This commit is contained in:
@@ -2,153 +2,59 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, cause error, simulated bool, singleSourceProtected bool) {
|
||||
code := clients.ErrorCode(cause)
|
||||
message := ""
|
||||
if cause != nil {
|
||||
message = cause.Error()
|
||||
func (s *Service) applyFailureDecision(ctx context.Context, taskID string, requestedModel string, candidate store.RuntimeModelCandidate, decision failureDecision, singleSourceProtection bool, simulated bool) (store.CandidateFailureEffectResult, error) {
|
||||
result, err := s.store.ApplyCandidateFailureEffect(ctx, store.CandidateFailureEffectInput{
|
||||
Effect: decision.Effect,
|
||||
Target: decision.Target,
|
||||
PlatformID: candidate.PlatformID,
|
||||
PlatformModelID: candidate.PlatformModelID,
|
||||
RequestedModel: requestedModel,
|
||||
ModelType: candidate.ModelType,
|
||||
CooldownSeconds: decision.CooldownSeconds,
|
||||
DemoteSteps: decision.DemoteSteps,
|
||||
SingleSourceProtection: singleSourceProtection,
|
||||
})
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
autoDisablePolicy := effectiveRuntimePolicy(candidate.AutoDisablePolicy, candidate.RuntimePolicyOverride, "autoDisablePolicy")
|
||||
if failurePolicyMatches(autoDisablePolicy, code, message) && intFromPolicy(autoDisablePolicy, "threshold") <= 1 {
|
||||
if singleSourceProtected {
|
||||
s.emitSingleSourceProtected(ctx, taskID, candidate, "auto_disable", code, simulated)
|
||||
} else if err := s.store.DisableCandidatePlatform(ctx, candidate.PlatformID); err == nil {
|
||||
_ = s.emit(ctx, taskID, "task.policy.auto_disabled", "running", "auto_disable", 0.48, "candidate platform disabled by failure policy", map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"code": code,
|
||||
}, simulated)
|
||||
}
|
||||
}
|
||||
|
||||
degradePolicy := effectiveRuntimePolicy(candidate.DegradePolicy, candidate.RuntimePolicyOverride, "degradePolicy")
|
||||
if failurePolicyMatches(degradePolicy, code, message) {
|
||||
cooldownSeconds := intFromPolicy(degradePolicy, "cooldownSeconds")
|
||||
if singleSourceProtected {
|
||||
s.emitSingleSourceProtected(ctx, taskID, candidate, "degrade_cooldown", code, simulated)
|
||||
} else if err := s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, cooldownSeconds); err == nil {
|
||||
_ = s.emit(ctx, taskID, "task.policy.degraded", "running", "degrade", 0.5, "candidate model cooled down by failure policy", map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"cooldownSeconds": cooldownSeconds,
|
||||
"code": code,
|
||||
}, simulated)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) applyFailoverAction(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, requestedModel string, decision failoverDecision, simulated bool, singleSourceProtected bool) {
|
||||
switch decision.Action {
|
||||
case "disable_and_next":
|
||||
if singleSourceProtected {
|
||||
s.emitSingleSourceProtected(ctx, taskID, candidate, decision.Action, decision.Info.Code, simulated)
|
||||
return
|
||||
}
|
||||
target := decision.Target
|
||||
if target == "" {
|
||||
target = "platform"
|
||||
}
|
||||
var err error
|
||||
if target == "model" {
|
||||
err = s.store.DisableCandidatePlatformModel(ctx, candidate.PlatformModelID)
|
||||
} else {
|
||||
err = s.store.DisableCandidatePlatform(ctx, candidate.PlatformID)
|
||||
}
|
||||
if err == nil {
|
||||
_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate platform disabled by runner failover policy", addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"action": decision.Action,
|
||||
"target": target,
|
||||
"reason": decision.Reason,
|
||||
}, decision.Match, decision.Info), simulated)
|
||||
}
|
||||
case "cooldown_and_next":
|
||||
if singleSourceProtected {
|
||||
s.emitSingleSourceProtected(ctx, taskID, candidate, decision.Action, decision.Info.Code, simulated)
|
||||
return
|
||||
}
|
||||
target := decision.Target
|
||||
if target == "" {
|
||||
target = "model"
|
||||
}
|
||||
var err error
|
||||
if target == "platform" {
|
||||
err = s.store.CooldownCandidatePlatform(ctx, candidate.PlatformID, decision.CooldownSeconds)
|
||||
} else {
|
||||
err = s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, decision.CooldownSeconds)
|
||||
}
|
||||
if err == nil {
|
||||
_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate model cooled down by runner failover policy", addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"cooldownSeconds": decision.CooldownSeconds,
|
||||
"action": decision.Action,
|
||||
"target": target,
|
||||
"reason": decision.Reason,
|
||||
}, decision.Match, decision.Info), simulated)
|
||||
}
|
||||
case "demote_and_next":
|
||||
demoteSteps := decision.DemoteSteps
|
||||
if demoteSteps <= 0 {
|
||||
demoteSteps = 1
|
||||
}
|
||||
if dynamicPriority, err := s.store.DemoteCandidatePlatformPriorityBySteps(ctx, candidate.PlatformID, candidate.PlatformModelID, requestedModel, candidate.ModelType, demoteSteps); err == nil {
|
||||
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner failover policy", addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"dynamicPriority": dynamicPriority,
|
||||
"demoteSteps": demoteSteps,
|
||||
"action": decision.Action,
|
||||
"target": "platform",
|
||||
"reason": decision.Reason,
|
||||
"errorMessage": decision.Info.Message,
|
||||
}, decision.Match, decision.Info), simulated)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) emitSingleSourceProtected(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, action string, code string, simulated bool) {
|
||||
_ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"action": action,
|
||||
"code": code,
|
||||
}, simulated)
|
||||
}
|
||||
|
||||
func (s *Service) applyPriorityDemotePolicy(ctx context.Context, taskID string, attemptNo int, runnerPolicy store.RunnerPolicy, candidate store.RuntimeModelCandidate, requestedModel string, cause error, simulated bool) {
|
||||
if errors.Is(cause, store.ErrRateLimited) {
|
||||
return
|
||||
}
|
||||
decision := priorityDemoteDecisionForCandidate(runnerPolicy, cause)
|
||||
if !decision.Demote {
|
||||
return
|
||||
}
|
||||
demoteSteps := decision.DemoteSteps
|
||||
if demoteSteps <= 0 {
|
||||
demoteSteps = 99
|
||||
}
|
||||
if dynamicPriority, err := s.store.DemoteCandidatePlatformPriorityBySteps(ctx, candidate.PlatformID, candidate.PlatformModelID, requestedModel, candidate.ModelType, demoteSteps); err == nil {
|
||||
s.recordAttemptTrace(ctx, taskID, attemptNo, priorityDemoteTraceEntry(decision, candidate.PlatformID, candidate.PlatformModelID, dynamicPriority))
|
||||
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner policy", addPolicyTracePayload(map[string]any{
|
||||
if result.GuardReason == "single_source" {
|
||||
_ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"dynamicPriority": dynamicPriority,
|
||||
"demoteSteps": demoteSteps,
|
||||
"code": clients.ErrorCode(cause),
|
||||
"reason": decision.Reason,
|
||||
"errorMessage": messageFromError(cause),
|
||||
"effect": decision.Effect,
|
||||
"target": decision.Target,
|
||||
"guardReason": result.GuardReason,
|
||||
}, decision.Match, decision.Info), simulated)
|
||||
return result, nil
|
||||
}
|
||||
if !result.Applied {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
payload := addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"effect": decision.Effect,
|
||||
"target": decision.Target,
|
||||
"reason": decision.Reason,
|
||||
}, decision.Match, decision.Info)
|
||||
switch decision.Effect {
|
||||
case "disable":
|
||||
_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate disabled by runner failover policy", payload, simulated)
|
||||
case "cooldown":
|
||||
payload["cooldownSeconds"] = decision.CooldownSeconds
|
||||
_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate cooled down by runner failover policy", payload, simulated)
|
||||
case "demote":
|
||||
payload["demoteSteps"] = decision.DemoteSteps
|
||||
payload["dynamicPriority"] = result.DynamicPriority
|
||||
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner failover policy", payload, simulated)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func messageFromError(err error) string {
|
||||
|
||||
Reference in New Issue
Block a user