wangbo
31565af07a
将同平台重试、跨平台动作、健康副作用和冷却排队统一到单一失败决策，避免旧降级策略与 failover 重复执行。\n\n增加事务级单源保护、候选实时刷新、冷却错误契约、管理接口严格校验及兼容策略只读展示。\n\n验证：真实 Gateway HTTP/PostgreSQL 接受测试 10 项通过；go test ./...、pnpm openapi、pnpm lint、pnpm test、pnpm build 均通过。
109 lines
3.7 KiB
Go
109 lines
3.7 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
func (s *Service) applyFailureDecision(ctx context.Context, taskID string, requestedModel string, candidate store.RuntimeModelCandidate, decision failureDecision, singleSourceProtection bool, simulated bool) (store.CandidateFailureEffectResult, error) {
|
|
result, err := s.store.ApplyCandidateFailureEffect(ctx, store.CandidateFailureEffectInput{
|
|
Effect: decision.Effect,
|
|
Target: decision.Target,
|
|
PlatformID: candidate.PlatformID,
|
|
PlatformModelID: candidate.PlatformModelID,
|
|
RequestedModel: requestedModel,
|
|
ModelType: candidate.ModelType,
|
|
CooldownSeconds: decision.CooldownSeconds,
|
|
DemoteSteps: decision.DemoteSteps,
|
|
SingleSourceProtection: singleSourceProtection,
|
|
})
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if result.GuardReason == "single_source" {
|
|
_ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", addPolicyTracePayload(map[string]any{
|
|
"platformId": candidate.PlatformID,
|
|
"platformModelId": candidate.PlatformModelID,
|
|
"effect": decision.Effect,
|
|
"target": decision.Target,
|
|
"guardReason": result.GuardReason,
|
|
}, decision.Match, decision.Info), simulated)
|
|
return result, nil
|
|
}
|
|
if !result.Applied {
|
|
return result, nil
|
|
}
|
|
|
|
payload := addPolicyTracePayload(map[string]any{
|
|
"platformId": candidate.PlatformID,
|
|
"platformModelId": candidate.PlatformModelID,
|
|
"effect": decision.Effect,
|
|
"target": decision.Target,
|
|
"reason": decision.Reason,
|
|
}, decision.Match, decision.Info)
|
|
switch decision.Effect {
|
|
case "disable":
|
|
_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate disabled by runner failover policy", payload, simulated)
|
|
case "cooldown":
|
|
payload["cooldownSeconds"] = decision.CooldownSeconds
|
|
_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate cooled down by runner failover policy", payload, simulated)
|
|
case "demote":
|
|
payload["demoteSteps"] = decision.DemoteSteps
|
|
payload["dynamicPriority"] = result.DynamicPriority
|
|
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner failover policy", payload, simulated)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func messageFromError(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
func effectiveRuntimePolicy(base map[string]any, override map[string]any, key string) map[string]any {
|
|
policy := base
|
|
if nested, ok := override[key].(map[string]any); ok && len(nested) > 0 {
|
|
policy = mergeMap(policy, nested)
|
|
}
|
|
return policy
|
|
}
|
|
|
|
func failurePolicyMatches(policy map[string]any, code string, message string) bool {
|
|
if len(policy) == 0 || !boolFromMap(policy, "enabled") {
|
|
return false
|
|
}
|
|
keywords := stringListFromPolicy(policy, "keywords")
|
|
if len(keywords) == 0 {
|
|
return false
|
|
}
|
|
target := strings.ToLower(strings.TrimSpace(code + " " + message))
|
|
for _, keyword := range keywords {
|
|
keyword = strings.ToLower(strings.TrimSpace(keyword))
|
|
if keyword != "" && strings.Contains(target, keyword) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func stringListFromPolicy(values map[string]any, key string) []string {
|
|
raw, ok := values[key].([]any)
|
|
if !ok {
|
|
if typed, ok := values[key].([]string); ok {
|
|
return typed
|
|
}
|
|
return nil
|
|
}
|
|
out := make([]string, 0, len(raw))
|
|
for _, item := range raw {
|
|
if text, ok := item.(string); ok && strings.TrimSpace(text) != "" {
|
|
out = append(out, text)
|
|
}
|
|
}
|
|
return out
|
|
}
|