package runner import ( "context" "strings" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) func (s *Service) applyFailureDecision(ctx context.Context, taskID string, requestedModel string, candidate store.RuntimeModelCandidate, decision failureDecision, singleSourceProtection bool, simulated bool) (store.CandidateFailureEffectResult, error) { result, err := s.store.ApplyCandidateFailureEffect(ctx, store.CandidateFailureEffectInput{ Effect: decision.Effect, Target: decision.Target, PlatformID: candidate.PlatformID, PlatformModelID: candidate.PlatformModelID, RequestedModel: requestedModel, ModelType: candidate.ModelType, CooldownSeconds: decision.CooldownSeconds, DemoteSteps: decision.DemoteSteps, SingleSourceProtection: singleSourceProtection, }) if err != nil { return result, err } if result.GuardReason == "single_source" { _ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", addPolicyTracePayload(map[string]any{ "platformId": candidate.PlatformID, "platformModelId": candidate.PlatformModelID, "effect": decision.Effect, "target": decision.Target, "guardReason": result.GuardReason, }, decision.Match, decision.Info), simulated) return result, nil } if !result.Applied { return result, nil } payload := addPolicyTracePayload(map[string]any{ "platformId": candidate.PlatformID, "platformModelId": candidate.PlatformModelID, "effect": decision.Effect, "target": decision.Target, "reason": decision.Reason, }, decision.Match, decision.Info) switch decision.Effect { case "disable": _ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate disabled by runner failover policy", payload, simulated) case "cooldown": payload["cooldownSeconds"] = decision.CooldownSeconds _ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate cooled down by runner failover policy", payload, simulated) case "demote": payload["demoteSteps"] = decision.DemoteSteps payload["dynamicPriority"] = result.DynamicPriority _ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner failover policy", payload, simulated) } return result, nil } func messageFromError(err error) string { if err == nil { return "" } return err.Error() } func effectiveRuntimePolicy(base map[string]any, override map[string]any, key string) map[string]any { policy := base if nested, ok := override[key].(map[string]any); ok && len(nested) > 0 { policy = mergeMap(policy, nested) } return policy } func failurePolicyMatches(policy map[string]any, code string, message string) bool { if len(policy) == 0 || !boolFromMap(policy, "enabled") { return false } keywords := stringListFromPolicy(policy, "keywords") if len(keywords) == 0 { return false } target := strings.ToLower(strings.TrimSpace(code + " " + message)) for _, keyword := range keywords { keyword = strings.ToLower(strings.TrimSpace(keyword)) if keyword != "" && strings.Contains(target, keyword) { return true } } return false } func stringListFromPolicy(values map[string]any, key string) []string { raw, ok := values[key].([]any) if !ok { if typed, ok := values[key].([]string); ok { return typed } return nil } out := make([]string, 0, len(raw)) for _, item := range raw { if text, ok := item.(string); ok && strings.TrimSpace(text) != "" { out = append(out, text) } } return out }