feat: add priority demotion controls
This commit is contained in:
@@ -43,7 +43,6 @@ type failoverDecision struct {
|
||||
type priorityDemoteDecision struct {
|
||||
Demote bool
|
||||
Reason string
|
||||
Step int
|
||||
Match policyRuleMatch
|
||||
Info failureInfo
|
||||
}
|
||||
@@ -110,9 +109,9 @@ func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate sto
|
||||
return failoverDecision{Retry: false, Action: "stop", Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
|
||||
}
|
||||
|
||||
func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) (bool, int) {
|
||||
func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) bool {
|
||||
decision := priorityDemoteDecisionForCandidate(runnerPolicy, err)
|
||||
return decision.Demote, decision.Step
|
||||
return decision.Demote
|
||||
}
|
||||
|
||||
func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err error) priorityDemoteDecision {
|
||||
@@ -128,11 +127,7 @@ func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err err
|
||||
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_disabled", Info: info}
|
||||
}
|
||||
if match, ok := priorityDemotePolicyMatch(policy, info); ok {
|
||||
step := intFromPolicy(policy, "demoteStep")
|
||||
if step <= 0 {
|
||||
step = 100
|
||||
}
|
||||
return priorityDemoteDecision{Demote: true, Reason: "priority_demote_policy", Step: step, Match: match, Info: info}
|
||||
return priorityDemoteDecision{Demote: true, Reason: "priority_demote_policy", Match: match, Info: info}
|
||||
}
|
||||
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_no_match", Info: info}
|
||||
}
|
||||
|
||||
@@ -171,19 +171,16 @@ func TestPriorityDemotePolicyIsKeywordGatedAndHardStopSafe(t *testing.T) {
|
||||
"categories": []any{"request_error"},
|
||||
},
|
||||
PriorityDemotePolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"demoteStep": 25,
|
||||
"keywords": []any{"rate_limit"},
|
||||
"enabled": true,
|
||||
"keywords": []any{"rate_limit"},
|
||||
},
|
||||
}
|
||||
|
||||
shouldDemote, step := shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true})
|
||||
if !shouldDemote || step != 25 {
|
||||
t.Fatalf("priority demotion should be enabled only by matched policy, got shouldDemote=%v step=%d", shouldDemote, step)
|
||||
if !shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true}) {
|
||||
t.Fatal("priority demotion should be enabled only by matched policy")
|
||||
}
|
||||
|
||||
shouldDemote, _ = shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true})
|
||||
if shouldDemote {
|
||||
if shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true}) {
|
||||
t.Fatal("priority demotion should not run for hard-stop request errors")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,17 +73,26 @@ func (s *Service) applyPriorityDemotePolicy(ctx context.Context, taskID string,
|
||||
if !decision.Demote {
|
||||
return
|
||||
}
|
||||
if err := s.store.DemoteCandidatePlatformPriority(ctx, candidate.PlatformID, decision.Step); err == nil {
|
||||
s.recordAttemptTrace(ctx, taskID, attemptNo, priorityDemoteTraceEntry(decision, candidate.PlatformID, candidate.PlatformModelID))
|
||||
if dynamicPriority, err := s.store.DemoteCandidatePlatformPriority(ctx, candidate.PlatformID); err == nil {
|
||||
s.recordAttemptTrace(ctx, taskID, attemptNo, priorityDemoteTraceEntry(decision, candidate.PlatformID, candidate.PlatformModelID, dynamicPriority))
|
||||
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner policy", addPolicyTracePayload(map[string]any{
|
||||
"platformId": candidate.PlatformID,
|
||||
"platformModelId": candidate.PlatformModelID,
|
||||
"demoteStep": decision.Step,
|
||||
"dynamicPriority": dynamicPriority,
|
||||
"code": clients.ErrorCode(cause),
|
||||
"reason": decision.Reason,
|
||||
"errorMessage": messageFromError(cause),
|
||||
}, decision.Match, decision.Info), simulated)
|
||||
}
|
||||
}
|
||||
|
||||
func messageFromError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func effectiveRuntimePolicy(base map[string]any, override map[string]any, key string) map[string]any {
|
||||
policy := base
|
||||
if nested, ok := override[key].(map[string]any); ok && len(nested) > 0 {
|
||||
|
||||
@@ -34,10 +34,12 @@ func failoverTraceEntry(decision failoverDecision) map[string]any {
|
||||
return entry
|
||||
}
|
||||
|
||||
func priorityDemoteTraceEntry(decision priorityDemoteDecision, platformID string, platformModelID string) map[string]any {
|
||||
func priorityDemoteTraceEntry(decision priorityDemoteDecision, platformID string, platformModelID string, dynamicPriority int) map[string]any {
|
||||
entry := policyTraceEntry("priority_demoted", "priority_demote", "demote", decision.Reason, decision.Match, decision.Info)
|
||||
entry["demote"] = decision.Demote
|
||||
entry["demoteStep"] = decision.Step
|
||||
if dynamicPriority > 0 {
|
||||
entry["dynamicPriority"] = dynamicPriority
|
||||
}
|
||||
entry["platformId"] = platformID
|
||||
entry["platformModelId"] = platformModelID
|
||||
return entry
|
||||
|
||||
Reference in New Issue
Block a user