feat: add priority demotion controls

This commit is contained in:
2026-05-12 18:43:20 +08:00
parent 3917b84b5d
commit 98abd247d6
22 changed files with 917 additions and 85 deletions
+3 -8
View File
@@ -43,7 +43,6 @@ type failoverDecision struct {
type priorityDemoteDecision struct {
Demote bool
Reason string
Step int
Match policyRuleMatch
Info failureInfo
}
@@ -110,9 +109,9 @@ func failoverDecisionForCandidate(runnerPolicy store.RunnerPolicy, candidate sto
return failoverDecision{Retry: false, Action: "stop", Reason: "client_non_retryable", Match: policyRuleMatch{Source: "provider_client", Policy: "ClientError", Rule: "Retryable", Value: "false"}, Info: info}
}
func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) (bool, int) {
func shouldDemoteCandidatePriority(runnerPolicy store.RunnerPolicy, err error) bool {
decision := priorityDemoteDecisionForCandidate(runnerPolicy, err)
return decision.Demote, decision.Step
return decision.Demote
}
func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err error) priorityDemoteDecision {
@@ -128,11 +127,7 @@ func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err err
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_disabled", Info: info}
}
if match, ok := priorityDemotePolicyMatch(policy, info); ok {
step := intFromPolicy(policy, "demoteStep")
if step <= 0 {
step = 100
}
return priorityDemoteDecision{Demote: true, Reason: "priority_demote_policy", Step: step, Match: match, Info: info}
return priorityDemoteDecision{Demote: true, Reason: "priority_demote_policy", Match: match, Info: info}
}
return priorityDemoteDecision{Demote: false, Reason: "priority_demote_no_match", Info: info}
}
@@ -171,19 +171,16 @@ func TestPriorityDemotePolicyIsKeywordGatedAndHardStopSafe(t *testing.T) {
"categories": []any{"request_error"},
},
PriorityDemotePolicy: map[string]any{
"enabled": true,
"demoteStep": 25,
"keywords": []any{"rate_limit"},
"enabled": true,
"keywords": []any{"rate_limit"},
},
}
shouldDemote, step := shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true})
if !shouldDemote || step != 25 {
t.Fatalf("priority demotion should be enabled only by matched policy, got shouldDemote=%v step=%d", shouldDemote, step)
if !shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "rate_limit", Message: "rate_limit from upstream", Retryable: true}) {
t.Fatal("priority demotion should be enabled only by matched policy")
}
shouldDemote, _ = shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true})
if shouldDemote {
if shouldDemoteCandidatePriority(runnerPolicy, &clients.ClientError{Code: "bad_request", StatusCode: 400, Retryable: true}) {
t.Fatal("priority demotion should not run for hard-stop request errors")
}
}
+12 -3
View File
@@ -73,17 +73,26 @@ func (s *Service) applyPriorityDemotePolicy(ctx context.Context, taskID string,
if !decision.Demote {
return
}
if err := s.store.DemoteCandidatePlatformPriority(ctx, candidate.PlatformID, decision.Step); err == nil {
s.recordAttemptTrace(ctx, taskID, attemptNo, priorityDemoteTraceEntry(decision, candidate.PlatformID, candidate.PlatformModelID))
if dynamicPriority, err := s.store.DemoteCandidatePlatformPriority(ctx, candidate.PlatformID); err == nil {
s.recordAttemptTrace(ctx, taskID, attemptNo, priorityDemoteTraceEntry(decision, candidate.PlatformID, candidate.PlatformModelID, dynamicPriority))
_ = s.emit(ctx, taskID, "task.policy.priority_demoted", "running", "priority_demote", 0.52, "candidate platform priority demoted by runner policy", addPolicyTracePayload(map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"demoteStep": decision.Step,
"dynamicPriority": dynamicPriority,
"code": clients.ErrorCode(cause),
"reason": decision.Reason,
"errorMessage": messageFromError(cause),
}, decision.Match, decision.Info), simulated)
}
}
func messageFromError(err error) string {
if err == nil {
return ""
}
return err.Error()
}
func effectiveRuntimePolicy(base map[string]any, override map[string]any, key string) map[string]any {
policy := base
if nested, ok := override[key].(map[string]any); ok && len(nested) > 0 {
+4 -2
View File
@@ -34,10 +34,12 @@ func failoverTraceEntry(decision failoverDecision) map[string]any {
return entry
}
func priorityDemoteTraceEntry(decision priorityDemoteDecision, platformID string, platformModelID string) map[string]any {
func priorityDemoteTraceEntry(decision priorityDemoteDecision, platformID string, platformModelID string, dynamicPriority int) map[string]any {
entry := policyTraceEntry("priority_demoted", "priority_demote", "demote", decision.Reason, decision.Match, decision.Info)
entry["demote"] = decision.Demote
entry["demoteStep"] = decision.Step
if dynamicPriority > 0 {
entry["dynamicPriority"] = dynamicPriority
}
entry["platformId"] = platformID
entry["platformModelId"] = platformModelID
return entry