feat: add runner failover policies and traces

2026-05-12 02:16:42 +08:00
parent be31923e74
commit 05632172d0
26 changed files with 2033 additions and 140 deletions
@@ -0,0 +1,128 @@
+package runner
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+func failureTraceEntry(err error, retryable bool) map[string]any {
+	info := failureInfoFromError(err)
+	entry := policyTraceEntry("failure", "client", "failed", "client_call_failed", policyRuleMatch{}, info)
+	entry["retryable"] = retryable
+	return entry
+}
+
+func retryTraceEntry(decision retryDecision, action string, clientAttempt int, maxAttempts int) map[string]any {
+	entry := policyTraceEntry("same_client_retry", "same_client", action, decision.Reason, decision.Match, decision.Info)
+	entry["retry"] = decision.Retry
+	entry["clientAttempt"] = clientAttempt
+	entry["maxAttempts"] = maxAttempts
+	return entry
+}
+
+func failoverTraceEntry(decision failoverDecision) map[string]any {
+	event := "failover_next"
+	if !decision.Retry {
+		event = "failover_stop"
+	}
+	entry := policyTraceEntry(event, "next_platform", decision.Action, decision.Reason, decision.Match, decision.Info)
+	entry["retry"] = decision.Retry
+	if decision.CooldownSeconds > 0 {
+		entry["cooldownSeconds"] = decision.CooldownSeconds
+	}
+	return entry
+}
+
+func priorityDemoteTraceEntry(decision priorityDemoteDecision, platformID string, platformModelID string) map[string]any {
+	entry := policyTraceEntry("priority_demoted", "priority_demote", "demote", decision.Reason, decision.Match, decision.Info)
+	entry["demote"] = decision.Demote
+	entry["demoteStep"] = decision.Step
+	entry["platformId"] = platformID
+	entry["platformModelId"] = platformModelID
+	return entry
+}
+
+func failoverTimeBudgetTraceEntry(elapsedSeconds int, maxDurationSeconds int, info failureInfo) map[string]any {
+	entry := policyTraceEntry("failover_stop", "next_platform", "stop", "failover_time_budget_exceeded", policyRuleMatch{
+		Source: "gateway_runner_policies.failover_policy",
+		Policy: "failoverPolicy",
+		Rule:   "maxDurationSeconds",
+		Value:  fmt.Sprintf("%d", maxDurationSeconds),
+	}, info)
+	entry["elapsedSeconds"] = elapsedSeconds
+	entry["maxDurationSeconds"] = maxDurationSeconds
+	return entry
+}
+
+func policyTraceEntry(event string, scope string, action string, reason string, match policyRuleMatch, info failureInfo) map[string]any {
+	entry := map[string]any{
+		"event":     event,
+		"scope":     scope,
+		"action":    action,
+		"reason":    reason,
+		"createdAt": time.Now().Format(time.RFC3339Nano),
+	}
+	if info.Code != "" {
+		entry["errorCode"] = info.Code
+	}
+	if info.Message != "" {
+		entry["message"] = info.Message
+	}
+	if info.Status > 0 {
+		entry["statusCode"] = info.Status
+	}
+	if info.Category != "" {
+		entry["category"] = info.Category
+	}
+	if match.Source != "" {
+		entry["policySource"] = match.Source
+	}
+	if match.Policy != "" {
+		entry["policy"] = match.Policy
+	}
+	if match.Rule != "" {
+		entry["policyRule"] = match.Rule
+	}
+	if match.Value != "" {
+		entry["matchedValue"] = match.Value
+	}
+	return entry
+}
+
+func addPolicyTracePayload(payload map[string]any, match policyRuleMatch, info failureInfo) map[string]any {
+	if payload == nil {
+		payload = map[string]any{}
+	}
+	if info.Code != "" {
+		payload["errorCode"] = info.Code
+	}
+	if info.Status > 0 {
+		payload["statusCode"] = info.Status
+	}
+	if info.Category != "" {
+		payload["category"] = info.Category
+	}
+	if match.Source != "" {
+		payload["policySource"] = match.Source
+	}
+	if match.Policy != "" {
+		payload["policy"] = match.Policy
+	}
+	if match.Rule != "" {
+		payload["policyRule"] = match.Rule
+	}
+	if match.Value != "" {
+		payload["matchedValue"] = match.Value
+	}
+	return payload
+}
+
+func (s *Service) recordAttemptTrace(ctx context.Context, taskID string, attemptNo int, entry map[string]any) {
+	if attemptNo <= 0 || len(entry) == 0 {
+		return
+	}
+	if err := s.store.AppendTaskAttemptTrace(ctx, taskID, attemptNo, entry); err != nil {
+		s.logger.Warn("append task attempt trace failed", "taskID", taskID, "attempt", attemptNo, "error", err)
+	}
+}