easyai-ai-gateway/apps/api/internal/runner/trace.go

package runner

import (
	"context"
	"fmt"
	"time"

	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)

func failureTraceEntry(err error, retryable bool) map[string]any {
	return failureTraceEntryWithReason(err, retryable, "client", "client_call_failed")
}

func failureTraceEntryWithReason(err error, retryable bool, scope string, reason string) map[string]any {
	info := failureInfoFromError(err)
	entry := policyTraceEntry("failure", scope, "failed", reason, policyRuleMatch{}, info)
	entry["retryable"] = retryable
	return entry
}

func retryTraceEntry(candidate store.RuntimeModelCandidate, decision retryDecision, action string, clientAttempt int, maxAttempts int) map[string]any {
	entry := policyTraceEntry("same_client_retry", "same_client", action, decision.Reason, decision.Match, decision.Info)
	entry["retry"] = decision.Retry
	entry["clientAttempt"] = clientAttempt
	entry["maxAttempts"] = maxAttempts
	addCandidatePriorityTraceFields(entry, candidate)
	return entry
}

func failoverTraceEntry(decision failoverDecision, candidate store.RuntimeModelCandidate) map[string]any {
	event := "failover_next"
	if !decision.Retry {
		event = "failover_stop"
	}
	entry := policyTraceEntry(event, "next_platform", decision.Action, decision.Reason, decision.Match, decision.Info)
	entry["retry"] = decision.Retry
	addCandidatePriorityTraceFields(entry, candidate)
	if decision.CooldownSeconds > 0 {
		entry["cooldownSeconds"] = decision.CooldownSeconds
	}
	if decision.Target != "" {
		entry["target"] = decision.Target
	}
	if decision.DemoteSteps > 0 {
		entry["demoteSteps"] = decision.DemoteSteps
	}
	return entry
}

func failureDecisionTraceEntry(decision failureDecision, candidate store.RuntimeModelCandidate, clientAttempt int, maxAttempts int, applied bool, guardReason string) map[string]any {
	entry := policyTraceEntry("failure_decision", "candidate_failure", decision.Route, decision.Reason, decision.Match, decision.Info)
	entry["route"] = decision.Route
	entry["effect"] = decision.Effect
	entry["applied"] = applied
	entry["guardReason"] = guardReason
	entry["clientAttempt"] = clientAttempt
	entry["maxAttempts"] = maxAttempts
	entry["rule"] = decision.Match.Rule
	entry["value"] = decision.Match.Value
	if decision.Target != "" {
		entry["target"] = decision.Target
	}
	if decision.CooldownSeconds > 0 {
		entry["cooldownSeconds"] = decision.CooldownSeconds
	}
	if decision.DemoteSteps > 0 {
		entry["demoteSteps"] = decision.DemoteSteps
	}
	addCandidatePriorityTraceFields(entry, candidate)
	return entry
}

func priorityDemoteTraceEntry(decision priorityDemoteDecision, platformID string, platformModelID string, dynamicPriority int) map[string]any {
	entry := policyTraceEntry("priority_demoted", "priority_demote", "demote", decision.Reason, decision.Match, decision.Info)
	entry["demote"] = decision.Demote
	if decision.DemoteSteps > 0 {
		entry["demoteSteps"] = decision.DemoteSteps
	}
	if dynamicPriority > 0 {
		entry["dynamicPriority"] = dynamicPriority
	}
	entry["platformId"] = platformID
	entry["platformModelId"] = platformModelID
	return entry
}

func failoverTimeBudgetTraceEntry(elapsedSeconds int, maxDurationSeconds int, info failureInfo, candidate store.RuntimeModelCandidate) map[string]any {
	entry := policyTraceEntry("failover_stop", "next_platform", "stop", "failover_time_budget_exceeded", policyRuleMatch{
		Source: "gateway_runner_policies.failover_policy",
		Policy: "failoverPolicy",
		Rule:   "maxDurationSeconds",
		Value:  fmt.Sprintf("%d", maxDurationSeconds),
	}, info)
	entry["elapsedSeconds"] = elapsedSeconds
	entry["maxDurationSeconds"] = maxDurationSeconds
	addCandidatePriorityTraceFields(entry, candidate)
	return entry
}

func addCandidatePriorityTraceFields(entry map[string]any, candidate store.RuntimeModelCandidate) {
	if entry == nil {
		return
	}
	entry["currentPriority"] = candidate.PlatformPriority
	entry["platformPriority"] = candidate.PlatformPriority
	if candidate.PlatformID != "" {
		entry["currentPlatformId"] = candidate.PlatformID
	}
	if candidate.PlatformName != "" {
		entry["currentPlatformName"] = candidate.PlatformName
	}
	if candidate.CacheAffinity.Key != "" {
		entry["cacheAffinityKey"] = candidate.CacheAffinity.Key
		entry["cacheAffinityScore"] = candidate.CacheAffinity.Score
		entry["cacheAffinityConfidence"] = candidate.CacheAffinity.Confidence
		entry["cacheAffinityHitRatio"] = candidate.CacheAffinity.EMAHitRatio
		entry["cacheAffinityLastHitRatio"] = candidate.CacheAffinity.LastHitRatio
		entry["cacheAffinityCachedInputTokens"] = candidate.CacheAffinity.CachedInputTokens
		entry["cacheAffinityBoost"] = candidate.CacheAffinity.Boost
		entry["cacheAffinityApplied"] = candidate.CacheAffinity.Applied
	}
}

func policyTraceEntry(event string, scope string, action string, reason string, match policyRuleMatch, info failureInfo) map[string]any {
	entry := map[string]any{
		"event":     event,
		"scope":     scope,
		"action":    action,
		"reason":    reason,
		"createdAt": time.Now().Format(time.RFC3339Nano),
	}
	if info.Code != "" {
		entry["errorCode"] = info.Code
	}
	if info.Message != "" {
		entry["message"] = info.Message
	}
	if info.Status > 0 {
		entry["statusCode"] = info.Status
	}
	if info.Category != "" {
		entry["category"] = info.Category
	}
	if match.Source != "" {
		entry["policySource"] = match.Source
	}
	if match.Policy != "" {
		entry["policy"] = match.Policy
	}
	if match.Rule != "" {
		entry["policyRule"] = match.Rule
	}
	if match.Value != "" {
		entry["matchedValue"] = match.Value
	}
	return entry
}

func addPolicyTracePayload(payload map[string]any, match policyRuleMatch, info failureInfo) map[string]any {
	if payload == nil {
		payload = map[string]any{}
	}
	if info.Code != "" {
		payload["errorCode"] = info.Code
	}
	if info.Status > 0 {
		payload["statusCode"] = info.Status
	}
	if info.Category != "" {
		payload["category"] = info.Category
	}
	if match.Source != "" {
		payload["policySource"] = match.Source
	}
	if match.Policy != "" {
		payload["policy"] = match.Policy
	}
	if match.Rule != "" {
		payload["policyRule"] = match.Rule
	}
	if match.Value != "" {
		payload["matchedValue"] = match.Value
	}
	return payload
}

func (s *Service) recordAttemptTrace(ctx context.Context, taskID string, attemptNo int, entry map[string]any) {
	if attemptNo <= 0 || len(entry) == 0 {
		return
	}
	if err := s.store.AppendTaskAttemptTrace(ctx, taskID, attemptNo, entry); err != nil {
		s.logger.Warn("append task attempt trace failed", "taskID", taskID, "attempt", attemptNo, "error", err)
	}
}