feat: add runner failover policies and traces

2026-05-12 02:16:42 +08:00
parent be31923e74
commit 05632172d0
26 changed files with 2033 additions and 140 deletions
@@ -229,6 +229,20 @@ type RuntimePolicySet struct {
 	UpdatedAt         time.Time      `json:"updatedAt"`
 }

+type RunnerPolicy struct {
+	ID                   string         `json:"id"`
+	PolicyKey            string         `json:"policyKey"`
+	Name                 string         `json:"name"`
+	Description          string         `json:"description,omitempty"`
+	FailoverPolicy       map[string]any `json:"failoverPolicy,omitempty"`
+	HardStopPolicy       map[string]any `json:"hardStopPolicy,omitempty"`
+	PriorityDemotePolicy map[string]any `json:"priorityDemotePolicy,omitempty"`
+	Metadata             map[string]any `json:"metadata,omitempty"`
+	Status               string         `json:"status"`
+	CreatedAt            time.Time      `json:"createdAt"`
+	UpdatedAt            time.Time      `json:"updatedAt"`
+}
+
 type PricingRule struct {
 	ID              string         `json:"id"`
 	RuleSetID       string         `json:"ruleSetId,omitempty"`
@@ -446,6 +460,7 @@ type TaskAttempt struct {
 	Retryable          bool           `json:"retryable"`
 	Simulated          bool           `json:"simulated"`
 	RequestID          string         `json:"requestId,omitempty"`
+	StatusCode         int            `json:"statusCode,omitempty"`
 	Usage              map[string]any `json:"usage,omitempty"`
 	Metrics            map[string]any `json:"metrics,omitempty"`
 	RequestSnapshot    map[string]any `json:"requestSnapshot,omitempty"`
@@ -1796,6 +1811,11 @@ func IsUniqueViolation(err error) bool {
 	return isUniqueViolation(err)
 }

+func IsUndefinedDatabaseObject(err error) bool {
+	var pgErr *pgconn.PgError
+	return errors.As(err, &pgErr) && (pgErr.Code == "42P01" || pgErr.Code == "42703")
+}
+
 func isUniqueViolation(err error) bool {
 	var pgErr *pgconn.PgError
 	return errors.As(err, &pgErr) && pgErr.Code == "23505"
@@ -0,0 +1,168 @@
+package store
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+const runnerPolicyColumns = `
+id::text, policy_key, name, COALESCE(description, ''), failover_policy,
+hard_stop_policy, priority_demote_policy, metadata, status, created_at, updated_at`
+
+type RunnerPolicyInput struct {
+	PolicyKey            string         `json:"policyKey"`
+	Name                 string         `json:"name"`
+	Description          string         `json:"description"`
+	FailoverPolicy       map[string]any `json:"failoverPolicy"`
+	HardStopPolicy       map[string]any `json:"hardStopPolicy"`
+	PriorityDemotePolicy map[string]any `json:"priorityDemotePolicy"`
+	Metadata             map[string]any `json:"metadata"`
+	Status               string         `json:"status"`
+}
+
+type runnerPolicyScanner interface {
+	Scan(dest ...any) error
+}
+
+func (s *Store) GetActiveRunnerPolicy(ctx context.Context) (RunnerPolicy, error) {
+	item, err := scanRunnerPolicy(s.pool.QueryRow(ctx, `
+SELECT `+runnerPolicyColumns+`
+FROM gateway_runner_policies
+ORDER BY CASE WHEN policy_key = 'default-runner-v1' THEN 0 ELSE 1 END,
+         CASE WHEN status = 'active' THEN 0 ELSE 1 END,
+         updated_at DESC
+LIMIT 1`))
+	if err != nil {
+		if err == pgx.ErrNoRows || IsUndefinedDatabaseObject(err) {
+			return defaultRunnerPolicy(), nil
+		}
+		return RunnerPolicy{}, err
+	}
+	return item, nil
+}
+
+func (s *Store) UpsertDefaultRunnerPolicy(ctx context.Context, input RunnerPolicyInput) (RunnerPolicy, error) {
+	input = normalizeRunnerPolicyInput(input)
+	failoverPolicy, _ := json.Marshal(emptyObjectIfNil(input.FailoverPolicy))
+	hardStopPolicy, _ := json.Marshal(emptyObjectIfNil(input.HardStopPolicy))
+	priorityDemotePolicy, _ := json.Marshal(emptyObjectIfNil(input.PriorityDemotePolicy))
+	metadata, _ := json.Marshal(emptyObjectIfNil(input.Metadata))
+	return scanRunnerPolicy(s.pool.QueryRow(ctx, `
+INSERT INTO gateway_runner_policies (
+  policy_key, name, description, failover_policy, hard_stop_policy, priority_demote_policy, metadata, status
+)
+VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, $7, $8)
+ON CONFLICT (policy_key) DO UPDATE
+SET name = EXCLUDED.name,
+    description = EXCLUDED.description,
+    failover_policy = EXCLUDED.failover_policy,
+    hard_stop_policy = EXCLUDED.hard_stop_policy,
+    priority_demote_policy = EXCLUDED.priority_demote_policy,
+    metadata = EXCLUDED.metadata,
+    status = EXCLUDED.status,
+    updated_at = now()
+RETURNING `+runnerPolicyColumns,
+		input.PolicyKey, input.Name, input.Description, failoverPolicy, hardStopPolicy, priorityDemotePolicy, metadata, input.Status,
+	))
+}
+
+func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
+	var item RunnerPolicy
+	var failoverPolicy []byte
+	var hardStopPolicy []byte
+	var priorityDemotePolicy []byte
+	var metadata []byte
+	if err := scanner.Scan(
+		&item.ID,
+		&item.PolicyKey,
+		&item.Name,
+		&item.Description,
+		&failoverPolicy,
+		&hardStopPolicy,
+		&priorityDemotePolicy,
+		&metadata,
+		&item.Status,
+		&item.CreatedAt,
+		&item.UpdatedAt,
+	); err != nil {
+		return RunnerPolicy{}, err
+	}
+	item.FailoverPolicy = decodeObject(failoverPolicy)
+	item.HardStopPolicy = decodeObject(hardStopPolicy)
+	item.PriorityDemotePolicy = decodeObject(priorityDemotePolicy)
+	item.Metadata = decodeObject(metadata)
+	return item, nil
+}
+
+func normalizeRunnerPolicyInput(input RunnerPolicyInput) RunnerPolicyInput {
+	input.PolicyKey = strings.TrimSpace(input.PolicyKey)
+	if input.PolicyKey == "" {
+		input.PolicyKey = "default-runner-v1"
+	}
+	input.Name = strings.TrimSpace(input.Name)
+	if input.Name == "" {
+		input.Name = "默认全局调度策略"
+	}
+	input.Description = strings.TrimSpace(input.Description)
+	input.Status = strings.TrimSpace(input.Status)
+	if input.Status == "" {
+		input.Status = "active"
+	}
+	return input
+}
+
+func defaultRunnerPolicy() RunnerPolicy {
+	now := time.Now()
+	return RunnerPolicy{
+		PolicyKey:            "default-runner-v1",
+		Name:                 "默认全局调度策略",
+		Description:          "控制多个候选平台之间的故障切换；模型运行策略只可覆盖 failoverPolicy，不能覆盖 hardStopPolicy。",
+		FailoverPolicy:       defaultRunnerFailoverPolicy(),
+		HardStopPolicy:       defaultRunnerHardStopPolicy(),
+		PriorityDemotePolicy: defaultRunnerPriorityDemotePolicy(),
+		Metadata:             map[string]any{"source": "code-default"},
+		Status:               "active",
+		CreatedAt:            now,
+		UpdatedAt:            now,
+	}
+}
+
+func defaultRunnerPriorityDemotePolicy() map[string]any {
+	return map[string]any{
+		"enabled":     true,
+		"demoteStep":  100,
+		"categories":  []any{"network", "timeout", "stream_error", "rate_limit", "provider_5xx", "provider_overloaded"},
+		"codes":       []any{"network", "timeout", "stream_read_error", "rate_limit", "server_error", "overloaded"},
+		"statusCodes": []any{408, 429, 500, 502, 503, 504},
+		"keywords":    []any{"timeout", "network", "rate_limit", "overloaded", "temporarily_unavailable", "server_error", "429", "5xx"},
+	}
+}
+
+func defaultRunnerFailoverPolicy() map[string]any {
+	return map[string]any{
+		"enabled":            true,
+		"maxPlatforms":       99,
+		"maxDurationSeconds": 600,
+		"allowCategories":    []any{"network", "timeout", "stream_error", "rate_limit", "provider_5xx", "provider_overloaded", "auth_error"},
+		"denyCategories":     []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
+		"allowCodes":         []any{"auth_failed", "invalid_api_key", "missing_credentials"},
+		"allowKeywords":      []any{"timeout", "network", "rate_limit", "overloaded", "temporarily_unavailable", "server_error", "auth_failed", "invalid_api_key", "missing_credentials", "unauthorized", "forbidden", "429", "5xx"},
+		"denyKeywords":       []any{"invalid_parameter", "missing required", "bad request"},
+		"allowStatusCodes":   []any{401, 403, 408, 429, 500, 502, 503, 504},
+		"denyStatusCodes":    []any{},
+	}
+}
+
+func defaultRunnerHardStopPolicy() map[string]any {
+	return map[string]any{
+		"enabled":     true,
+		"categories":  []any{"request_error", "unsupported_model", "user_permission", "insufficient_balance"},
+		"codes":       []any{"bad_request", "invalid_request", "invalid_parameter", "missing_required", "unsupported_kind", "unsupported_model", "insufficient_balance", "permission_denied"},
+		"statusCodes": []any{},
+		"keywords":    []any{"invalid_parameter", "missing required", "bad request", "insufficient balance"},
+	}
+}
@@ -134,6 +134,28 @@ WHERE id = $1::uuid`, platformID, cooldownSeconds)
 	return err
 }

+func (s *Store) DemoteCandidatePlatformPriority(ctx context.Context, platformID string, demoteStep int) error {
+	if strings.TrimSpace(platformID) == "" {
+		return nil
+	}
+	if demoteStep <= 0 {
+		demoteStep = 100
+	}
+	_, err := s.pool.Exec(ctx, `
+UPDATE integration_platforms target
+SET dynamic_priority = GREATEST(
+      COALESCE(target.dynamic_priority, target.priority),
+      COALESCE((
+        SELECT MAX(COALESCE(peer.dynamic_priority, peer.priority))
+        FROM integration_platforms peer
+        WHERE peer.deleted_at IS NULL
+      ), target.priority) + $2::int
+    ),
+    updated_at = now()
+WHERE target.id = $1::uuid`, platformID, demoteStep)
+	return err
+}
+
 func scanRuntimePolicySet(scanner runtimePolicyScanner) (RuntimePolicySet, error) {
 	var item RuntimePolicySet
 	var rateLimitPolicy []byte
@@ -3,6 +3,7 @@ package store
 import (
 	"context"
 	"encoding/json"
+	"strconv"
 	"strings"
 	"time"

@@ -234,6 +235,27 @@ func (s *Store) ListTaskAttempts(ctx context.Context, taskID string) ([]TaskAtte
 	return attemptsByTaskID[taskID], nil
 }

+func (s *Store) AppendTaskAttemptTrace(ctx context.Context, taskID string, attemptNo int, entry map[string]any) error {
+	entryJSON, _ := json.Marshal(emptyObjectIfNil(entry))
+	_, err := s.pool.Exec(ctx, `
+UPDATE gateway_task_attempts
+SET metrics = jsonb_set(
+      COALESCE(metrics, '{}'::jsonb),
+      '{trace}',
+      (
+        CASE
+          WHEN jsonb_typeof(COALESCE(metrics->'trace', '[]'::jsonb)) = 'array'
+          THEN COALESCE(metrics->'trace', '[]'::jsonb)
+          ELSE '[]'::jsonb
+        END
+      ) || jsonb_build_array($3::jsonb),
+      true
+    )
+WHERE task_id = $1::uuid
+  AND attempt_no = $2`, taskID, attemptNo, string(entryJSON))
+	return err
+}
+
 func (s *Store) listTaskAttemptsByTaskIDs(ctx context.Context, taskIDs []string) (map[string][]TaskAttempt, error) {
 	itemsByTaskID := map[string][]TaskAttempt{}
 	if len(taskIDs) == 0 {
@@ -331,6 +353,7 @@ func enrichTaskAttemptFromMetrics(item *TaskAttempt) {
 	item.ModelAlias = firstNonEmpty(item.ModelAlias, taskAttemptMetricString(item.Metrics, "modelAlias"))
 	item.ModelType = firstNonEmpty(item.ModelType, taskAttemptMetricString(item.Metrics, "modelType"))
 	item.ClientID = firstNonEmpty(item.ClientID, taskAttemptMetricString(item.Metrics, "clientId"))
+	item.StatusCode = taskAttemptMetricInt(item.Metrics, "statusCode")
 }

 func taskAttemptMetricString(metrics map[string]any, key string) string {
@@ -338,6 +361,25 @@ func taskAttemptMetricString(metrics map[string]any, key string) string {
 	return strings.TrimSpace(value)
 }

+func taskAttemptMetricInt(metrics map[string]any, key string) int {
+	switch value := metrics[key].(type) {
+	case int:
+		return value
+	case int64:
+		return int(value)
+	case float64:
+		return int(value)
+	case json.Number:
+		next, _ := value.Int64()
+		return int(next)
+	case string:
+		next, _ := strconv.Atoi(strings.TrimSpace(value))
+		return next
+	default:
+		return 0
+	}
+}
+
 func (s *Store) FinishTaskAttempt(ctx context.Context, input FinishTaskAttemptInput) error {
 	responseJSON, _ := json.Marshal(emptyObjectIfNil(input.ResponseSnapshot))
 	usageJSON, _ := json.Marshal(emptyObjectIfNil(input.Usage))