增加单一源保护全局策略

apps/api/internal/httpapi/core_flow_integration_test.go

@@ -1282,6 +1282,41 @@ WHERE m.platform_id = $1::uuid
 	if err := testPool.QueryRow(ctx, `SELECT status FROM integration_platforms WHERE id = $1::uuid`, invalidKeyPlatform.ID).Scan(&invalidKeyPlatformStatus); err != nil {
 		t.Fatalf("read invalid key platform status: %v", err)
 	}
+	if invalidKeyPlatformStatus != "enabled" {
+		t.Fatalf("single source protection should keep the only source enabled, got %q", invalidKeyPlatformStatus)
+	}
+	var validKeyPlatform struct {
+		ID string `json:"id"`
+	}
+	doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms", loginResponse.AccessToken, map[string]any{
+		"provider":    "openai",
+		"platformKey": "openai-auto-disable-success-" + suffixText,
+		"name":        "OpenAI Auto Disable Success",
+		"baseUrl":     "https://api.openai.com/v1",
+		"authType":    "bearer",
+		"credentials": map[string]any{"mode": "simulation"},
+		"priority":    60,
+	}, http.StatusCreated, &validKeyPlatform)
+	doJSON(t, server.URL, http.MethodPost, "/api/admin/platforms/"+validKeyPlatform.ID+"/models", loginResponse.AccessToken, map[string]any{
+		"canonicalModelKey": "openai:gpt-4o-mini",
+		"modelName":         autoDisableModel,
+		"modelAlias":        autoDisableModel,
+		"modelType":         []string{"text_generate"},
+		"displayName":       "Auto Disable Smoke Success",
+	}, http.StatusCreated, nil)
+	doAPIV1ChatCompletionAndLoadTask(t, ctx, testPool, server.URL, apiKeyResponse.Secret, map[string]any{
+		"model":                autoDisableModel,
+		"runMode":              "simulation",
+		"simulation":           true,
+		"simulationDurationMs": 5,
+		"messages":             []map[string]any{{"role": "user", "content": "disable with fallback please"}},
+	}, "auto-disable-with-fallback-chat-"+suffixText, http.StatusOK, nil, &autoDisableTask.Task)
+	if autoDisableTask.Task.Status != "succeeded" {
+		t.Fatalf("auto disable task should fail over after disabling the bad source: %+v", autoDisableTask.Task)
+	}
+	if err := testPool.QueryRow(ctx, `SELECT status FROM integration_platforms WHERE id = $1::uuid`, invalidKeyPlatform.ID).Scan(&invalidKeyPlatformStatus); err != nil {
+		t.Fatalf("read invalid key platform status after fallback: %v", err)
+	}
 	if invalidKeyPlatformStatus != "disabled" {
 		t.Fatalf("auto disable policy should disable platform, got %q", invalidKeyPlatformStatus)
 	}

apps/api/internal/runner/retry_decision_test.go

@@ -37,6 +37,40 @@ func TestFailoverBudgetDefaults(t *testing.T) {
 	}
 }
 
+func TestSingleSourceProtectionDefaultsOn(t *testing.T) {
+	runnerPolicy := store.RunnerPolicy{Status: "active"}
+
+	if !singleSourceProtectionActive([]store.RuntimeModelCandidate{{PlatformID: "only-source"}}, 99, runnerPolicy) {
+		t.Fatal("single source protection should default to enabled")
+	}
+}
+
+func TestSingleSourceProtectionCanBeDisabled(t *testing.T) {
+	runnerPolicy := store.RunnerPolicy{
+		Status:             "active",
+		SingleSourcePolicy: map[string]any{"enabled": false},
+	}
+
+	if singleSourceProtectionActive([]store.RuntimeModelCandidate{{PlatformID: "only-source"}}, 99, runnerPolicy) {
+		t.Fatal("single source protection should respect the global toggle")
+	}
+}
+
+func TestSingleSourceProtectionUsesEffectivePlatformBudget(t *testing.T) {
+	candidates := []store.RuntimeModelCandidate{
+		{PlatformID: "first-source"},
+		{PlatformID: "second-source"},
+	}
+	runnerPolicy := store.RunnerPolicy{Status: "active"}
+
+	if !singleSourceProtectionActive(candidates, 1, runnerPolicy) {
+		t.Fatal("maxPlatforms=1 should make this execution a single-source run")
+	}
+	if singleSourceProtectionActive(candidates, 2, runnerPolicy) {
+		t.Fatal("multiple effective sources should allow disable and cooldown actions")
+	}
+}
+
 func TestFailoverTimeBudgetExceeded(t *testing.T) {
 	if !failoverTimeBudgetExceeded(time.Now().Add(-601*time.Second), 10*time.Minute) {
 		t.Fatal("failover time budget should stop retries after the configured duration")

apps/api/internal/runner/runtime_policy.go

@@ -9,7 +9,7 @@ import (
 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
 )
 
-func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, cause error, simulated bool) {
+func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, cause error, simulated bool, singleSourceProtected bool) {
 	code := clients.ErrorCode(cause)
 	message := ""
 	if cause != nil {
@@ -18,7 +18,9 @@ func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID stri
 
 	autoDisablePolicy := effectiveRuntimePolicy(candidate.AutoDisablePolicy, candidate.RuntimePolicyOverride, "autoDisablePolicy")
 	if failurePolicyMatches(autoDisablePolicy, code, message) && intFromPolicy(autoDisablePolicy, "threshold") <= 1 {
-		if err := s.store.DisableCandidatePlatform(ctx, candidate.PlatformID); err == nil {
+		if singleSourceProtected {
+			s.emitSingleSourceProtected(ctx, taskID, candidate, "auto_disable", code, simulated)
+		} else if err := s.store.DisableCandidatePlatform(ctx, candidate.PlatformID); err == nil {
 			_ = s.emit(ctx, taskID, "task.policy.auto_disabled", "running", "auto_disable", 0.48, "candidate platform disabled by failure policy", map[string]any{
 				"platformId":      candidate.PlatformID,
 				"platformModelId": candidate.PlatformModelID,
@@ -30,7 +32,9 @@ func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID stri
 	degradePolicy := effectiveRuntimePolicy(candidate.DegradePolicy, candidate.RuntimePolicyOverride, "degradePolicy")
 	if failurePolicyMatches(degradePolicy, code, message) {
 		cooldownSeconds := intFromPolicy(degradePolicy, "cooldownSeconds")
-		if err := s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, cooldownSeconds); err == nil {
+		if singleSourceProtected {
+			s.emitSingleSourceProtected(ctx, taskID, candidate, "degrade_cooldown", code, simulated)
+		} else if err := s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, cooldownSeconds); err == nil {
 			_ = s.emit(ctx, taskID, "task.policy.degraded", "running", "degrade", 0.5, "candidate model cooled down by failure policy", map[string]any{
 				"platformId":      candidate.PlatformID,
 				"platformModelId": candidate.PlatformModelID,
@@ -41,9 +45,13 @@ func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID stri
 	}
 }
 
-func (s *Service) applyFailoverAction(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, decision failoverDecision, simulated bool) {
+func (s *Service) applyFailoverAction(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, decision failoverDecision, simulated bool, singleSourceProtected bool) {
 	switch decision.Action {
 	case "disable_and_next":
+		if singleSourceProtected {
+			s.emitSingleSourceProtected(ctx, taskID, candidate, decision.Action, decision.Info.Code, simulated)
+			return
+		}
 		if err := s.store.DisableCandidatePlatform(ctx, candidate.PlatformID); err == nil {
 			_ = s.emit(ctx, taskID, "task.policy.failover_disabled", "running", "failover", 0.51, "candidate platform disabled by runner failover policy", addPolicyTracePayload(map[string]any{
 				"platformId":      candidate.PlatformID,
@@ -53,6 +61,10 @@ func (s *Service) applyFailoverAction(ctx context.Context, taskID string, candid
 			}, decision.Match, decision.Info), simulated)
 		}
 	case "cooldown_and_next":
+		if singleSourceProtected {
+			s.emitSingleSourceProtected(ctx, taskID, candidate, decision.Action, decision.Info.Code, simulated)
+			return
+		}
 		if err := s.store.CooldownCandidatePlatformModel(ctx, candidate.PlatformModelID, decision.CooldownSeconds); err == nil {
 			_ = s.emit(ctx, taskID, "task.policy.failover_cooled_down", "running", "failover", 0.51, "candidate model cooled down by runner failover policy", addPolicyTracePayload(map[string]any{
 				"platformId":      candidate.PlatformID,
@@ -65,6 +77,15 @@ func (s *Service) applyFailoverAction(ctx context.Context, taskID string, candid
 	}
 }
 
+func (s *Service) emitSingleSourceProtected(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, action string, code string, simulated bool) {
+	_ = s.emit(ctx, taskID, "task.policy.single_source_protected", "running", "policy_guard", 0.5, "single source protected from disable or cooldown", map[string]any{
+		"platformId":      candidate.PlatformID,
+		"platformModelId": candidate.PlatformModelID,
+		"action":          action,
+		"code":            code,
+	}, simulated)
+}
+
 func (s *Service) applyPriorityDemotePolicy(ctx context.Context, taskID string, attemptNo int, runnerPolicy store.RunnerPolicy, candidate store.RuntimeModelCandidate, requestedModel string, cause error, simulated bool) {
 	if errors.Is(cause, store.ErrRateLimited) {
 		return

apps/api/internal/runner/service.go

@@ -240,6 +240,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
 	}
 	maxPlatforms := maxPlatformsForCandidates(candidates, runnerPolicy)
 	maxFailoverDuration := maxFailoverDurationForCandidates(candidates, runnerPolicy)
+	singleSourceProtected := singleSourceProtectionActive(candidates, maxPlatforms, runnerPolicy)
 	var lastErr error
 	var lastCandidate store.RuntimeModelCandidate
 	var lastPreprocessing *parameterPreprocessingLog
@@ -275,7 +276,7 @@ candidatesLoop:
 				break candidatesLoop
 			}
 			candidateBody := preprocessing.Body
-			response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta)
+			response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta, singleSourceProtected)
 			if err == nil {
 				attemptNo = nextAttemptNo
 				billings := s.billings(ctx, user, task.Kind, candidateBody, candidate, response, isSimulation(task, candidate))
@@ -445,7 +446,7 @@ candidatesLoop:
 		if !decision.Retry {
 			break
 		}
-		s.applyFailoverAction(ctx, task.ID, candidate, decision, isSimulation(task, candidate))
+		s.applyFailoverAction(ctx, task.ID, candidate, decision, isSimulation(task, candidate), singleSourceProtected)
 		if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.55, "retrying next client", addPolicyTracePayload(map[string]any{
 			"attempt": attemptNo,
 			"action":  decision.Action,
@@ -486,7 +487,7 @@ candidatesLoop:
 	return Result{Task: failed, Output: failed.Result}, lastErr
 }
 
-func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta) (clients.Response, error) {
+func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta, singleSourceProtected bool) (clients.Response, error) {
 	simulated := isSimulation(task, candidate)
 	baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing))
 	reservations := s.rateLimitReservations(ctx, user, candidate, body)
@@ -638,7 +639,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 			ErrorMessage:       err.Error(),
 		})
 		_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
-		s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated)
+		s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated, singleSourceProtected)
 		return clients.Response{}, err
 	}
 	uploadedResult, err := s.uploadGeneratedAssets(ctx, task.ID, task.Kind, response.Result)
@@ -1099,6 +1100,42 @@ func maxPlatformsForCandidates(candidates []store.RuntimeModelCandidate, runnerP
 	return maxPlatforms
 }
 
+func singleSourceProtectionActive(candidates []store.RuntimeModelCandidate, maxPlatforms int, runnerPolicy store.RunnerPolicy) bool {
+	if !runnerSingleSourceProtectionEnabled(runnerPolicy) {
+		return false
+	}
+	return runtimeCandidateSourceCount(candidates, maxPlatforms) <= 1
+}
+
+func runnerSingleSourceProtectionEnabled(runnerPolicy store.RunnerPolicy) bool {
+	if enabled, ok := runnerPolicy.SingleSourcePolicy["enabled"].(bool); ok {
+		return enabled
+	}
+	return true
+}
+
+func runtimeCandidateSourceCount(candidates []store.RuntimeModelCandidate, maxPlatforms int) int {
+	limit := len(candidates)
+	if maxPlatforms > 0 && maxPlatforms < limit {
+		limit = maxPlatforms
+	}
+	sources := make(map[string]bool, limit)
+	for index := 0; index < limit; index++ {
+		key := strings.TrimSpace(candidates[index].PlatformID)
+		if key == "" {
+			key = strings.TrimSpace(candidates[index].PlatformModelID)
+		}
+		if key == "" {
+			key = strings.TrimSpace(candidates[index].ClientID)
+		}
+		if key == "" {
+			key = strconv.Itoa(index)
+		}
+		sources[key] = true
+	}
+	return len(sources)
+}
+
 func maxFailoverDurationForCandidates(candidates []store.RuntimeModelCandidate, runnerPolicy store.RunnerPolicy) time.Duration {
 	seconds := intFromPolicy(runnerPolicy.FailoverPolicy, "maxDurationSeconds")
 	if seconds <= 0 {

apps/api/internal/store/postgres.go

@@ -277,6 +277,7 @@ type RunnerPolicy struct {
 	FailoverPolicy       map[string]any `json:"failoverPolicy,omitempty"`
 	HardStopPolicy       map[string]any `json:"hardStopPolicy,omitempty"`
 	PriorityDemotePolicy map[string]any `json:"priorityDemotePolicy,omitempty"`
+	SingleSourcePolicy   map[string]any `json:"singleSourcePolicy,omitempty"`
 	Metadata             map[string]any `json:"metadata,omitempty"`
 	Status               string         `json:"status"`
 	CreatedAt            time.Time      `json:"createdAt"`

apps/api/internal/store/runner_policies.go

@@ -11,7 +11,7 @@ import (
 
 const runnerPolicyColumns = `
 id::text, policy_key, name, COALESCE(description, ''), failover_policy,
-hard_stop_policy, priority_demote_policy, metadata, status, created_at, updated_at`
+hard_stop_policy, priority_demote_policy, single_source_policy, metadata, status, created_at, updated_at`
 
 type RunnerPolicyInput struct {
 	PolicyKey            string         `json:"policyKey"`
@@ -20,6 +20,7 @@ type RunnerPolicyInput struct {
 	FailoverPolicy       map[string]any `json:"failoverPolicy"`
 	HardStopPolicy       map[string]any `json:"hardStopPolicy"`
 	PriorityDemotePolicy map[string]any `json:"priorityDemotePolicy"`
+	SingleSourcePolicy   map[string]any `json:"singleSourcePolicy"`
 	Metadata             map[string]any `json:"metadata"`
 	Status               string         `json:"status"`
 }
@@ -50,23 +51,25 @@ func (s *Store) UpsertDefaultRunnerPolicy(ctx context.Context, input RunnerPolic
 	failoverPolicy, _ := json.Marshal(emptyObjectIfNil(input.FailoverPolicy))
 	hardStopPolicy, _ := json.Marshal(emptyObjectIfNil(input.HardStopPolicy))
 	priorityDemotePolicy, _ := json.Marshal(emptyObjectIfNil(input.PriorityDemotePolicy))
+	singleSourcePolicy, _ := json.Marshal(emptyObjectIfNil(input.SingleSourcePolicy))
 	metadata, _ := json.Marshal(emptyObjectIfNil(input.Metadata))
 	return scanRunnerPolicy(s.pool.QueryRow(ctx, `
 INSERT INTO gateway_runner_policies (
-  policy_key, name, description, failover_policy, hard_stop_policy, priority_demote_policy, metadata, status
+  policy_key, name, description, failover_policy, hard_stop_policy, priority_demote_policy, single_source_policy, metadata, status
 )
-VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, $7, $8)
+VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, $7, $8, $9)
 ON CONFLICT (policy_key) DO UPDATE
 SET name = EXCLUDED.name,
     description = EXCLUDED.description,
     failover_policy = EXCLUDED.failover_policy,
     hard_stop_policy = EXCLUDED.hard_stop_policy,
     priority_demote_policy = EXCLUDED.priority_demote_policy,
+    single_source_policy = EXCLUDED.single_source_policy,
     metadata = EXCLUDED.metadata,
     status = EXCLUDED.status,
     updated_at = now()
 RETURNING `+runnerPolicyColumns,
-		input.PolicyKey, input.Name, input.Description, failoverPolicy, hardStopPolicy, priorityDemotePolicy, metadata, input.Status,
+		input.PolicyKey, input.Name, input.Description, failoverPolicy, hardStopPolicy, priorityDemotePolicy, singleSourcePolicy, metadata, input.Status,
 	))
 }
 
@@ -75,6 +78,7 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
 	var failoverPolicy []byte
 	var hardStopPolicy []byte
 	var priorityDemotePolicy []byte
+	var singleSourcePolicy []byte
 	var metadata []byte
 	if err := scanner.Scan(
 		&item.ID,
@@ -84,6 +88,7 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
 		&failoverPolicy,
 		&hardStopPolicy,
 		&priorityDemotePolicy,
+		&singleSourcePolicy,
 		&metadata,
 		&item.Status,
 		&item.CreatedAt,
@@ -94,6 +99,7 @@ func scanRunnerPolicy(scanner runnerPolicyScanner) (RunnerPolicy, error) {
 	item.FailoverPolicy = decodeObject(failoverPolicy)
 	item.HardStopPolicy = decodeObject(hardStopPolicy)
 	item.PriorityDemotePolicy = decodeObject(priorityDemotePolicy)
+	item.SingleSourcePolicy = decodeObject(singleSourcePolicy)
 	item.Metadata = decodeObject(metadata)
 	return item, nil
 }
@@ -124,6 +130,7 @@ func defaultRunnerPolicy() RunnerPolicy {
 		FailoverPolicy:       defaultRunnerFailoverPolicy(),
 		HardStopPolicy:       defaultRunnerHardStopPolicy(),
 		PriorityDemotePolicy: defaultRunnerPriorityDemotePolicy(),
+		SingleSourcePolicy:   defaultRunnerSingleSourcePolicy(),
 		Metadata:             map[string]any{"source": "code-default"},
 		Status:               "active",
 		CreatedAt:            now,
@@ -141,6 +148,12 @@ func defaultRunnerPriorityDemotePolicy() map[string]any {
 	}
 }
 
+func defaultRunnerSingleSourcePolicy() map[string]any {
+	return map[string]any{
+		"enabled": true,
+	}
+}
+
 func defaultRunnerFailoverPolicy() map[string]any {
 	return map[string]any{
 		"enabled":            true,