Decouple stream cancellation and clarify failover rules

apps/api/internal/httpapi/core_flow_integration_test.go

@@ -351,6 +351,48 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
 		t.Fatalf("unexpected compatible chat response: %+v", compatChat)
 	}
 
+	cancelMarker := "cancel-stream-" + suffixText
+	cancelCtx, cancelRequest := context.WithCancel(context.Background())
+	cancelPayload := map[string]any{
+		"model":                defaultTextModel,
+		"runMode":              "simulation",
+		"messages":             []map[string]any{{"role": "user", "content": "cancelled stream"}},
+		"stream":               true,
+		"simulation":           true,
+		"simulationDurationMs": 250,
+		"cancelTestId":         cancelMarker,
+	}
+	cancelRaw, err := json.Marshal(cancelPayload)
+	if err != nil {
+		t.Fatalf("marshal cancelled stream payload: %v", err)
+	}
+	cancelReq, err := http.NewRequestWithContext(cancelCtx, http.MethodPost, server.URL+"/v1/chat/completions", bytes.NewReader(cancelRaw))
+	if err != nil {
+		t.Fatalf("build cancelled stream request: %v", err)
+	}
+	cancelReq.Header.Set("Authorization", "Bearer "+apiKeyResponse.Secret)
+	cancelReq.Header.Set("Content-Type", "application/json")
+	cancelErrCh := make(chan error, 1)
+	go func() {
+		resp, err := http.DefaultClient.Do(cancelReq)
+		if resp != nil {
+			_, _ = io.ReadAll(resp.Body)
+			_ = resp.Body.Close()
+		}
+		cancelErrCh <- err
+	}()
+	cancelTaskID := waitForTaskIDByRequestMarker(t, ctx, testPool, cancelMarker, 2*time.Second)
+	cancelRequest()
+	select {
+	case <-cancelErrCh:
+	case <-time.After(time.Second):
+		t.Fatal("cancelled stream request did not return after client cancellation")
+	}
+	cancelledStreamTask := waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, cancelTaskID, []string{"succeeded"}, 2*time.Second)
+	if cancelledStreamTask.Status != "succeeded" {
+		t.Fatalf("client-cancelled compatible stream should keep backend task running to success, got %+v", cancelledStreamTask)
+	}
+
 	var imageResponse struct {
 		Task struct {
 			ID     string         `json:"id"`
@@ -1398,6 +1440,26 @@ func waitForTaskStatus(t *testing.T, baseURL string, token string, taskID string
 	return detail
 }
 
+func waitForTaskIDByRequestMarker(t *testing.T, ctx context.Context, pool *pgxpool.Pool, marker string, timeout time.Duration) string {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		var taskID string
+		err := pool.QueryRow(ctx, `
+SELECT id::text
+FROM gateway_tasks
+WHERE request->>'cancelTestId' = $1
+ORDER BY created_at DESC
+LIMIT 1`, marker).Scan(&taskID)
+		if err == nil && taskID != "" {
+			return taskID
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("task with request marker %s was not created within %s", marker, timeout)
+	return ""
+}
+
 func assertLoadAvoidanceSimulatedRetryChain(t *testing.T, ctx context.Context, testPool *pgxpool.Pool, baseURL string, adminToken string, runtimeToken string, suffixText string) {
 	t.Helper()
 	model := "load-avoidance-smoke-" + suffixText

apps/api/internal/httpapi/handlers.go

@@ -1,6 +1,7 @@
 package httpapi
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -564,10 +565,15 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
 			writeTaskAccepted(w, task)
 			return
 		}
+		runCtx, cancelRun := s.requestExecutionContext(r)
+		defer cancelRun()
 		if compatible {
 			if boolValue(body, "stream") {
 				flusher := prepareCompatibleStream(w)
-				result, runErr := s.runner.ExecuteStream(r.Context(), task, user, func(delta string) error {
+				result, runErr := s.runner.ExecuteStream(runCtx, task, user, func(delta string) error {
+					if !requestStillConnected(r) {
+						return nil
+					}
 					writeCompatibleDelta(w, kind, model, delta)
 					if flusher != nil {
 						flusher.Flush()
@@ -575,6 +581,9 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
 					return nil
 				})
 				if runErr != nil {
+					if !requestStillConnected(r) {
+						return
+					}
 					status := statusFromRunError(runErr)
 					errorPayload := map[string]any{
 						"code":    runErrorCode(runErr),
@@ -593,29 +602,66 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
 					}
 					return
 				}
+				if !requestStillConnected(r) {
+					return
+				}
 				writeCompatibleDone(w, kind, model, result.Output)
 				if flusher != nil {
 					flusher.Flush()
 				}
 				return
 			}
-			result, runErr := s.runner.Execute(r.Context(), task, user)
+			result, runErr := s.runner.Execute(runCtx, task, user)
 			if runErr != nil {
+				if !requestStillConnected(r) {
+					return
+				}
 				writeError(w, statusFromRunError(runErr), runErr.Error(), runErrorCode(runErr))
 				return
 			}
+			if !requestStillConnected(r) {
+				return
+			}
 			writeJSON(w, http.StatusOK, result.Output)
 			return
 		}
-		result, runErr := s.runner.Execute(r.Context(), task, user)
+		result, runErr := s.runner.Execute(runCtx, task, user)
 		if runErr != nil {
 			s.logger.Warn("task completed with failure", "kind", kind, "taskId", task.ID, "error", runErr)
 		}
 
+		if !requestStillConnected(r) {
+			return
+		}
 		writeTaskAccepted(w, result.Task)
 	})
 }
 
+func (s *Server) requestExecutionContext(r *http.Request) (context.Context, context.CancelFunc) {
+	base := context.WithoutCancel(r.Context())
+	if s.ctx == nil {
+		return base, func() {}
+	}
+	ctx, cancel := context.WithCancel(base)
+	go func() {
+		select {
+		case <-s.ctx.Done():
+			cancel()
+		case <-ctx.Done():
+		}
+	}()
+	return ctx, cancel
+}
+
+func requestStillConnected(r *http.Request) bool {
+	select {
+	case <-r.Context().Done():
+		return false
+	default:
+		return true
+	}
+}
+
 func asyncRequest(r *http.Request) bool {
 	value := strings.TrimSpace(strings.ToLower(r.Header.Get("x-async")))
 	return value == "1" || value == "true" || value == "yes" || value == "on"

apps/api/internal/store/rate_limit_status.go

@@ -24,6 +24,8 @@ type ModelRateLimitStatus struct {
 	PlatformID                string                   `json:"platformId"`
 	PlatformName              string                   `json:"platformName"`
 	Provider                  string                   `json:"provider"`
+	PlatformStatus            string                   `json:"platformStatus"`
+	PlatformDisabledReason    *PlatformPolicyEvent     `json:"platformDisabledReason,omitempty"`
 	PlatformPriority          int                      `json:"platformPriority"`
 	PlatformDynamicPriority   *int                     `json:"platformDynamicPriority,omitempty"`
 	PlatformEffectivePriority int                      `json:"platformEffectivePriority"`
@@ -62,9 +64,27 @@ type PriorityDemotionRecord struct {
 	CreatedAt       time.Time `json:"createdAt"`
 }
 
+type PlatformPolicyEvent struct {
+	ID              string    `json:"id"`
+	TaskID          string    `json:"taskId"`
+	PlatformID      string    `json:"platformId"`
+	PlatformModelID string    `json:"platformModelId,omitempty"`
+	EventType       string    `json:"eventType"`
+	Reason          string    `json:"reason,omitempty"`
+	ErrorCode       string    `json:"errorCode,omitempty"`
+	ErrorMessage    string    `json:"errorMessage,omitempty"`
+	Category        string    `json:"category,omitempty"`
+	StatusCode      int       `json:"statusCode,omitempty"`
+	PolicySource    string    `json:"policySource,omitempty"`
+	Policy          string    `json:"policy,omitempty"`
+	PolicyRule      string    `json:"policyRule,omitempty"`
+	MatchedValue    string    `json:"matchedValue,omitempty"`
+	CreatedAt       time.Time `json:"createdAt"`
+}
+
 func (s *Store) ListModelRateLimitStatuses(ctx context.Context) ([]ModelRateLimitStatus, error) {
 	rows, err := s.pool.Query(ctx, `
-		SELECT m.id::text, m.platform_id::text, p.name, p.provider,
+		SELECT m.id::text, m.platform_id::text, p.name, p.provider, p.status,
 		       p.priority, p.dynamic_priority, COALESCE(p.dynamic_priority, p.priority),
 		       m.model_name, COALESCE(NULLIF(m.provider_model_name, ''), m.model_name), COALESCE(m.model_alias, ''),
 	       m.model_type, m.display_name, m.enabled,
@@ -161,6 +181,7 @@ ORDER BY p.priority ASC, m.model_name ASC`)
 			&item.PlatformID,
 			&item.PlatformName,
 			&item.Provider,
+			&item.PlatformStatus,
 			&item.PlatformPriority,
 			&platformDynamicPriority,
 			&item.PlatformEffectivePriority,
@@ -212,8 +233,15 @@ ORDER BY p.priority ASC, m.model_name ASC`)
 	if err != nil {
 		return nil, err
 	}
+	disabledReasons, err := s.listLatestPlatformDisabledReasons(ctx, items)
+	if err != nil {
+		return nil, err
+	}
 	for index := range items {
 		items[index].RecentPriorityDemotions = demotions[items[index].PlatformID]
+		if items[index].PlatformStatus != "enabled" {
+			items[index].PlatformDisabledReason = disabledReasons[items[index].PlatformID]
+		}
 	}
 	sort.SliceStable(items, func(i, j int) bool {
 		if items[i].LoadRatio == items[j].LoadRatio {
@@ -309,6 +337,101 @@ func priorityDemotionRecordFromEventPayload(id string, taskID string, message st
 	}
 }
 
+func (s *Store) listLatestPlatformDisabledReasons(ctx context.Context, statuses []ModelRateLimitStatus) (map[string]*PlatformPolicyEvent, error) {
+	out := map[string]*PlatformPolicyEvent{}
+	seen := map[string]bool{}
+	platformIDs := make([]string, 0, len(statuses))
+	for _, status := range statuses {
+		platformID := strings.TrimSpace(status.PlatformID)
+		if platformID == "" || status.PlatformStatus == "enabled" || seen[platformID] {
+			continue
+		}
+		seen[platformID] = true
+		platformIDs = append(platformIDs, platformID)
+	}
+	if len(platformIDs) == 0 {
+		return out, nil
+	}
+	rows, err := s.pool.Query(ctx, `
+SELECT id::text, task_id::text, event_type, COALESCE(message, ''), payload, COALESCE(attempt_error_message, ''), created_at
+FROM (
+  SELECT e.*,
+         a.error_message AS attempt_error_message,
+         row_number() OVER (
+           PARTITION BY e.payload->>'platformId'
+           ORDER BY e.created_at DESC, e.seq DESC
+         ) AS disabled_rank
+  FROM gateway_task_events e
+  LEFT JOIN LATERAL (
+    SELECT error_message
+    FROM gateway_task_attempts attempt
+    WHERE attempt.task_id = e.task_id
+      AND attempt.platform_id::text = e.payload->>'platformId'
+    ORDER BY attempt.attempt_no DESC, attempt.started_at DESC
+    LIMIT 1
+  ) a ON TRUE
+  WHERE e.event_type IN ('task.policy.failover_disabled', 'task.policy.auto_disabled')
+    AND e.payload->>'platformId' = ANY($1::text[])
+) ranked
+WHERE disabled_rank = 1`, platformIDs)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var id string
+		var taskID string
+		var eventType string
+		var message string
+		var payloadBytes []byte
+		var attemptErrorMessage string
+		var createdAt time.Time
+		if err := rows.Scan(&id, &taskID, &eventType, &message, &payloadBytes, &attemptErrorMessage, &createdAt); err != nil {
+			return nil, err
+		}
+		record := platformPolicyEventFromPayload(id, taskID, eventType, message, attemptErrorMessage, decodeObject(payloadBytes), createdAt)
+		if record.PlatformID == "" {
+			continue
+		}
+		out[record.PlatformID] = &record
+	}
+	return out, rows.Err()
+}
+
+func platformPolicyEventFromPayload(id string, taskID string, eventType string, message string, attemptErrorMessage string, payload map[string]any, createdAt time.Time) PlatformPolicyEvent {
+	errorMessage := stringValue(payload["errorMessage"])
+	if errorMessage == "" {
+		errorMessage = stringValue(payload["message"])
+	}
+	if errorMessage == "" {
+		errorMessage = strings.TrimSpace(attemptErrorMessage)
+	}
+	if errorMessage == "" {
+		errorMessage = strings.TrimSpace(message)
+	}
+	errorCode := stringValue(payload["errorCode"])
+	if errorCode == "" {
+		errorCode = stringValue(payload["code"])
+	}
+	return PlatformPolicyEvent{
+		ID:              id,
+		TaskID:          taskID,
+		PlatformID:      stringValue(payload["platformId"]),
+		PlatformModelID: stringValue(payload["platformModelId"]),
+		EventType:       eventType,
+		Reason:          stringValue(payload["reason"]),
+		ErrorCode:       errorCode,
+		ErrorMessage:    errorMessage,
+		Category:        stringValue(payload["category"]),
+		StatusCode:      intValue(payload["statusCode"]),
+		PolicySource:    stringValue(payload["policySource"]),
+		Policy:          stringValue(payload["policy"]),
+		PolicyRule:      stringValue(payload["policyRule"]),
+		MatchedValue:    stringValue(payload["matchedValue"]),
+		CreatedAt:       createdAt,
+	}
+}
+
 func effectiveModelRateLimitPolicy(platformPolicy map[string]any, runtimePolicy map[string]any, runtimeOverride map[string]any, modelPolicy map[string]any) map[string]any {
 	policy := platformPolicy
 	if hasRateLimitRules(runtimePolicy) {

apps/api/internal/store/rate_limit_status_test.go

@@ -61,3 +61,26 @@ func TestPriorityDemotionRecordFromEventPayloadKeepsReason(t *testing.T) {
 		t.Fatalf("expected createdAt %s, got %s", createdAt, record.CreatedAt)
 	}
 }
+
+func TestPlatformPolicyEventFromPayloadUsesAttemptErrorMessage(t *testing.T) {
+	createdAt := time.Date(2026, 5, 12, 10, 30, 0, 0, time.UTC)
+	record := platformPolicyEventFromPayload("event-1", "task-1", "task.policy.failover_disabled", "fallback event message", "upstream invalid api key", map[string]any{
+		"platformId":      "platform-1",
+		"platformModelId": "platform-model-1",
+		"reason":          "failover_allow_policy",
+		"errorCode":       "auth_failed",
+		"category":        "auth_error",
+		"statusCode":      float64(401),
+		"policySource":    "gateway_runner_policies.failover_policy",
+		"policy":          "failoverPolicy",
+		"policyRule":      "allowCategories",
+		"matchedValue":    "auth_error",
+	}, createdAt)
+
+	if record.EventType != "task.policy.failover_disabled" || record.Reason != "failover_allow_policy" {
+		t.Fatalf("expected disabled event identity to survive, got %+v", record)
+	}
+	if record.ErrorMessage != "upstream invalid api key" || record.StatusCode != 401 {
+		t.Fatalf("expected disabled reason details from attempt, got %+v", record)
+	}
+}