Decouple stream cancellation and clarify failover rules

apps/api/internal/httpapi/core_flow_integration_test.go

@@ -351,6 +351,48 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
 		t.Fatalf("unexpected compatible chat response: %+v", compatChat)
 	}
 
+	cancelMarker := "cancel-stream-" + suffixText
+	cancelCtx, cancelRequest := context.WithCancel(context.Background())
+	cancelPayload := map[string]any{
+		"model":                defaultTextModel,
+		"runMode":              "simulation",
+		"messages":             []map[string]any{{"role": "user", "content": "cancelled stream"}},
+		"stream":               true,
+		"simulation":           true,
+		"simulationDurationMs": 250,
+		"cancelTestId":         cancelMarker,
+	}
+	cancelRaw, err := json.Marshal(cancelPayload)
+	if err != nil {
+		t.Fatalf("marshal cancelled stream payload: %v", err)
+	}
+	cancelReq, err := http.NewRequestWithContext(cancelCtx, http.MethodPost, server.URL+"/v1/chat/completions", bytes.NewReader(cancelRaw))
+	if err != nil {
+		t.Fatalf("build cancelled stream request: %v", err)
+	}
+	cancelReq.Header.Set("Authorization", "Bearer "+apiKeyResponse.Secret)
+	cancelReq.Header.Set("Content-Type", "application/json")
+	cancelErrCh := make(chan error, 1)
+	go func() {
+		resp, err := http.DefaultClient.Do(cancelReq)
+		if resp != nil {
+			_, _ = io.ReadAll(resp.Body)
+			_ = resp.Body.Close()
+		}
+		cancelErrCh <- err
+	}()
+	cancelTaskID := waitForTaskIDByRequestMarker(t, ctx, testPool, cancelMarker, 2*time.Second)
+	cancelRequest()
+	select {
+	case <-cancelErrCh:
+	case <-time.After(time.Second):
+		t.Fatal("cancelled stream request did not return after client cancellation")
+	}
+	cancelledStreamTask := waitForTaskStatus(t, server.URL, apiKeyResponse.Secret, cancelTaskID, []string{"succeeded"}, 2*time.Second)
+	if cancelledStreamTask.Status != "succeeded" {
+		t.Fatalf("client-cancelled compatible stream should keep backend task running to success, got %+v", cancelledStreamTask)
+	}
+
 	var imageResponse struct {
 		Task struct {
 			ID     string         `json:"id"`
@@ -1398,6 +1440,26 @@ func waitForTaskStatus(t *testing.T, baseURL string, token string, taskID string
 	return detail
 }
 
+func waitForTaskIDByRequestMarker(t *testing.T, ctx context.Context, pool *pgxpool.Pool, marker string, timeout time.Duration) string {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		var taskID string
+		err := pool.QueryRow(ctx, `
+SELECT id::text
+FROM gateway_tasks
+WHERE request->>'cancelTestId' = $1
+ORDER BY created_at DESC
+LIMIT 1`, marker).Scan(&taskID)
+		if err == nil && taskID != "" {
+			return taskID
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("task with request marker %s was not created within %s", marker, timeout)
+	return ""
+}
+
 func assertLoadAvoidanceSimulatedRetryChain(t *testing.T, ctx context.Context, testPool *pgxpool.Pool, baseURL string, adminToken string, runtimeToken string, suffixText string) {
 	t.Helper()
 	model := "load-avoidance-smoke-" + suffixText

apps/api/internal/httpapi/handlers.go

@@ -1,6 +1,7 @@
 package httpapi
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -564,10 +565,15 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
 			writeTaskAccepted(w, task)
 			return
 		}
+		runCtx, cancelRun := s.requestExecutionContext(r)
+		defer cancelRun()
 		if compatible {
 			if boolValue(body, "stream") {
 				flusher := prepareCompatibleStream(w)
-				result, runErr := s.runner.ExecuteStream(r.Context(), task, user, func(delta string) error {
+				result, runErr := s.runner.ExecuteStream(runCtx, task, user, func(delta string) error {
+					if !requestStillConnected(r) {
+						return nil
+					}
 					writeCompatibleDelta(w, kind, model, delta)
 					if flusher != nil {
 						flusher.Flush()
@@ -575,6 +581,9 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
 					return nil
 				})
 				if runErr != nil {
+					if !requestStillConnected(r) {
+						return
+					}
 					status := statusFromRunError(runErr)
 					errorPayload := map[string]any{
 						"code":    runErrorCode(runErr),
@@ -593,29 +602,66 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
 					}
 					return
 				}
+				if !requestStillConnected(r) {
+					return
+				}
 				writeCompatibleDone(w, kind, model, result.Output)
 				if flusher != nil {
 					flusher.Flush()
 				}
 				return
 			}
-			result, runErr := s.runner.Execute(r.Context(), task, user)
+			result, runErr := s.runner.Execute(runCtx, task, user)
 			if runErr != nil {
+				if !requestStillConnected(r) {
+					return
+				}
 				writeError(w, statusFromRunError(runErr), runErr.Error(), runErrorCode(runErr))
 				return
 			}
+			if !requestStillConnected(r) {
+				return
+			}
 			writeJSON(w, http.StatusOK, result.Output)
 			return
 		}
-		result, runErr := s.runner.Execute(r.Context(), task, user)
+		result, runErr := s.runner.Execute(runCtx, task, user)
 		if runErr != nil {
 			s.logger.Warn("task completed with failure", "kind", kind, "taskId", task.ID, "error", runErr)
 		}
 
+		if !requestStillConnected(r) {
+			return
+		}
 		writeTaskAccepted(w, result.Task)
 	})
 }
 
+func (s *Server) requestExecutionContext(r *http.Request) (context.Context, context.CancelFunc) {
+	base := context.WithoutCancel(r.Context())
+	if s.ctx == nil {
+		return base, func() {}
+	}
+	ctx, cancel := context.WithCancel(base)
+	go func() {
+		select {
+		case <-s.ctx.Done():
+			cancel()
+		case <-ctx.Done():
+		}
+	}()
+	return ctx, cancel
+}
+
+func requestStillConnected(r *http.Request) bool {
+	select {
+	case <-r.Context().Done():
+		return false
+	default:
+		return true
+	}
+}
+
 func asyncRequest(r *http.Request) bool {
 	value := strings.TrimSpace(strings.ToLower(r.Header.Get("x-async")))
 	return value == "1" || value == "true" || value == "yes" || value == "on"