fix(security): harden CSRF with Content-Type gate and expand E2E coverage (#2818)
Publish to PyPI / build-and-publish (push) Waiting to run
Python Linting / Run Ruff (push) Waiting to run

Defense-in-depth over GET→POST alone: reject the three CORS-safelisted
simple-form Content-Types (x-www-form-urlencoded, multipart/form-data,
text/plain) on 16 no-body POST handlers (glob + legacy) to block
<form method=POST> CSRF that bypasses method-only gating. Move
comfyui_switch_version to a JSON body so the preflight requirement applies.
Split db_mode/policy/update/channel_url_list into GET(read) + POST(write).
Tighten do_fix (high → high+) and gate three previously-ungated config
setters at middle. Resynchronize openapi.yaml (27 paths, 30 operations,
ComfyUISwitchVersionParams as a shared $ref component). Add E2E harness
variants, Playwright config, CSRF/secgate suites, 39-endpoint coverage,
and a CHANGELOG.

Breaking: legacy per-op POST routes (install/uninstall/fix/disable/update/
reinstall/abort_current) are removed; callers already use queue/batch.
Legacy /manager/notice (v1) is removed; /v2/manager/notice is retained.

Reported-by: XlabAI Team of Tencent Xuanwu Lab
CVSS: 8.1 (AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H)
This commit is contained in:
Dr.Lt.Data
2026-04-22 05:04:30 +09:00
committed by GitHub
parent 49e205acd4
commit 4410ebc6a6
70 changed files with 13638 additions and 434 deletions
@@ -189,65 +189,44 @@ class TestReinstall:
"""cm-cli reinstall --uv-compile"""
def test_reinstall_with_uv_compile(self):
"""Reinstall an existing pack with --uv-compile."""
# Install first
"""Reinstall an existing pack with --uv-compile — resolver MUST run."""
_run_cm_cli("install", REPO_TEST1)
assert _pack_exists(PACK_TEST1)
# Reinstall with --uv-compile
r = _run_cm_cli("reinstall", "--uv-compile", REPO_TEST1)
combined = r.stdout + r.stderr
# Reinstall should re-resolve or report the pack exists
# Note: Manager's reinstall may fail to remove the existing directory
# before re-cloning (known issue — purge_node_state bug)
assert _pack_exists(PACK_TEST1)
assert "Resolving dependencies" in combined or "Already exists" in combined
assert "Resolving dependencies" in combined, (
f"Expected resolver to run on reinstall but output had: {combined[:500]!r}"
)
class TestUpdate:
"""cm-cli update --uv-compile"""
class TestUvCompileVerbs:
"""cm-cli verbs that support --uv-compile.
def test_update_single_with_uv_compile(self):
"""Update an installed pack with --uv-compile."""
WI-NN Cluster 5 (bloat-sweep dev:ci-004/005/006/007/011 B9 copy-paste):
consolidates 5 previously-separate test functions that all assert the same
"Resolving dependencies" emission after install+verb. Parametrized across
the 5 supported verb/target combinations.
"""
@pytest.mark.parametrize(
"cm_args",
[
pytest.param(("update", "--uv-compile", REPO_TEST1), id="update-single"),
pytest.param(("update", "--uv-compile", "all"), id="update-all"),
pytest.param(("fix", "--uv-compile", REPO_TEST1), id="fix-single"),
pytest.param(("fix", "--uv-compile", "all"), id="fix-all"),
pytest.param(("restore-dependencies", "--uv-compile"), id="restore-dependencies"),
],
)
def test_verb_with_uv_compile_runs_resolver(self, cm_args):
"""Every --uv-compile-aware verb triggers dependency resolution."""
_run_cm_cli("install", REPO_TEST1)
assert _pack_exists(PACK_TEST1)
r = _run_cm_cli("update", "--uv-compile", REPO_TEST1)
combined = r.stdout + r.stderr
assert "Resolving dependencies" in combined
def test_update_all_with_uv_compile(self):
"""update all --uv-compile runs uv-compile after updating."""
_run_cm_cli("install", REPO_TEST1)
assert _pack_exists(PACK_TEST1)
r = _run_cm_cli("update", "--uv-compile", "all")
combined = r.stdout + r.stderr
assert "Resolving dependencies" in combined
class TestFix:
"""cm-cli fix --uv-compile"""
def test_fix_single_with_uv_compile(self):
"""Fix an installed pack with --uv-compile."""
_run_cm_cli("install", REPO_TEST1)
assert _pack_exists(PACK_TEST1)
r = _run_cm_cli("fix", "--uv-compile", REPO_TEST1)
combined = r.stdout + r.stderr
assert "Resolving dependencies" in combined
def test_fix_all_with_uv_compile(self):
"""fix all --uv-compile runs uv-compile after fixing."""
_run_cm_cli("install", REPO_TEST1)
assert _pack_exists(PACK_TEST1)
r = _run_cm_cli("fix", "--uv-compile", "all")
r = _run_cm_cli(*cm_args)
combined = r.stdout + r.stderr
assert "Resolving dependencies" in combined
@@ -256,14 +235,41 @@ class TestFix:
class TestUvCompileStandalone:
"""cm-cli uv-sync (standalone command, formerly uv-compile)"""
def test_uv_compile_no_packs(self):
"""uv-compile with no node packs → 'No custom node packs found'."""
def test_uv_compile_no_test_packs_exits_zero(self):
"""uv-sync without test packs must exit rc==0 (clean success).
WI-OO Item 2 (bloat dev:ci-008 B5): split from the previous OR-fallback
test. This half pins the exit-code contract.
"""
r = _run_cm_cli("uv-sync")
assert r.returncode == 0, (
f"uv-sync should exit 0 when no test packs are installed; "
f"got rc={r.returncode}. Output: {(r.stdout + r.stderr)[:500]!r}"
)
def test_uv_compile_no_test_packs_emits_signal(self):
"""uv-sync emits a definitive signal — never silent success.
WI-OO Item 2 (bloat dev:ci-008 B5): split from the previous OR-fallback
test. This half pins the output-signal contract. The emitted marker
depends on what's installed in the E2E sandbox at the moment — either
'No custom node packs' (empty tree with no resolvable requirements) or
'Resolved' (non-empty tree with successful resolution). Asserting the
disjunction here is narrower than the original OR (which also accepted
rc==0 with completely silent output); this test requires an actual
human-readable marker in the output stream.
"""
r = _run_cm_cli("uv-sync")
combined = r.stdout + r.stderr
# Only ComfyUI-Manager exists (no requirements.txt in it normally)
# so either "No custom node packs found" or resolves 0
assert r.returncode == 0 or "No custom node packs" in combined
# Precondition: exit success is verified by the sibling test above.
assert r.returncode == 0, f"Precondition failed: uv-sync rc={r.returncode}"
empty_marker = "No custom node packs" in combined
resolved_marker = "Resolved" in combined
assert empty_marker or resolved_marker, (
f"Expected 'No custom node packs' (empty tree) or 'Resolved' "
f"(non-empty tree) marker; output was silent or unrecognized: "
f"{combined[:500]!r}"
)
def test_uv_compile_with_packs(self):
"""uv-compile after installing test pack → resolves."""
@@ -276,33 +282,6 @@ class TestUvCompileStandalone:
assert "Resolving dependencies" in combined
assert "Resolved" in combined
def test_uv_compile_conflict_attribution(self):
"""uv-compile with conflicting packs → shows attribution."""
_run_cm_cli("install", REPO_TEST1)
_run_cm_cli("install", REPO_TEST2)
r = _run_cm_cli("uv-sync")
combined = r.stdout + r.stderr
assert r.returncode != 0
assert "Conflicting packages (by node pack):" in combined
assert PACK_TEST1 in combined
assert PACK_TEST2 in combined
class TestRestoreDependencies:
"""cm-cli restore-dependencies --uv-compile"""
def test_restore_dependencies_with_uv_compile(self):
"""restore-dependencies --uv-compile runs resolver after restore."""
_run_cm_cli("install", REPO_TEST1)
assert _pack_exists(PACK_TEST1)
r = _run_cm_cli("restore-dependencies", "--uv-compile")
combined = r.stdout + r.stderr
assert "Resolving dependencies" in combined
class TestConflictAttributionDetail:
"""Verify conflict attribution output details."""
+8 -3
View File
@@ -202,9 +202,14 @@ uv pip install \
-r "$E2E_ROOT/comfyui/requirements.txt" \
--extra-index-url "$PYTORCH_CPU_INDEX"
# Step 4: Install ComfyUI-Manager (non-editable, production-like)
log "Step 4/8: Installing ComfyUI-Manager..."
uv pip install --python "$VENV_PY" "$MANAGER_ROOT"
# Step 4: Install ComfyUI-Manager (editable — venv tracks workspace edits)
# Editable install prevents silent drift between the workspace source and the
# installed package: any change to comfyui_manager/** is visible to E2E
# immediately without re-running this script. The 2026-04-18 junk_value-rejection
# regression (surfaced in WI-E/WI-G, root-caused in WI-I) was masked for weeks by
# a non-editable snapshot — this flag closes that failure mode.
log "Step 4/8: Installing ComfyUI-Manager (editable)..."
uv pip install --python "$VENV_PY" -e "$MANAGER_ROOT"
# Step 5: Create symlink for custom_nodes discovery
log "Step 5/8: Creating custom_nodes symlink..."
+34 -6
View File
@@ -6,9 +6,15 @@
# Claude's Bash tool — the call returns only when ComfyUI is accepting requests.
#
# Input env vars:
# E2E_ROOT — (required) path to E2E environment from setup_e2e_env.sh
# PORT — ComfyUI listen port (default: 8199)
# TIMEOUT — max seconds to wait for readiness (default: 120)
# E2E_ROOT — (required) path to E2E environment from setup_e2e_env.sh
# PORT — ComfyUI listen port (default: 8199)
# TIMEOUT — max seconds to wait for readiness (default: 120)
# ENABLE_LEGACY_UI — if set to "1"/"true"/"yes", add --enable-manager-legacy-ui
# (for Playwright legacy-UI tests; pytest suites should
# leave this unset because glob and legacy manager_server
# modules are mutex-loaded and several pytest suites hit
# glob-only v2 endpoints such as /v2/manager/queue/task).
# The convenience wrapper start_comfyui_legacy.sh sets it.
#
# Output (last line on success):
# COMFYUI_PID=<pid> PORT=<port>
@@ -36,7 +42,11 @@ PY="$E2E_ROOT/venv/bin/python"
COMFY_DIR="$E2E_ROOT/comfyui"
LOG_DIR="$E2E_ROOT/logs"
LOG_FILE="$LOG_DIR/comfyui.log"
PID_FILE="$LOG_DIR/comfyui.pid"
# Port-namespaced PID file — prevents concurrent tests on different ports
# (e.g., teammate running pytest on 8199 while Playwright runs on 8200)
# from overwriting each other's PID, which would cause stop_comfyui.sh to
# kill the wrong process (observed in WI-CC: 8200 stop killed 8199 PID 2979469).
PID_FILE="$LOG_DIR/comfyui.${PORT}.pid"
mkdir -p "$LOG_DIR"
@@ -69,12 +79,30 @@ log "Starting ComfyUI on port $PORT..."
# Create empty log file (ensures tail -f works from the start)
: > "$LOG_FILE"
# Launch with unbuffered Python output so log lines appear immediately
# Assemble manager flags. ENABLE_LEGACY_UI toggles --enable-manager-legacy-ui
# without forcing every caller to care — pytest leaves it unset (glob mode),
# start_comfyui_legacy.sh sets it (legacy UI mode).
MANAGER_FLAGS=(--enable-manager)
case "${ENABLE_LEGACY_UI:-}" in
1|true|TRUE|yes|YES)
MANAGER_FLAGS+=(--enable-manager-legacy-ui)
log "Legacy UI enabled via ENABLE_LEGACY_UI=${ENABLE_LEGACY_UI}"
;;
esac
# Launch with unbuffered Python output so log lines appear immediately.
# COMFYUI_MANAGER_SKIP_MANAGER_REQUIREMENTS=1 is the WI-WW safety belt:
# any install/update/reinstall path that would normally run
# `pip install -r manager_requirements.txt` becomes a no-op log line.
# Essential for WI-YY real-E2E tests that trigger install/update flows
# — without it, a real update_comfyui task could run unbounded pip
# installs on the test venv.
PYTHONUNBUFFERED=1 \
HOME="$E2E_ROOT/home" \
COMFYUI_MANAGER_SKIP_MANAGER_REQUIREMENTS=1 \
nohup "$PY" "$COMFY_DIR/main.py" \
--cpu \
--enable-manager \
"${MANAGER_FLAGS[@]}" \
--port "$PORT" \
> "$LOG_FILE" 2>&1 &
COMFYUI_PID=$!
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# start_comfyui_legacy.sh — Thin wrapper that launches ComfyUI in LEGACY UI mode.
#
# Delegates to start_comfyui.sh with ENABLE_LEGACY_UI=1. The underlying script
# translates that into --enable-manager-legacy-ui on main.py, which registers
# the legacy Manager dialog frontend and routes POST /v2/manager/queue/* to
# the legacy handler module (legacy/manager_server.py).
#
# Use this wrapper for Playwright legacy-UI tests (tests/playwright/legacy-ui-*).
# Do NOT use for pytest suites that hit glob-only v2 endpoints (e.g.
# /v2/manager/queue/task), because glob/manager_server and legacy/manager_server
# are mutex-loaded — see comfyui_manager/__init__.py::start().
#
# Input env vars (forwarded to start_comfyui.sh):
# E2E_ROOT — required
# PORT — default 8199
# TIMEOUT — default 120
#
# Output (last line on success, inherited from start_comfyui.sh):
# COMFYUI_PID=<pid> PORT=<port>
#
# Exit: 0=ready, 1=timeout/failure
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec env ENABLE_LEGACY_UI=1 bash "$SCRIPT_DIR/start_comfyui.sh" "$@"
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env bash
# start_comfyui_permissive.sh — Launch ComfyUI in PERMISSIVE security mode
# for WI-YY real-E2E tests of `high+` gated endpoints.
#
# Patches `security_level = normal-` into the manager config.ini before
# launching (with backup of original value), then delegates to
# start_comfyui.sh with ENABLE_LEGACY_UI=1 (wi-037 and wi-038 are
# legacy-only routes). The corresponding stop_comfyui.sh teardown should
# be paired with restore_config() inside the pytest fixture — this
# script does NOT restore on its own, so fixture teardown MUST cleanup.
#
# Why permissive mode is needed:
# Three endpoints check is_allowed_security_level('high+')
# (security_utils.py:20-26): at is_local_mode=True (127.0.0.1 listen)
# the gate requires security_level ∈ {weak, normal-}. Default
# `security_level = normal` fails, so the POST returns 403.
# - wi-014 POST /v2/comfyui_manager/comfyui_switch_version
# - wi-037 POST /v2/customnode/install/git_url
# - wi-038 POST /v2/customnode/install/pip
# Setting security_level = normal- allows real E2E execution of these
# endpoints with fixed, trusted inputs (never test-input-derived URLs).
#
# SECURITY NOTE:
# The endpoints are gated at high+ because they execute arbitrary remote
# code (git clone / pip install / version switch). This harness opens
# the gate ONLY in the E2E sandbox with HARDCODED trusted inputs
# (ComfyUI_examples repo; text-unidecode package). Never use with
# user-input-derived inputs — the 403 contract at default security is
# the positive-path security behavior we want to preserve in production.
#
# Input env vars (forwarded to start_comfyui.sh):
# E2E_ROOT — required
# PORT — default 8199
# TIMEOUT — default 120
#
# Output (last line on success, inherited from start_comfyui.sh):
# COMFYUI_PID=<pid> PORT=<port>
#
# Exit: 0=ready, 1=timeout/failure
#
# Side effect: $E2E_ROOT/comfyui/user/__manager/config.ini gets
# `security_level = normal-`. The original value is preserved at
# config.ini.before-permissive for the fixture to restore on teardown.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
[[ -n "${E2E_ROOT:-}" ]] || { echo "[start_comfyui_permissive] ERROR: E2E_ROOT is not set" >&2; exit 1; }
CONFIG="$E2E_ROOT/comfyui/user/__manager/config.ini"
BACKUP="$CONFIG.before-permissive"
[[ -f "$CONFIG" ]] || { echo "[start_comfyui_permissive] ERROR: config not found at $CONFIG" >&2; exit 1; }
# Preserve original config so the fixture can restore it on teardown.
# If a previous run left a backup, do NOT overwrite.
if [[ ! -f "$BACKUP" ]]; then
cp "$CONFIG" "$BACKUP"
echo "[start_comfyui_permissive] Backed up original config to $BACKUP"
fi
# Patch security_level to normal- (idempotent).
if grep -qE '^security_level\s*=' "$CONFIG"; then
sed -i -E 's/^security_level\s*=.*/security_level = normal-/' "$CONFIG"
else
sed -i -E '/^\[default\]/a security_level = normal-' "$CONFIG"
fi
echo "[start_comfyui_permissive] Patched security_level = normal- in $CONFIG"
exec env ENABLE_LEGACY_UI=1 bash "$SCRIPT_DIR/start_comfyui.sh" "$@"
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
# start_comfyui_strict.sh — Launch ComfyUI in STRICT security mode for SECGATE tests.
#
# Patches `security_level = strong` into the manager config.ini before launching
# (with backup of original value), then delegates to start_comfyui.sh. The
# corresponding stop_comfyui.sh teardown should be paired with restore_config()
# inside the pytest fixture (this script does NOT restore on its own — restore
# happens at fixture teardown to keep this wrapper symmetric with
# start_comfyui_legacy.sh).
#
# Why strict mode is needed:
# Several state-changing endpoints (snapshot/remove [middle], snapshot/restore
# [middle+], reboot [middle], queue/update_all [middle+]) check
# is_allowed_security_level(<gate>). At the default `security_level = normal`
# (and is_local_mode = True since we listen on 127.0.0.1), middle and middle+
# operations are ALLOWED — so the 403 path is unreachable. Setting
# security_level = strong puts NORMAL out of the allowed sets and makes the
# 403 contract observable.
#
# At-or-below `normal` configurations cannot test the 403 path for these gates;
# `strong` is required.
#
# Input env vars (forwarded to start_comfyui.sh):
# E2E_ROOT — required
# PORT — default 8199
# TIMEOUT — default 120
#
# Output (last line on success, inherited from start_comfyui.sh):
# COMFYUI_PID=<pid> PORT=<port>
#
# Exit: 0=ready, 1=timeout/failure
#
# Side effect: $E2E_ROOT/comfyui/user/__manager/config.ini gets
# `security_level = strong`. The original value is preserved at
# config.ini.before-strict for the fixture to restore on teardown.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
[[ -n "${E2E_ROOT:-}" ]] || { echo "[start_comfyui_strict] ERROR: E2E_ROOT is not set" >&2; exit 1; }
CONFIG="$E2E_ROOT/comfyui/user/__manager/config.ini"
BACKUP="$CONFIG.before-strict"
[[ -f "$CONFIG" ]] || { echo "[start_comfyui_strict] ERROR: config not found at $CONFIG" >&2; exit 1; }
# Preserve original config so the fixture can restore it on teardown.
# If a previous run left a backup, do NOT overwrite (preserves the *true*
# pre-strict baseline across crashed test runs).
if [[ ! -f "$BACKUP" ]]; then
cp "$CONFIG" "$BACKUP"
echo "[start_comfyui_strict] Backed up original config to $BACKUP"
fi
# Patch security_level to strong (idempotent — works whether the line is
# already `strong`, `normal`, or any other value).
if grep -qE '^security_level\s*=' "$CONFIG"; then
sed -i -E 's/^security_level\s*=.*/security_level = strong/' "$CONFIG"
else
# security_level missing entirely (unusual) — append under [default]
sed -i -E '/^\[default\]/a security_level = strong' "$CONFIG"
fi
echo "[start_comfyui_strict] Patched security_level = strong in $CONFIG"
exec bash "$SCRIPT_DIR/start_comfyui.sh" "$@"
+9 -1
View File
@@ -23,7 +23,15 @@ die() { err "$@"; exit 1; }
# --- Validate ---
[[ -n "${E2E_ROOT:-}" ]] || die "E2E_ROOT is not set"
PID_FILE="$E2E_ROOT/logs/comfyui.pid"
PID_FILE="$E2E_ROOT/logs/comfyui.${PORT}.pid"
# Legacy single-port path — warn if encountered so concurrent tests on
# different ports don't overwrite each other's PID file (observed during
# WI-CC: stop_comfyui.sh on port 8200 accidentally killed another teammate's
# PID 2979469 running on port 8199 because both shared $E2E_ROOT/logs/comfyui.pid).
LEGACY_PID_FILE="$E2E_ROOT/logs/comfyui.pid"
if [[ -f "$LEGACY_PID_FILE" ]] && [[ ! -f "$PID_FILE" ]]; then
log "WARN: found legacy unported PID file $LEGACY_PID_FILE but no ${PID_FILE}. Cross-port risk — ignoring legacy file."
fi
# --- Read PID ---
COMFYUI_PID=""
+720
View File
@@ -0,0 +1,720 @@
"""E2E tests for ComfyUI Manager configuration API endpoints.
Tests the dual GET (read) + POST (write) configuration endpoints:
- /v2/manager/db_mode
- /v2/manager/policy/update
- /v2/manager/channel_url_list
Each write test reads the original value, sets a new value via POST,
reads back via GET to verify, then restores the original to ensure
idempotency.
Requires a pre-built E2E environment (from setup_e2e_env.sh).
Set E2E_ROOT env var to point at it, or the tests will be skipped.
Usage:
E2E_ROOT=/tmp/e2e_full_test pytest tests/e2e/test_e2e_config_api.py -v
"""
from __future__ import annotations
import hashlib
import os
import subprocess
import time
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
MANAGER_CONFIG_INI = (
os.path.join(COMFYUI_PATH, "user", "__manager", "config.ini")
if COMFYUI_PATH
else ""
)
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
# Reboot recovery window (same as test_e2e_system_info TestReboot)
REBOOT_TIMEOUT = 60.0
REBOOT_INTERVAL = 2.0
def _read_config_ini_value(key: str) -> str | None:
"""Read a value directly from the manager config.ini (for disk-level assertion)."""
if not os.path.isfile(MANAGER_CONFIG_INI):
return None
import configparser
cp = configparser.ConfigParser()
cp.read(MANAGER_CONFIG_INI)
for section in cp.sections():
if cp.has_option(section, key):
return cp.get(section, key)
return None
# ---------------------------------------------------------------------------
# Disk-persistence helpers (Stage2 WI-E PoC — reusable by the WEAK-rated 5
# tests listed in reports/e2e_verification_audit.md §4 once a follow-up WI
# propagates them). Callable with `None` expected value for "absent" checks.
# ---------------------------------------------------------------------------
def _assert_config_ini_contains(key: str, expected: str | None) -> None:
"""Assert the manager config.ini has ``key`` set to ``expected`` on disk.
Interface:
key — config.ini option name (searched across all sections)
expected — the exact string value the key must hold, OR None to
assert the key is absent / config.ini missing.
Raises AssertionError with pre/post hashes + on-disk value for diagnosis.
The helper verifies persistence independently from the HTTP API —
catches no-op handlers that return 200 without writing to disk.
"""
actual = _read_config_ini_value(key)
if expected is None:
assert actual is None, (
f"config.ini[{key}] expected absent, found {actual!r}"
)
return
# For diagnosis on failure, capture a hash of the file so reviewers can
# tell whether ANY mutation happened vs. the wrong-value case.
file_hash = "<missing>"
if os.path.isfile(MANAGER_CONFIG_INI):
with open(MANAGER_CONFIG_INI, "rb") as fh:
file_hash = hashlib.sha256(fh.read()).hexdigest()[:12]
assert actual == expected, (
f"config.ini[{key}] disk mismatch: expected {expected!r}, "
f"got {actual!r} (file sha256[:12]={file_hash}, path={MANAGER_CONFIG_INI})"
)
def _assert_config_ini_persists_across_reboot(
key: str,
expected: str,
timeout: float = REBOOT_TIMEOUT,
) -> None:
"""Assert ``key=expected`` survives a ComfyUI reboot on disk AND via API.
Interface:
key — config.ini option name
expected — value the key must still hold post-reboot
timeout — max seconds to wait for the server to come back healthy
Behavior:
1. Issue POST /v2/manager/reboot (tolerates ConnectionError mid-
response — server drops the connection during shutdown).
2. Poll /system_stats until the server answers 200 or timeout.
3. Re-read config.ini from disk → must equal ``expected``.
4. Re-read the value via the appropriate GET endpoint (derived
from the key) → must equal ``expected`` as well.
Note: This helper WILL replace the ComfyUI process. Any fixture that
pins a PID should treat the post-reboot PID as unknown. The
module-scoped ``comfyui`` fixture's teardown calls stop_comfyui.sh,
which kills by port rather than stored PID, so teardown continues
to work.
"""
try:
resp = requests.post(f"{BASE_URL}/v2/manager/reboot", timeout=10)
if resp.status_code == 403:
pytest.skip(
"reboot denied by security policy "
"(E2E_SECURITY_LEVEL does not permit 'middle')"
)
assert resp.status_code == 200, (
f"reboot returned unexpected status {resp.status_code}: {resp.text}"
)
except requests.ConnectionError:
# Server closed the socket mid-reboot response. Expected on some
# platforms; treat as success-so-far and rely on healthcheck below.
pass
time.sleep(2) # grace period for shutdown
deadline = time.monotonic() + timeout
recovered = False
while time.monotonic() < deadline:
try:
r = requests.get(f"{BASE_URL}/system_stats", timeout=5)
if r.status_code == 200:
recovered = True
break
except (requests.ConnectionError, requests.Timeout):
pass
time.sleep(REBOOT_INTERVAL)
assert recovered, (
f"server did not recover within {timeout}s after reboot — "
f"cannot verify {key!r} persistence"
)
# Disk side: config.ini preserved the value.
_assert_config_ini_contains(key, expected)
# API side: the restarted server re-read config.ini and serves the value.
api_path = {
"db_mode": "/v2/manager/db_mode",
"update_policy": "/v2/manager/policy/update",
"channel_url": "/v2/manager/channel_url_list",
}.get(key)
if api_path is None:
return # caller responsible for API verification for non-standard keys
api_resp = requests.get(f"{BASE_URL}{api_path}", timeout=10)
api_resp.raise_for_status()
if api_path.endswith("channel_url_list"):
# channel_url asymmetry: config.ini stores the full URL, API returns the
# reverse-mapped channel NAME. When caller passes the URL as `expected`,
# translate URL→NAME via the API's own `list` (`name::url` entries).
# Callers passing a NAME (legacy path) continue to work unchanged.
body = api_resp.json()
actual_api = body.get("selected")
expected_to_compare = expected
if isinstance(expected, str) and "://" in expected:
for entry in body.get("list", []):
if isinstance(entry, str) and "::" in entry:
name, url = entry.split("::", 1)
if url == expected:
expected_to_compare = name
break
else:
# URL not in the known list → server reports "custom"
expected_to_compare = "custom"
else:
actual_api = api_resp.text
expected_to_compare = expected
assert actual_api == expected_to_compare, (
f"post-reboot API mismatch for {key}: "
f"config.ini has {expected!r} but GET {api_path} returned {actual_api!r}"
)
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready (run setup_e2e_env.sh first)",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui() -> int:
"""Start ComfyUI and return its PID."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI:\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID from start_comfyui output:\n{r.stdout}")
def _stop_comfyui():
"""Stop ComfyUI."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def comfyui():
"""Start ComfyUI once for the module, stop after all tests."""
pid = _start_comfyui()
yield pid
_stop_comfyui()
@pytest.fixture(scope="module", autouse=True)
def config_snapshot(comfyui):
"""Snapshot config values at module start, restore at module teardown.
Guards against state leak if any in-module test fails mid-mutation
(leaving config.ini in a corrupted/unexpected state that would poison
"original" reads in subsequent tests).
"""
snapshot = {
"db_mode": requests.get(f"{BASE_URL}/v2/manager/db_mode", timeout=10).text,
"update_policy": requests.get(
f"{BASE_URL}/v2/manager/policy/update", timeout=10
).text,
"channel_selected": requests.get(
f"{BASE_URL}/v2/manager/channel_url_list", timeout=10
).json().get("selected"),
}
yield snapshot
# Best-effort restore; log but don't fail if restore hits issues
for path, key, value in (
("/v2/manager/db_mode", "db_mode", snapshot["db_mode"]),
("/v2/manager/policy/update", "update_policy", snapshot["update_policy"]),
("/v2/manager/channel_url_list", "channel_selected", snapshot["channel_selected"]),
):
try:
resp = requests.post(
f"{BASE_URL}{path}",
json={"value": value},
timeout=10,
)
if not resp.ok:
print(
f"[config_snapshot] restore FAILED for {key}={value!r}: "
f"status={resp.status_code}",
)
except Exception as e: # noqa: BLE001
print(f"[config_snapshot] restore EXCEPTION for {key}: {e}")
# ---------------------------------------------------------------------------
# Tests — db_mode
# ---------------------------------------------------------------------------
class TestConfigDbMode:
"""Test GET/POST /v2/manager/db_mode round-trip."""
DB_MODE_VALUES = ("cache", "channel", "local", "remote")
def test_read_db_mode(self, comfyui):
"""GET /v2/manager/db_mode returns a valid db mode string."""
resp = requests.get(f"{BASE_URL}/v2/manager/db_mode", timeout=10)
assert resp.status_code == 200, f"Expected 200, got {resp.status_code}"
assert resp.text in self.DB_MODE_VALUES, (
f"Unexpected db_mode value: {resp.text!r}"
)
def test_set_and_restore_db_mode(self, comfyui):
"""POST sets db_mode, GET reads it back, disk + reboot persistence proven, then original is restored.
Stage2 WI-E PoC — demonstrates the two disk-persistence helpers:
* _assert_config_ini_contains → disk-level verification
* _assert_config_ini_persists_across_reboot → restart-survival verification
This test is the first of the six §4 WEAK round-trip tests (per
reports/e2e_verification_audit.md) to gain independent disk state
assertions. Propagation to the other five is tracked as a follow-up
WI — see the completion report accompanying this change.
"""
# Read original — baseline for both round-trip and restore verification.
resp = requests.get(f"{BASE_URL}/v2/manager/db_mode", timeout=10)
resp.raise_for_status()
original = resp.text
# Pick a different value so the mutation is observable.
new_mode = "local" if original != "local" else "remote"
# Snapshot config.ini BEFORE mutation — reviewers can tell from
# pre_hash vs. post_hash whether the POST actually touched the file.
pre_hash = (
hashlib.sha256(open(MANAGER_CONFIG_INI, "rb").read()).hexdigest()[:12]
if os.path.isfile(MANAGER_CONFIG_INI)
else "<missing>"
)
try:
# Set new value via POST.
resp = requests.post(
f"{BASE_URL}/v2/manager/db_mode",
json={"value": new_mode},
timeout=10,
)
assert resp.status_code == 200, (
f"POST db_mode failed: {resp.status_code} {resp.text}"
)
# (1) API round-trip — the existing WEAK check.
resp = requests.get(f"{BASE_URL}/v2/manager/db_mode", timeout=10)
resp.raise_for_status()
assert resp.text == new_mode, (
f"db_mode not updated: expected {new_mode!r}, got {resp.text!r}"
)
# (2) Disk persistence — helper #1 asserts config.ini on disk
# reflects the new value. This defeats a "no-op handler that
# caches in memory but never writes" regression.
_assert_config_ini_contains("db_mode", new_mode)
# Capture post-POST hash — assertion diagnostic only; failing the
# above already reports the mismatch. Required for AC-5c evidence.
post_hash = (
hashlib.sha256(open(MANAGER_CONFIG_INI, "rb").read()).hexdigest()[:12]
if os.path.isfile(MANAGER_CONFIG_INI)
else "<missing>"
)
assert pre_hash != post_hash or pre_hash == "<missing>", (
f"config.ini hash unchanged after POST: {pre_hash}; "
f"server may be caching without writing to disk"
)
# (3) Reboot persistence — helper #2 restarts ComfyUI and
# re-verifies both disk and API still report new_mode. This
# defeats a "value only in memory, lost on restart" regression.
# NOTE: this helper replaces the ComfyUI process; downstream
# tests in this module will hit the fresh instance. The
# module-scoped `comfyui` fixture's teardown kills by port, so
# cleanup still works regardless of the new PID.
_assert_config_ini_persists_across_reboot("db_mode", new_mode)
finally:
# Restore original value on whichever server instance is live
# (pre- or post-reboot — the restored value also persists to
# disk via the restarted handler).
requests.post(
f"{BASE_URL}/v2/manager/db_mode",
json={"value": original},
timeout=10,
)
# Verify restoration end-to-end: API + disk.
resp = requests.get(f"{BASE_URL}/v2/manager/db_mode", timeout=10)
resp.raise_for_status()
assert resp.text == original, (
f"Failed to restore db_mode: expected {original!r}, got {resp.text!r}"
)
_assert_config_ini_contains("db_mode", original)
# ---------------------------------------------------------------------------
# Tests — update policy
# ---------------------------------------------------------------------------
class TestConfigUpdatePolicy:
"""Test GET/POST /v2/manager/policy/update round-trip."""
POLICY_VALUES = ("stable", "stable-comfyui", "nightly", "nightly-comfyui")
def test_read_update_policy(self, comfyui):
"""GET /v2/manager/policy/update returns a valid policy string."""
resp = requests.get(
f"{BASE_URL}/v2/manager/policy/update", timeout=10
)
assert resp.status_code == 200, f"Expected 200, got {resp.status_code}"
assert resp.text in self.POLICY_VALUES, (
f"Unexpected policy value: {resp.text!r}"
)
def test_set_and_restore_update_policy(self, comfyui):
"""POST sets update policy, disk + reboot persistence proven, then original restored (WI-G).
WI-G full-helper application (mirrors test_set_and_restore_db_mode PoC):
* _assert_config_ini_contains → disk-level verification
* _assert_config_ini_persists_across_reboot → restart-survival verification
"""
# Read original — baseline for both round-trip and restore verification.
resp = requests.get(
f"{BASE_URL}/v2/manager/policy/update", timeout=10
)
resp.raise_for_status()
original = resp.text
# Pick a different value
new_policy = "nightly" if original != "nightly" else "stable"
try:
# Set new value via POST
resp = requests.post(
f"{BASE_URL}/v2/manager/policy/update",
json={"value": new_policy},
timeout=10,
)
assert resp.status_code == 200, (
f"POST policy/update failed: {resp.status_code} {resp.text}"
)
# (1) API round-trip — existing WEAK check retained.
resp = requests.get(
f"{BASE_URL}/v2/manager/policy/update", timeout=10
)
resp.raise_for_status()
assert resp.text == new_policy, (
f"Policy not updated: expected {new_policy!r}, got {resp.text!r}"
)
# (2) Disk persistence — helper #1 proves config.ini on disk was mutated.
_assert_config_ini_contains("update_policy", new_policy)
# (3) Reboot persistence — helper #2 proves the value survives a
# full ComfyUI restart on both disk and via API.
_assert_config_ini_persists_across_reboot("update_policy", new_policy)
finally:
# Restore original value on whichever server instance is live.
requests.post(
f"{BASE_URL}/v2/manager/policy/update",
json={"value": original},
timeout=10,
)
# Verify restoration end-to-end: API + disk.
resp = requests.get(
f"{BASE_URL}/v2/manager/policy/update", timeout=10
)
resp.raise_for_status()
assert resp.text == original, (
f"Failed to restore policy: expected {original!r}, got {resp.text!r}"
)
_assert_config_ini_contains("update_policy", original)
# ---------------------------------------------------------------------------
# Tests — channel_url_list
# ---------------------------------------------------------------------------
class TestConfigChannelUrlList:
"""Test GET/POST /v2/manager/channel_url_list round-trip."""
def test_read_channel_url_list(self, comfyui):
"""GET /v2/manager/channel_url_list returns {selected, list} structure."""
resp = requests.get(
f"{BASE_URL}/v2/manager/channel_url_list", timeout=10
)
assert resp.status_code == 200, f"Expected 200, got {resp.status_code}"
data = resp.json()
assert "selected" in data, "Response missing 'selected' field"
assert "list" in data, "Response missing 'list' field"
assert isinstance(data["list"], list), (
f"'list' should be an array, got {type(data['list']).__name__}"
)
assert isinstance(data["selected"], str), (
f"'selected' should be a string, got {type(data['selected']).__name__}"
)
def test_channel_list_entries_are_name_url_strings(self, comfyui):
"""Each entry in channel list is a 'name::url' string."""
resp = requests.get(
f"{BASE_URL}/v2/manager/channel_url_list", timeout=10
)
resp.raise_for_status()
data = resp.json()
for i, entry in enumerate(data["list"]):
assert isinstance(entry, str), (
f"Entry {i} should be a string, got {type(entry).__name__}"
)
assert "::" in entry, (
f"Entry {i} should contain '::' separator: {entry!r}"
)
def test_set_and_restore_channel(self, comfyui):
"""POST sets channel, disk + reboot persistence proven, then original restored (WI-G).
WI-G full-helper application. Notes on the channel_url asymmetry:
* config.ini stores the full URL under key `channel_url`
* GET /channel_url_list returns the NAME (reverse-mapped from URL)
* POST /channel_url_list accepts {value: NAME} and maps to URL
The helpers resolve URL↔NAME internally when key == "channel_url".
"""
# Read original — capture both NAME (for API round-trip) and URL (for disk checks).
resp = requests.get(
f"{BASE_URL}/v2/manager/channel_url_list", timeout=10
)
resp.raise_for_status()
original_data = resp.json()
original_selected = original_data["selected"]
channel_map = {} # name -> url
for entry in original_data["list"]:
if isinstance(entry, str) and "::" in entry:
name, url = entry.split("::", 1)
channel_map[name] = url
original_url = channel_map.get(original_selected)
available_channels = list(channel_map.keys())
if len(available_channels) < 2:
pytest.skip("Only one channel available, cannot test switching")
# Pick a different channel (name + its URL)
new_channel = next(
(ch for ch in available_channels if ch != original_selected),
None,
)
if new_channel is None or original_url is None:
pytest.skip("No alternative channel found or original URL unresolved")
new_channel_url = channel_map[new_channel]
try:
# Set new channel via POST (server maps NAME → URL internally)
resp = requests.post(
f"{BASE_URL}/v2/manager/channel_url_list",
json={"value": new_channel},
timeout=10,
)
assert resp.status_code == 200, (
f"POST channel_url_list failed: {resp.status_code} {resp.text}"
)
# (1) API round-trip — existing WEAK check retained (verifies NAME).
resp = requests.get(
f"{BASE_URL}/v2/manager/channel_url_list", timeout=10
)
resp.raise_for_status()
data = resp.json()
assert data["selected"] == new_channel, (
f"Channel not updated: expected {new_channel!r}, "
f"got {data['selected']!r}"
)
# (2) Disk persistence — helper asserts config.ini holds the URL.
_assert_config_ini_contains("channel_url", new_channel_url)
# (3) Reboot persistence — helper reboots and re-verifies disk URL
# + API NAME (internal URL→NAME translation handles the asymmetry).
_assert_config_ini_persists_across_reboot("channel_url", new_channel_url)
finally:
# Restore original channel on whichever server instance is live.
requests.post(
f"{BASE_URL}/v2/manager/channel_url_list",
json={"value": original_selected},
timeout=10,
)
# Verify restoration end-to-end: API NAME + disk URL.
resp = requests.get(
f"{BASE_URL}/v2/manager/channel_url_list", timeout=10
)
resp.raise_for_status()
data = resp.json()
assert data["selected"] == original_selected, (
f"Failed to restore channel: expected {original_selected!r}, "
f"got {data['selected']!r}"
)
_assert_config_ini_contains("channel_url", original_url)
# ---------------------------------------------------------------------------
# Parametrized consolidations (WI-NN bloat Priority 3)
# ---------------------------------------------------------------------------
#
# Two parametrized tests below consolidate the 6 copy-paste tests that
# previously lived on the 3 per-endpoint TestConfig* classes:
# * test_set_*_invalid_body (×3 endpoints) → one parametrized
# * test_set_*_junk_value_rejected / unknown_name → one parametrized
#
# Cluster 1 (roundtrip, set_and_restore_* ×3) remained unparametrized:
# the channel_url_list case carries URL↔NAME asymmetry that the helpers
# resolve internally only when `key == "channel_url"`, and the channel-
# map extraction step has no counterpart in the db_mode/policy bodies.
# Forcing a single parametrized body produced a ~100-line branch soup;
# the three per-endpoint tests remain distinct functions for readability.
# Config endpoints that accept/reject via {"value": ...} JSON body + config.ini
# on-disk persistence. Each descriptor supplies the config.ini key AND the
# junk-value payload; the valid-values whitelist is used to both pick a
# valid restore target and to sanity-check post-rejection disk state.
_CONFIG_POST_ENDPOINTS = [
pytest.param(
"/v2/manager/db_mode",
"db_mode",
"pwned_junk_value_xyz",
("cache", "channel", "local", "remote"),
id="db_mode",
),
pytest.param(
"/v2/manager/policy/update",
"update_policy",
"pwned_junk_policy_xyz",
("stable", "stable-comfyui", "nightly", "nightly-comfyui"),
id="update_policy",
),
pytest.param(
"/v2/manager/channel_url_list",
"channel_url",
"pwned_unknown_channel_xyz",
None, # channel uses dynamic whitelist (name→url map); see _read_channel_selected
id="channel_url_list",
),
]
def _read_channel_selected() -> str | None:
resp = requests.get(f"{BASE_URL}/v2/manager/channel_url_list", timeout=10)
if not resp.ok:
return None
return resp.json().get("selected")
class TestConfigPostNegativeContracts:
"""Parametrized negative-path tests for the 3 config POST endpoints.
WI-NN Cluster 2 (invalid body) + Cluster 3 (junk value) consolidate the
6 previous copy-paste tests. Each parametrize case exercises one endpoint;
the two test functions cover the two negative contracts separately so
failures still point at the correct contract.
"""
@pytest.mark.parametrize("endpoint,key,_junk,_values", _CONFIG_POST_ENDPOINTS)
def test_malformed_body_returns_400(self, comfyui, endpoint, key, _junk, _values):
"""WI-NN Cluster 2 (teng:ci-003/ci-008/ci-015 B9): malformed JSON → 400 + disk unchanged."""
before = _read_config_ini_value(key)
resp = requests.post(
f"{BASE_URL}{endpoint}",
data="not-json",
headers={"Content-Type": "application/json"},
timeout=10,
)
assert resp.status_code == 400, (
f"Expected 400 for malformed JSON on {endpoint}, got {resp.status_code}"
)
# Disk-state invariant — malformed POST must not touch config.ini.
_assert_config_ini_contains(key, before)
@pytest.mark.parametrize("endpoint,key,junk,values", _CONFIG_POST_ENDPOINTS)
def test_junk_value_rejected(self, comfyui, endpoint, key, junk, values):
"""WI-NN Cluster 3 (teng:ci-004/ci-009/ci-014 B9): unknown/junk value → 400 + disk/API unchanged.
For db_mode/policy the whitelist is static and verifiable directly
via `_read_config_ini_value`. For channel_url_list the whitelist is
dynamic (server-built name→url map), so we compare the API-level
`selected` string before/after instead.
"""
# Capture pre-state that the endpoint's own API exposes. Also capture
# disk state for the static-whitelist endpoints.
pre_disk = _read_config_ini_value(key)
pre_api_selected = _read_channel_selected() if key == "channel_url" else None
resp = requests.post(
f"{BASE_URL}{endpoint}",
json={"value": junk},
timeout=10,
)
assert resp.status_code == 400, (
f"Unknown/junk value on {endpoint} should return 400, got {resp.status_code}"
)
if values is not None:
# Static whitelist: on-disk value must still be a whitelisted
# value (server did not write junk).
post_disk = _read_config_ini_value(key)
assert post_disk in values, (
f"config.ini {key} corrupted with junk value: {post_disk!r}"
)
else:
# Dynamic whitelist (channel): API-level NAME must be unchanged.
post_api_selected = _read_channel_selected()
assert pre_api_selected == post_api_selected, (
f"{endpoint} selected mutated on invalid request: "
f"{pre_api_selected!r} -> {post_api_selected!r}"
)
# Also check config.ini URL is unchanged (if pre was present).
assert _read_config_ini_value(key) == pre_disk, (
f"config.ini {key} changed on invalid {endpoint} POST"
)
+315
View File
@@ -0,0 +1,315 @@
"""E2E tests for the GET-rejection contract on state-changing endpoints.
SCOPE — important clarification:
This suite verifies ONE specific CSRF mitigation layer: that state-changing
endpoints reject HTTP GET requests (so that <img src="..."> / link-click /
redirect-based cross-origin triggers cannot mutate server state). This is
the contract established in commit 99caef55 which converted 12+ endpoints
from GET to POST.
NOT COVERED by this suite:
- Origin / Referer header validation
- Same-site cookie enforcement
- Anti-CSRF token verification
- Cross-site form POST defense
Those remaining CSRF defenses are handled separately (e.g., via the
origin_only_middleware at the aiohttp layer) and are the subject of
other test layers. Do NOT read PASS here as "CSRF fully solved" — read
it as "the method-conversion contract holds".
Requires a pre-built E2E environment (from setup_e2e_env.sh).
"""
from __future__ import annotations
import os
import subprocess
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui() -> int:
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI:\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID:\n{r.stdout}")
def _stop_comfyui():
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
@pytest.fixture(scope="module")
def comfyui():
pid = _start_comfyui()
yield pid
_stop_comfyui()
# ---------------------------------------------------------------------------
# State-changing endpoints that MUST reject GET per CSRF mitigation contract
# ---------------------------------------------------------------------------
# (method, path, description) — derived from commit 99caef55 scope
STATE_CHANGING_POST_ENDPOINTS = [
("/v2/manager/queue/start", "start worker"),
("/v2/manager/queue/reset", "reset queue"),
("/v2/manager/queue/update_all", "update all packs"),
("/v2/manager/queue/update_comfyui", "update ComfyUI core"),
("/v2/manager/queue/install_model", "queue model download"),
("/v2/manager/queue/task", "enqueue task"),
("/v2/snapshot/save", "save snapshot"),
("/v2/snapshot/remove", "remove snapshot"),
("/v2/snapshot/restore", "restore snapshot"),
("/v2/manager/reboot", "reboot server"),
("/v2/comfyui_manager/comfyui_switch_version", "switch ComfyUI version"),
("/v2/customnode/import_fail_info", "import fail info"),
("/v2/customnode/import_fail_info_bulk", "bulk import fail info"),
]
class TestStateChangingEndpointsRejectGet:
"""Every state-changing endpoint MUST reject HTTP GET.
This is the narrow CSRF-mitigation contract established by the
GET→POST conversion (commit 99caef55). It blocks <img>-tag,
link-click, and redirect-based cross-origin triggers. Full origin
verification is a separate layer and is NOT tested here.
"""
@pytest.mark.parametrize(
"path,description",
STATE_CHANGING_POST_ENDPOINTS,
ids=[p for p, _ in STATE_CHANGING_POST_ENDPOINTS],
)
def test_get_is_rejected(self, comfyui, path, description):
resp = requests.get(
f"{BASE_URL}{path}",
timeout=10,
allow_redirects=False,
)
# GET must NOT succeed with any 2xx or redirect status on a
# state-changing endpoint. Prior assertion had a Python operator-
# precedence bug (`A or (X is False)` → dead code). Use explicit
# membership check instead.
assert resp.status_code not in range(200, 400), (
f"CSRF-CONTRACT BYPASS: GET {path} returned {resp.status_code} "
f"(2xx/3xx indicates accept or redirect — endpoint must reject): "
f"{description}"
)
# Narrow the accepted rejection statuses to method-not-allowed /
# not-found / forbidden / bad-request. Other 4xx/5xx codes are
# suspicious and should be investigated.
assert resp.status_code in (400, 403, 404, 405), (
f"GET {path} returned unexpected status {resp.status_code} "
f"(expected 400/403/404/405): {resp.text[:200]}"
)
class TestCsrfPostWorks:
"""Sanity check: the POST counterparts actually work (CSRF fix didn't break the API)."""
def test_queue_reset_post_works(self, comfyui):
"""POST queue/reset should succeed (the same path rejects GET)."""
resp = requests.post(f"{BASE_URL}/v2/manager/queue/reset", timeout=10)
assert resp.status_code == 200, (
f"POST queue/reset should succeed, got {resp.status_code}: {resp.text[:200]}"
)
def test_snapshot_save_post_works(self, comfyui):
"""POST snapshot/save should succeed."""
resp = requests.post(f"{BASE_URL}/v2/snapshot/save", timeout=30)
assert resp.status_code == 200, (
f"POST snapshot/save should succeed, got {resp.status_code}: {resp.text[:200]}"
)
# Cleanup — remove the snapshot we just created
list_resp = requests.get(f"{BASE_URL}/v2/snapshot/getlist", timeout=10)
if list_resp.ok:
items = list_resp.json().get("items", [])
if items:
requests.post(
f"{BASE_URL}/v2/snapshot/remove",
params={"target": items[0]},
timeout=10,
)
class TestCsrfReadEndpointsStillAllowGet:
"""Negative control: read-only endpoints should still allow GET.
Ensures the CSRF fix didn't over-correct by making pure-read endpoints
POST-only, which would break the UI.
"""
@pytest.mark.parametrize(
"path",
[
"/v2/manager/version",
"/v2/manager/db_mode",
"/v2/manager/policy/update",
"/v2/manager/channel_url_list",
"/v2/manager/queue/status",
"/v2/manager/queue/history_list",
"/v2/manager/is_legacy_manager_ui",
"/v2/customnode/installed",
"/v2/snapshot/getlist",
"/v2/snapshot/get_current",
"/v2/comfyui_manager/comfyui_versions",
],
)
def test_get_read_endpoint_succeeds(self, comfyui, path):
resp = requests.get(f"{BASE_URL}{path}", timeout=10)
assert resp.status_code == 200, (
f"Read endpoint GET {path} should succeed, got {resp.status_code}: "
f"{resp.text[:200]}"
)
# ---------------------------------------------------------------------------
# Content-Type gate — second CSRF mitigation layer
# ---------------------------------------------------------------------------
#
# GET→POST conversion alone does NOT block <form method=POST> from a malicious
# cross-origin page, because browsers mark form submissions with one of three
# CORS "simple request" Content-Types (Fetch spec §3.2.3) and do NOT preflight
# them. These 9 high-risk state mutation endpoints therefore additionally
# reject those three MIME types at the handler entry. Bare POST (no body) and
# application/json remain accepted — same-origin fetch() and existing
# manager-core JS callers are unaffected.
#
# Source of truth for the gated handler list:
# comfyui_manager/common/manager_security.py :: reject_simple_form_post
# comfyui_manager/glob/manager_server.py :: 9 handlers call it
FORM_REJECTED_POST_ENDPOINTS = [
"/v2/manager/queue/update_all",
"/v2/snapshot/remove",
"/v2/snapshot/restore",
"/v2/snapshot/save",
"/v2/manager/queue/reset",
"/v2/manager/queue/start",
"/v2/manager/queue/update_comfyui",
# "/v2/comfyui_manager/comfyui_switch_version" — removed in WI #258:
# migrated from query-string to JSON body, now a body-reading handler.
# Per the module policy in common/manager_security.py, body-reading
# handlers are NOT gated (CORS preflight on application/json already
# blocks cross-origin form POST forgery).
"/v2/manager/reboot",
]
SIMPLE_FORM_CONTENT_TYPES = [
"application/x-www-form-urlencoded",
"multipart/form-data; boundary=----WebKitFormBoundaryTest",
"text/plain",
]
class TestFormContentTypeRejected:
"""Every gated state-changing endpoint MUST reject CORS simple-request
Content-Types to block preflight-less <form method=POST> CSRF.
Matrix: 9 endpoints × 3 Content-Types = 27 assertions.
"""
@pytest.mark.parametrize(
"content_type",
SIMPLE_FORM_CONTENT_TYPES,
ids=["urlencoded", "multipart", "textplain"],
)
@pytest.mark.parametrize(
"path",
FORM_REJECTED_POST_ENDPOINTS,
ids=FORM_REJECTED_POST_ENDPOINTS,
)
def test_form_content_type_rejected(self, comfyui, path, content_type):
"""POST with a simple-form Content-Type must be rejected with 400.
The handler's Content-Type gate runs BEFORE the security_level check,
so the expected status is 400 even under security levels that would
otherwise return 403.
"""
resp = requests.post(
f"{BASE_URL}{path}",
headers={"Content-Type": content_type},
data="", # empty body still counts: browsers would not preflight
timeout=10,
)
assert resp.status_code == 400, (
f"CSRF FORM-POST GATE BYPASS: POST {path} with "
f"Content-Type={content_type!r} returned {resp.status_code} "
f"(expected 400): {resp.text[:200]}"
)
class TestNoBodyPostStillAccepted:
"""Positive control: bare POST (no body, no Content-Type) must still pass
the form-content-type gate.
The existing frontend (snapshot.js, comfyui-manager.js, etc.) issues
fetch()/XHR POSTs without an explicit body for these idempotent-ish
operations; a regression here would break those callers.
"""
@pytest.mark.parametrize(
"path",
# Subset — pick endpoints whose happy path is deterministic under the
# default E2E environment. queue/update_comfyui, snapshot/restore etc.
# have side effects (spawn worker, stage restore file) or require
# a specific security level that isn't guaranteed here.
[
"/v2/manager/queue/reset",
"/v2/manager/queue/start",
"/v2/snapshot/save",
],
ids=["queue-reset", "queue-start", "snapshot-save"],
)
def test_no_body_post_still_accepted(self, comfyui, path):
"""Bare POST (no Content-Type, no body) must NOT be rejected by the
form-content-type gate. Any response other than 400-with-form-text
proves the gate did not fire."""
resp = requests.post(f"{BASE_URL}{path}", timeout=30)
# The gate returns 400 with a very specific text. Non-gate 400s
# (validation errors, etc.) are allowed — we only assert the gate
# itself didn't trigger.
if resp.status_code == 400:
assert "Invalid Content-Type for this endpoint" not in resp.text, (
f"POST {path} (bare) hit the form-content-type gate: "
f"{resp.text[:200]}"
)
+388
View File
@@ -0,0 +1,388 @@
"""E2E tests for the GET-rejection contract on legacy-mode state-changing endpoints.
SCOPE — important clarification:
This suite is the LEGACY-MODE counterpart to test_e2e_csrf.py. It verifies the
same CSRF mitigation contract — that state-changing endpoints reject HTTP GET —
but against the legacy manager_server module loaded via --enable-manager-legacy-ui.
Why a separate file:
comfyui_manager/__init__.py loads `glob.manager_server` XOR `legacy.manager_server`
(mutex via args.enable_manager_legacy_ui). So a single ComfyUI process exposes
either the glob route table or the legacy route table, never both. Verifying
legacy CSRF mitigation requires its own server lifecycle with the legacy flag
set, which is incompatible with the module-scoped glob fixture in test_e2e_csrf.py.
Coverage gap closed by this file:
Commit 99caef55 ("fix(security): mitigate CSRF on state-changing endpoints")
applied the GET→POST conversion to BOTH glob/manager_server.py (91 line diff)
and legacy/manager_server.py (92 line diff). However, test_e2e_csrf.py only
exercises glob mode (start_comfyui.sh uses --enable-manager without
--enable-manager-legacy-ui). Without this file, anyone reverting a legacy
@routes.post back to @routes.get would not be caught by CI.
Endpoint list — derived empirically from the working tree (NOT statically from
the 99caef55 diff), because subsequent legacy refactoring removed several
endpoints that were initially in scope (e.g., queue/abort_current). The list
mirrors test_e2e_csrf.py's STATE_CHANGING_POST_ENDPOINTS for parity, with three
adjustments:
- Drop /v2/manager/queue/task (glob-only; legacy uses queue/batch instead)
- Add /v2/manager/queue/batch (legacy task enqueue, mirrors queue/task role)
- Drop /v2/manager/db_mode, /v2/manager/policy/update, /v2/manager/channel_url_list
from REJECT-GET. These are split into @routes.get (read) + @routes.post
(write) in BOTH glob and legacy. The CSRF mitigation contract applies only
to the POST half — GET legitimately serves the current value. They remain
in the ALLOW-GET list below. (test_e2e_csrf.py erroneously includes them in
both lists, so the equivalent assertions fail there too — see follow-up note
in WI-FF completion_report.)
Legacy-parity additions (WI-JJ):
- /v2/customnode/install/git_url, /v2/customnode/install/pip — legacy-only
install endpoints. Added to LEGACY_STATE_CHANGING_POST_ENDPOINTS for
GET-rejection coverage; happy-path install E2E remains out of scope.
- /v2/manager/is_legacy_manager_ui legacy-side flag value asserted True via
TestLegacyIsLegacyManagerUIReturnsTrue — symmetric to the glob-side
False assertion in test_e2e_system_info.py::test_returns_boolean_field.
NOT COVERED by this suite (same caveats as test_e2e_csrf.py):
- Origin / Referer header validation
- Same-site cookie enforcement
- Anti-CSRF token verification
- Cross-site form POST defense
Requires a pre-built E2E environment (from setup_e2e_env.sh).
"""
from __future__ import annotations
import os
import subprocess
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready",
)
# ---------------------------------------------------------------------------
# Helpers — start_comfyui_legacy.sh wrapper sets ENABLE_LEGACY_UI=1 which
# translates to --enable-manager-legacy-ui inside start_comfyui.sh.
# ---------------------------------------------------------------------------
def _start_comfyui_legacy() -> int:
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui_legacy.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI (legacy):\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID:\n{r.stdout}")
def _stop_comfyui():
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
@pytest.fixture(scope="module")
def comfyui_legacy():
pid = _start_comfyui_legacy()
yield pid
_stop_comfyui()
# ---------------------------------------------------------------------------
# State-changing legacy endpoints that MUST reject GET per CSRF mitigation contract
# ---------------------------------------------------------------------------
# (path, description) — mirrors test_e2e_csrf.py STATE_CHANGING_POST_ENDPOINTS,
# with queue/task replaced by queue/batch (legacy task-enqueue equivalent).
LEGACY_STATE_CHANGING_POST_ENDPOINTS = [
("/v2/manager/queue/start", "start worker"),
("/v2/manager/queue/reset", "reset queue"),
("/v2/manager/queue/update_all", "update all packs"),
("/v2/manager/queue/update_comfyui", "update ComfyUI core"),
("/v2/manager/queue/install_model", "queue model download"),
("/v2/manager/queue/batch", "enqueue task batch (legacy)"),
("/v2/snapshot/save", "save snapshot"),
("/v2/snapshot/remove", "remove snapshot"),
("/v2/snapshot/restore", "restore snapshot"),
("/v2/manager/reboot", "reboot server"),
("/v2/comfyui_manager/comfyui_switch_version", "switch ComfyUI version"),
# NOTE: db_mode, policy/update, channel_url_list have a legitimate GET handler
# for reading the current value; only POST mutates state. Verified separately
# in TestLegacyCsrfReadEndpointsStillAllowGet below.
("/v2/customnode/import_fail_info", "import fail info"),
("/v2/customnode/import_fail_info_bulk", "bulk import fail info"),
# Legacy-only install endpoints (no glob counterpart). Added in WI-JJ to
# extend CSRF GET-rejection coverage — these are state-changing (they
# enqueue install tasks) and must not be triggerable via <img>/link.
("/v2/customnode/install/git_url", "install custom node by git URL (legacy-only)"),
("/v2/customnode/install/pip", "install pip package for custom node (legacy-only)"),
]
class TestLegacyStateChangingEndpointsRejectGet:
"""Every legacy state-changing endpoint MUST reject HTTP GET.
Verifies the CSRF-mitigation contract on the legacy server module
under --enable-manager-legacy-ui. Mirrors the glob-side test in
test_e2e_csrf.py::TestStateChangingEndpointsRejectGet.
"""
@pytest.mark.parametrize(
"path,description",
LEGACY_STATE_CHANGING_POST_ENDPOINTS,
ids=[p for p, _ in LEGACY_STATE_CHANGING_POST_ENDPOINTS],
)
def test_get_is_rejected(self, comfyui_legacy, path, description):
resp = requests.get(
f"{BASE_URL}{path}",
timeout=10,
allow_redirects=False,
)
assert resp.status_code not in range(200, 400), (
f"CSRF-CONTRACT BYPASS (legacy): GET {path} returned "
f"{resp.status_code} (2xx/3xx indicates accept or redirect — "
f"endpoint must reject): {description}"
)
assert resp.status_code in (400, 403, 404, 405), (
f"GET {path} returned unexpected status {resp.status_code} "
f"(expected 400/403/404/405): {resp.text[:200]}"
)
class TestLegacyCsrfPostWorks:
"""Sanity check: the legacy POST counterparts actually work."""
def test_queue_reset_post_works(self, comfyui_legacy):
"""POST queue/reset should succeed (the same path rejects GET)."""
resp = requests.post(f"{BASE_URL}/v2/manager/queue/reset", timeout=10)
assert resp.status_code == 200, (
f"POST queue/reset should succeed, got {resp.status_code}: {resp.text[:200]}"
)
def test_snapshot_save_post_works(self, comfyui_legacy):
"""POST snapshot/save should succeed on legacy."""
resp = requests.post(f"{BASE_URL}/v2/snapshot/save", timeout=30)
assert resp.status_code == 200, (
f"POST snapshot/save should succeed, got {resp.status_code}: {resp.text[:200]}"
)
# Cleanup — remove the snapshot we just created
list_resp = requests.get(f"{BASE_URL}/v2/snapshot/getlist", timeout=10)
if list_resp.ok:
items = list_resp.json().get("items", [])
if items:
requests.post(
f"{BASE_URL}/v2/snapshot/remove",
params={"target": items[0]},
timeout=10,
)
class TestLegacyCsrfReadEndpointsStillAllowGet:
"""Negative control: read-only legacy endpoints should still allow GET.
Ensures the CSRF fix didn't over-correct on legacy by making pure-read
endpoints POST-only, which would break the legacy UI.
"""
@pytest.mark.parametrize(
"path",
[
"/v2/manager/version",
"/v2/manager/db_mode",
"/v2/manager/policy/update",
"/v2/manager/channel_url_list",
"/v2/manager/queue/status",
"/v2/manager/queue/history_list",
"/v2/manager/is_legacy_manager_ui",
"/v2/customnode/installed",
"/v2/snapshot/getlist",
"/v2/snapshot/get_current",
"/v2/comfyui_manager/comfyui_versions",
],
)
def test_get_read_endpoint_succeeds(self, comfyui_legacy, path):
resp = requests.get(f"{BASE_URL}{path}", timeout=10)
assert resp.status_code == 200, (
f"Legacy read endpoint GET {path} should succeed, got "
f"{resp.status_code}: {resp.text[:200]}"
)
class TestLegacyIsLegacyManagerUIReturnsTrue:
"""Legacy-mode parity for TestIsLegacyManagerUI in test_e2e_system_info.py.
The glob-side test (`system_info.py::test_returns_boolean_field`) asserts
the flag returns False under start_comfyui.sh (which omits
--enable-manager-legacy-ui). This test asserts the symmetric contract:
under start_comfyui_legacy.sh the handler must return True.
Launcher-deterministic: `tests/e2e/scripts/start_comfyui_legacy.sh` sets
ENABLE_LEGACY_UI=1, which start_comfyui.sh translates to
--enable-manager-legacy-ui. `action='store_true'` makes the flag True,
so the handler at legacy/manager_server.py:995-999 must return
`{"is_legacy_manager_ui": True}`.
Without this assertion, a regression that silently drops the CLI flag
(e.g., mis-edited MANAGER_FLAGS in start_comfyui.sh) would leave the
legacy route table in place while the flag-value response reverted to
False — breaking UI mode detection for any frontend code that keys off
this endpoint.
"""
def test_returns_true_under_legacy_mode(self, comfyui_legacy):
resp = requests.get(
f"{BASE_URL}/v2/manager/is_legacy_manager_ui", timeout=10
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}"
)
data = resp.json()
assert "is_legacy_manager_ui" in data, (
f"Response missing 'is_legacy_manager_ui' field: {data}"
)
assert data["is_legacy_manager_ui"] is True, (
f"Legacy launcher sets --enable-manager-legacy-ui; expected True, "
f"got {data['is_legacy_manager_ui']!r}. "
f"If start_comfyui_legacy.sh stopped propagating ENABLE_LEGACY_UI=1, "
f"fix the wrapper rather than relaxing this assertion."
)
# ---------------------------------------------------------------------------
# Content-Type gate — legacy-side parity with test_e2e_csrf.py
# ---------------------------------------------------------------------------
#
# The 9 legacy handlers mirrored from glob each call
# manager_security.reject_simple_form_post() before their security-level
# check. This suite verifies the legacy route table enforces the same
# CORS-simple-request Content-Type rejection contract.
LEGACY_FORM_REJECTED_POST_ENDPOINTS = [
"/v2/manager/queue/update_all",
"/v2/snapshot/remove",
"/v2/snapshot/restore",
"/v2/snapshot/save",
"/v2/manager/queue/reset",
"/v2/manager/queue/start",
"/v2/manager/queue/update_comfyui",
# "/v2/comfyui_manager/comfyui_switch_version" — removed in WI #258:
# migrated from query-string to JSON body on legacy + glob in parallel,
# body-reading handler is not Content-Type-gated (see module policy in
# common/manager_security.py).
"/v2/manager/reboot",
]
LEGACY_SIMPLE_FORM_CONTENT_TYPES = [
"application/x-www-form-urlencoded",
"multipart/form-data; boundary=----WebKitFormBoundaryTest",
"text/plain",
]
class TestLegacyFormContentTypeRejected:
"""Legacy counterpart to TestFormContentTypeRejected in test_e2e_csrf.py.
Matrix: 9 endpoints × 3 Content-Types = 27 assertions against the legacy
route table.
"""
@pytest.mark.parametrize(
"content_type",
LEGACY_SIMPLE_FORM_CONTENT_TYPES,
ids=["urlencoded", "multipart", "textplain"],
)
@pytest.mark.parametrize(
"path",
LEGACY_FORM_REJECTED_POST_ENDPOINTS,
ids=LEGACY_FORM_REJECTED_POST_ENDPOINTS,
)
def test_form_content_type_rejected(
self, comfyui_legacy, path, content_type
):
"""POST with a simple-form Content-Type must be rejected with 400
on the legacy route table as well."""
resp = requests.post(
f"{BASE_URL}{path}",
headers={"Content-Type": content_type},
data="",
timeout=10,
)
assert resp.status_code == 400, (
f"CSRF FORM-POST GATE BYPASS (legacy): POST {path} with "
f"Content-Type={content_type!r} returned {resp.status_code} "
f"(expected 400): {resp.text[:200]}"
)
class TestLegacyNoBodyPostStillAccepted:
"""Positive control for the legacy route table: bare POST still works."""
@pytest.mark.parametrize(
"path",
[
"/v2/manager/queue/reset",
"/v2/manager/queue/start",
"/v2/snapshot/save",
],
ids=["queue-reset", "queue-start", "snapshot-save"],
)
def test_no_body_post_still_accepted(self, comfyui_legacy, path):
"""Bare POST (no Content-Type, no body) must not hit the gate on
legacy either."""
resp = requests.post(f"{BASE_URL}{path}", timeout=30)
if resp.status_code == 400:
assert "Invalid Content-Type for this endpoint" not in resp.text, (
f"POST {path} (bare, legacy) hit the form-content-type gate: "
f"{resp.text[:200]}"
)
class TestLegacySetChannelUrlRejectsInvalid:
"""Legacy set_channel_url must reject unknown channel names with 400
(MAJOR fix — previously silently returned 200).
Parity with glob set_channel_url and with set_db_mode /
set_update_policy whitelist enforcement.
"""
def test_invalid_channel_returns_400(self, comfyui_legacy):
resp = requests.post(
f"{BASE_URL}/v2/manager/channel_url_list",
json={"value": "definitely-not-a-real-channel"},
timeout=10,
)
assert resp.status_code == 400, (
f"Legacy set_channel_url accepted unknown channel silently: "
f"status={resp.status_code}, body={resp.text[:300]}"
)
assert "Invalid channel name" in resp.text, (
f"Expected 'Invalid channel name' in rejection text; got: "
f"{resp.text[:300]}"
)
+385
View File
@@ -0,0 +1,385 @@
"""E2E tests for ComfyUI Manager custom node information endpoints.
Tests the custom node information and mapping endpoints:
- GET /v2/customnode/getmappings — node-to-package mappings
- GET /v2/customnode/fetch_updates — update check (deprecated, 410)
- GET /v2/customnode/installed — installed packages dict
- POST /v2/customnode/import_fail_info — single node failure info
- POST /v2/customnode/import_fail_info_bulk — bulk node failure info
Requires a pre-built E2E environment (from setup_e2e_env.sh).
Set E2E_ROOT env var to point at it, or the tests will be skipped.
Usage:
E2E_ROOT=/tmp/e2e_full_test pytest tests/e2e/test_e2e_customnode_info.py -v
"""
from __future__ import annotations
import os
import subprocess
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready (run setup_e2e_env.sh first)",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui() -> int:
"""Start ComfyUI and return its PID."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI:\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID from start_comfyui output:\n{r.stdout}")
def _stop_comfyui():
"""Stop ComfyUI."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def comfyui():
"""Start ComfyUI once for the module, stop after all tests."""
pid = _start_comfyui()
yield pid
_stop_comfyui()
# ---------------------------------------------------------------------------
# Tests — getmappings
# ---------------------------------------------------------------------------
class TestCustomNodeMappings:
"""Test GET /v2/customnode/getmappings."""
def test_getmappings_returns_dict(self, comfyui):
"""GET /v2/customnode/getmappings?mode=local returns non-empty mapping with valid per-entry schema.
WI-M strengthening: previously only dict-type check. Now verifies
content-level invariants: non-empty DB (the manager ships with the
full custom-node mappings baked in), and every entry conforms to
the documented `[node_list: list, metadata: dict]` shape on a
random sample. Defeats a regression where the DB loader returns
an empty `{}` (dict type PASS, zero-utility content).
"""
resp = requests.get(
f"{BASE_URL}/v2/customnode/getmappings",
params={"mode": "local"},
timeout=30,
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"Expected dict response, got {type(data).__name__}"
)
# Content: at least 1 entry (E2E env ships the stock DB with thousands
# of mappings; anything < 100 suggests DB load regression).
assert len(data) >= 100, (
f"getmappings returned only {len(data)} entries — DB load regression?"
)
# Structural sample: first 5 entries must conform to [node_list, metadata].
for i, (key, entry) in enumerate(list(data.items())[:5]):
assert isinstance(entry, list) and len(entry) >= 2, (
f"Entry {i} ({key!r}) not [node_list, metadata]: {entry!r}"
)
assert isinstance(entry[0], list), (
f"Entry {i} node_list is not a list: {type(entry[0]).__name__}"
)
assert isinstance(entry[1], dict), (
f"Entry {i} metadata is not a dict: {type(entry[1]).__name__}"
)
# ---------------------------------------------------------------------------
# Tests — fetch_updates (deprecated)
# ---------------------------------------------------------------------------
class TestFetchUpdates:
"""Test GET /v2/customnode/fetch_updates (deprecated endpoint)."""
def test_fetch_updates_returns_deprecated(self, comfyui):
"""GET /v2/customnode/fetch_updates returns 410 Gone with deprecation notice."""
resp = requests.get(
f"{BASE_URL}/v2/customnode/fetch_updates",
params={"mode": "local"},
timeout=10,
)
assert resp.status_code == 410, (
f"Expected 410 (Gone) for deprecated endpoint, got {resp.status_code}"
)
data = resp.json()
assert data.get("deprecated") is True, (
"Response should include 'deprecated: true'"
)
# ---------------------------------------------------------------------------
# Tests — installed
# ---------------------------------------------------------------------------
class TestInstalledPacks:
"""Test GET /v2/customnode/installed."""
def test_installed_returns_dict(self, comfyui):
"""GET /v2/customnode/installed returns dict containing seeded E2E pack with valid per-entry schema.
WI-M strengthening: previously only dict-type check. The E2E setup
seeds `ComfyUI_SigmoidOffsetScheduler` (the test package used across
task_operations/endpoint tests); its presence is a hard precondition
for most other tests. We now assert it's in the installed dict AND
that its entry has the documented InstalledPack fields
(cnr_id/ver/enabled). Defeats a regression where `installed` returns
an empty dict despite packs existing on disk.
"""
resp = requests.get(
f"{BASE_URL}/v2/customnode/installed", timeout=10
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"Expected dict response, got {type(data).__name__}"
)
# Content: E2E seed pack must be present.
seed_pack = "ComfyUI_SigmoidOffsetScheduler"
assert seed_pack in data, (
f"Seeded E2E pack {seed_pack!r} missing from installed dict. "
f"Keys: {list(data.keys())}"
)
# Schema: the seed pack's entry must carry the documented fields.
entry = data[seed_pack]
assert isinstance(entry, dict), (
f"{seed_pack} entry should be a dict, got {type(entry).__name__}"
)
for required_key in ("cnr_id", "ver", "enabled"):
assert required_key in entry, (
f"{seed_pack} entry missing required key {required_key!r}: {entry!r}"
)
def test_installed_imported_mode(self, comfyui):
"""GET ?mode=imported returns the frozen startup snapshot with schema.
WI-T Cluster G target 4 (research-cluster-g.md Strategy A):
(a) status 200 + dict body (contract)
(b) E2E seed pack `ComfyUI_SigmoidOffsetScheduler` is in the snapshot
(c) each entry carries the documented InstalledPack schema —
cnr_id / ver / enabled (aux_id is Optional)
(d) frozen-at-startup invariant (cheap form) — no install has run
since server start, so imported keys == default keys.
Design intent (glob/manager_server.py:1510-1520): `imported` returns
the module-level `startup_time_installed_node_packs` captured once at
import; `default` re-scans the filesystem. At test time they must
agree on keys. Divergence post-install is covered by the
[E2E-DEBT] companion below.
"""
# (a) Frozen snapshot
resp_imp = requests.get(
f"{BASE_URL}/v2/customnode/installed",
params={"mode": "imported"},
timeout=10,
)
assert resp_imp.status_code == 200, (
f"Expected 200 for imported mode, got {resp_imp.status_code}"
)
imported = resp_imp.json()
assert isinstance(imported, dict), (
f"Expected dict response, got {type(imported).__name__}"
)
# (b) E2E seed pack must appear in the startup snapshot
seed = "ComfyUI_SigmoidOffsetScheduler"
assert seed in imported, (
f"seed pack {seed!r} missing from imported snapshot; "
f"keys={list(imported)}"
)
# (c) Schema: each entry carries cnr_id / ver / enabled
entry = imported[seed]
assert isinstance(entry, dict), (
f"{seed} entry should be dict, got {type(entry).__name__}: {entry!r}"
)
for required in ("cnr_id", "ver", "enabled"):
assert required in entry, (
f"{seed} entry missing required field {required!r}: {entry!r}"
)
# (d) Frozen invariant (cheap form): no install has run since startup,
# so imported keys must equal default keys at this point.
resp_def = requests.get(
f"{BASE_URL}/v2/customnode/installed", timeout=10,
)
assert resp_def.status_code == 200
default = resp_def.json()
assert set(imported.keys()) == set(default.keys()), (
f"imported != default at startup (no install has run): "
f"only-imported={set(imported) - set(default)}, "
f"only-default={set(default) - set(imported)}"
)
# WI-OO Item 4 (bloat reviewer:ci-013 B7 stale-skip): removed
# `test_imported_mode_is_frozen_after_install` — the body was a TODO stub
# masked by a skip marker. With no install trigger between the two
# imported-mode GETs, `snap_before == snap_after` held trivially; the test
# could not prove the frozen-invariant it claimed. The E2E-DEBT for a true
# mid-session install (Strategy B) remains — when revisited, add a fresh
# test that actually exercises /v2/customnode/install or FS manipulation
# between the two snapshots. Strategy A (cheap equality at startup) is
# already covered by `test_installed_imported_mode` above.
# ---------------------------------------------------------------------------
# Tests — import_fail_info
# ---------------------------------------------------------------------------
class TestImportFailInfo:
"""Test POST /v2/customnode/import_fail_info."""
def test_unknown_cnr_id_returns_400(self, comfyui):
"""POST with unknown cnr_id returns 400 (no failure info available)."""
resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info",
json={"cnr_id": "nonexistent_pack_that_does_not_exist_12345"},
timeout=10,
)
assert resp.status_code == 400, (
f"Expected 400 for unknown cnr_id, got {resp.status_code}"
)
def test_missing_fields_returns_400(self, comfyui):
"""POST without cnr_id or url returns 400."""
resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info",
json={"invalid_field": "value"},
timeout=10,
)
assert resp.status_code == 400, (
f"Expected 400 for missing fields, got {resp.status_code}"
)
def test_invalid_body_returns_error(self, comfyui):
"""POST with non-dict body returns 400."""
resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info",
json="not-a-dict",
timeout=10,
)
assert resp.status_code == 400, (
f"Expected 400 for non-dict body, got {resp.status_code}"
)
# ---------------------------------------------------------------------------
# Tests — import_fail_info_bulk
# ---------------------------------------------------------------------------
class TestImportFailInfoBulk:
"""Test POST /v2/customnode/import_fail_info_bulk."""
def test_bulk_with_cnr_ids_returns_dict(self, comfyui):
"""POST with cnr_ids list returns 200 with results dict."""
resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info_bulk",
json={"cnr_ids": ["nonexistent_pack_12345"]},
timeout=30,
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"Expected dict response, got {type(data).__name__}"
)
# Unknown pack should have null value (no error info)
assert "nonexistent_pack_12345" in data, (
"Response should contain entry for requested cnr_id"
)
assert data["nonexistent_pack_12345"] is None, (
"Unknown pack should map to null (no import failure info)"
)
def test_bulk_empty_lists_returns_400(self, comfyui):
"""POST with empty cnr_ids and no urls returns 400."""
resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info_bulk",
json={"cnr_ids": [], "urls": []},
timeout=10,
)
assert resp.status_code == 400, (
f"Expected 400 for empty lists, got {resp.status_code}"
)
def test_bulk_with_urls_returns_dict(self, comfyui):
"""POST with urls list returns 200 + per-url result of None (unknown) or dict (found).
WI-M strengthening: previously only dict-type check. Now verifies
per-url result correctness: each requested URL MUST appear as a key,
and the value is either `None` (unknown URL — expected for the fake
URL we send) or a `dict` (populated fail-info). Anything else
(e.g. a bare string, a list, or missing-key) is a schema violation.
"""
fake_url = "https://github.com/nonexistent/nonexistent-node-pack"
resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info_bulk",
json={"urls": [fake_url]},
timeout=30,
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"Expected dict response, got {type(data).__name__}"
)
# Content: the URL we queried must be a key in the response.
assert fake_url in data, (
f"Requested URL missing from bulk response. Expected key {fake_url!r}, "
f"got keys: {list(data.keys())}"
)
# Per-URL value must be None (unknown, expected here) or dict (populated).
result = data[fake_url]
assert result is None or isinstance(result, dict), (
f"bulk[{fake_url!r}] must be None or dict, got {type(result).__name__}: {result!r}"
)
+48 -63
View File
@@ -98,7 +98,7 @@ def _queue_task(task: dict) -> None:
timeout=10,
)
resp.raise_for_status()
requests.get(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
requests.post(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
def _remove_pack(name: str) -> None:
@@ -192,14 +192,28 @@ class TestEndpointInstallUninstall:
def test_installed_list_shows_pack(self, comfyui):
"""GET /v2/customnode/installed includes the installed pack."""
# Self-contained precondition: ensure pack installed (don't rely on prior test)
if not _pack_exists(PACK_DIR_NAME):
pytest.skip("Pack not installed (previous test may have failed)")
_queue_task({
"ui_id": "e2e-setup",
"client_id": "e2e-setup",
"kind": "install",
"params": {
"id": PACK_ID,
"version": PACK_VERSION,
"selected_version": "latest",
"mode": "remote",
"channel": "default",
},
})
assert _wait_for(lambda: _pack_exists(PACK_DIR_NAME)), (
"Setup failed: pack not installed"
)
resp = requests.get(f"{BASE_URL}/v2/customnode/installed", timeout=10)
resp.raise_for_status()
installed = resp.json()
# Match by cnr_id (case-insensitive) following main repo pattern
package_found = any(
pkg.get("cnr_id", "").lower() == PACK_CNR_ID.lower()
for pkg in installed.values()
@@ -210,9 +224,32 @@ class TestEndpointInstallUninstall:
)
def test_uninstall_via_endpoint(self, comfyui):
"""POST /v2/manager/queue/task (uninstall) -> pack removed from disk."""
"""POST /v2/manager/queue/task (uninstall) -> pack removed from disk AND absent from API.
WI-N strengthening: previously FS-only (`not _pack_exists`). The API
`installed` endpoint is the authoritative contract surface: a pack is
"uninstalled" only when both the filesystem entry is gone AND the
Manager's in-memory installed-index no longer lists its cnr_id.
Defeats a regression where the FS delete succeeds but the installed
cache still reports the pack (e.g. cache-invalidation bug).
"""
# Self-contained: ensure pack is installed before testing uninstall
if not _pack_exists(PACK_DIR_NAME):
pytest.skip("Pack not installed (previous test may have failed)")
_queue_task({
"ui_id": "e2e-uninstall-setup",
"client_id": "e2e-uninstall-setup",
"kind": "install",
"params": {
"id": PACK_ID,
"version": PACK_VERSION,
"selected_version": "latest",
"mode": "remote",
"channel": "default",
},
})
assert _wait_for(lambda: _pack_exists(PACK_DIR_NAME)), (
"Setup failed: cannot install pack for uninstall test"
)
_queue_task({
"ui_id": "e2e-uninstall",
@@ -226,45 +263,7 @@ class TestEndpointInstallUninstall:
lambda: not _pack_exists(PACK_DIR_NAME),
), f"{PACK_DIR_NAME} still exists after uninstall ({POLL_TIMEOUT}s timeout)"
def test_installed_list_after_uninstall(self, comfyui):
"""After uninstall, pack no longer appears in installed list."""
if _pack_exists(PACK_DIR_NAME):
pytest.skip("Pack still exists (previous test may have failed)")
resp = requests.get(f"{BASE_URL}/v2/customnode/installed", timeout=10)
resp.raise_for_status()
installed = resp.json()
package_found = any(
pkg.get("cnr_id", "").lower() == PACK_CNR_ID.lower()
for pkg in installed.values()
if isinstance(pkg, dict) and pkg.get("cnr_id")
)
assert not package_found, f"{PACK_CNR_ID} still in installed list after uninstall"
def test_install_uninstall_cycle(self, comfyui):
"""Complete install/uninstall cycle in a single test."""
_remove_pack(PACK_DIR_NAME)
# Install
_queue_task({
"ui_id": "e2e-cycle-install",
"client_id": "e2e-cycle",
"kind": "install",
"params": {
"id": PACK_ID,
"version": PACK_VERSION,
"selected_version": "latest",
"mode": "remote",
"channel": "default",
},
})
assert _wait_for(
lambda: _pack_exists(PACK_DIR_NAME),
), f"Pack not installed within {POLL_TIMEOUT}s"
assert _has_tracking(PACK_DIR_NAME), "Pack missing .tracking"
# Verify in installed list
# API cross-check: cnr_id must be absent from /v2/customnode/installed.
resp = requests.get(f"{BASE_URL}/v2/customnode/installed", timeout=10)
resp.raise_for_status()
installed = resp.json()
@@ -273,30 +272,16 @@ class TestEndpointInstallUninstall:
for pkg in installed.values()
if isinstance(pkg, dict) and pkg.get("cnr_id")
)
assert package_found, f"{PACK_CNR_ID} not in installed list"
# Uninstall
_queue_task({
"ui_id": "e2e-cycle-uninstall",
"client_id": "e2e-cycle",
"kind": "uninstall",
"params": {
"node_name": PACK_CNR_ID,
},
})
assert _wait_for(
lambda: not _pack_exists(PACK_DIR_NAME),
), f"Pack not uninstalled within {POLL_TIMEOUT}s"
assert not package_found, (
f"FS delete succeeded but {PACK_CNR_ID} still present in "
f"/v2/customnode/installed — cache-invalidation regression. "
f"Keys: {list(installed.keys())}"
)
class TestEndpointStartup:
"""Verify ComfyUI startup with unified resolver."""
def test_comfyui_started(self, comfyui):
"""ComfyUI is running and responds to health check."""
resp = requests.get(f"{BASE_URL}/system_stats", timeout=10)
assert resp.status_code == 200
def test_startup_resolver_ran(self, comfyui):
"""Startup log contains unified resolver output."""
log_path = os.path.join(E2E_ROOT, "logs", "comfyui.log")
+105 -33
View File
@@ -79,43 +79,56 @@ def _start_comfyui() -> int:
global _comfyui_proc # noqa: PLW0603
log_dir = os.path.join(E2E_ROOT, "logs")
os.makedirs(log_dir, exist_ok=True)
log_file = open(os.path.join(log_dir, "comfyui.log"), "w") # noqa: SIM115
log_path = os.path.join(log_dir, "comfyui.log")
# Open the log file for the subprocess. We must keep this fd open while
# the process runs, but MUST close it on any exit path to avoid handle
# leaks (particularly on Windows where open handles block rmtree).
log_file = open(log_path, "w") # noqa: SIM115
def _read_tail() -> str:
with open(log_path) as f:
return f.read()[-2000:]
env = {
**os.environ,
"COMFYUI_PATH": COMFYUI_PATH,
"PYTHONUNBUFFERED": "1",
}
_comfyui_proc = subprocess.Popen(
[_venv_python(), "-u", os.path.join(COMFYUI_PATH, "main.py"),
"--listen", "127.0.0.1", "--port", str(PORT),
"--cpu", "--enable-manager"],
stdout=log_file, stderr=subprocess.STDOUT,
env=env,
)
# Wait for server to be ready.
# Manager may restart ComfyUI after startup dependency install (exit 0 → re-launch).
# If the process exits with code 0, keep polling — the restarted process will bind the port.
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
try:
_comfyui_proc = subprocess.Popen(
[_venv_python(), "-u", os.path.join(COMFYUI_PATH, "main.py"),
"--listen", "127.0.0.1", "--port", str(PORT),
"--cpu", "--enable-manager"],
stdout=log_file, stderr=subprocess.STDOUT,
env=env,
)
# Wait for server to be ready.
# Manager may restart ComfyUI after startup dependency install (exit 0 → re-launch).
# If the process exits with code 0, keep polling — the restarted process will bind the port.
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
try:
r = requests.get(f"{BASE_URL}/system_stats", timeout=2)
if r.status_code == 200:
return _comfyui_proc.pid
except requests.ConnectionError:
pass
if _comfyui_proc.poll() is not None:
if _comfyui_proc.returncode != 0:
log_file.close()
raise RuntimeError(
f"ComfyUI exited with code {_comfyui_proc.returncode}:\n{_read_tail()}"
)
# exit 0 = Manager restart. Keep polling for the restarted process.
time.sleep(1)
raise RuntimeError(f"ComfyUI did not start within 120s. Log:\n{_read_tail()}")
except Exception:
# Ensure log_file handle is released on any failure
try:
r = requests.get(f"{BASE_URL}/system_stats", timeout=2)
if r.status_code == 200:
return _comfyui_proc.pid
except requests.ConnectionError:
log_file.close()
except Exception: # noqa: BLE001
pass
if _comfyui_proc.poll() is not None:
if _comfyui_proc.returncode != 0:
log_file.close()
log_content = open(os.path.join(log_dir, "comfyui.log")).read()[-2000:] # noqa: SIM115
raise RuntimeError(
f"ComfyUI exited with code {_comfyui_proc.returncode}:\n{log_content}"
)
# exit 0 = Manager restart. Keep polling for the restarted process.
time.sleep(1)
log_file.close()
log_content = open(os.path.join(log_dir, "comfyui.log")).read()[-2000:] # noqa: SIM115
raise RuntimeError(f"ComfyUI did not start within 120s. Log:\n{log_content}")
raise
def _stop_comfyui():
@@ -135,7 +148,7 @@ def _queue_task(task: dict) -> None:
"""Queue a Manager task and start the worker."""
resp = requests.post(f"{BASE_URL}/v2/manager/queue/task", json=task, timeout=10)
resp.raise_for_status()
requests.get(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
requests.post(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
def _remove_pack(name: str) -> None:
@@ -222,7 +235,16 @@ class TestNightlyInstallCycle:
"""
def test_01_nightly_install(self, comfyui):
"""Nightly install should git-clone the repo into custom_nodes."""
"""Nightly install should git-clone the requested repo AND .git/config remote.origin.url matches.
WI-N strengthening: previously only `pack_exists + .git/ dir`. The
`.git/` directory alone proves *some* git artifact exists but NOT
that the clone targeted the requested URL — a regression where the
manager clones the wrong repo (or a cached stale one) would still
pass the old test. We now parse `.git/config` and assert the
`[remote "origin"] url = ...` line matches REPO_TEST1 (with or
without `.git` suffix — git accepts both forms).
"""
_remove_pack(PACK_TEST1)
assert not _pack_exists(PACK_TEST1), (
f"Failed to clean {PACK_TEST1} — file locks may be holding the directory"
@@ -240,9 +262,31 @@ class TestNightlyInstallCycle:
)
# Verify .git directory exists (git clone, not zip download)
git_dir = os.path.join(CUSTOM_NODES, PACK_TEST1, ".git")
pack_dir = os.path.join(CUSTOM_NODES, PACK_TEST1)
git_dir = os.path.join(pack_dir, ".git")
assert os.path.isdir(git_dir), "No .git directory — not a git clone"
# Parse .git/config for [remote "origin"] url and cross-check against
# the requested repo URL. Git stores either `REPO` or `REPO.git`.
git_config = os.path.join(git_dir, "config")
assert os.path.isfile(git_config), (
f".git/config missing at {git_config} — corrupted clone?"
)
import configparser
cp = configparser.ConfigParser()
cp.read(git_config)
origin_section = 'remote "origin"'
assert origin_section in cp, (
f"[{origin_section}] section missing from .git/config: sections={cp.sections()!r}"
)
remote_url = cp[origin_section].get("url", "").rstrip("/")
acceptable = {REPO_TEST1, REPO_TEST1 + ".git", REPO_TEST1.rstrip("/"), REPO_TEST1.rstrip("/") + ".git"}
assert remote_url in acceptable or remote_url.rstrip(".git") == REPO_TEST1.rstrip(".git"), (
f".git/config remote.origin.url mismatch — expected one of "
f"{sorted(acceptable)}, got {remote_url!r}. The clone targeted "
f"the wrong repository!"
)
def test_02_no_module_error(self, comfyui):
"""Server log must not contain ModuleNotFoundError (Phase 1 regression)."""
log_path = os.path.join(E2E_ROOT, "logs", "comfyui.log")
@@ -256,7 +300,13 @@ class TestNightlyInstallCycle:
)
def test_03_nightly_uninstall(self, comfyui):
"""Uninstall the nightly-installed pack."""
"""Uninstall the nightly-installed pack from disk AND from API installed-index.
WI-N strengthening: previously FS-only. The installed-index API is the
authoritative Manager contract; FS-deletion alone is insufficient to
call the operation "uninstalled". Cross-check that the pack key is
absent from `/v2/customnode/installed` response.
"""
if not _pack_exists(PACK_TEST1):
pytest.skip("Pack not installed (previous test may have failed)")
@@ -271,3 +321,25 @@ class TestNightlyInstallCycle:
assert _wait_for(lambda: not _pack_exists(PACK_TEST1)), (
f"{PACK_TEST1} still exists after uninstall ({POLL_TIMEOUT}s timeout)"
)
# API cross-check: pack must be absent from /v2/customnode/installed.
# Nightly packs appear keyed by directory name (no cnr_id for git-URL installs),
# so membership check uses the pack's dir name.
resp = requests.get(f"{BASE_URL}/v2/customnode/installed", timeout=10)
resp.raise_for_status()
installed = resp.json()
assert PACK_TEST1 not in installed, (
f"FS delete succeeded but {PACK_TEST1!r} still present in "
f"/v2/customnode/installed — cache-invalidation regression. "
f"Keys: {list(installed.keys())}"
)
# Defensive: also check by aux_id / cnr_id in case of schema variation.
for pkg_key, pkg in installed.items():
if isinstance(pkg, dict):
assert (
pkg.get("cnr_id") != PACK_TEST1
and pkg.get("aux_id") != PACK_TEST1
), (
f"Installed entry {pkg_key!r} still references {PACK_TEST1!r} "
f"via cnr_id/aux_id: {pkg!r}"
)
+308
View File
@@ -0,0 +1,308 @@
"""E2E positive-path tests for legacy-only GET endpoints.
SCOPE — closes the 6 pytest-N gaps from reports/api-coverage-matrix.md
(WI-TT). Each target endpoint is registered ONLY in
`comfyui_manager/legacy/manager_server.py` and thus reachable only when
ComfyUI runs with `--enable-manager-legacy-ui`.
Endpoints covered:
- GET /customnode/alternatives (legacy L1072)
- GET /v2/customnode/disabled_versions/{node_name} (legacy L1273)
- GET /v2/customnode/getlist (legacy L1018)
- GET /v2/customnode/versions/{node_name} (legacy L1262)
- GET /v2/externalmodel/getlist (legacy L1143)
- GET /v2/manager/notice (legacy L1747)
Why a separate file:
comfyui_manager/__init__.py loads `glob.manager_server` XOR
`legacy.manager_server` (mutex via args.enable_manager_legacy_ui), so
these routes do not exist under the glob-mode fixture used by most
other E2E suites. Mirrors the fixture pattern in
`test_e2e_csrf_legacy.py` — separate module-scoped `comfyui_legacy`
fixture that launches via `start_comfyui_legacy.sh`.
Handler-shape notes (for reviewers):
- disabled_versions returns HTTP 400 when the given node has NO
disabled versions (handler L1283-1286). This is not a param-validation
error — it is the handler's convention for "empty result". The seed
E2E pack (`ComfyUI_SigmoidOffsetScheduler`) installs cleanly with no
disabled versions, so the positive path here asserts the endpoint is
reachable and the param parses — status ∈ {200, 400} with per-branch
body schema checks. Documented upstream as a handler design quirk.
- notice returns `text/html` (not JSON); handler L1787 returns
`web.Response(text=markdown_content, status=200)`.
Requires a pre-built E2E environment (from setup_e2e_env.sh).
"""
from __future__ import annotations
import os
import subprocess
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
SEED_PACK = "ComfyUI_SigmoidOffsetScheduler"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready",
)
def _start_comfyui_legacy() -> int:
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui_legacy.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI (legacy):\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID:\n{r.stdout}")
def _stop_comfyui():
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
@pytest.fixture(scope="module")
def comfyui_legacy():
pid = _start_comfyui_legacy()
yield pid
_stop_comfyui()
class TestLegacyCustomNodeAlternatives:
"""GET /customnode/alternatives?mode=local (wi-031)."""
def test_returns_dict_of_alternatives(self, comfyui_legacy):
resp = requests.get(
f"{BASE_URL}/customnode/alternatives",
params={"mode": "local"},
timeout=30,
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"alternatives response should be a dict keyed by unified id, "
f"got {type(data).__name__}"
)
class TestLegacyCustomNodeDisabledVersions:
"""GET /v2/customnode/disabled_versions/{node_name} (wi-032).
The handler returns 200 + list[{version}] when disabled versions exist,
and 400 otherwise (empty-result convention — not a validation error).
Seed pack has no disabled versions, so positive path here is:
endpoint reachable + param parsed correctly + response-shape on each
branch.
"""
def test_endpoint_reachable_and_parses_param(self, comfyui_legacy):
resp = requests.get(
f"{BASE_URL}/v2/customnode/disabled_versions/{SEED_PACK}",
timeout=10,
)
assert resp.status_code in (200, 400), (
f"disabled_versions should return 200 (has disabled) or 400 "
f"(none), got {resp.status_code}: {resp.text[:200]}"
)
if resp.status_code == 200:
data = resp.json()
assert isinstance(data, list), (
f"disabled_versions 200 body should be a list, "
f"got {type(data).__name__}"
)
for entry in data:
assert "version" in entry, (
f"each entry should have 'version' key, got {entry!r}"
)
class TestLegacyCustomNodeGetList:
"""GET /v2/customnode/getlist?mode=local (wi-033)."""
def test_returns_channel_and_node_packs(self, comfyui_legacy):
resp = requests.get(
f"{BASE_URL}/v2/customnode/getlist",
params={"mode": "local", "skip_update": "true"},
timeout=60,
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"getlist response should be a dict, got {type(data).__name__}"
)
assert "channel" in data, (
f"getlist response missing 'channel' field: keys={list(data)}"
)
assert "node_packs" in data, (
f"getlist response missing 'node_packs' field: keys={list(data)}"
)
assert isinstance(data["node_packs"], dict), (
f"node_packs should be a dict, got {type(data['node_packs']).__name__}"
)
class TestLegacyCustomNodeVersions:
"""GET /v2/customnode/versions/{node_name} (wi-034).
The seed pack is a CNR pack and should have at least one version.
"""
def test_returns_versions_list_for_seed_pack(self, comfyui_legacy):
resp = requests.get(
f"{BASE_URL}/v2/customnode/versions/{SEED_PACK}",
timeout=15,
)
assert resp.status_code == 200, (
f"Expected 200 for known CNR pack {SEED_PACK!r}, "
f"got {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, list), (
f"versions response should be a list, got {type(data).__name__}"
)
assert len(data) > 0, (
f"CNR pack {SEED_PACK!r} should report at least one version, "
f"got empty list"
)
class TestLegacyExternalModelGetList:
"""GET /v2/externalmodel/getlist?mode=local (wi-035)."""
def test_returns_models_payload(self, comfyui_legacy):
resp = requests.get(
f"{BASE_URL}/v2/externalmodel/getlist",
params={"mode": "local"},
timeout=30,
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"externalmodel/getlist should return a dict, "
f"got {type(data).__name__}"
)
assert "models" in data, (
f"externalmodel/getlist missing 'models' field: keys={list(data)}"
)
assert isinstance(data["models"], list), (
f"'models' should be a list, got {type(data['models']).__name__}"
)
class TestLegacyManagerNotice:
"""GET /v2/manager/notice (wi-036).
Returns text/html (not JSON) — handler concatenates markdown fragments
or a fallback 'Unable to retrieve Notice' string. Both branches return
HTTP 200, so the positive-path assertion is status + non-empty body.
"""
def test_returns_text_body(self, comfyui_legacy):
resp = requests.get(f"{BASE_URL}/v2/manager/notice", timeout=30)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
)
assert resp.text, "notice body should be non-empty (markdown or fallback)"
class TestLegacyQueueBatch:
"""POST /v2/manager/queue/batch (wi-039).
Handler shape (legacy/manager_server.py:740-801):
- Request: JSON dict whose top-level keys select actions —
update_all | reinstall | install | uninstall | update |
update_comfyui | disable | install_model | fix.
Unrecognized keys are silently skipped (no-match falls through
the if/elif chain).
- Response: always `{"failed": [list of failed ids]}`, status 200.
- Side effect: `finalize_temp_queue_batch(json_data, failed)` writes
a batch snapshot to the history store IFF the action helpers
populated `temp_queue_batch`. With an empty or unrecognized-key
payload, `temp_queue_batch` stays empty and no history is written
(guard: `if len(temp_queue_batch):` at L444).
- `_queue_start()` is called unconditionally to nudge the worker.
Safe-payload choice: empty JSON body `{}`. Rationale —
(a) exercises the full handler path (request parse → action
for-loop no-op → finalize-with-empty → queue_start → 200 json),
(b) leaves zero side effects on installed packs / disk state,
(c) still round-trips through the aiohttp handler lock and
`temp_queue_batch` snapshot guard so a future regression
(e.g., unconditional history write, lock deadlock) would be
caught.
The dispatch's side-effect verification is covered indirectly: the
test asserts queue/status is still 200 after POST, proving the lock
released and the worker nudge completed cleanly. History-growth
verification would require an expensive mutating batch, which the
dispatch explicitly discourages.
"""
def test_accepts_empty_payload_returns_failed_list(self, comfyui_legacy):
resp = requests.post(
f"{BASE_URL}/v2/manager/queue/batch",
json={},
timeout=15,
)
assert resp.status_code == 200, (
f"Expected 200 for empty-batch payload, got "
f"{resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"queue/batch response should be a dict, "
f"got {type(data).__name__}"
)
assert "failed" in data, (
f"queue/batch response missing 'failed' key: {data!r}"
)
assert isinstance(data["failed"], list), (
f"'failed' should be a list, got {type(data['failed']).__name__}"
)
assert data["failed"] == [], (
f"no actions performed → 'failed' should be empty, "
f"got {data['failed']!r}"
)
# Side-effect liveness check: queue/status still 200 after POST,
# proving the worker lock was released cleanly.
status_resp = requests.get(
f"{BASE_URL}/v2/manager/queue/status", timeout=10
)
assert status_resp.status_code == 200, (
f"queue/status should remain callable after queue/batch POST, "
f"got {status_resp.status_code}: {status_resp.text[:200]}"
)
+687
View File
@@ -0,0 +1,687 @@
"""E2E REAL-execution tests for legacy-UI state-changing endpoints (WI-YY).
SCOPE — closes 6 Playwright-mock gaps by executing the real backend
operation via HTTP:
Default-security fixture (middle+ / no-gate endpoints):
- wi-020 POST /v2/manager/queue/install_model (tiny TAEF1 model, <5MB)
- wi-024 POST /v2/manager/queue/update_comfyui (safe via env var)
Permissive-security fixture (high+ endpoints — normal- harness):
- wi-014 POST /v2/comfyui_manager/comfyui_switch_version (no-op self-switch)
- wi-037 POST /v2/customnode/install/git_url (nodepack-test1-do-not-install)
- wi-038 POST /v2/customnode/install/pip (text-unidecode)
Pre-seeded broken-pack fixture (no-gate endpoint, needs scan-time state):
- wi-015 POST /v2/customnode/import_fail_info (pre-seeded broken pack)
Safety belt — COMFYUI_MANAGER_SKIP_MANAGER_REQUIREMENTS=1 is exported in
tests/e2e/scripts/start_comfyui.sh (WI-YY change). Any install/update path
that would normally run `pip install -r manager_requirements.txt` becomes
a no-op log line — essential for WI-YY real E2E: without this, triggering
the update_comfyui queue worker could run unbounded pip installs against
the test venv.
Permissive harness security rationale:
wi-014/037/038 execute arbitrary remote code (version switch, git
clone, pip install) and are gated at `high+` precisely to prevent
such operations at default security. The permissive harness
(start_comfyui_permissive.sh) reflects the production use case
these endpoints exist to serve — operators in a trusted environment
lower security_level to normal-/weak to enable these features. The
200 path IS a supported feature, and testing it requires exactly
this configuration. Permissive harness uses HARDCODED trusted inputs:
- wi-014: the CURRENT ComfyUI version (self-switch no-op)
- wi-037: https://github.com/ltdrdata/nodepack-test1-do-not-install
(project's test-fixture repo, also used by tests/cli/test_uv_compile.py)
- wi-038: text-unidecode (pure-Python, ~8KB, idempotent)
User-input-derived values MUST NEVER be substituted. The 403 contract
at default security remains the positive-path security behavior in
production — verified by test_e2e_csrf_legacy.py and
test_e2e_secgate_default.py.
WI-YY.3 (wi-015) real-E2E strategy:
The import_fail_info endpoint returns info for packs that failed to
import during the ComfyUI custom_nodes/ startup scan. To exercise
the 200 path with real state, we pre-seed a known-broken pack via
git clone (skip pip install). On server start, the scan attempts
to import the pack, it fails, prestartup_script.py L302-305
captures the stderr traceback into
cm_global.error_dict[<module_name>], and the pack is registered
in core.unified_manager (manager_core.py:541-561).
Pack selection: ComfyUI-YoloWorld-EfficientSAM — user-suggested.
Its production failure mode is that requirements.txt pins
UNINSTALLABLE packages (unresolvable versions / removed-from-index
/ etc), so even the normal install flow
(`/v2/customnode/install/git_url` → gitclone_install → pip install
-r requirements.txt) leaves the pack dir present but dependencies
unsatisfied → scan import fails → the exact state this endpoint
is meant to report on. The pre-seed fixture (git clone, skip pip)
reproduces the END STATE that the production install path reaches
after pip-failure, without the cost and non-determinism of running
the failing pip. This is NOT "missing deps we chose not to
install" — it is the genuine production failure mode packaged as
a deterministic fixture. Tiny clone (few KB of Python).
Requires a pre-built E2E environment (from setup_e2e_env.sh).
"""
from __future__ import annotations
import os
import subprocess
import time
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
# Small, well-known VAE-approx model from the whitelist (verified via
# GET /v2/externalmodel/getlist?mode=cache — 4.71MB, public raw URL).
# Selected for minimal download time + deterministic whitelist membership.
TAEF1_MODEL_SPEC = {
"name": "TAEF1 Decoder",
"type": "TAESD",
"base": "FLUX.1",
"save_path": "vae_approx",
"description": "WI-YY real-E2E install — TAEF1 decoder",
"reference": "https://github.com/madebyollin/taesd",
"filename": "taef1_decoder.pth",
"url": "https://github.com/madebyollin/taesd/raw/main/taef1_decoder.pth",
}
# HARDCODED TRUSTED INPUTS for the permissive-harness suite.
# Never substitute user-derived values — these constants exist to test
# the supported 200-path of features the operator explicitly enables by
# lowering security_level to normal-.
# TRUSTED_GIT_URL: same purpose-built test fixture used by
# tests/cli/test_uv_compile.py (REPO_TEST1 at L41). The 'do-not-install'
# suffix in the repo name is the project's convention for
# test-fixture-only packs — safe to install and delete repeatedly in E2E.
TRUSTED_GIT_URL = "https://github.com/ltdrdata/nodepack-test1-do-not-install"
TRUSTED_GIT_DIRNAME = "nodepack-test1-do-not-install"
TRUSTED_PIP_PKG = "text-unidecode" # pure-Python, ~8KB, idempotent
# Broken-pack pre-seed target for wi-015 real E2E.
# User-suggested: ZHO-ZHO-ZHO/ComfyUI-YoloWorld-EfficientSAM. In
# production, this pack's requirements.txt pins uninstallable
# packages, so gitclone_install → pip install -r leaves the pack dir
# present but deps unsatisfied. Our pre-seed (git clone, skip pip)
# reproduces that END STATE deterministically — see module docstring
# §"WI-YY.3 (wi-015) real-E2E strategy".
BROKEN_PACK_URL = "https://github.com/ZHO-ZHO-ZHO/ComfyUI-YoloWorld-EfficientSAM"
BROKEN_PACK_DIRNAME = "ComfyUI-YoloWorld-EfficientSAM"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready",
)
def _start_comfyui_legacy() -> int:
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui_legacy.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI (legacy):\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID:\n{r.stdout}")
def _stop_comfyui():
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
@pytest.fixture(scope="module")
def comfyui_legacy():
pid = _start_comfyui_legacy()
yield pid
_stop_comfyui()
def _start_comfyui_permissive() -> int:
"""Launch via start_comfyui_permissive.sh — patches config.ini to
`security_level = normal-` (backup at config.ini.before-permissive)
then delegates to start_comfyui.sh with ENABLE_LEGACY_UI=1.
The permissive fixture MUST restore config on teardown.
"""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui_permissive.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI (permissive):\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID:\n{r.stdout}")
def _restore_permissive_config():
"""Restore $CONFIG from $CONFIG.before-permissive. Safe to call even
if the backup is missing (idempotent). Mirrors the pattern used by
test_e2e_secgate_strict.py for the strict harness.
"""
config = os.path.join(
COMFYUI_PATH, "user", "__manager", "config.ini"
)
backup = config + ".before-permissive"
if os.path.isfile(backup):
import shutil
shutil.move(backup, config)
@pytest.fixture(scope="module")
def comfyui_permissive():
"""Module-scoped fixture: start server with security_level=normal-,
tear down with config restore. Use for wi-014/037/038 which require
`high+` (security_utils.py:20-26 allows weak/normal- at is_local_mode).
"""
pid = _start_comfyui_permissive()
try:
yield pid
finally:
_stop_comfyui()
_restore_permissive_config()
def _seed_broken_pack() -> str:
"""Pre-seed the broken pack via git clone --depth 1. Returns the
absolute path to the cloned directory. pip install is skipped —
this reproduces the production end-state where gitclone_install +
pip install -r requirements.txt leaves the pack dir in place
despite pip failing on uninstallable package pins (see module
docstring §"WI-YY.3 real-E2E strategy"). The ImportError at scan
time populates cm_global.error_dict (prestartup_script.py:
302-305) and registers the pack in unified_manager
(manager_core.py:541-561).
"""
target = os.path.join(COMFYUI_PATH, "custom_nodes", BROKEN_PACK_DIRNAME)
if os.path.isdir(target):
import shutil
shutil.rmtree(target)
r = subprocess.run(
["git", "clone", "--depth", "1", BROKEN_PACK_URL, target],
capture_output=True,
text=True,
timeout=60,
)
if r.returncode != 0:
raise RuntimeError(
f"Failed to clone broken-pack seed {BROKEN_PACK_URL!r}: "
f"rc={r.returncode} stderr={r.stderr!r}"
)
assert os.path.isdir(os.path.join(target, ".git")), (
f"Clone reported success but {target}/.git missing"
)
return target
def _remove_broken_pack():
target = os.path.join(COMFYUI_PATH, "custom_nodes", BROKEN_PACK_DIRNAME)
if os.path.isdir(target):
import shutil
shutil.rmtree(target, ignore_errors=True)
@pytest.fixture(scope="module")
def comfyui_with_broken_pack():
"""Module-scoped fixture: pre-seed a broken pack, start the legacy
server so its scan captures the import failure, yield, then stop
server + remove the pack.
Uses the default-security legacy launcher because
/v2/customnode/import_fail_info has no security gate
(legacy/manager_server.py:1289-1303).
"""
_seed_broken_pack()
try:
pid = _start_comfyui_legacy()
try:
yield pid
finally:
_stop_comfyui()
finally:
_remove_broken_pack()
def _wait_for_file(target: str, timeout: float, poll_interval: float = 1.0) -> bool:
"""Poll for target file existence up to `timeout` seconds. Returns
True if the file appears (and has non-zero size) before timeout.
Relying on disk artifact rather than queue/status counters because
task_batch_queue drains to empty post-completion, so
`queue/status.total_count` returns to 0 once the worker is idle —
it is not a persistent completion counter.
"""
deadline = time.time() + timeout
while time.time() < deadline:
if os.path.exists(target) and os.path.getsize(target) > 0:
return True
time.sleep(poll_interval)
return False
class TestUpdateComfyuiQueued:
"""Real E2E for wi-024 POST /v2/manager/queue/update_comfyui.
At default `security_level = normal` the handler has no security
gate (legacy/manager_server.py:1572-1576 — unlike install/git_url
and install/pip which require `high+`). The handler appends an
("update-comfyui", (...)) entry to temp_queue_batch and returns 200.
It does NOT start the worker — that is triggered separately by
POST /v2/manager/queue/batch at handler L797-799 (the UI batches
update_comfyui:true via queue/batch in production, per
comfyui-manager.js:478-480).
Therefore the real-E2E contract for this endpoint is:
(a) HTTP 200 return,
(b) temp_queue_batch mutation observable via subsequent queue/status
delta once the worker processes (when batch is called later).
We verify (a) — the direct endpoint's immediate contract. Triggering
the worker with a real git pull would risk advancing the test-env
ComfyUI git state; the env var only protects against pip install
runaway, not against HEAD advancing.
"""
def test_direct_endpoint_returns_200(self, comfyui_legacy):
resp = requests.post(
f"{BASE_URL}/v2/manager/queue/update_comfyui", timeout=15
)
assert resp.status_code == 200, (
f"update_comfyui should return 200 at default security_level, "
f"got {resp.status_code}: {resp.text[:200]}"
)
class TestInstallModelRealDownload:
"""Real E2E for wi-020 POST /v2/manager/queue/install_model.
Flow:
1. Clean any pre-existing target file (test isolation).
2. POST install_model with the TAEF1 Decoder spec (whitelisted,
~4.71MB from github.com/madebyollin/taesd raw).
3. POST /v2/manager/queue/batch with empty body — this nudges
_queue_start() (handler L797-799) to drain temp_queue_batch.
4. Poll for the .pth file to land at models/vae_approx/ with
non-zero size (primary completion signal — task_batch_queue
drains to empty post-completion so total_count returns to 0,
making queue/status an unreliable completion proxy).
5. Verify the downloaded file size matches expected ~4.7MB.
6. Teardown deletes the file.
Handler security: middle+ at local_mode (is_loopback 127.0.0.1) allows
`normal` → request accepted. Whitelist check at L1649 passes because
the model entry IS in model-list.json (verified via
GET /v2/externalmodel/getlist). Non-safetensors check at L1653 is
bypassed because is_allowed_security_level('high+') is false — falls
into the whitelist-url branch which DOES find a match.
Safety belt: COMFYUI_MANAGER_SKIP_MANAGER_REQUIREMENTS=1 is exported
at server startup. Download itself is direct HTTP (no pip run), so
the env var is a belt+suspenders for any transitive install that
might trigger.
"""
def _target_path(self) -> str:
return os.path.join(
COMFYUI_PATH, "models", "vae_approx", TAEF1_MODEL_SPEC["filename"]
)
def test_install_model_downloads_file(self, comfyui_legacy):
target = self._target_path()
# (1) Pre-clean — idempotent test setup.
if os.path.exists(target):
os.remove(target)
assert not os.path.exists(target), (
f"Pre-condition: {target} should not exist before install"
)
try:
# (2) Queue the install.
post_resp = requests.post(
f"{BASE_URL}/v2/manager/queue/install_model",
json=TAEF1_MODEL_SPEC,
timeout=15,
)
assert post_resp.status_code == 200, (
f"install_model should return 200 for whitelisted model, "
f"got {post_resp.status_code}: {post_resp.text[:300]}"
)
# (3) Nudge the worker via an empty batch call. This triggers
# _queue_start() at L797-799 which drains temp_queue_batch.
batch_resp = requests.post(
f"{BASE_URL}/v2/manager/queue/batch",
json={},
timeout=15,
)
assert batch_resp.status_code == 200, (
f"queue/batch nudge should return 200, "
f"got {batch_resp.status_code}"
)
# (4) Poll for the target file to appear. Primary completion
# signal — disk artifact is the durable proof of a real
# download (HTTP body was written to the expected location).
appeared = _wait_for_file(target, timeout=120.0, poll_interval=2.0)
assert appeared, (
f"Install task did not produce {target} within 120s; "
f"last queue status: "
f"{requests.get(f'{BASE_URL}/v2/manager/queue/status', timeout=5).json()!r}"
)
# (5) Verify the file is the real model (not a placeholder /
# error HTML).
size = os.path.getsize(target)
assert size > 1_000_000, (
f"Downloaded file {target} is suspiciously small ({size} bytes); "
f"expected ~4.7MB for TAEF1 decoder"
)
finally:
# (6) Teardown — delete the downloaded file regardless of
# assertion outcome, so re-runs start clean.
if os.path.exists(target):
os.remove(target)
# ---------------------------------------------------------------------------
# Permissive-harness suite (high+ gated endpoints)
# ---------------------------------------------------------------------------
class TestSwitchComfyuiSelfSwitch:
"""Real E2E for wi-014 POST /v2/comfyui_manager/comfyui_switch_version.
Strategy: GET /v2/comfyui_manager/comfyui_versions to discover the
CURRENTLY checked-out version, then POST switch to that same version
— a no-op self-switch that exercises the 200 branch of the handler
(legacy/manager_server.py:1590-1604) without advancing the test-env
ComfyUI git HEAD. core.switch_comfyui is called with the current
version; any git checkout/fetch of the already-checked-out ref is
idempotent.
"""
def test_self_switch_returns_200(self, comfyui_permissive):
versions_resp = requests.get(
f"{BASE_URL}/v2/comfyui_manager/comfyui_versions", timeout=30
)
assert versions_resp.status_code == 200, (
f"comfyui_versions GET should return 200, "
f"got {versions_resp.status_code}: {versions_resp.text[:200]}"
)
current = versions_resp.json().get("current")
assert current, (
f"comfyui_versions response missing 'current' field: "
f"{versions_resp.json()!r}"
)
# WI #258: migrated from query-string (params=) to JSON body (json=).
# Legacy handler only reads `ver` from the body; client_id/ui_id are
# tolerated if present but not required by legacy.
switch_resp = requests.post(
f"{BASE_URL}/v2/comfyui_manager/comfyui_switch_version",
json={"ver": current},
timeout=60,
)
assert switch_resp.status_code == 200, (
f"comfyui_switch_version to current={current!r} should return "
f"200 at security_level=normal- (high+ allowed), "
f"got {switch_resp.status_code}: {switch_resp.text[:300]}"
)
class TestInstallViaGitUrlRealClone:
"""Real E2E for wi-037 POST /v2/customnode/install/git_url.
Strategy: POST with body=TRUSTED_GIT_URL (plain text). Handler
(legacy/manager_server.py:1502-1519) runs core.gitclone_install(url)
synchronously; 200 on success + 'After restarting ComfyUI' log;
'skip' action on already-installed → also 200.
Teardown: rm -rf custom_nodes/ComfyUI_examples so re-runs are clean.
"""
def _target_dir(self) -> str:
return os.path.join(COMFYUI_PATH, "custom_nodes", TRUSTED_GIT_DIRNAME)
def test_install_via_git_url_clones_repo(self, comfyui_permissive):
target = self._target_dir()
# Pre-clean — idempotent test setup.
if os.path.isdir(target):
import shutil
shutil.rmtree(target)
assert not os.path.isdir(target), (
f"Pre-condition: {target} should not exist before install"
)
try:
resp = requests.post(
f"{BASE_URL}/v2/customnode/install/git_url",
data=TRUSTED_GIT_URL,
headers={"Content-Type": "text/plain"},
timeout=180,
)
assert resp.status_code == 200, (
f"install/git_url with trusted URL {TRUSTED_GIT_URL!r} "
f"should return 200 at security_level=normal-, got "
f"{resp.status_code}: {resp.text[:300]}"
)
assert os.path.isdir(target), (
f"Clone reported success but directory missing at {target}"
)
# Verify this looks like a real clone (has .git), not a stub.
assert os.path.isdir(os.path.join(target, ".git")), (
f"{target} exists but has no .git subdir — not a real clone"
)
finally:
if os.path.isdir(target):
import shutil
shutil.rmtree(target, ignore_errors=True)
class TestInstallPipRealExecute:
"""Real E2E for wi-038 POST /v2/customnode/install/pip.
Strategy: POST with body=TRUSTED_PIP_PKG. Handler
(legacy/manager_server.py:1522-1531) runs core.pip_install(pkgs)
which builds a `['#FORCE', 'pip', 'install', '-U', <pkg>]` command
and calls try_install_script. The `#FORCE` prefix marks the command
as LAZY — reserve_script appends it to
`user/__manager/startup-scripts/install-scripts.txt`, to be
executed by ComfyUI on the NEXT startup (legacy/manager_core.py:
1830-1837, 1871-1876). The command does NOT run synchronously.
Therefore the real-E2E contract here is:
(a) POST returns 200 immediately,
(b) install-scripts.txt contains a newly-appended line referencing
the trusted package name AND the 'pip install' verb.
Verifying the eventual pip install actually runs would require
restarting ComfyUI and waiting for the startup hook — out of scope
for this suite's module-scoped fixture. Contract (b) is the
durable on-disk evidence that the handler correctly scheduled the
install.
Teardown: truncate the script file so re-runs start with a clean
lazy-queue.
"""
def _script_path(self) -> str:
return os.path.join(
COMFYUI_PATH,
"user", "__manager", "startup-scripts", "install-scripts.txt",
)
def test_install_pip_schedules_lazy_install(self, comfyui_permissive):
script_path = self._script_path()
# Capture pre-state so we can assert a NEW line was appended.
pre_lines: list[str] = []
if os.path.isfile(script_path):
with open(script_path, "r") as f:
pre_lines = f.readlines()
try:
resp = requests.post(
f"{BASE_URL}/v2/customnode/install/pip",
data=TRUSTED_PIP_PKG,
headers={"Content-Type": "text/plain"},
timeout=30,
)
assert resp.status_code == 200, (
f"install/pip with trusted pkg {TRUSTED_PIP_PKG!r} should "
f"return 200 at security_level=normal-, got "
f"{resp.status_code}: {resp.text[:300]}"
)
# Verify a NEW line was appended AND it references the
# trusted package + the pip install verb.
assert os.path.isfile(script_path), (
f"install-scripts.txt not created at {script_path}"
)
with open(script_path, "r") as f:
post_lines = f.readlines()
new_lines = post_lines[len(pre_lines):]
assert new_lines, (
f"No new entry appended to {script_path} after POST"
)
joined = "".join(new_lines)
assert TRUSTED_PIP_PKG in joined, (
f"New entry does not reference {TRUSTED_PIP_PKG!r}: "
f"{joined!r}"
)
assert "pip" in joined and "install" in joined, (
f"New entry does not look like a pip install command: "
f"{joined!r}"
)
finally:
# Restore pre-state so other tests / re-runs are unaffected.
if os.path.isfile(script_path):
with open(script_path, "w") as f:
f.writelines(pre_lines)
# ---------------------------------------------------------------------------
# Broken-pack pre-seed suite (wi-015 real E2E)
# ---------------------------------------------------------------------------
class TestImportFailInfoReal:
"""Real E2E for wi-015 POST /v2/customnode/import_fail_info (WI-YY.3).
See module docstring for strategy. Two assertions:
1. POST with `{cnr_id: BROKEN_PACK_DIRNAME}` returns 200 + dict
body containing the captured error info (at minimum a `msg`
field per prestartup_script.py:303; the handler forwards the
full `{name, path, msg}` record).
2. POST with an unrelated cnr_id returns 400 (control — verifies
the handler doesn't leak info for packs that never failed).
Identifier discovery (empirical — verified via import_fail_info_bulk
probe during implementation): for a non-CNR git-cloned pack, the
lookup key is the DIRECTORY BASENAME as `cnr_id`. The repo URL and
the aux_id form (`author/repo`) both return 400 because
get_module_name (manager_core.py:398-407) does NOT match on URL
for active_nodes entries; URL matching only happens via
unknown_active_nodes which requires a different registration path
than what a plain `git clone` produces (non-CNR packs without aux_id
metadata land in active_nodes keyed by directory basename). The
cnr_id=basename route is the supported single-endpoint lookup for
this class of pack.
"""
def test_import_fail_info_returns_error(self, comfyui_with_broken_pack):
# Warm-up: /v2/customnode/import_fail_info (single) does NOT call
# unified_manager.reload — it assumes state is already loaded.
# The paired BULK endpoint at manager_server.py:1320-1321 calls
# `reload('cache')` + `get_custom_nodes('default', 'cache')`
# which runs update_cache_at_path (manager_core.py:742) per
# directory — this is what registers our broken pack in
# unified_manager via its directory-basename cnr_id
# (manager_core.py:557-561; aux_id form = author/repo,
# cnr_id form = directory basename). Without this warmup, the
# single-endpoint POST sees an empty active_nodes/
# unknown_active_nodes and returns 400.
warm_resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info_bulk",
json={"cnr_ids": ["__warmup__"]},
timeout=60,
)
assert warm_resp.status_code == 200, (
f"warmup bulk failed: {warm_resp.status_code} "
f"{warm_resp.text[:200]}"
)
# Identifier discovery (per dispatch): the key that unlocks the
# error_dict lookup for a non-CNR git-cloned pack is the
# DIRECTORY BASENAME, NOT the repo URL and NOT the aux_id.
# Verified empirically via the bulk probe — full-URL and
# author/repo both returned null, only the basename cnr_id key
# returned the captured error info. Route: handler calls
# unified_manager.get_module_name(cnr_id) which reads
# active_nodes[cnr_id] → (version, fullpath), returning
# basename(fullpath) == BROKEN_PACK_DIRNAME.
resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info",
json={"cnr_id": BROKEN_PACK_DIRNAME},
timeout=15,
)
assert resp.status_code == 200, (
f"import_fail_info should return 200 for pre-seeded broken "
f"pack cnr_id={BROKEN_PACK_DIRNAME!r}; got {resp.status_code}: "
f"{resp.text[:300]}"
)
data = resp.json()
assert isinstance(data, dict), (
f"Response body should be a dict, got {type(data).__name__}"
)
# prestartup_script.py:303 stores {'name', 'path', 'msg'} — `msg`
# accumulates the captured stderr output from the failing import.
assert "msg" in data, (
f"Response dict missing 'msg' field — import error info not "
f"captured. keys={list(data)}"
)
assert data["msg"], (
f"'msg' field is empty — expected captured stderr from the "
f"failing import. Full response: {data!r}"
)
def test_import_fail_info_unknown_cnr_id_returns_400(self, comfyui_with_broken_pack):
# Control: for a cnr_id NOT in active_nodes/unknown_active_nodes,
# get_module_name returns None and the handler returns 400
# (manager_server.py:1303). Distinguishes "info available" from
# "no matching pack registered".
resp = requests.post(
f"{BASE_URL}/v2/customnode/import_fail_info",
json={"cnr_id": "nonexistent_pack_xyz_123"},
timeout=15,
)
assert resp.status_code == 400, (
f"import_fail_info for unknown cnr_id should return 400; got "
f"{resp.status_code}: {resp.text[:200]}"
)
+561
View File
@@ -0,0 +1,561 @@
"""E2E tests for ComfyUI Manager queue lifecycle endpoints.
Exercises the queue management endpoints on a running ComfyUI instance:
- GET /v2/manager/queue/status — queue status JSON
- GET /v2/manager/queue/history — task history (filterable)
- GET /v2/manager/queue/history_list — batch history IDs
- POST /v2/manager/queue/reset — reset queue
- POST /v2/manager/queue/start — start processing
Scenario:
reset → verify clean status → queue a task → start → wait for
completion → check history → verify history_list → reset → verify
status returns to clean state.
Requires a pre-built E2E environment (from setup_e2e_env.sh).
Set E2E_ROOT env var to point at it, or the tests will be skipped.
Usage:
E2E_ROOT=/tmp/e2e_full_test pytest tests/e2e/test_e2e_queue_lifecycle.py -v
"""
from __future__ import annotations
import os
import subprocess
import time
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
HISTORY_DIR = (
os.path.join(COMFYUI_PATH, "user", "__manager", "batch_history")
if COMFYUI_PATH
else ""
)
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
# Polling configuration
POLL_TIMEOUT = 30
POLL_INTERVAL = 0.5
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready (run setup_e2e_env.sh first)",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui() -> int:
"""Start ComfyUI and return its PID."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI:\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID from start_comfyui output:\n{r.stdout}")
def _stop_comfyui():
"""Stop ComfyUI."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
def _wait_for(predicate, timeout=POLL_TIMEOUT, interval=POLL_INTERVAL):
"""Poll *predicate* until it returns True or *timeout* seconds elapse."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return True
time.sleep(interval)
return False
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def comfyui():
"""Start ComfyUI once for the module, stop after all tests."""
pid = _start_comfyui()
yield pid
_stop_comfyui()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestQueueLifecycle:
"""Queue management lifecycle: reset → status → task → start → history."""
def test_reset_queue(self, comfyui):
"""POST /v2/manager/queue/reset returns 200 AND all queue counts are zeroed.
WI-L strengthening: previously status-only. Now verifies the post-reset
status payload reports a fully-quiescent queue. `wipe_queue()` only
clears `pending_tasks` unconditionally (see manager_server.py:396-403),
but at the start of this module-scoped fixture no task has been run, so
all counts — pending / in_progress / total / done / is_processing —
must be 0/False. Any non-zero count here would indicate a leak from an
earlier test module or a reset-handler regression.
"""
resp = requests.post(f"{BASE_URL}/v2/manager/queue/reset", timeout=10)
assert resp.status_code == 200, (
f"Queue reset failed with status {resp.status_code}"
)
status = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=10)
assert status.status_code == 200
data = status.json()
assert data["pending_count"] == 0, (
f"pending_count != 0 after reset: {data['pending_count']}"
)
assert data["in_progress_count"] == 0, (
f"in_progress_count != 0 after reset: {data['in_progress_count']}"
)
assert data["total_count"] == 0, (
f"total_count != 0 after reset: {data['total_count']}"
)
assert data["done_count"] == 0, (
f"done_count != 0 after reset (fresh module): {data['done_count']}"
)
assert data["is_processing"] is False, (
f"is_processing should be False after reset, got {data['is_processing']!r}"
)
def test_status_with_client_id_filter(self, comfyui):
"""GET /v2/manager/queue/status?client_id=X returns client-scoped counts."""
resp = requests.get(
f"{BASE_URL}/v2/manager/queue/status",
params={"client_id": "e2e-queue-test"},
timeout=10,
)
assert resp.status_code == 200
data = resp.json()
assert data["client_id"] == "e2e-queue-test", (
f"Expected client_id echo, got {data.get('client_id')}"
)
assert "total_count" in data, "Filtered response missing 'total_count'"
assert "pending_count" in data, "Filtered response missing 'pending_count'"
def test_start_queue_already_idle(self, comfyui):
"""POST /v2/manager/queue/start on empty queue returns 200 or 201 AND worker stabilizes to idle.
WI-L strengthening: previously status-only. The contract:
- 200 = worker thread newly started; on empty queue it finds no work,
calls `task_queue.finalize()` no-op (done_count==0), and exits.
- 201 = worker already alive; same stabilization expected.
In either case the post-condition is: pending==0, in_progress==0,
is_processing eventually False (worker exits within a few seconds
when there's nothing to do). This defeats a regression where
`start_worker()` accidentally spawns a hot-loop that never exits.
"""
# Ensure idle baseline (pending empty before start).
requests.post(f"{BASE_URL}/v2/manager/queue/reset", timeout=10)
pre = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=10).json()
assert pre["pending_count"] == 0, (
f"Pre-condition: pending_count must be 0 before idle-start test, got {pre['pending_count']}"
)
resp = requests.post(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
assert resp.status_code in (200, 201), (
f"Queue start returned unexpected status {resp.status_code}"
)
# Worker observation: either it never flipped (201 + already-idle worker
# that stays idle) or it flipped True→False (200 + brief run). Poll
# until is_processing is False with stable pending/in_progress==0.
deadline = time.monotonic() + 10.0
final = None
while time.monotonic() < deadline:
r = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=5)
if r.status_code == 200:
final = r.json()
if (
final["pending_count"] == 0
and final["in_progress_count"] == 0
and final["is_processing"] is False
):
break
time.sleep(0.3)
assert final is not None, "queue/status never returned 200 during poll"
assert final["pending_count"] == 0, (
f"pending_count non-zero after start on empty queue: {final['pending_count']}"
)
assert final["in_progress_count"] == 0, (
f"in_progress_count non-zero after start on empty queue: {final['in_progress_count']}"
)
assert final["is_processing"] is False, (
f"worker did not stabilize to idle within 10s: is_processing={final['is_processing']!r}"
f"possible hot-loop regression in start_worker() / task_worker"
)
def test_queue_task_and_history(self, comfyui):
"""Full lifecycle: queue task → start → wait → verify history."""
# Reset to clean slate. Note: reset wipes pending/running but not
# file-based batch history, so we track completion by our OWN ui_id
# rather than a global done_count which can reflect unrelated tasks.
requests.post(f"{BASE_URL}/v2/manager/queue/reset", timeout=10)
UI_ID = "e2e-queue-lifecycle"
# Queue a lightweight task (install a small CNR package)
task_payload = {
"ui_id": UI_ID,
"client_id": UI_ID,
"kind": "install",
"params": {
"id": "ComfyUI_SigmoidOffsetScheduler",
"version": "1.0.1",
"selected_version": "latest",
"mode": "remote",
"channel": "default",
},
}
resp = requests.post(
f"{BASE_URL}/v2/manager/queue/task",
json=task_payload,
timeout=10,
)
assert resp.status_code == 200, (
f"Queue task failed with status {resp.status_code}: {resp.text}"
)
# Start processing
resp = requests.post(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
assert resp.status_code in (200, 201), (
f"Queue start failed with status {resp.status_code}"
)
# Wait for OUR task to complete: status.pending_count filtered by
# this client_id drops to 0 AND is_processing for our client is false.
# This avoids the global-done_count race from stale history.
def _our_task_completed():
r = requests.get(
f"{BASE_URL}/v2/manager/queue/status",
params={"client_id": UI_ID},
timeout=10,
)
if r.status_code != 200:
return False
d = r.json()
return (
d.get("pending_count", 1) == 0
and d.get("in_progress_count", 1) == 0
and d.get("done_count", 0) >= 1
)
assert _wait_for(_our_task_completed, timeout=120), (
"Our queued task did not complete within timeout"
)
# Check history. If the server returns 400, it is the known
# TaskHistoryItem serialization bug — surface it via pytest.skip
# with a specific reason rather than silently passing, so the bug
# remains visible.
resp = requests.get(f"{BASE_URL}/v2/manager/queue/history", timeout=10)
# Server-side fix: TaskHistoryItem now serializes via model_dump(mode='json').
# Any 400 is a genuine failure, not a tolerated server bug.
assert resp.status_code == 200, (
f"Queue history unexpected status {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert "history" in data, "History response missing 'history' field"
def test_history_with_ui_id_filter(self, comfyui):
"""GET /v2/manager/queue/history?ui_id=X returns entries matching the filter.
WI-T Cluster C target 1: previously 200-or-400-accept without filter-
semantic check. Now asserts:
- 200 OK
- response contains 'history' field (dict)
- every returned entry's ui_id actually matches the filter value.
Seed strategy: query unfiltered first — if history is non-empty, pick
an existing ui_id; otherwise enqueue + wait on a lightweight install
to seed one. Defeats regressions where the server accepts the ui_id
param but returns the full unfiltered history.
"""
# Discover an existing ui_id via unfiltered call; seed if empty.
all_resp = requests.get(f"{BASE_URL}/v2/manager/queue/history", timeout=10)
assert all_resp.status_code == 200, (
f"Unfiltered history unexpected status {all_resp.status_code}: {all_resp.text[:200]}"
)
all_history = all_resp.json().get("history", {})
def _extract_ui_ids(h):
"""Shape-resilient ui_id extractor (WI-P pattern)."""
if isinstance(h, dict):
# Either {ui_id: task_data} map or a single task-dict with 'ui_id'
if "ui_id" in h and ("kind" in h or "params" in h):
uid = h.get("ui_id")
return [uid] if uid is not None else []
return list(h.keys())
if isinstance(h, list):
return [e.get("ui_id") for e in h if isinstance(e, dict) and "ui_id" in e]
return []
ids = _extract_ui_ids(all_history) if isinstance(all_history, (dict, list)) else []
if ids:
target_ui_id = ids[0]
else:
# Seed a lightweight install so the filter has a target.
target_ui_id = "e2e-hist-filter-seed"
seed_payload = {
"ui_id": target_ui_id,
"client_id": target_ui_id,
"kind": "install",
"params": {
"id": "ComfyUI_SigmoidOffsetScheduler",
"version": "1.0.1",
"selected_version": "latest",
"mode": "remote",
"channel": "default",
},
}
r = requests.post(f"{BASE_URL}/v2/manager/queue/task", json=seed_payload, timeout=10)
assert r.status_code == 200, f"seed queue/task failed: {r.status_code} {r.text[:200]}"
r = requests.post(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
assert r.status_code in (200, 201), f"seed queue/start failed: {r.status_code}"
def _seed_done():
s = requests.get(
f"{BASE_URL}/v2/manager/queue/status",
params={"client_id": target_ui_id},
timeout=5,
)
if s.status_code != 200:
return False
d = s.json()
return (
d.get("pending_count", 1) == 0
and d.get("in_progress_count", 1) == 0
and d.get("done_count", 0) >= 1
)
assert _wait_for(_seed_done, timeout=120), (
"seed task for ui_id filter did not complete within timeout"
)
# Filter request (action under test)
resp = requests.get(
f"{BASE_URL}/v2/manager/queue/history",
params={"ui_id": target_ui_id},
timeout=10,
)
assert resp.status_code == 200, (
f"Filtered history unexpected status {resp.status_code}: {resp.text[:200]}"
)
data = resp.json()
assert "history" in data, "Filtered history response missing 'history' field"
# Filter semantics: every returned entry must match target_ui_id.
returned_ids = _extract_ui_ids(data["history"])
assert returned_ids, (
f"filter for ui_id={target_ui_id!r} returned empty; "
f"unfiltered showed ids={ids!r}"
)
for uid in returned_ids:
assert uid == target_ui_id, (
f"filter leaked other ui_id: got {uid!r}, expected {target_ui_id!r} "
f"(full response ids={returned_ids!r})"
)
def test_history_with_pagination(self, comfyui):
"""GET /v2/manager/queue/history honors max_items + offset consistently.
WI-T Cluster C target 2: previously only checked the cap (max_items=1).
Now also verifies:
- unfiltered total is stable (reference count N),
- max_items=1 → len ≤ 1 (cap),
- max_items ≥ N → len == N (no silent truncation),
- when N ≥ 2: offset=0 and offset=1 return different keys
(offset actually advances through the list).
"""
# Reference: full unfiltered count
full_resp = requests.get(f"{BASE_URL}/v2/manager/queue/history", timeout=10)
assert full_resp.status_code == 200, (
f"Unfiltered history unexpected status {full_resp.status_code}"
)
full_history = full_resp.json().get("history", {})
assert isinstance(full_history, dict), (
f"expected dict for unfiltered history, got {type(full_history).__name__}"
)
full_n = len(full_history)
# (1) Cap: max_items=1 → ≤1 entry
r1 = requests.get(
f"{BASE_URL}/v2/manager/queue/history",
params={"max_items": "1", "offset": "0"},
timeout=10,
)
assert r1.status_code == 200, (
f"Paginated history unexpected status {r1.status_code}: {r1.text[:200]}"
)
h1 = r1.json().get("history", {})
assert isinstance(h1, dict), f"expected dict, got {type(h1).__name__}"
assert len(h1) <= 1, (
f"Pagination cap violated: max_items=1 but got {len(h1)} entries"
)
# (2) Large max_items returns everything available (no silent truncation)
large_cap = max(full_n, 1) + 100
r_all = requests.get(
f"{BASE_URL}/v2/manager/queue/history",
params={"max_items": str(large_cap), "offset": "0"},
timeout=10,
)
assert r_all.status_code == 200
h_all = r_all.json().get("history", {})
assert isinstance(h_all, dict)
assert len(h_all) == full_n, (
f"Pagination inconsistency: unfiltered={full_n} entries, "
f"max_items={large_cap} returned {len(h_all)}"
)
# (3) Offset progression (only when ≥2 entries exist)
if full_n >= 2:
r_off0 = requests.get(
f"{BASE_URL}/v2/manager/queue/history",
params={"max_items": "1", "offset": "0"},
timeout=10,
).json().get("history", {})
r_off1 = requests.get(
f"{BASE_URL}/v2/manager/queue/history",
params={"max_items": "1", "offset": "1"},
timeout=10,
).json().get("history", {})
if r_off0 and r_off1:
assert set(r_off0.keys()) != set(r_off1.keys()), (
f"offset progression failed: offset=0 keys={list(r_off0.keys())!r} == "
f"offset=1 keys={list(r_off1.keys())!r}"
)
def test_history_list(self, comfyui):
"""GET /v2/manager/queue/history_list returns batch IDs matching disk state.
WI-T Cluster C target 3: previously shape-only. Now cross-references
the API response against the filesystem batch_history directory —
Manager stores one JSON file per batch under user/__manager/batch_history/,
and the handler returns their basenames (without .json) sorted by mtime.
API ∩ FS must be equal: no phantom ids, no missing ids.
"""
# API
resp = requests.get(f"{BASE_URL}/v2/manager/queue/history_list", timeout=10)
assert resp.status_code == 200, (
f"History list failed with status {resp.status_code}"
)
data = resp.json()
assert "ids" in data, "History list response missing 'ids' field"
api_ids = data["ids"]
assert isinstance(api_ids, list), (
f"'ids' should be a list, got {type(api_ids)}"
)
# Filesystem cross-reference
if not os.path.isdir(HISTORY_DIR):
# If dir absent, API must also be empty (no phantom entries).
assert not api_ids, (
f"API returned {len(api_ids)} ids but {HISTORY_DIR} does not exist"
)
return
fs_ids = {
f[:-5]
for f in os.listdir(HISTORY_DIR)
if f.endswith(".json")
and os.path.isfile(os.path.join(HISTORY_DIR, f))
}
api_id_set = set(api_ids)
assert api_id_set == fs_ids, (
f"API history_list diverges from filesystem:\n"
f" only-in-api={api_id_set - fs_ids}\n"
f" only-on-fs ={fs_ids - api_id_set}\n"
f" HISTORY_DIR={HISTORY_DIR}"
)
def test_history_path_traversal_rejected(self, comfyui):
"""GET /v2/manager/queue/history with path-traversal id is rejected.
Security boundary: id must stay within manager_batch_history_path.
Defense in depth:
1. 400 status (server rejects the request)
2. Response body contains no file content (leak check)
3. Sentinel file outside the history dir is NOT read/touched
4. Multiple traversal variants (bare, encoded, backslash, absolute)
"""
import pathlib
# Sentinel in E2E_ROOT — target with various traversal encodings
sentinel = pathlib.Path(E2E_ROOT) / "_history_sentinel_must_not_read.txt"
sentinel_content = "sentinel-content-secret-xyz-12345"
sentinel.write_text(sentinel_content)
try:
traversal_ids = [
"../../../etc/passwd",
"../../secret",
"/etc/passwd",
# Sentinel-targeted variants (would reach _history_sentinel... if traversal works)
"../../../_history_sentinel_must_not_read",
"../../_history_sentinel_must_not_read",
# URL-encoded traversal
"%2e%2e%2f%2e%2e%2fetc%2fpasswd",
"%2e%2e/etc/passwd",
# Backslash (Windows-style)
"..\\..\\etc\\passwd",
# Null byte injection attempt (classic bypass)
"legit_id\x00/../../etc/passwd",
]
for bad_id in traversal_ids:
resp = requests.get(
f"{BASE_URL}/v2/manager/queue/history",
params={"id": bad_id},
timeout=10,
)
assert resp.status_code == 400, (
f"Path traversal id {bad_id!r} should return 400, got {resp.status_code}"
)
# Content leak check — no /etc/passwd OR sentinel leaked
body = resp.text
assert "root:" not in body, (
f"Traversal id {bad_id!r} leaked /etc/passwd content"
)
assert sentinel_content not in body, (
f"Traversal id {bad_id!r} leaked sentinel file content — "
f"path traversal actually succeeded!"
)
# Sentinel file must still exist (no accidental writes/deletes via traversal)
assert sentinel.exists(), "Sentinel file was deleted — traversal side-effect!"
assert sentinel.read_text() == sentinel_content, (
"Sentinel file was modified — traversal side-effect!"
)
finally:
sentinel.unlink(missing_ok=True)
+140
View File
@@ -0,0 +1,140 @@
"""E2E demonstration that the high+ T2 SECGATE-PENDING Goals are testable
at the DEFAULT security_level=normal — no strict-mode harness needed.
WI-KK research finding (see decision-trail wi-kk-t2-secgate-harness-...):
The 8 T2 SECGATE-PENDING Goals listed in reports/e2e_verification_audit.md
were assumed to all need a restricted-security-level harness. After reading
comfyui_manager/glob/utils/security_utils.py:14-40 the actual gate semantics
become clear:
- is_local_mode = is_loopback(args.listen) → True for our 127.0.0.1 setup
- For Risk=high+: returns True iff security_level in [WEAK, NORMAL_]
- The default normal IS NOT in that set → high+ operations return False → 403
So the high+ Goals are ALREADY 403-testable at default config:
- CV4 (comfyui_switch_version) ← THIS file proves it
- IM4 (install_model non-safetensors) ← deferred (see note below)
- LGU2 (customnode/install/git_url) ← deferred (legacy-only endpoint)
- LPP2 (customnode/install/pip) ← deferred (legacy-only endpoint)
CV4 is the cleanest demonstration: it is registered in glob and has a
synchronous is_allowed_security_level('high+') guard at
comfyui_manager/glob/manager_server.py:1856 that returns 403 directly.
Goals deferred to follow-up WIs (with notes for the audit-reflect WI):
- IM4: the non-safetensors check happens DEEP in the install pipeline (in
get_risky_level + the worker), not at the HTTP handler. There is NO
synchronous 403 from POST /v2/manager/queue/install_model — the handler
accepts the JSON and queues a task; rejection only surfaces during task
execution. This requires a queue-observation test pattern, not a simple
HTTP 403 check.
- LGU2 / LPP2: registered ONLY in legacy/manager_server.py (1502, 1522),
not in glob. Testing them requires the legacy fixture
(start_comfyui_legacy.sh) — fits naturally into a follow-up
test_e2e_secgate_legacy_default.py module.
This file therefore demonstrates the harness-not-needed insight with the
single Goal where it cleanly applies (CV4) and documents the audit-reflect
implications inline.
Requires a pre-built E2E environment (from setup_e2e_env.sh).
"""
from __future__ import annotations
import os
import subprocess
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui_default() -> int:
"""Launch ComfyUI at the default security_level (normal) — glob mode."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI (default):\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID:\n{r.stdout}")
def _stop_comfyui():
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
@pytest.fixture(scope="module")
def comfyui_default():
pid = _start_comfyui_default()
try:
yield pid
finally:
_stop_comfyui()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestSecurityGate403_CV4:
"""Goal CV4 — POST /v2/comfyui_manager/comfyui_switch_version must
return 403 below `high+`. At the default security_level=normal +
is_local_mode=True, NORMAL is NOT in the allowed-set [WEAK, NORMAL_]
for the high+ check, so the 403 path triggers WITHOUT any harness.
Handler: comfyui_manager/glob/manager_server.py:1854-1858
Gate: is_allowed_security_level("high+")
"""
def test_switch_version_returns_403_at_default(self, comfyui_default):
# We deliberately send a syntactically valid JSON body so the
# request would reach the Pydantic validation step IF the gate
# were broken. The gate is the FIRST check in the handler, so
# 403 must precede any 400-from-validation outcome.
# Body transport (JSON) per WI #258 — previously query string.
resp = requests.post(
f"{BASE_URL}/v2/comfyui_manager/comfyui_switch_version",
json={"ver": "v0.12.1", "client_id": "secgate-cv4", "ui_id": "secgate-cv4"},
timeout=10,
)
assert resp.status_code == 403, (
f"CV4 SECURITY-GATE BYPASS: POST comfyui_switch_version returned "
f"{resp.status_code} at security_level=normal (expected 403). "
f"This means the high+ gate is broken — version downgrade attacks "
f"would succeed. Response: {resp.text[:200]}"
)
+242
View File
@@ -0,0 +1,242 @@
"""E2E PoC for the strict-mode security gate (T2 SECGATE-PENDING harness).
SCOPE — important clarification:
This suite is the PoC for the security_level=strong harness needed to verify
the 403 contract on middle/middle+ gates. The default E2E config
(security_level=normal, is_local_mode=True) puts NORMAL inside the allowed
set for both middle and middle+ checks (see comfyui_manager/glob/utils/
security_utils.py:32-38), so the 403 path on these gates is unreachable
without elevating the security_level to STRONG.
The 4 T2 SECGATE-PENDING Goals at middle/middle+ that depend on this harness:
- SR4 (snapshot/remove, gate=middle) ← THIS PoC
- SR6 (snapshot/restore, gate=middle+) ← follow-up
- V5 (manager/reboot, gate=middle) ← follow-up
- UA2 (manager/queue/update_all, gate=middle+) ← follow-up
WI-KK established the harness pattern (start_comfyui_strict.sh + a fixture
that backs up + restores config.ini). Once this PoC lands, the remaining
3 strict-mode tests are mechanical additions to this file.
Why a separate file (not test_e2e_csrf*.py-style mixed):
The strict-mode server lifecycle is heavyweight (config patch + restart). It
must NOT contaminate normal-mode test suites where security_level=normal is
expected. By keeping strict tests in their own module with their own fixture,
we keep the cost contained and the contracts unambiguous.
Negative-check coverage (per verification_design.md §7.3 Security Boundary
Template):
- 403 status
- target snapshot file UNCHANGED on disk
- (log substring check is OPTIONAL and not asserted here — observable in
server logs but cumbersome to scrape from pytest)
Requires a pre-built E2E environment (from setup_e2e_env.sh).
"""
from __future__ import annotations
import os
import shutil
import subprocess
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
MANAGER_CONFIG = (
os.path.join(COMFYUI_PATH, "user", "__manager", "config.ini")
if COMFYUI_PATH else ""
)
SNAPSHOT_DIR = (
os.path.join(COMFYUI_PATH, "user", "__manager", "snapshots")
if COMFYUI_PATH else ""
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui_strict() -> int:
"""Launch ComfyUI with security_level=strong (start_comfyui_strict.sh)."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui_strict.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI (strict):\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID:\n{r.stdout}")
def _stop_comfyui():
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
def _restore_config_from_backup():
"""Restore config.ini from the .before-strict backup left by the script."""
backup = MANAGER_CONFIG + ".before-strict"
if os.path.isfile(backup):
shutil.copyfile(backup, MANAGER_CONFIG)
os.remove(backup)
@pytest.fixture(scope="module")
def comfyui_strict():
"""Start ComfyUI in strict mode for the duration of the module.
Teardown order matters:
1. Stop the server (so it releases the config file lock).
2. Restore the original config.ini (so subsequent test modules see
the default security_level=normal again).
"""
pid = _start_comfyui_strict()
try:
yield pid
finally:
_stop_comfyui()
_restore_config_from_backup()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestSecurityGate403_SR4:
"""Goal SR4 — POST /v2/snapshot/remove must return 403 below `middle`.
Handler: comfyui_manager/glob/manager_server.py:1535-1554
Gate: is_allowed_security_level("middle")
Strict (security_level=strong) → False → 403.
Negative check (verification_design.md §7.3): the target snapshot file
on disk must NOT be removed when the gate rejects the request.
"""
def test_remove_returns_403(self, comfyui_strict):
# Seed a snapshot file directly on disk so we have something the
# handler MIGHT delete if the gate were broken. We do not call
# /v2/snapshot/save here because that path also requires server
# state; touching a file is sufficient for the negative check.
os.makedirs(SNAPSHOT_DIR, exist_ok=True)
seed_name = "secgate-sr4-seed"
seed_path = os.path.join(SNAPSHOT_DIR, f"{seed_name}.json")
with open(seed_path, "w") as f:
f.write('{"snapshot": "seed for SR4 negative check"}')
try:
assert os.path.isfile(seed_path), "test seed file missing"
resp = requests.post(
f"{BASE_URL}/v2/snapshot/remove",
params={"target": seed_name},
timeout=10,
)
# Primary assertion: 403 from the security gate
assert resp.status_code == 403, (
f"SR4 SECURITY-GATE BYPASS: POST snapshot/remove?target="
f"{seed_name} returned {resp.status_code} at "
f"security_level=strong (expected 403). Response: "
f"{resp.text[:200]}"
)
# Negative-side assertion: the target file was NOT deleted
assert os.path.isfile(seed_path), (
f"SR4 NEGATIVE-CHECK FAILURE: snapshot file {seed_path} was "
f"deleted despite a 403 response — gate failed to block the "
f"side effect."
)
finally:
# Cleanup — remove the seed file we created
if os.path.isfile(seed_path):
os.remove(seed_path)
# Positive counterpart (POST works at default after teardown) is covered
# by test_e2e_secgate_default.py via its own ComfyUI startup; a
# skip-placeholder was previously parked here for documentation but
# removed in WI-MM because it added no verification and created a
# stale-skip row. See reports/e2e_verification_audit.md § SECGATE.
# ---------------------------------------------------------------------------
# Config setter 403 contract (added in WI #255)
# ---------------------------------------------------------------------------
#
# Before WI #255 the three config setters below had NO security_level gate,
# so any caller could mutate db_mode / update_policy / channel_url even at
# security_level=strong. WI #255 added `is_allowed_security_level('middle')`
# to all three (glob + legacy) at the same risk tier as uninstall/update /
# snapshot/remove.
#
# These tests belong in the STRICT harness — at default `security_level =
# normal`, the 'middle' allow-set is [weak, normal, normal-] which INCLUDES
# normal, so the 403 path is not reachable. Strong excludes normal, so the
# rejection is observable.
CONFIG_SETTER_ENDPOINTS = [
("/v2/manager/db_mode", {"value": "local"}, "set_db_mode_api"),
("/v2/manager/policy/update", {"value": "nightly-comfyui"}, "set_update_policy_api"),
("/v2/manager/channel_url_list", {"value": "default"}, "set_channel_url"),
]
class TestConfigSetterRequiresMiddle:
"""Each of the 3 config-mutation POST handlers must return 403 under
security_level=strong. Before WI #255 they returned 200 unconditionally.
Handlers (glob):
- comfyui_manager/glob/manager_server.py :: set_db_mode_api (L1954)
- comfyui_manager/glob/manager_server.py :: set_update_policy_api (L1972)
- comfyui_manager/glob/manager_server.py :: set_channel_url (L2000)
Gate: is_allowed_security_level('middle')
Strong → normal not in [weak, normal, normal-] → False → 403.
Negative-check aspect: a 403 response means the config file was NOT
mutated. We don't re-read config.ini here because the strict harness
also patches it — checking for absence of mutation would require
a pre/post diff that is brittle; the 403 status is the contract.
"""
@pytest.mark.parametrize(
"path,body,handler_name",
CONFIG_SETTER_ENDPOINTS,
ids=[p for p, _, _ in CONFIG_SETTER_ENDPOINTS],
)
def test_config_setter_returns_403(self, comfyui_strict, path, body, handler_name):
resp = requests.post(
f"{BASE_URL}{path}",
json=body,
timeout=10,
)
assert resp.status_code == 403, (
f"CONFIG-SETTER SECURITY-GATE BYPASS: POST {path} "
f"(handler {handler_name}) returned {resp.status_code} at "
f"security_level=strong (expected 403). Config mutation must "
f"require middle+. Response: {resp.text[:200]}"
)
+348
View File
@@ -0,0 +1,348 @@
"""E2E tests for ComfyUI Manager snapshot lifecycle endpoints.
Exercises the snapshot management endpoints on a running ComfyUI instance:
- GET /v2/snapshot/get_current — current system state as snapshot
- POST /v2/snapshot/save — save current state
- GET /v2/snapshot/getlist — list available snapshots
- POST /v2/snapshot/remove — remove a snapshot
Scenario:
get_current → save → getlist (verify new snapshot) → remove →
getlist (verify removed).
NOTE: /v2/snapshot/restore is intentionally NOT tested — it is a
destructive operation that alters installed node state.
Requires a pre-built E2E environment (from setup_e2e_env.sh).
Set E2E_ROOT env var to point at it, or the tests will be skipped.
Usage:
E2E_ROOT=/tmp/e2e_full_test pytest tests/e2e/test_e2e_snapshot_lifecycle.py -v
"""
from __future__ import annotations
import os
import subprocess
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SNAPSHOT_DIR = (
os.path.join(COMFYUI_PATH, "user", "__manager", "snapshots")
if COMFYUI_PATH
else ""
)
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready (run setup_e2e_env.sh first)",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui() -> int:
"""Start ComfyUI and return its PID."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI:\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID from start_comfyui output:\n{r.stdout}")
def _stop_comfyui():
"""Stop ComfyUI."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def comfyui():
"""Start ComfyUI once for the module, stop after all tests."""
pid = _start_comfyui()
yield pid
_stop_comfyui()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestSnapshotLifecycle:
"""Snapshot management lifecycle: get_current → save → list → remove."""
def test_get_current_snapshot(self, comfyui):
"""GET /v2/snapshot/get_current returns documented schema AND cross-refs installed state.
WI-M strengthening: previously dict-type only. Now verifies:
(a) the documented top-level keys are all present —
comfyui / git_custom_nodes / cnr_custom_nodes /
file_custom_nodes / pips;
(b) each list-valued field is actually a list (type-level schema);
(c) cross-reference — the E2E seed CNR pack
`ComfyUI_SigmoidOffsetScheduler` must appear in
`cnr_custom_nodes` if it exists on the filesystem.
Defeats regressions that return an empty dict or drop the
cnr_custom_nodes field while keeping 200 OK.
"""
resp = requests.get(f"{BASE_URL}/v2/snapshot/get_current", timeout=10)
assert resp.status_code == 200, (
f"get_current failed with status {resp.status_code}"
)
data = resp.json()
assert isinstance(data, dict), (
f"Expected dict from get_current, got {type(data)}"
)
# (a) Documented top-level keys.
required_keys = (
"comfyui",
"git_custom_nodes",
"cnr_custom_nodes",
"file_custom_nodes",
"pips",
)
for key in required_keys:
assert key in data, (
f"snapshot missing required top-level key {key!r}. "
f"Got keys: {list(data.keys())}"
)
# (b) cnr_custom_nodes is a dict mapping pack_name → version — that's
# the field we cross-ref below. Other collection-valued fields
# (git_custom_nodes, pips, file_custom_nodes) carry environment-
# dependent shapes (dict/list/mixed) and are intentionally not
# constrained at the type level here — only their presence is required.
assert isinstance(data["cnr_custom_nodes"], dict), (
f"snapshot['cnr_custom_nodes'] should be a dict (pack_name → version), "
f"got {type(data['cnr_custom_nodes']).__name__}"
)
# (c) Cross-reference installed state: if the E2E seed pack is on disk,
# it MUST appear in cnr_custom_nodes.
seed_pack = "ComfyUI_SigmoidOffsetScheduler"
custom_nodes_dir = os.path.join(COMFYUI_PATH, "custom_nodes") if COMFYUI_PATH else ""
seed_on_disk = (
bool(custom_nodes_dir)
and os.path.isdir(os.path.join(custom_nodes_dir, seed_pack))
)
if seed_on_disk:
assert seed_pack in data["cnr_custom_nodes"], (
f"Seed pack {seed_pack!r} exists on disk but missing from "
f"snapshot.cnr_custom_nodes={data['cnr_custom_nodes']!r}"
)
def test_save_snapshot(self, comfyui):
"""POST /v2/snapshot/save — full disk + content verification (WI-Q strengthening).
Defeats regressions where the endpoint returns 200 but (a) no file lands on
disk or (b) the file drifts from the live runtime state.
Verifies:
(a) a new *.json file appears under SNAPSHOT_DIR;
(b) the saved file's `cnr_custom_nodes` dict matches the live
GET /v2/snapshot/get_current response — same keys, same
versions (pack_name → version). This catches cases where
the save endpoint writes a stale or stub snapshot while
the live API reports the true runtime state.
"""
files_before = set()
if os.path.isdir(SNAPSHOT_DIR):
files_before = {f for f in os.listdir(SNAPSHOT_DIR) if f.endswith(".json")}
resp = requests.post(f"{BASE_URL}/v2/snapshot/save", timeout=30)
assert resp.status_code == 200, (
f"Snapshot save failed with status {resp.status_code}"
)
# (a) Effect verification: new file appears in snapshot directory
assert os.path.isdir(SNAPSHOT_DIR), (
f"Snapshot dir not created: {SNAPSHOT_DIR}"
)
files_after = {f for f in os.listdir(SNAPSHOT_DIR) if f.endswith(".json")}
new_files = files_after - files_before
assert len(new_files) >= 1, (
f"No new snapshot file created on disk: before={files_before}, after={files_after}"
)
# Content verification: new file is valid JSON dict
import json
new_file = next(iter(new_files))
with open(os.path.join(SNAPSHOT_DIR, new_file)) as f:
saved = json.load(f)
assert isinstance(saved, dict), (
f"Snapshot file content should be dict, got {type(saved).__name__}"
)
# (b) Content cross-reference: saved snapshot must match the live
# GET /v2/snapshot/get_current response on the cnr_custom_nodes
# field (the deterministic pack_name → version mapping). Other
# fields like `pips` are environment-dependent and drift fast;
# cnr_custom_nodes is the stable contract.
live_resp = requests.get(f"{BASE_URL}/v2/snapshot/get_current", timeout=10)
assert live_resp.status_code == 200, (
f"get_current failed with status {live_resp.status_code}"
)
live = live_resp.json()
assert "cnr_custom_nodes" in saved, (
f"Saved snapshot missing 'cnr_custom_nodes' field. "
f"Got keys: {list(saved.keys())}"
)
assert "cnr_custom_nodes" in live, (
f"Live get_current missing 'cnr_custom_nodes' field. "
f"Got keys: {list(live.keys())}"
)
assert saved["cnr_custom_nodes"] == live["cnr_custom_nodes"], (
f"Saved snapshot cnr_custom_nodes does not match live state.\n"
f" saved={saved['cnr_custom_nodes']!r}\n"
f" live ={live['cnr_custom_nodes']!r}"
)
def test_getlist_after_save(self, comfyui):
"""GET /v2/snapshot/getlist shows at least one snapshot after save."""
resp = requests.get(f"{BASE_URL}/v2/snapshot/getlist", timeout=10)
assert resp.status_code == 200, (
f"Snapshot getlist failed with status {resp.status_code}"
)
data = resp.json()
assert "items" in data, "Snapshot list response missing 'items' field"
assert isinstance(data["items"], list), (
f"'items' should be a list, got {type(data['items'])}"
)
assert len(data["items"]) > 0, (
"Expected at least one snapshot after save, but list is empty"
)
def test_remove_snapshot(self, comfyui):
"""POST /v2/snapshot/remove removes a specific snapshot.
Test is INDEPENDENT: creates its own snapshot as setup, removes it,
asserts. Does not depend on prior tests in this module.
"""
# SETUP: create a snapshot so we have a deterministic target
save_resp = requests.post(f"{BASE_URL}/v2/snapshot/save", timeout=30)
assert save_resp.status_code == 200, "setup save failed"
# Find the newly created snapshot by diffing against pre-save list
resp = requests.get(f"{BASE_URL}/v2/snapshot/getlist", timeout=10)
assert resp.status_code == 200
all_items = resp.json().get("items", [])
# The newest snapshot is at items[0] (desc-sorted)
assert all_items, "setup snapshot missing from getlist"
target = all_items[0]
count_before_remove = len(all_items)
# Remove (action under test)
resp = requests.post(
f"{BASE_URL}/v2/snapshot/remove",
params={"target": target},
timeout=10,
)
assert resp.status_code == 200, (
f"Snapshot remove failed with status {resp.status_code}"
)
# Verify removal
resp = requests.get(f"{BASE_URL}/v2/snapshot/getlist", timeout=10)
assert resp.status_code == 200
data_after = resp.json()
assert target not in data_after["items"], (
f"Snapshot '{target}' still in list after removal"
)
assert len(data_after["items"]) == count_before_remove - 1, (
f"Expected {count_before_remove - 1} snapshots after removal, "
f"got {len(data_after['items'])}"
)
def test_remove_nonexistent_snapshot(self, comfyui):
"""POST /v2/snapshot/remove with nonexistent target returns 200 (no-op)."""
resp = requests.post(
f"{BASE_URL}/v2/snapshot/remove",
params={"target": "nonexistent_snapshot_99999"},
timeout=10,
)
# Server returns 200 even when file doesn't exist (no-op behavior)
assert resp.status_code == 200, (
f"Remove nonexistent snapshot returned {resp.status_code}"
)
def test_remove_path_traversal_rejected(self, comfyui):
"""POST /v2/snapshot/remove with path-traversal target returns 400.
Security boundary: target must stay within snapshot dir.
"""
# Capture state before (any file that must NOT be deleted)
import pathlib
sentinel = pathlib.Path(E2E_ROOT) / "_sentinel_must_not_delete.txt"
sentinel.write_text("sentinel")
# Path traversal attempts
traversal_targets = [
"../../_sentinel_must_not_delete",
"../../../etc/passwd",
"/etc/passwd",
]
for target in traversal_targets:
resp = requests.post(
f"{BASE_URL}/v2/snapshot/remove",
params={"target": target},
timeout=10,
)
assert resp.status_code == 400, (
f"Path traversal target {target!r} should return 400, got {resp.status_code}"
)
# Sentinel file must still exist (no traversal succeeded)
assert sentinel.exists(), "Sentinel file was deleted — path traversal succeeded!"
sentinel.unlink()
class TestSnapshotGetCurrentSchema:
"""Verify get_current snapshot response structure."""
# WI-M dedup: `test_get_current_returns_dict` REMOVED — it was a strict
# subset of TestSnapshotLifecycle::test_get_current_snapshot (which now
# asserts the full documented schema + cross-ref with installed state
# on disk). Keeping both after the upgrade would be pure duplication.
# Audit §7 row count reduces 7 → 6 to reflect the removal.
def test_getlist_items_are_strings(self, comfyui):
"""Each item in the snapshot list is a string (filename stem)."""
resp = requests.get(f"{BASE_URL}/v2/snapshot/getlist", timeout=10)
assert resp.status_code == 200
data = resp.json()
for item in data.get("items", []):
assert isinstance(item, str), (
f"Snapshot item should be a string, got {type(item)}: {item}"
)
+239
View File
@@ -0,0 +1,239 @@
"""E2E tests for ComfyUI Manager system information endpoints.
Tests the system-level endpoints:
- GET /v2/manager/version — manager version string
- GET /v2/manager/is_legacy_manager_ui — legacy UI flag
- POST /v2/manager/reboot — server reboot (last test)
The reboot test is intentionally placed LAST because it triggers a
server restart. After POST, the test polls until the server comes back.
Requires a pre-built E2E environment (from setup_e2e_env.sh).
Set E2E_ROOT env var to point at it, or the tests will be skipped.
Usage:
E2E_ROOT=/tmp/e2e_full_test pytest tests/e2e/test_e2e_system_info.py -v
"""
from __future__ import annotations
import os
import subprocess
import time
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
# Reboot requires longer polling — server must fully restart
REBOOT_TIMEOUT = 60
REBOOT_INTERVAL = 2.0
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready (run setup_e2e_env.sh first)",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui() -> int:
"""Start ComfyUI and return its PID."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI:\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID from start_comfyui output:\n{r.stdout}")
def _stop_comfyui():
"""Stop ComfyUI."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
def _wait_for(predicate, timeout=30, interval=0.5):
"""Poll *predicate* until it returns True or *timeout* seconds elapse."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return True
time.sleep(interval)
return False
def _server_is_healthy():
"""Check if the ComfyUI server responds to a health endpoint."""
try:
resp = requests.get(f"{BASE_URL}/system_stats", timeout=5)
return resp.status_code == 200
except requests.ConnectionError:
return False
except requests.Timeout:
return False
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def comfyui():
"""Start ComfyUI once for the module, stop after all tests."""
pid = _start_comfyui()
yield pid
_stop_comfyui()
# ---------------------------------------------------------------------------
# Tests — version
# ---------------------------------------------------------------------------
class TestManagerVersion:
"""Test GET /v2/manager/version."""
def test_version_returns_string(self, comfyui):
"""GET /v2/manager/version returns a non-empty version string."""
resp = requests.get(f"{BASE_URL}/v2/manager/version", timeout=10)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}"
)
version = resp.text
assert isinstance(version, str), (
f"Expected string response, got {type(version).__name__}"
)
assert len(version.strip()) > 0, "Version string should not be empty"
def test_version_is_stable(self, comfyui):
"""Consecutive calls return the same version (no mutation)."""
resp1 = requests.get(f"{BASE_URL}/v2/manager/version", timeout=10)
resp1.raise_for_status()
resp2 = requests.get(f"{BASE_URL}/v2/manager/version", timeout=10)
resp2.raise_for_status()
assert resp1.text == resp2.text, (
f"Version changed between calls: {resp1.text!r} vs {resp2.text!r}"
)
# ---------------------------------------------------------------------------
# Tests — is_legacy_manager_ui
# ---------------------------------------------------------------------------
class TestIsLegacyManagerUI:
"""Test GET /v2/manager/is_legacy_manager_ui."""
def test_returns_boolean_field(self, comfyui):
"""GET /v2/manager/is_legacy_manager_ui returns False in E2E env.
WI-T Cluster G target 5 (research-cluster-g.md Target 2):
Strengthened from type-only `isinstance(bool)` to exact-value `is False`.
Launcher-deterministic: `tests/e2e/scripts/start_comfyui.sh` passes
only `--cpu --enable-manager --port` — NO `--enable-manager-legacy-ui`.
`action='store_true'` on that flag defaults to False, so the handler
at `glob/manager_server.py:1500-1506` must return
`{"is_legacy_manager_ui": False}`.
If the E2E launcher ever starts passing `--enable-manager-legacy-ui`,
this assertion fails loudly with a clear pointer — correct behavior.
"""
resp = requests.get(
f"{BASE_URL}/v2/manager/is_legacy_manager_ui", timeout=10
)
assert resp.status_code == 200, (
f"Expected 200, got {resp.status_code}"
)
data = resp.json()
assert "is_legacy_manager_ui" in data, (
f"Response missing 'is_legacy_manager_ui' field: {data}"
)
assert data["is_legacy_manager_ui"] is False, (
f"E2E launcher omits --enable-manager-legacy-ui; expected False, "
f"got {data['is_legacy_manager_ui']!r}. "
f"If start_comfyui.sh was changed to pass that flag, update this assertion."
)
# ---------------------------------------------------------------------------
# Tests — reboot (MUST BE LAST — server restarts)
# ---------------------------------------------------------------------------
class TestReboot:
"""Test POST /v2/manager/reboot.
This test MUST run last in the module because a successful reboot
terminates or replaces the server process. The test polls until the
server comes back (or times out).
"""
def test_reboot_and_recovery(self, comfyui):
"""POST /v2/manager/reboot triggers restart; server comes back."""
# Verify server is running before reboot
assert _server_is_healthy(), "Server not healthy before reboot test"
# Record pre-reboot version for comparison
pre_version = requests.get(
f"{BASE_URL}/v2/manager/version", timeout=10
).text
# Trigger reboot — server may drop connection mid-response
try:
resp = requests.post(f"{BASE_URL}/v2/manager/reboot", timeout=10)
if resp.status_code == 403:
pytest.skip(
"Reboot denied by security policy "
"(security_level does not allow 'middle')"
)
assert resp.status_code == 200, (
f"Expected 200 or 403 from reboot, got {resp.status_code}"
)
except requests.ConnectionError:
# Server dropped connection during reboot — expected behavior
pass
# Give the server a moment to begin shutdown
time.sleep(2)
# Poll until server comes back
recovered = _wait_for(
_server_is_healthy,
timeout=REBOOT_TIMEOUT,
interval=REBOOT_INTERVAL,
)
assert recovered, (
f"Server did not recover within {REBOOT_TIMEOUT}s after reboot"
)
# Verify server is functional after reboot
post_version = requests.get(
f"{BASE_URL}/v2/manager/version", timeout=10
).text
assert post_version == pre_version, (
f"Version changed after reboot: {pre_version!r} -> {post_version!r}"
)
File diff suppressed because it is too large Load Diff
+242
View File
@@ -0,0 +1,242 @@
"""E2E tests for ComfyUI Manager version management endpoints.
Exercises the version management endpoints on a running ComfyUI instance:
- GET /v2/comfyui_manager/comfyui_versions — list versions + current
- POST /v2/comfyui_manager/comfyui_switch_version — switch version (negative tests only)
Scenario:
List versions → verify response has 'versions' array and 'current'
string. For switch_version: test missing params returns 400 (actual
version switching is destructive and NOT tested here).
Requires a pre-built E2E environment (from setup_e2e_env.sh).
Set E2E_ROOT env var to point at it, or the tests will be skipped.
Usage:
E2E_ROOT=/tmp/e2e_full_test pytest tests/e2e/test_e2e_version_mgmt.py -v
"""
from __future__ import annotations
import os
import subprocess
import pytest
import requests
E2E_ROOT = os.environ.get("E2E_ROOT", "")
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
SCRIPTS_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "scripts"
)
PORT = 8199
BASE_URL = f"http://127.0.0.1:{PORT}"
pytestmark = pytest.mark.skipif(
not E2E_ROOT
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
reason="E2E_ROOT not set or E2E environment not ready (run setup_e2e_env.sh first)",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _start_comfyui() -> int:
"""Start ComfyUI and return its PID."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
r = subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
capture_output=True,
text=True,
timeout=180,
env=env,
)
if r.returncode != 0:
raise RuntimeError(f"Failed to start ComfyUI:\n{r.stderr}")
for part in r.stdout.strip().split():
if part.startswith("COMFYUI_PID="):
return int(part.split("=")[1])
raise RuntimeError(f"Could not parse PID from start_comfyui output:\n{r.stdout}")
def _stop_comfyui():
"""Stop ComfyUI."""
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
subprocess.run(
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
capture_output=True,
text=True,
timeout=30,
env=env,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def comfyui():
"""Start ComfyUI once for the module, stop after all tests."""
pid = _start_comfyui()
yield pid
_stop_comfyui()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestComfyUIVersions:
"""Verify /v2/comfyui_manager/comfyui_versions response structure."""
def test_versions_response_contract(self, comfyui):
"""GET /v2/comfyui_manager/comfyui_versions — full response contract.
Merged by WI-NN (bloat Priority 3, Cluster 7): absorbs the four previous
single-GET tests (test_versions_endpoint + test_versions_list_not_empty +
test_versions_items_are_strings + test_current_is_in_versions) into one
contract block. All 4 original tests hit the same endpoint; merging
removes 3 redundant round-trips and keeps every unique assertion.
"""
resp = requests.get(
f"{BASE_URL}/v2/comfyui_manager/comfyui_versions", timeout=10
)
# (a) status + top-level schema (was test_versions_endpoint)
assert resp.status_code == 200, (
f"comfyui_versions failed with status {resp.status_code}"
)
data = resp.json()
assert "versions" in data, "Response missing 'versions' field"
assert "current" in data, "Response missing 'current' field"
assert isinstance(data["versions"], list), (
f"'versions' should be a list, got {type(data['versions'])}"
)
assert isinstance(data["current"], str), (
f"'current' should be a string, got {type(data['current'])}"
)
# (b) versions list is non-empty (was test_versions_list_not_empty)
assert len(data["versions"]) > 0, (
"Expected at least one version in the list"
)
# (c) every entry is a string (was test_versions_items_are_strings)
for v in data["versions"]:
assert isinstance(v, str), (
f"Version entry should be a string, got {type(v)}: {v}"
)
# (d) current appears in versions list (was test_current_is_in_versions).
# Keep the "empty current" guard — handler emits "" if git state can't
# resolve a tag, which is non-ideal but not a contract violation.
if data["current"] and data["versions"]:
assert data["current"] in data["versions"], (
f"Current version '{data['current']}' not found in versions list"
)
class TestSwitchVersionNegative:
"""Negative tests for /v2/comfyui_manager/comfyui_switch_version.
Actual version switching is destructive and NOT exercised.
Only error paths (missing params, validation failures) are tested.
"""
@pytest.mark.parametrize(
"req_params",
[
pytest.param(None, id="no-params"),
pytest.param({"ver": "v1.0.0"}, id="partial-params-ver-only"),
],
)
def test_switch_version_missing_required_params_rejected(self, comfyui, req_params):
"""POST without full (ver, client_id, ui_id) must be rejected.
WI-OO Item 5 (bloat dbg:ci-018 B9+B1): merges the previously-separate
`missing_all_params` and `missing_client_id` tests. At the default
security_level=normal the high+ gate returns 403 BEFORE any param
validation runs, so both fully-empty and partial-param requests
exercise the same rejection path. Parametrized across both input
equivalence classes — keeps both inputs exercised as distinct
pytest invocations for diagnostics, without duplicating the body.
WI #258: Migrated from query-string (params=) to JSON body (json=).
When req_params is None we send no body at all (bare POST).
"""
url = f"{BASE_URL}/v2/comfyui_manager/comfyui_switch_version"
if req_params is None:
resp = requests.post(url, timeout=10)
else:
resp = requests.post(url, json=req_params, timeout=10)
assert resp.status_code in (400, 403), (
f"Expected 400 or 403 for missing/partial params "
f"(req_params={req_params!r}), got {resp.status_code}"
)
def test_switch_version_validation_error_body(self, comfyui):
"""Validation error (400) returns structured Pydantic error body.
WI-L strengthening: previously accepted 'error field present OR plain
text'. The contract is stricter — the ValidationError path emits
exactly:
{"error": "Validation error", "details": [<pydantic error entries>]}
We now assert the full schema: the `error` sentinel string, the
`details` list, and that each detail entry carries the Pydantic
triplet (loc / msg / type). This defeats a regression where the server
falls through to the generic `except Exception` branch (which returns
status=400 with an EMPTY body — would currently still pass old check).
WI #258: Send a well-formed JSON body with required fields missing
to reach the Pydantic validator (not the json.JSONDecodeError branch,
which produces a plain-text 400). An empty JSON object {} fails the
required-field check for `ver`/`client_id`/`ui_id` uniformly.
"""
resp = requests.post(
f"{BASE_URL}/v2/comfyui_manager/comfyui_switch_version",
json={},
timeout=10,
)
if resp.status_code == 403:
pytest.skip(
"Server security level blocks switch_version with 403 before "
"validation runs; validation-error-body contract not reachable"
)
assert resp.status_code == 400, (
f"Expected 400 validation error, got {resp.status_code}: {resp.text[:200]}"
)
# Pydantic validation returns JSON with 'error' + 'details' list.
try:
data = resp.json()
except requests.exceptions.JSONDecodeError:
pytest.fail(
f"400 response should be JSON but got plain text: {resp.text[:200]}"
)
assert "error" in data, (
f"400 response must include 'error' field, got: {data!r}"
)
assert data["error"] == "Validation error", (
f"'error' field must be the exact 'Validation error' sentinel, got {data['error']!r}"
)
assert "details" in data, (
f"400 response must include 'details' list, got: {data!r}"
)
details = data["details"]
assert isinstance(details, list), (
f"'details' must be a list, got {type(details).__name__}"
)
assert len(details) >= 1, (
"'details' must contain at least one Pydantic error entry, got empty list"
)
# Each entry is a Pydantic error dict with canonical keys.
for i, entry in enumerate(details):
assert isinstance(entry, dict), (
f"details[{i}] must be a dict, got {type(entry).__name__}"
)
for required_key in ("loc", "msg", "type"):
assert required_key in entry, (
f"details[{i}] missing Pydantic key {required_key!r}: {entry!r}"
)
+44
View File
@@ -0,0 +1,44 @@
# Playwright E2E Tests — Legacy Manager UI
Browser-based E2E tests for the ComfyUI-Manager legacy UI.
## Prerequisites
1. **E2E environment** built via `python tests/e2e/scripts/setup_e2e_env.py`
2. **Playwright installed**: `npx playwright install chromium`
3. **ComfyUI running** with legacy UI enabled:
```bash
E2E_ROOT=/tmp/e2e_full_test
PORT=8199
$E2E_ROOT/venv/bin/python $E2E_ROOT/comfyui/main.py \
--listen 127.0.0.1 --port $PORT \
--enable-manager-legacy-ui \
--cpu
```
## Running Tests
```bash
# With server already running:
PORT=8199 npx playwright test
# Single file:
PORT=8199 npx playwright test tests/playwright/legacy-ui-manager-menu.spec.ts
# Headed (visible browser):
PORT=8199 npx playwright test --headed
# Debug mode:
PORT=8199 npx playwright test --debug
```
## Test Files
| File | Scenarios |
|------|-----------|
| `legacy-ui-manager-menu.spec.ts` | Menu dialog rendering, settings dropdowns, API round-trip |
| `legacy-ui-custom-nodes.spec.ts` | Node list grid, filter, search, footer buttons |
| `legacy-ui-model-manager.spec.ts` | Model list grid, filter, search |
| `legacy-ui-snapshot.spec.ts` | Snapshot list, save, remove |
| `legacy-ui-navigation.spec.ts` | Dialog open/close, nested navigation, no duplicates |
+202
View File
@@ -0,0 +1,202 @@
# Legacy UI Playwright E2E — Test Scenarios
Scenario list based on the actual API call flow of the Legacy UI (runtime-verified).
## URL Convention
ComfyUI's `api.fetchApi()` automatically prepends the `/api` prefix to every path.
- JS call: `fetchApi('/v2/manager/db_mode')` → actual request: `GET /api/v2/manager/db_mode`
- When intercepted by Playwright `route`, the URL is captured in the `/api/v2/...` form.
## Install Flow (Runtime-verified)
```
[Install button click]
→ GET /api/v2/customnode/versions/{id} ← list available versions
→ Version-selection dialog (<select multiple> + "Select"/"Cancel")
→ "Select" click
→ POST /api/v2/manager/queue/batch ← actual install request
body: {"install":[{...nodeData, selected_version:"latest"}], "batch_id":"uuid"}
→ WebSocket push: cm-queue-status
{status:"in_progress", done_count, total_count}
{status:"batch-done", nodepack_result:{"hash":"success"}}
{status:"all-done"}
```
---
## 1. API calls at Manager Menu initialization
**File**: `legacy-ui-manager-menu.spec.ts` (existing + enhanced)
5 API calls made concurrently when the Manager Menu opens (runtime-verified):
| # | Scenario | Assertion | Endpoint |
|---|---------|------|----------|
| 1-1 | DB mode loading | Dropdown value shown | `GET /api/v2/manager/db_mode` |
| 1-2 | Channel list loading | Dropdown options shown | `GET /api/v2/manager/channel_url_list` |
| 1-3 | Update policy loading | Dropdown value shown | `GET /api/v2/manager/policy/update` |
| 1-4 | Notice loading | Right panel text is non-empty | `GET /api/v2/manager/notice` |
| 1-5 | DB mode change round-trip | POST → GET verification | `POST /api/v2/manager/db_mode` |
| 1-6 | Policy change round-trip | POST → GET verification | `POST /api/v2/manager/policy/update` |
| 1-7 | Channel change round-trip | POST → GET verification | `POST /api/v2/manager/channel_url_list` |
## 2. Custom Nodes Manager — list retrieval
**File**: `legacy-ui-custom-nodes.spec.ts` (existing + enhanced)
2 API calls made when the Custom Nodes Manager opens (runtime-verified):
| # | Scenario | UI action | Assertion | Endpoint |
|---|---------|---------|------|----------|
| 2-1 | List loading (cache) | Open Custom Nodes Manager | Grid rows > 0 | `GET /api/v2/customnode/getlist?mode=cache&skip_update=true` |
| 2-2 | Mapping loading | (concurrent with list) | Request observed | `GET /api/v2/customnode/getmappings?mode=cache` |
| 2-3 | Installed filter | Filter → "Installed" | rows ≤ All | Client-side filter |
| 2-4 | Not Installed filter | Filter → "Not Installed" | rows > 0 | Client-side filter |
| 2-5 | Import Failed filter | Filter → "Import Failed" | Filter works | Client-side filter |
| 2-6 | Check Update | "Check Update" button | Filter flips to "Update", API re-called | `GET /api/v2/customnode/getlist?mode=cache` (no `skip_update`) |
| 2-7 | Check Missing | "Check Missing" button | Filter flips to "Missing" | `GET /api/v2/customnode/getmappings?mode=cache` |
| 2-8 | Alternatives filter | Filter → "Alternatives of A1111" | Data loads | `GET /api/customnode/alternatives?mode=cache` |
## 3. Full node install lifecycle
**File**: `legacy-ui-node-lifecycle.spec.ts` (new)
Install flow: Install button → versions API → version-selection dialog → Select → queue/batch → WebSocket status push
| # | Scenario | UI action | Assertion | Endpoint |
|---|---------|---------|------|----------|
| 3-1 | Install — version query | "Not Installed" → "Install" click | Version-list dialog appears | `GET /api/v2/customnode/versions/{id}` |
| 3-2 | Install — select version + send batch | Pick version in `<select>` → "Select" click | `queue/batch` called with `install` key in body | `POST /api/v2/manager/queue/batch` |
| 3-3 | Install — WebSocket status | Install runs | `cm-queue-status` messages: in_progress → batch-done → all-done | WebSocket |
| 3-4 | Uninstall | "Installed" → "Uninstall" click → confirm dialog "OK" | `uninstall` key in batch body | `POST /api/v2/manager/queue/batch` |
| 3-5 | Disable | "Installed" → "Disable" click | `disable` key in batch body | `POST /api/v2/manager/queue/batch` |
| 3-6 | Enable | "Disabled" → "Enable" click → pick version | `install` key + `skip_post_install:true` in batch body | `GET disabled_versions/{id}``POST queue/batch` |
| 3-7 | Update | "Installed" → "Try update" click | `update` key in batch body | `POST /api/v2/manager/queue/batch` |
| 3-8 | Fix | import-fail node → "Try fix" click | `fix` key in batch body (skip if none) | `POST /api/v2/manager/queue/batch` |
## 4. Version management
**File**: `legacy-ui-node-versions.spec.ts` (new)
| # | Scenario | UI action | Assertion | Endpoint |
|---|---------|---------|------|----------|
| 4-1 | Installed node version list | "Installed" → "Switch Ver" click | Version list shown in `<select multiple>` dialog | `GET /api/v2/customnode/versions/{id}` |
| 4-2 | Disabled node version list | "Disabled" → "Enable" click | `disabled_versions` call observed | `GET /api/v2/customnode/disabled_versions/{id}` |
## 5. Batch operations + stop
**File**: `legacy-ui-batch-operations.spec.ts` (new)
| # | Scenario | UI action | Assertion | Endpoint |
|---|---------|---------|------|----------|
| 5-1 | Update All | Manager Menu → "Update All" click | `update_all` key in batch body | `POST /api/v2/manager/queue/batch` |
| 5-2 | Update ComfyUI | Manager Menu → "Update ComfyUI" click | `update_comfyui` key in batch body | `POST /api/v2/manager/queue/batch` |
| 5-3 | Stop (Manager Menu) | "Restart" toggle → "Stop" click | `queue/reset` invoked | `POST /api/v2/manager/queue/reset` |
| 5-4 | Stop (Custom Nodes Manager) | "Stop" button click | `queue/reset` invoked | `POST /api/v2/manager/queue/reset` |
Note: `queue/abort_current` is not called directly from JS (server-only). Stop uses `queue/reset`.
## 6. Git URL / PIP install
**File**: `legacy-ui-install-methods.spec.ts` (new)
| # | Scenario | UI action | Assertion | Endpoint |
|---|---------|---------|------|----------|
| 6-1 | Git URL install | "Install via Git URL" → enter URL → confirm | 200 or 403 | `POST /api/v2/customnode/install/git_url` |
| 6-2 | Git URL cancel | "Install via Git URL" → cancel | No API call | — |
| 6-3 | PIP package install | "Install PIP packages" → enter package name | 200 or 403 | `POST /api/v2/customnode/install/pip` |
## 7. Import failure details
**File**: `legacy-ui-custom-nodes.spec.ts` (enhanced)
| # | Scenario | UI action | Assertion | Endpoint |
|---|---------|---------|------|----------|
| 7-1 | Import failure details | "Import Failed" filter → "IMPORT FAILED ↗" click | Error dialog appears | `POST /api/v2/customnode/import_fail_info` |
Note: skipped when no import-failed nodes are present.
## 8. Model management
**File**: `legacy-ui-model-manager.spec.ts` (existing + enhanced)
| # | Scenario | UI action | Assertion | Endpoint |
|---|---------|---------|------|----------|
| 8-1 | Model list loading | Open Model Manager | Grid rows > 0 | `GET /api/v2/externalmodel/getlist?mode=cache` |
| 8-2 | Model search | Enter query | Grid filtered | Client-side filter |
| 8-3 | Model install | Row's "Install" click | `install_model` key in batch body | `POST /api/v2/manager/queue/batch` |
## 9. Full snapshot lifecycle
**File**: `legacy-ui-snapshot.spec.ts` (existing + enhanced)
| # | Scenario | UI action | Assertion | Endpoint |
|---|---------|---------|------|----------|
| 9-1 | Snapshot list | Open Snapshot Manager | Table rows shown | `GET /api/v2/snapshot/getlist` |
| 9-2 | Snapshot save | "Save snapshot" click | "Current snapshot saved" message | `POST /api/v2/snapshot/save` |
| 9-3 | Snapshot restore | "Restore" click | 200 or 403, RESTART button shown | `POST /api/v2/snapshot/restore?target=X` |
| 9-4 | Snapshot remove | "Remove" click | Row removed from list | `POST /api/v2/snapshot/remove?target=X` |
## 10. Dialog navigation
**File**: `legacy-ui-navigation.spec.ts` (existing)
| # | Scenario | Assertion |
|---|---------|------|
| 10-1 | Manager → Custom Nodes → close → re-open Manager | Dialog transitions cleanly |
| 10-2 | Manager → Model Manager → close → re-open | Dialog transitions cleanly |
| 10-3 | API call while dialog is open | Server responds normally |
| 10-4 | Legacy UI enabled check | `is_legacy_manager_ui: true` |
---
## File composition summary
| File | New/Enhanced | Scenarios | Target Endpoints |
|------|----------|:-------:|---------------|
| `legacy-ui-manager-menu.spec.ts` | enhanced | 7 | db_mode, channel_url_list, policy/update, notice |
| `legacy-ui-custom-nodes.spec.ts` | enhanced | 9 | getlist, getmappings, alternatives, import_fail_info |
| `legacy-ui-node-lifecycle.spec.ts` | **new** | 8 | versions/{id}, disabled_versions/{id}, queue/batch (install/uninstall/update/fix/disable/enable) |
| `legacy-ui-node-versions.spec.ts` | **new** | 2 | versions/{id}, disabled_versions/{id} |
| `legacy-ui-batch-operations.spec.ts` | **new** | 4 | queue/batch (update_all, update_comfyui), queue/reset |
| `legacy-ui-install-methods.spec.ts` | **new** | 3 | install/git_url, install/pip |
| `legacy-ui-model-manager.spec.ts` | enhanced | 3 | externalmodel/getlist, queue/batch (install_model) |
| `legacy-ui-snapshot.spec.ts` | enhanced | 4 | snapshot/getlist, save, restore, remove |
| `legacy-ui-navigation.spec.ts` | existing | 4 | is_legacy_manager_ui, version |
**Total: 44 scenarios**
## Legacy-only endpoint coverage
| Endpoint | Scenarios |
|----------|---------|
| `GET /api/v2/customnode/getlist` | 2-1, 2-6 |
| `GET /api/v2/customnode/getmappings` | 2-2, 2-7 |
| `GET /api/customnode/alternatives` | 2-8 |
| `GET /api/v2/customnode/versions/{id}` | 3-1, 4-1 |
| `GET /api/v2/customnode/disabled_versions/{id}` | 3-6, 4-2 |
| `POST /api/v2/customnode/import_fail_info` | 7-1 |
| `POST /api/v2/customnode/install/git_url` | 6-1 |
| `POST /api/v2/customnode/install/pip` | 6-3 |
| `GET /api/v2/externalmodel/getlist` | 8-1 |
| `POST /api/v2/manager/queue/batch` | 3-2, 3-4~3-8, 5-1, 5-2, 8-3 |
| `POST /api/v2/manager/queue/reset` | 5-3, 5-4 |
| `GET /api/v2/manager/notice` | 1-4 |
| `GET /api/v2/snapshot/getlist` | 9-1 |
| `POST /api/v2/snapshot/save` | 9-2 |
| `POST /api/v2/snapshot/restore` | 9-3 |
| `POST /api/v2/snapshot/remove` | 9-4 |
## Exclusions
- `share_option` — per user instruction
- **External-service auth/integration** (9 endpoints) — external services are unreachable from E2E
- **Individual queue endpoints** (6) — unused by JS; delegated internally through `queue/batch`
- `queue/abort_current` — unused by JS (Stop uses `queue/reset`)
- `/manager/notice` (v1) — superseded by v2
## API call verification method
Capture the actual API call sequence via Playwright `page.route('**/*')` interception.
Verify job progress/completion via the `cm-queue-status` WebSocket event.
+128
View File
@@ -0,0 +1,128 @@
/**
* Shared helpers for ComfyUI Manager Playwright E2E tests.
*
* The legacy UI is dialog-based: a "Manager" menu button on the ComfyUI
* top-bar opens ManagerMenuDialog, from which sub-dialogs (CustomNodes,
* Model, Snapshot) are launched.
*/
import { type Page, expect } from '@playwright/test';
/** Wait for the ComfyUI page to be fully loaded (queue ready). */
export async function waitForComfyUI(page: Page) {
// ComfyUI shows the canvas once the app is ready. Wait for the
// system_stats endpoint to respond — same check the Python E2E uses.
await page.waitForFunction(
async () => {
try {
const r = await fetch('/system_stats');
return r.ok;
} catch {
return false;
}
},
{ timeout: 30_000, polling: 1_000 },
);
// Give the extensions a moment to register their menu items.
await page.waitForTimeout(3_000);
// Close any overlay that might be covering the toolbar.
// Press Escape to dismiss popups/modals/sidebars.
await page.keyboard.press('Escape');
await page.waitForTimeout(1_000);
await page.keyboard.press('Escape');
await page.waitForTimeout(500);
}
/** Open the Manager Menu dialog via the top-bar button. */
export async function openManagerMenu(page: Page) {
// The legacy UI registers a "Manager" button via ComfyButton (new style)
// or a plain <button> (old style). The new-style button uses the
// "puzzle" icon and has tooltip "ComfyUI Manager" / content "Manager".
//
// ComfyButton renders as a structure like:
// <button class="comfyui-button" title="ComfyUI Manager">
// <span class="icon">...</span>
// <span>Manager</span>
// </button>
//
// We try multiple selectors to handle both old and new ComfyUI layouts.
const selectors = [
'button[title="ComfyUI Manager"]', // new-style ComfyButton
'button.comfyui-button:has-text("Manager")', // new-style fallback
'button:has-text("Manager")', // old-style plain button
];
for (const sel of selectors) {
const btn = page.locator(sel).first();
if (await btn.isVisible({ timeout: 3_000 }).catch(() => false)) {
await btn.click();
await page.waitForSelector('#cm-manager-dialog, .comfy-modal', { timeout: 10_000 });
return;
}
}
// Last resort: find any button with "Manager" in tooltip or text via DOM
const found = await page.evaluate(() => {
const buttons = document.querySelectorAll('button');
for (const btn of buttons) {
const text = btn.textContent?.toLowerCase() || '';
const title = btn.getAttribute('title')?.toLowerCase() || '';
if (text.includes('manager') || title.includes('manager')) {
(btn as HTMLElement).click();
return true;
}
}
return false;
});
if (found) {
// Wait for the dialog by polling for the element in DOM
await page.waitForFunction(
() => !!document.getElementById('cm-manager-dialog'),
{ timeout: 10_000, polling: 500 },
);
return;
}
await page.screenshot({ path: 'test-results/debug-manager-btn-not-found.png' });
throw new Error('Could not find Manager button in ComfyUI toolbar');
}
/** Click a button inside the Manager Menu dialog by its visible text. */
export async function clickMenuButton(page: Page, text: string) {
const dialog = page.locator('#cm-manager-dialog').first();
await dialog.locator(`button:has-text("${text}")`).click();
}
/** Close the topmost dialog via its X (close) button or Escape. */
export async function closeDialog(page: Page) {
// Try clicking close buttons on visible dialogs. The manager-menu dialog
// (`#cm-manager-dialog`) is a ComfyDialog with `.p-dialog-close-button` (X),
// while sub-dialogs use `.cm-close-btn`. Try both.
for (const sel of [
'#cn-manager-dialog button.cm-close-btn',
'#cmm-manager-dialog button.cm-close-btn',
'#snapshot-manager-dialog button.cm-close-btn',
'#cm-manager-dialog button.cm-close-btn',
'#cm-manager-dialog .p-dialog-close-button',
'.cm-close-btn',
'.p-dialog-close-button',
]) {
const btn = page.locator(sel).last();
if (await btn.isVisible({ timeout: 500 }).catch(() => false)) {
await btn.click();
await page.waitForTimeout(300);
return;
}
}
// Fallback: press Escape (ComfyDialog may not honor this reliably)
await page.keyboard.press('Escape');
await page.waitForTimeout(300);
}
/** Assert the Manager Menu dialog is visible and contains expected sections. */
export async function assertManagerMenuVisible(page: Page) {
const dialog = page.locator('#cm-manager-dialog').first();
await expect(dialog).toBeVisible();
}
@@ -0,0 +1,152 @@
/**
* E2E tests: Legacy Custom Nodes Manager dialog.
*
* Tests the TurboGrid-based custom node list, filters, search,
* and basic row interactions.
*
* Requires ComfyUI running with --enable-manager-legacy-ui on PORT.
*/
import { test, expect } from '@playwright/test';
import { waitForComfyUI, openManagerMenu, clickMenuButton } from './helpers';
test.describe('Custom Nodes Manager', () => {
test.beforeEach(async ({ page }) => {
await page.goto('/');
await waitForComfyUI(page);
await openManagerMenu(page);
});
test('opens from Manager menu and renders grid', async ({ page }) => {
await clickMenuButton(page, 'Custom Nodes Manager');
// Wait for the custom nodes dialog to appear
await page.waitForSelector('#cn-manager-dialog', {
timeout: 10_000,
});
// The grid should be present
const grid = page.locator('.cn-manager-grid, .tg-body').first();
await expect(grid).toBeVisible({ timeout: 15_000 });
});
test('loads custom node list (non-empty)', async ({ page }) => {
await clickMenuButton(page, 'Custom Nodes Manager');
await page.waitForSelector('.cn-manager-grid, .tg-body', { timeout: 15_000 });
// Wait for data to load — grid rows should appear
await page.waitForFunction(
() => {
const rows = document.querySelectorAll('.tg-body .tg-row, .cn-manager-grid tr');
return rows.length > 0;
},
{ timeout: 30_000, polling: 1_000 },
);
const rows = page.locator('.tg-body .tg-row, .cn-manager-grid tr');
const count = await rows.count();
expect(count).toBeGreaterThan(0);
});
test('filter dropdown changes displayed nodes', async ({ page }) => {
await clickMenuButton(page, 'Custom Nodes Manager');
await page.waitForSelector('.cn-manager-grid, .tg-body', { timeout: 15_000 });
// Wait for initial data load
await page.waitForFunction(
() => document.querySelectorAll('.tg-body .tg-row, .cn-manager-grid tr').length > 0,
{ timeout: 30_000, polling: 1_000 },
);
// Find the filter select (class: cn-manager-filter) and switch to "Installed"
const filterSelect = page.locator('select.cn-manager-filter').first();
// Hard-fail if filter UI missing — that's a regression, not a skip condition
await expect(filterSelect).toBeVisible({ timeout: 5_000 });
const initialCount = await page.locator('.tg-body .tg-row').count();
await filterSelect.selectOption({ label: 'Installed' });
// Wait for row count to actually CHANGE (state-based, not wall-clock).
// If filter is broken and returns everything, this will fail within 10s.
await expect
.poll(async () => page.locator('.tg-body .tg-row').count(), { timeout: 10_000 })
.not.toBe(initialCount);
// Installed count should be <= total
const filteredCount = await page.locator('.tg-body .tg-row').count();
expect(filteredCount).toBeLessThanOrEqual(initialCount);
});
test('search input filters the grid', async ({ page }) => {
await clickMenuButton(page, 'Custom Nodes Manager');
await page.waitForSelector('.cn-manager-grid, .tg-body', { timeout: 15_000 });
await page.waitForFunction(
() => document.querySelectorAll('.tg-body .tg-row, .cn-manager-grid tr').length > 0,
{ timeout: 30_000, polling: 1_000 },
);
// Find search input
const searchInput = page.locator('.cn-manager-keywords, input[type="text"][placeholder*="earch"], input[type="search"]').first();
await expect(searchInput).toBeVisible({ timeout: 5_000 });
const initialCount = await page.locator('.tg-body .tg-row, .cn-manager-grid tr').count();
await searchInput.fill('ComfyUI-Manager');
// State-based wait: count must actually narrow (or become 0)
await expect
.poll(
async () => page.locator('.tg-body .tg-row, .cn-manager-grid tr').count(),
{ timeout: 10_000 },
)
.toBeLessThan(initialCount);
const filteredCount = await page.locator('.tg-body .tg-row, .cn-manager-grid tr').count();
expect(filteredCount).toBeLessThanOrEqual(initialCount);
});
test('footer buttons are present', async ({ page }) => {
// Wave3 WI-U Cluster H target 4: strengthen from OR-of-2 to AND-of-all-
// always-visible-admin-buttons. js/custom-nodes-manager.js:26-34 defines 6
// footer buttons, but `.cn-manager-restart` and `.cn-manager-stop` are
// `display: none` by default in custom-nodes-manager.css:47-62 (shown only
// via showRestart()/showStop() — conditional on restart-required /
// task-running state). In a clean Manager state, neither is visible.
//
// The 4 ALWAYS-visible footer admin buttons are:
// - "Install via Git URL" — primary install entrypoint
// - "Used In Workflow" — filter to workflow-referenced nodes
// - "Check Update" — refresh available-update list
// - "Check Missing" — scan for missing nodes
//
// We assert all 4 are visible (AND semantics). Hidden-by-default Restart/
// Stop are checked structurally — exist in DOM but may be hidden.
await clickMenuButton(page, 'Custom Nodes Manager');
await page.waitForSelector('#cn-manager-dialog', {
timeout: 15_000,
});
const dialog = page.locator('#cn-manager-dialog').last();
// AND semantics: every always-visible footer button MUST be visible.
const alwaysVisibleButtons = [
'Install via Git URL',
'Used In Workflow',
'Check Update',
'Check Missing',
];
for (const label of alwaysVisibleButtons) {
await expect(
dialog.locator(`button:has-text("${label}")`).first(),
`always-visible footer button "${label}" must be present and visible`,
).toBeVisible();
}
// Structural presence for conditional buttons — they exist in the DOM but
// are hidden until showRestart()/showStop() toggles `display: block`.
for (const cls of ['.cn-manager-restart', '.cn-manager-stop']) {
await expect(
dialog.locator(cls),
`conditional footer button ${cls} must be present in DOM (may be hidden)`,
).toHaveCount(1);
}
});
});
+294
View File
@@ -0,0 +1,294 @@
/**
* E2E tests: UI-driven install/uninstall effect verification.
*
* Contract: LEGACY UI tests must drive the action via UI elements (no direct API calls).
* Effect is observed through backend state (queue/status, installed list) and/or UI badges.
*
* Requires ComfyUI running with --enable-manager-legacy-ui on PORT.
* Test pack: ComfyUI_SigmoidOffsetScheduler (ltdrdata's test pack).
*/
import { test, expect } from '@playwright/test';
import { waitForComfyUI, openManagerMenu, clickMenuButton } from './helpers';
const PACK_CNR_ID = 'comfyui_sigmoidoffsetscheduler';
async function waitForAllDone(page: import('@playwright/test').Page, timeoutMs = 90_000): Promise<void> {
// Three-phase polling with DETERMINISTIC baseline:
// Phase 0 — snapshot baseline. To make the baseline deterministic across
// runs (and immune to leaking history from prior tests in the
// session), we FETCH the baseline immediately after the caller
// has triggered the UI action. The caller is expected to have
// called /v2/manager/queue/reset at the start of its test flow
// so that done_count starts at 0 for this test's session.
// Phase 1 — wait for task acceptance:
// total_count > 0 OR is_processing=true OR done_count > baseline
// Phase 2 — wait for drain (total_count === 0 && is_processing=false)
const deadline = Date.now() + timeoutMs;
// Phase 0: baseline. If fetch fails, treat as 0 but log so the test signal
// isn't silently degraded.
let baselineDone = 0;
const baselineResp = await page.request
.get('/v2/manager/queue/status')
.catch(() => null);
if (baselineResp && baselineResp.ok()) {
const baseline = await baselineResp.json();
baselineDone = baseline?.done_count ?? 0;
} else {
console.warn('[waitForAllDone] baseline fetch failed — treating as 0');
}
// Phase 1: task acceptance
const acceptDeadline = Math.min(Date.now() + 15_000, deadline);
let accepted = false;
while (Date.now() < acceptDeadline) {
const status = await page.request
.get('/v2/manager/queue/status')
.then((r) => r.json())
.catch(() => null);
if (
status &&
((status.total_count ?? 0) > 0 ||
status.is_processing === true ||
(status.done_count ?? 0) > baselineDone)
) {
accepted = true;
break;
}
await page.waitForTimeout(500);
}
if (!accepted) {
throw new Error('Queue never accepted the task (empty queue for 15s after UI action)');
}
// Phase 2: drain
while (Date.now() < deadline) {
const status = await page.request
.get('/v2/manager/queue/status')
.then((r) => r.json())
.catch(() => null);
if (status && status.is_processing === false && (status.total_count ?? 0) === 0) {
return;
}
await page.waitForTimeout(1_500);
}
throw new Error(`Queue did not drain within ${timeoutMs}ms`);
}
async function isPackInstalled(page: import('@playwright/test').Page): Promise<boolean> {
const resp = await page.request.get('/v2/customnode/installed');
if (!resp.ok()) return false;
const data = await resp.json();
for (const pkg of Object.values<unknown>(data)) {
if (
pkg &&
typeof pkg === 'object' &&
(pkg as { cnr_id?: string }).cnr_id?.toLowerCase() === PACK_CNR_ID
) {
return true;
}
}
return false;
}
test.describe('UI-driven install/uninstall', () => {
test.beforeEach(async ({ page }) => {
await page.goto('/');
await waitForComfyUI(page);
});
test('LB1 Install button triggers install effect', async ({ page }) => {
// Reset queue at start for deterministic done_count baseline in waitForAllDone
await page.request.post('/v2/manager/queue/reset');
// Precondition: pack must NOT be installed. If the seed pack is already
// installed (prior pytest runs, pre-seeded E2E environment), the
// "Not Installed" filter applied below would correctly exclude its row and
// `packRow.toBeVisible` would fail with "element(s) not found". Uninstall
// via API as test SETUP (not verification) — mirrors LB2's inverse pattern
// that API-installs if the pack is absent. queue/batch is used here (not
// queue/task) because queue/batch is the legacy manager_server endpoint
// for task enqueueing; queue/task is glob-only — under
// --enable-manager-legacy-ui (which this spec requires) POST /queue/task
// falls through to aiohttp's GET-only static catch-all and returns 405.
if (await isPackInstalled(page)) {
const queueResp = await page.request.post('/v2/manager/queue/batch', {
data: JSON.stringify({
batch_id: 'lb1-setup-uninstall',
uninstall: [{
id: 'ComfyUI_SigmoidOffsetScheduler',
ui_id: 'lb1-setup-uninstall',
version: '1.0.1',
selected_version: 'latest',
mode: 'local',
channel: 'default',
}],
}),
headers: { 'Content-Type': 'application/json' },
});
expect(queueResp.ok()).toBe(true);
await page.request.post('/v2/manager/queue/start');
await waitForAllDone(page, 60_000);
// Hard fail if setup itself couldn't uninstall the pack
expect(await isPackInstalled(page)).toBe(false);
}
// UI flow: open Manager → Custom Nodes Manager
await openManagerMenu(page);
await clickMenuButton(page, 'Custom Nodes Manager');
await page.waitForSelector('#cn-manager-dialog', { timeout: 15_000 });
// Wait for grid to populate before applying filter (avoids race on empty grid)
await expect(page.locator('.tg-body .tg-row').first()).toBeVisible({ timeout: 30_000 });
const initialRowCount = await page.locator('.tg-body .tg-row').count();
// Filter to Not Installed to make install buttons visible. Wait for
// filtered row count to actually change (DOM state, not wall-clock).
const filterSelect = page.locator('select.cn-manager-filter').first();
if (await filterSelect.isVisible().catch(() => false)) {
await filterSelect.selectOption({ value: 'not-installed' });
await expect
.poll(async () => page.locator('.tg-body .tg-row').count(), { timeout: 10_000 })
.not.toBe(initialRowCount);
}
// Search for the specific test pack. Wait for search to narrow results.
// Search matches title/author/description per custom-nodes-manager.js:605
// (NOT id). The pack's title is "ComfyUI Sigmoid Offset Scheduler" (with
// spaces), so "SigmoidOffsetScheduler" (no spaces) would miss — use
// "Sigmoid Offset Scheduler" to match the title substring.
const searchInput = page
.locator('.cn-manager-keywords, input[type="search"], input[type="text"][placeholder*="earch"]')
.first();
if (await searchInput.isVisible().catch(() => false)) {
await searchInput.fill('Sigmoid Offset Scheduler');
// Wait for search to settle — row count stabilizes
await expect
.poll(async () => page.locator('.tg-body .tg-row').count(), { timeout: 10_000 })
.toBeLessThanOrEqual(5);
}
// Scope button to the row containing the pack name (not arbitrary first row).
// Row DOM renders the title column, which reads "ComfyUI Sigmoid Offset
// Scheduler" — match the substring that appears there, not the id.
// TurboGrid splits each logical row into TWO DOM .tg-row elements (left
// frozen-column pane with the title + right scrollable-column pane with
// Version/Action/etc.). The Install button lives in the right pane, so
// filtering by title-text picks the left pane which has no Install button.
// Use `.tg-body` scope + `button[mode="install"]` directly, then assert
// only one such button exists (single search result narrows to 1 row).
const packRow = page.locator('.tg-body .tg-row', { hasText: 'Sigmoid Offset Scheduler' }).first();
await expect(packRow).toBeVisible({ timeout: 10_000 });
const installBtn = page.locator('.tg-body button[mode="install"]').first();
// Hard fail if the Install button isn't visible in the filtered result
await expect(installBtn).toBeVisible({ timeout: 5_000 });
await installBtn.click();
// Version selector dialog appears
const selectBtn = page.locator('.comfy-modal button:has-text("Select")').first();
await selectBtn.waitFor({ timeout: 10_000 });
await selectBtn.click();
// Effect verification: wait for queue to drain then check installed state
await waitForAllDone(page, 120_000);
const installed = await isPackInstalled(page);
expect(installed).toBe(true);
});
test('LB2 Uninstall button triggers uninstall effect', async ({ page }) => {
// Reset queue at start for deterministic done_count baseline in waitForAllDone
await page.request.post('/v2/manager/queue/reset');
// Precondition: pack must be installed. Install via API as test SETUP
// (not verification). This makes LB2 independent of LB1 — hard-failing
// on a UI bug rather than skipping on a missing precondition. queue/batch
// is the legacy manager_server endpoint (see LB1 comment above); install
// is async, so waitForAllDone is still required after queue/start.
const preinstalled = await isPackInstalled(page);
if (!preinstalled) {
await page.request.post('/v2/manager/queue/reset');
const queueResp = await page.request.post('/v2/manager/queue/batch', {
data: JSON.stringify({
batch_id: 'lb2-setup-install',
install: [{
id: 'ComfyUI_SigmoidOffsetScheduler',
ui_id: 'lb2-setup-install',
version: '1.0.1',
selected_version: 'latest',
mode: 'remote',
channel: 'default',
}],
}),
headers: { 'Content-Type': 'application/json' },
});
expect(queueResp.ok()).toBe(true);
const queueBody = await queueResp.json();
expect(queueBody.failed ?? []).toEqual([]);
await page.request.post('/v2/manager/queue/start');
// Poll the terminal state directly: isPackInstalled returning true is
// the unambiguous success signal. Using waitForAllDone here is racy —
// fast-path installs (pack already on disk / cached CNR artifacts) can
// complete before waitForAllDone's Phase 0 baseline fetch runs, leaving
// Phase 1 unable to distinguish "already done" from "never queued".
// Polling isPackInstalled avoids that ambiguity entirely.
await expect.poll(() => isPackInstalled(page), { timeout: 120_000 }).toBe(true);
}
await openManagerMenu(page);
await clickMenuButton(page, 'Custom Nodes Manager');
await page.waitForSelector('#cn-manager-dialog', { timeout: 15_000 });
await expect(page.locator('.tg-body .tg-row').first()).toBeVisible({ timeout: 30_000 });
const initialRowCount = await page.locator('.tg-body .tg-row').count();
// Filter to Installed to make Uninstall buttons visible
const filterSelect = page.locator('select.cn-manager-filter').first();
if (await filterSelect.isVisible().catch(() => false)) {
await filterSelect.selectOption({ label: 'Installed' });
await expect
.poll(async () => page.locator('.tg-body .tg-row').count(), { timeout: 10_000 })
.not.toBe(initialRowCount);
}
// Search matches title/author/description per custom-nodes-manager.js:605
// (NOT id). Pack title is "ComfyUI Sigmoid Offset Scheduler" (spaces) —
// use the space-separated form to match (WI-CC pattern).
const searchInput = page
.locator('.cn-manager-keywords, input[type="search"], input[type="text"][placeholder*="earch"]')
.first();
if (await searchInput.isVisible().catch(() => false)) {
await searchInput.fill('Sigmoid Offset Scheduler');
await expect
.poll(async () => page.locator('.tg-body .tg-row').count(), { timeout: 10_000 })
.toBeLessThanOrEqual(5);
}
// Scope packRow visibility to the specific pack title, but the Uninstall
// button lives in the right-pane .tg-row (TurboGrid dual-pane rendering),
// which is NOT a child of the title-bearing left-pane row. Scope the
// button lookup to the grid body + search-narrowed result set (WI-CC pattern).
const packRow = page.locator('.tg-body .tg-row', { hasText: 'Sigmoid Offset Scheduler' }).first();
await expect(packRow).toBeVisible({ timeout: 10_000 });
const uninstallBtn = page.locator('.tg-body button[mode="uninstall"]').first();
await expect(uninstallBtn).toBeVisible({ timeout: 5_000 });
await uninstallBtn.click();
// A confirmation dialog appears — custom-nodes-manager.js uses
// `customConfirm` (PrimeVue p-dialog), not `.comfy-modal`. The dialog
// is the last-opened one (on top of manager-menu + CustomNodes dialogs);
// its Confirm button accessible name has a leading space (icon + text),
// so match by visible text substring rather than exact name.
const confirmDialog = page.locator('dialog[open], [role="dialog"]').last();
const confirmBtn = confirmDialog.locator('button:has-text("Confirm"), button:has-text("Yes"), button:has-text("OK")').first();
if (await confirmBtn.isVisible({ timeout: 5_000 }).catch(() => false)) {
await confirmBtn.click();
}
// Poll isPackInstalled directly — the uninstall queue drains fast enough
// that waitForAllDone's Phase 0/1 baseline-vs-done race can miss
// acceptance. isPackInstalled==false is the unambiguous terminal signal.
await expect.poll(() => isPackInstalled(page), { timeout: 60_000 }).toBe(false);
});
});
@@ -0,0 +1,334 @@
/**
* E2E tests: Legacy Manager Menu Dialog.
*
* Verifies that the legacy UI manager menu opens correctly, renders
* all expected controls, and that settings dropdowns round-trip through
* the server API.
*
* Requires ComfyUI running with --enable-manager-legacy-ui on PORT.
*/
import { test, expect } from '@playwright/test';
import { waitForComfyUI, openManagerMenu, assertManagerMenuVisible, closeDialog } from './helpers';
test.describe('Manager Menu Dialog', () => {
test.beforeEach(async ({ page }) => {
await page.goto('/');
await waitForComfyUI(page);
});
test('opens via Manager button and shows 3-column layout', async ({ page }) => {
await openManagerMenu(page);
await assertManagerMenuVisible(page);
// The dialog should contain known buttons
const dialog = page.locator('#cm-manager-dialog').first();
await expect(dialog.locator('button:has-text("Custom Nodes Manager")')).toBeVisible();
await expect(dialog.locator('button:has-text("Model Manager")')).toBeVisible();
await expect(dialog.locator('button:has-text("Restart")')).toBeVisible();
});
test('shows DB mode and Update Policy dropdowns', async ({ page }) => {
// WI-OO Item 3 (bloat dev:ci-022 B8 title-mismatch): renamed from
// "shows settings dropdowns (DB, Channel, Policy)". The original title
// promised three dropdowns but the body only asserted DB + Policy; in
// this legacy-UI build the channel combo is populated via a separate
// code path and is not reliably surfaced as a <select> in `#cm-manager-dialog`
// at open time (the DB-mode combo's options overlap with channel names via
// the "Channel" entry, which is what the original filter regex accidentally
// caught). Renaming makes the test's actual contract match its name;
// channel-dropdown coverage belongs in a dedicated test once the combo's
// stable selector is established.
await openManagerMenu(page);
const dialog = page.locator('#cm-manager-dialog').first();
// DB mode combo — options include cache/local/channel/remote.
const dbCombo = dialog.locator('select').filter({ hasText: /Cache|Local|Channel/ }).first();
await expect(dbCombo).toBeVisible();
// Update policy combo — options include Stable/Nightly variants.
const policyCombo = dialog.locator('select').filter({ hasText: /Stable|Nightly/ }).first();
await expect(policyCombo).toBeVisible();
});
test('DB mode dropdown persists via UI (close-reopen verification)', async ({ page }) => {
// Wave3 WI-U Cluster H target 1: UI-only contract.
// No page.request / page.waitForResponse — pure UI interaction + dialog
// close-reopen as the persistence proof. networkidle is used only as a
// settle barrier (wait), never as assertion input. Close via the dialog's
// own `.p-dialog-close-button` (X button) because Escape doesn't close
// ComfyDialog reliably.
await openManagerMenu(page);
const dialog = page.locator('#cm-manager-dialog').first();
const dbCombo = dialog.locator('select').filter({ hasText: /Cache|Local|Channel/ }).first();
const original = await dbCombo.inputValue();
const newValue = original !== 'local' ? 'local' : 'cache';
try {
// Select via UI — the onchange handler fires the save. Wait for
// network quiescence so the save completes before we close.
await dbCombo.selectOption(newValue);
await page.waitForLoadState('networkidle');
// Close + reopen (UI-only persistence proof)
await dialog.locator('.p-dialog-close-button').first().click();
// ComfyDialog.close() sets display:none but keeps the element in DOM,
// so check visibility (toBeHidden), not presence (toHaveCount 0).
await expect(page.locator('#cm-manager-dialog').first()).toBeHidden({ timeout: 5_000 });
await openManagerMenu(page);
const reopenedDialog = page.locator('#cm-manager-dialog').first();
const reopenedCombo = reopenedDialog
.locator('select')
.filter({ hasText: /Cache|Local|Channel/ })
.first();
const persistedValue = await reopenedCombo.inputValue();
expect(persistedValue).toBe(newValue);
} finally {
// UI-only restore: reopen if needed + selectOption back to original.
// ComfyDialog keeps the element in DOM on close (display:none), so
// test visibility rather than presence.
const existing = page.locator('#cm-manager-dialog').first();
if ((await existing.count()) === 0 || !(await existing.isVisible().catch(() => false))) {
await openManagerMenu(page);
}
const cleanupDialog = page.locator('#cm-manager-dialog').first();
const cleanupCombo = cleanupDialog
.locator('select')
.filter({ hasText: /Cache|Local|Channel/ })
.first();
// selectOption is idempotent; if the value is already `original` this
// is a no-op. networkidle guarantees the save settles before
// subsequent tests run.
await cleanupCombo.selectOption(original);
await page.waitForLoadState('networkidle');
}
});
test('Update Policy dropdown persists via UI (close-reopen verification)', async ({ page }) => {
// Wave3 WI-U Cluster H target 2: same UI-only pattern as the DB mode test.
await openManagerMenu(page);
const dialog = page.locator('#cm-manager-dialog').first();
const policyCombo = dialog.locator('select').filter({ hasText: /Stable|Nightly/ }).first();
const original = await policyCombo.inputValue();
const newValue = original !== 'nightly-comfyui' ? 'nightly-comfyui' : 'stable-comfyui';
try {
await policyCombo.selectOption(newValue);
await page.waitForLoadState('networkidle');
await dialog.locator('.p-dialog-close-button').first().click();
// ComfyDialog.close() sets display:none but keeps the element in DOM,
// so check visibility (toBeHidden), not presence (toHaveCount 0).
await expect(page.locator('#cm-manager-dialog').first()).toBeHidden({ timeout: 5_000 });
await openManagerMenu(page);
const reopenedDialog = page.locator('#cm-manager-dialog').first();
const reopenedCombo = reopenedDialog
.locator('select')
.filter({ hasText: /Stable|Nightly/ })
.first();
const persistedValue = await reopenedCombo.inputValue();
expect(persistedValue).toBe(newValue);
} finally {
// UI-only restore
if ((await page.locator('#cm-manager-dialog').count()) === 0) {
await openManagerMenu(page);
}
const cleanupDialog = page.locator('#cm-manager-dialog').first();
const cleanupCombo = cleanupDialog
.locator('select')
.filter({ hasText: /Stable|Nightly/ })
.first();
await cleanupCombo.selectOption(original);
await page.waitForLoadState('networkidle');
}
});
test('closes and reopens without duplicating', async ({ page }) => {
await openManagerMenu(page);
await assertManagerMenuVisible(page);
await closeDialog(page);
// ComfyDialog.close() sets display:none but keeps the element in DOM —
// assert the (single) instance is now hidden instead of detached.
await expect(page.locator('#cm-manager-dialog').first()).toBeHidden({ timeout: 5_000 });
// Reopen
await openManagerMenu(page);
await assertManagerMenuVisible(page);
// Exactly one dialog instance expected. `=== 1` guards against real
// duplication bugs (ComfyDialog reuses the element, so a duplicate
// instance would be a real regression).
await expect(page.locator('#cm-manager-dialog')).toHaveCount(1);
});
// WI-VV coverage — close 4 LOW-risk Playwright P-gaps from
// reports/api-coverage-matrix.md. Each test exercises a UI trigger that
// the spec suite previously missed, without destructive action.
test('WI-VV wi-001: Switch ComfyUI button fetches comfyui_versions', async ({ page }) => {
// Clicking 'Switch ComfyUI' triggers GET /v2/comfyui_manager/comfyui_versions
// (comfyui-manager.js:612) and opens a secondary version-selector dialog.
// We assert the GET response populated with a non-empty version list
// and DO NOT select a version (selection would trigger the downstream
// POST /v2/comfyui_manager/comfyui_switch_version — out of scope for
// safe P-closure).
await openManagerMenu(page);
const dialog = page.locator('#cm-manager-dialog').first();
const switchBtn = dialog.locator('button:has-text("Switch ComfyUI")').first();
await expect(switchBtn).toBeVisible();
// Race the click with the response interception so we capture the GET
// that the click fires.
const [resp] = await Promise.all([
page.waitForResponse(
(r) =>
r.url().includes('/v2/comfyui_manager/comfyui_versions') &&
r.request().method() === 'GET',
{ timeout: 15_000 },
),
switchBtn.click(),
]);
expect(resp.status()).toBe(200);
const payload = await resp.json();
expect(payload).toHaveProperty('versions');
expect(Array.isArray(payload.versions)).toBe(true);
expect(payload.versions.length).toBeGreaterThan(0);
// Dismiss the secondary version-selector dialog without selecting by
// navigating away. Reloading the page collapses all ComfyDialogs and
// restores a clean slate for subsequent tests.
await page.goto('/');
await waitForComfyUI(page);
});
test('WI-VV wi-005: channel dropdown populates from channel_url_list GET', async ({ page }) => {
// Opening the manager menu triggers GET /v2/manager/channel_url_list
// (comfyui-manager.js:963) which async-populates the channel combo.
// Stable selector per reports/legacy-ui-channel-combo-dom-mapping.md:
// select[title^="Configure the channel"]
// Options are appended from `data.list` after the fetch resolves;
// `expect.poll` waits for population without racing the async fetch.
await openManagerMenu(page);
const dialog = page.locator('#cm-manager-dialog').first();
const channelCombo = dialog.locator(
'select[title^="Configure the channel"]',
);
await expect(channelCombo).toBeVisible();
await expect
.poll(
async () => (await channelCombo.locator('option').count()),
{ timeout: 10_000, message: 'channel combo should populate from GET /v2/manager/channel_url_list' },
)
.toBeGreaterThan(0);
// Current selection should be a non-empty string (the server-side
// `selected` field from the endpoint response).
const value = await channelCombo.inputValue();
expect(value).not.toBe('');
});
test('WI-VV wi-017: changing channel combo POSTs channel_url_list', async ({ page }) => {
// Changing the channel dropdown fires the onchange handler at
// comfyui-manager.js:975-977 which POSTs the new value to
// /v2/manager/channel_url_list. Teardown in finally restores the
// original selection to keep downstream tests clean.
await openManagerMenu(page);
const dialog = page.locator('#cm-manager-dialog').first();
const channelCombo = dialog.locator(
'select[title^="Configure the channel"]',
);
await expect(channelCombo).toBeVisible();
// Wait for options to populate before reading values.
await expect
.poll(async () => (await channelCombo.locator('option').count()), {
timeout: 10_000,
})
.toBeGreaterThan(0);
const original = await channelCombo.inputValue();
const optionValues = await channelCombo
.locator('option')
.evaluateAll((opts) => opts.map((o) => (o as HTMLOptionElement).value));
const alternative = optionValues.find((v) => v !== original && v !== '');
// If the server exposes only one channel, skip with reason — we
// cannot exercise the POST without a different selectable option.
if (!alternative) {
test.skip(
true,
`channel combo only offers one value (${original}); POST path unreachable in this env`,
);
}
try {
const [postResp] = await Promise.all([
page.waitForResponse(
(r) =>
r.url().includes('/v2/manager/channel_url_list') &&
r.request().method() === 'POST',
{ timeout: 10_000 },
),
channelCombo.selectOption(alternative!),
]);
expect(postResp.status()).toBe(200);
} finally {
// Restore — accept the POST but do not re-assert; a failure here
// should not mask the assertion failure above.
const restoreCombo = page
.locator('#cm-manager-dialog')
.first()
.locator('select[title^="Configure the channel"]');
if ((await restoreCombo.count()) > 0 && (await restoreCombo.inputValue()) !== original) {
await restoreCombo.selectOption(original).catch(() => undefined);
await page.waitForLoadState('networkidle').catch(() => undefined);
}
}
});
test('WI-VV wi-021: queue/reset POST succeeds at idle (API-level Playwright)', async ({ page, request }) => {
// UI-click path is NOT feasible at idle: comfyui-manager.js:795-802
// restart_stop_button reads "Restart" when no tasks are in progress and
// invokes rebootAPI() (server reboot) — clicking it at idle would
// kill the test server mid-run. The `.cn-manager-stop` /
// `.model-manager-stop` buttons that DO call `/v2/manager/queue/reset`
// (custom-nodes-manager.js:465, model-manager.js:173) are display:none
// at idle via CSS. Inducing in-progress state would require starting a
// real install — explicitly out-of-scope for this LOW-risk P-closure.
//
// Fallback: exercise the endpoint via page.request (Playwright's
// browser-context HTTP client). This verifies endpoint availability +
// idempotency at idle, which is the essential contract the UI-click
// would assert. The UI-wiring of the button is trivially visible from
// JS-source grep (3 callers, all with identical `fetchApi` POST).
await page.goto('/');
await waitForComfyUI(page);
// Pre-check: queue should be empty so reset is a true no-op.
const statusBefore = await request.get('/v2/manager/queue/status');
expect(statusBefore.status()).toBe(200);
const statusJson = await statusBefore.json();
const resetResp = await request.post('/v2/manager/queue/reset');
expect(resetResp.status()).toBe(200);
// Post-check: queue/status still callable (handler released locks
// cleanly) and the reset did not break queue introspection.
const statusAfter = await request.get('/v2/manager/queue/status');
expect(statusAfter.status()).toBe(200);
// Sanity: is_processing (or equivalent flag) should remain stable
// when reset was called on an empty queue — we don't strictly assert
// the flag here because the exact field name differs across Manager
// versions; the 200-on-status is the portable contract.
expect(await statusAfter.json()).toBeDefined();
void statusJson; // retained for debug, not asserted (pre/post shapes are impl-detail)
});
});
@@ -0,0 +1,135 @@
/**
* E2E tests: Legacy Model Manager dialog.
*
* Tests the model list grid, filters, and search.
*
* Requires ComfyUI running with --enable-manager-legacy-ui on PORT.
*/
import { test, expect } from '@playwright/test';
import { waitForComfyUI, openManagerMenu, clickMenuButton } from './helpers';
test.describe('Model Manager', () => {
test.beforeEach(async ({ page }) => {
await page.goto('/');
await waitForComfyUI(page);
await openManagerMenu(page);
});
test('opens from Manager menu and renders grid', async ({ page }) => {
await clickMenuButton(page, 'Model Manager');
await page.waitForSelector('#cmm-manager-dialog', {
timeout: 10_000,
});
const grid = page.locator('.cmm-manager-grid, .tg-body').first();
await expect(grid).toBeVisible({ timeout: 15_000 });
});
test('loads model list (non-empty)', async ({ page }) => {
// Wave3 WI-U Cluster H target 3 (LM1): previously rows>0 only. Now also
// verifies the install-state column is rendered for every logical model row.
//
// TurboGrid renders each logical row as TWO DOM `.tg-row` elements (left
// frozen-column pane + right scrollable-column pane). Only the right pane
// carries the "installed" column, which `model-manager.js:342-345` formats
// as EITHER `<div class="cmm-icon-passed">...</div>` (installed===True) OR
// `<button class="cmm-btn-install">Install</button>` (installed===False),
// with a `"Refresh Required"` fallback at :340.
//
// Invariant: the number of install-state indicators equals the number of
// logical rows, i.e. half the DOM-row count. This catches a regression
// where the installed column stops rendering for any model (partial or
// complete).
await clickMenuButton(page, 'Model Manager');
await page.waitForSelector('.cmm-manager-grid, .tg-body', { timeout: 15_000 });
await page.waitForFunction(
() => document.querySelectorAll('.tg-body .tg-row, .cmm-manager-grid tr').length > 0,
{ timeout: 30_000, polling: 1_000 },
);
const rows = page.locator('.tg-body .tg-row, .cmm-manager-grid tr');
const domRowCount = await rows.count();
expect(domRowCount).toBeGreaterThan(0);
// Count install indicators across the whole grid.
const installedCount = await page
.locator('.cmm-icon-passed, .cmm-btn-install')
.count();
const refreshCount = await page
.locator('.tg-body :text("Refresh Required"), .cmm-manager-grid :text("Refresh Required")')
.count();
const totalIndicators = installedCount + refreshCount;
// Each logical model row must expose an install-state indicator.
expect(totalIndicators, 'at least one row must have a valid install-state indicator').toBeGreaterThan(0);
// Expected indicator count: one per logical row. TurboGrid doubles DOM
// rows for the 2-pane layout, so logical_count = domRowCount / 2 when
// dual-pane. For single-pane (fallback) the ratio is 1:1. Accept either.
const logicalRowCount = domRowCount / 2;
const isDualPane = Number.isInteger(logicalRowCount) && totalIndicators === logicalRowCount;
const isSinglePane = totalIndicators === domRowCount;
expect(
isDualPane || isSinglePane,
`install-state indicator count mismatch: totalIndicators=${totalIndicators}, ` +
`domRowCount=${domRowCount}. Expected either ${logicalRowCount} (dual-pane) or ${domRowCount} (single-pane).`,
).toBe(true);
});
test('search input filters the model grid', async ({ page }) => {
await clickMenuButton(page, 'Model Manager');
await page.waitForSelector('.cmm-manager-grid, .tg-body', { timeout: 15_000 });
await page.waitForFunction(
() => document.querySelectorAll('.tg-body .tg-row, .cmm-manager-grid tr').length > 0,
{ timeout: 30_000, polling: 1_000 },
);
const searchInput = page.locator('.cmm-manager-keywords, input[type="text"][placeholder*="earch"], input[type="search"]').first();
await expect(searchInput).toBeVisible({ timeout: 5_000 });
const initialCount = await page.locator('.tg-body .tg-row, .cmm-manager-grid tr').count();
await searchInput.fill('stable diffusion');
// State-based wait: row count must change (or narrow). If the search
// is entirely broken and returns all rows, this will fail the poll.
await expect
.poll(
async () => page.locator('.tg-body .tg-row, .cmm-manager-grid tr').count(),
{ timeout: 10_000 },
)
.not.toBe(initialCount);
const filteredCount = await page.locator('.tg-body .tg-row, .cmm-manager-grid tr').count();
expect(filteredCount).toBeLessThanOrEqual(initialCount);
});
test('filter dropdown is present with expected options', async ({ page }) => {
// Wave3 WI-U Cluster H target 5: previously options.length>0 only.
// Now asserts the filter dropdown surfaces all 4 known states defined by
// ModelManager.initFilter() in js/model-manager.js:74-86 —
// `All`, `Installed`, `Not Installed`, `In Workflow`.
await clickMenuButton(page, 'Model Manager');
await page.waitForSelector('#cmm-manager-dialog', {
timeout: 15_000,
});
const dialog = page.locator('#cmm-manager-dialog').last();
const filterSelect = dialog.locator('select').filter({ hasText: /All|Installed/ }).first();
await expect(filterSelect).toBeVisible({ timeout: 5_000 });
const options = (await filterSelect.locator('option').allTextContents()).map((s) => s.trim());
// Exact set match (normalized): js/model-manager.js:74-86 defines this
// list. If labels change, update this assertion consciously.
const expected = ['All', 'Installed', 'Not Installed', 'In Workflow'];
const actual = new Set(options);
for (const label of expected) {
expect(
actual.has(label),
`filter dropdown missing expected option "${label}". Options=${JSON.stringify(options)}`,
).toBe(true);
}
});
});
@@ -0,0 +1,63 @@
/**
* E2E tests: Dialog navigation and lifecycle.
*
* Tests opening/closing dialogs, nested dialog navigation, and
* verifies no duplicate instances are created.
*
* Requires ComfyUI running with --enable-manager-legacy-ui on PORT.
*/
import { test, expect } from '@playwright/test';
import { waitForComfyUI, openManagerMenu, clickMenuButton, closeDialog, assertManagerMenuVisible } from './helpers';
test.describe('Dialog Navigation', () => {
test.beforeEach(async ({ page }) => {
await page.goto('/');
await waitForComfyUI(page);
});
test('Manager menu → Custom Nodes → close → Manager still visible', async ({ page }) => {
await openManagerMenu(page);
await assertManagerMenuVisible(page);
await clickMenuButton(page, 'Custom Nodes Manager');
await page.waitForSelector('#cn-manager-dialog', {
timeout: 15_000,
});
// Close the Custom Nodes dialog
await closeDialog(page);
await page.waitForTimeout(500);
// Manager menu should still be accessible (reopen if needed)
await openManagerMenu(page);
await assertManagerMenuVisible(page);
});
test('Manager menu → Model Manager → close → reopen', async ({ page }) => {
await openManagerMenu(page);
await clickMenuButton(page, 'Model Manager');
await page.waitForSelector('#cmm-manager-dialog', {
timeout: 15_000,
});
// Close the Model Manager dialog via its close button (p-dialog-close-button)
const mmMask = page.locator('.p-dialog-mask:has(#cmm-manager-dialog)');
const mmCloseBtn = mmMask.locator('button[aria-label="Close"], .p-dialog-close-button').first();
if (await mmCloseBtn.isVisible({ timeout: 2_000 }).catch(() => false)) {
await mmCloseBtn.click();
} else {
await page.keyboard.press('Escape');
}
await page.waitForTimeout(1_000);
// Reopen: need to open Manager menu first, then Model Manager
await openManagerMenu(page);
await clickMenuButton(page, 'Model Manager');
await page.waitForSelector('#cmm-manager-dialog', {
timeout: 15_000,
});
});
});
+124
View File
@@ -0,0 +1,124 @@
/**
* E2E tests: Legacy Snapshot Manager dialog.
*
* Tests UI-driven save and remove operations.
* Requires ComfyUI running with --enable-manager-legacy-ui on PORT.
*/
import { test, expect } from '@playwright/test';
import { waitForComfyUI, openManagerMenu, clickMenuButton } from './helpers';
const SNAPSHOT_ROW_SELECTOR = '#snapshot-manager-dialog tr, #snapshot-manager-dialog li';
async function getSnapshotNames(page: import('@playwright/test').Page): Promise<string[]> {
const resp = await page.request.get('/v2/snapshot/getlist');
if (!resp.ok()) return [];
const data = await resp.json();
return Array.isArray(data?.items) ? data.items : [];
}
test.describe('Snapshot Manager', () => {
// Track snapshots created during each test so afterEach can clean them up.
// Prevents test-run accumulation on disk across runs.
const createdDuringTest = new Set<string>();
test.beforeEach(async ({ page }) => {
createdDuringTest.clear();
await page.goto('/');
await waitForComfyUI(page);
await openManagerMenu(page);
});
test.afterEach(async ({ page }) => {
// Cleanup snapshots newly created during the test to avoid state leak.
for (const name of createdDuringTest) {
await page.request.post(`/v2/snapshot/remove?target=${encodeURIComponent(name)}`);
}
});
test('opens snapshot manager from Manager menu', async ({ page }) => {
await clickMenuButton(page, 'Snapshot Manager');
// Snapshot manager should appear
await page.waitForSelector('#snapshot-manager-dialog', {
timeout: 10_000,
});
});
test('SS1 Save button creates a new snapshot row', async ({ page }) => {
await clickMenuButton(page, 'Snapshot Manager');
await page.waitForSelector('#snapshot-manager-dialog', { timeout: 10_000 });
// Baseline snapshot names (not row count — more reliable)
const namesBefore = await getSnapshotNames(page);
// Click Save button (UI-driven). Hard fail if the button doesn't exist.
const saveBtn = page
.locator('#snapshot-manager-dialog button:has-text("Save"), #snapshot-manager-dialog button:has-text("Create")')
.first();
await expect(saveBtn).toBeVisible({ timeout: 5_000 });
await saveBtn.click();
// Wait for new snapshot to appear in backend list (UI row count may lag)
await expect
.poll(async () => (await getSnapshotNames(page)).length, { timeout: 15_000 })
.toBeGreaterThan(namesBefore.length);
const namesAfter = await getSnapshotNames(page);
const newNames = namesAfter.filter((n) => !namesBefore.includes(n));
expect(newNames.length).toBeGreaterThanOrEqual(1);
// Register for afterEach cleanup
newNames.forEach((n) => createdDuringTest.add(n));
// UI row count should also reflect the new snapshot
const rowsAfter = await page.locator(SNAPSHOT_ROW_SELECTOR).count();
expect(rowsAfter).toBeGreaterThan(0);
});
test('UI Remove button deletes a snapshot row', async ({ page }) => {
// SETUP: create a snapshot via API so we have a deterministic target
const saveResp = await page.request.post('/v2/snapshot/save');
expect(saveResp.ok()).toBe(true);
const namesAfterSave = await getSnapshotNames(page);
expect(namesAfterSave.length).toBeGreaterThan(0);
const targetName = namesAfterSave[0]; // desc-sorted — newest at [0]
// Open the Snapshot Manager via UI
await clickMenuButton(page, 'Snapshot Manager');
await page.waitForSelector('#snapshot-manager-dialog', { timeout: 10_000 });
// Locate the row containing our target snapshot
const targetRow = page
.locator(SNAPSHOT_ROW_SELECTOR, { hasText: targetName })
.first();
await expect(targetRow).toBeVisible({ timeout: 10_000 });
// Click the Remove/Delete button inside that row (UI-driven)
const removeBtn = targetRow.locator(
'button:has-text("Remove"), button:has-text("Delete"), button[title*="emove" i], button[title*="elete" i]',
);
if (!(await removeBtn.first().isVisible({ timeout: 2_000 }).catch(() => false))) {
// If the Remove UI is a right-click / hover / icon without text, register for
// cleanup via the afterEach and report a specific failure so the test surfaces
// the UI gap rather than pretending it verified deletion.
createdDuringTest.add(targetName);
throw new Error(
'Remove/Delete button not found in snapshot row — ' +
'UI regression or selector change; update selector to match current UI',
);
}
// Accept confirmation dialog if the UI raises one
page.once('dialog', async (d) => {
await d.accept();
});
await removeBtn.first().click();
// Effect verification: snapshot disappears from backend AND from UI
await expect
.poll(async () => (await getSnapshotNames(page)).includes(targetName), { timeout: 10_000 })
.toBe(false);
await expect(page.locator(SNAPSHOT_ROW_SELECTOR, { hasText: targetName })).toHaveCount(0);
});
});