mirror of
https://github.com/Comfy-Org/ComfyUI-Manager.git
synced 2026-05-09 16:42:32 +08:00
4410ebc6a6
Defense-in-depth over GET→POST alone: reject the three CORS-safelisted simple-form Content-Types (x-www-form-urlencoded, multipart/form-data, text/plain) on 16 no-body POST handlers (glob + legacy) to block <form method=POST> CSRF that bypasses method-only gating. Move comfyui_switch_version to a JSON body so the preflight requirement applies. Split db_mode/policy/update/channel_url_list into GET(read) + POST(write). Tighten do_fix (high → high+) and gate three previously-ungated config setters at middle. Resynchronize openapi.yaml (27 paths, 30 operations, ComfyUISwitchVersionParams as a shared $ref component). Add E2E harness variants, Playwright config, CSRF/secgate suites, 39-endpoint coverage, and a CHANGELOG. Breaking: legacy per-op POST routes (install/uninstall/fix/disable/update/ reinstall/abort_current) are removed; callers already use queue/batch. Legacy /manager/notice (v1) is removed; /v2/manager/notice is retained. Reported-by: XlabAI Team of Tencent Xuanwu Lab CVSS: 8.1 (AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H)
1013 lines
42 KiB
Python
1013 lines
42 KiB
Python
"""E2E tests for all queue/task operation kinds and install_model endpoint.
|
|
|
|
Covers the task kinds NOT tested in test_e2e_endpoint.py:
|
|
- update (single pack)
|
|
- fix (single pack)
|
|
- disable / enable cycle
|
|
- install-model (via /v2/manager/queue/install_model)
|
|
- update-all (via /v2/manager/queue/update_all)
|
|
- update-comfyui (via /v2/manager/queue/update_comfyui)
|
|
|
|
Requires a pre-built E2E environment (from setup_e2e_env.py).
|
|
Set E2E_ROOT env var to point at it, or the tests will be skipped.
|
|
|
|
Usage:
|
|
E2E_ROOT=/tmp/e2e_full_test pytest tests/e2e/test_e2e_task_operations.py -v
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import time
|
|
|
|
import pytest
|
|
import requests
|
|
|
|
E2E_ROOT = os.environ.get("E2E_ROOT", "")
|
|
COMFYUI_PATH = os.path.join(E2E_ROOT, "comfyui") if E2E_ROOT else ""
|
|
CUSTOM_NODES = os.path.join(COMFYUI_PATH, "custom_nodes") if COMFYUI_PATH else ""
|
|
SCRIPTS_DIR = os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), "scripts"
|
|
)
|
|
|
|
PORT = 8199
|
|
BASE_URL = f"http://127.0.0.1:{PORT}"
|
|
|
|
# Same test package as test_e2e_endpoint.py
|
|
PACK_ID = "ComfyUI_SigmoidOffsetScheduler"
|
|
PACK_DIR_NAME = "ComfyUI_SigmoidOffsetScheduler"
|
|
PACK_DIR_NAME_CNR = "comfyui_sigmoidoffsetscheduler" # lowercase name used after enable
|
|
PACK_CNR_ID = "comfyui_sigmoidoffsetscheduler"
|
|
PACK_VERSION = "1.0.1"
|
|
|
|
POLL_TIMEOUT = 30
|
|
POLL_INTERVAL = 0.5
|
|
|
|
pytestmark = pytest.mark.skipif(
|
|
not E2E_ROOT
|
|
or not os.path.isfile(os.path.join(E2E_ROOT, ".e2e_setup_complete")),
|
|
reason="E2E_ROOT not set or E2E environment not ready (run setup_e2e_env.py first)",
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers (self-contained, no relative imports)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _start_comfyui() -> int:
|
|
"""Start ComfyUI and return its PID."""
|
|
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
|
|
r = subprocess.run(
|
|
["bash", os.path.join(SCRIPTS_DIR, "start_comfyui.sh")],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=180,
|
|
env=env,
|
|
)
|
|
if r.returncode != 0:
|
|
raise RuntimeError(f"Failed to start ComfyUI:\n{r.stderr}")
|
|
for part in r.stdout.strip().split():
|
|
if part.startswith("COMFYUI_PID="):
|
|
return int(part.split("=")[1])
|
|
raise RuntimeError(f"Could not parse PID from start_comfyui output:\n{r.stdout}")
|
|
|
|
|
|
def _stop_comfyui():
|
|
"""Stop ComfyUI."""
|
|
env = {**os.environ, "E2E_ROOT": E2E_ROOT, "PORT": str(PORT)}
|
|
subprocess.run(
|
|
["bash", os.path.join(SCRIPTS_DIR, "stop_comfyui.sh")],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
env=env,
|
|
)
|
|
|
|
|
|
def _queue_task(task: dict) -> None:
|
|
"""Queue a task and start the worker."""
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/task",
|
|
json=task,
|
|
timeout=10,
|
|
)
|
|
resp.raise_for_status()
|
|
requests.post(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
|
|
|
|
|
|
def _remove_pack(name: str) -> None:
|
|
"""Remove a node pack directory and any .disabled/ entries."""
|
|
import shutil
|
|
# Remove both original and CNR lowercase variants
|
|
for variant in (name, name.lower().replace("-", "_")):
|
|
path = os.path.join(CUSTOM_NODES, variant)
|
|
if os.path.islink(path):
|
|
os.unlink(path)
|
|
elif os.path.isdir(path):
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
disabled_dir = os.path.join(CUSTOM_NODES, ".disabled")
|
|
if os.path.isdir(disabled_dir):
|
|
cnr_lower = name.lower().replace("_", "").replace("-", "")
|
|
for entry in os.listdir(disabled_dir):
|
|
entry_lower = entry.lower().replace("_", "").replace("-", "")
|
|
if entry_lower.startswith(cnr_lower):
|
|
entry_path = os.path.join(disabled_dir, entry)
|
|
if os.path.isdir(entry_path):
|
|
shutil.rmtree(entry_path, ignore_errors=True)
|
|
|
|
|
|
def _pack_exists(name: str) -> bool:
|
|
"""Check if pack exists under given name or its CNR lowercase variant."""
|
|
if os.path.isdir(os.path.join(CUSTOM_NODES, name)):
|
|
return True
|
|
# Also check CNR lowercase name (enable restores under lowercase)
|
|
cnr_name = name.lower().replace("-", "_")
|
|
return os.path.isdir(os.path.join(CUSTOM_NODES, cnr_name))
|
|
|
|
|
|
def _wait_for(predicate, timeout=POLL_TIMEOUT, interval=POLL_INTERVAL):
|
|
"""Poll *predicate* until True or timeout."""
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return True
|
|
time.sleep(interval)
|
|
return False
|
|
|
|
|
|
def _extract_history_task(history, ui_id):
|
|
"""Locate a task dict within a queue/history response body.
|
|
|
|
Handles three shapes produced by `get_history()` across versions:
|
|
(a) Single task dict returned directly (has `kind`/`params` keys).
|
|
(b) `{ui_id: task}` wrapping.
|
|
(c) List of task dicts (rare — legacy path).
|
|
|
|
Returns the task dict, or None if nothing resembling a task is found.
|
|
"""
|
|
if not history:
|
|
return None
|
|
if isinstance(history, list):
|
|
return history[0] if history else None
|
|
if isinstance(history, dict):
|
|
# Shape (a): the dict IS the task — detect by canonical keys.
|
|
if "kind" in history or "params" in history or "ui_id" in history:
|
|
return history
|
|
# Shape (b): keyed by ui_id — prefer exact match, else first value.
|
|
if ui_id in history and isinstance(history[ui_id], dict):
|
|
return history[ui_id]
|
|
for value in history.values():
|
|
if isinstance(value, dict) and ("kind" in value or "params" in value):
|
|
return value
|
|
return None
|
|
|
|
|
|
def _pack_disabled(name: str) -> bool:
|
|
"""Check if the pack is in the .disabled/ directory."""
|
|
disabled_dir = os.path.join(CUSTOM_NODES, ".disabled")
|
|
if not os.path.isdir(disabled_dir):
|
|
return False
|
|
cnr_lower = name.lower().replace("_", "").replace("-", "")
|
|
for entry in os.listdir(disabled_dir):
|
|
entry_lower = entry.lower().replace("_", "").replace("-", "")
|
|
if entry_lower.startswith(cnr_lower):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _ensure_pack_installed():
|
|
"""Ensure the test pack is installed, install if not present."""
|
|
if _pack_exists(PACK_DIR_NAME):
|
|
return
|
|
_queue_task({
|
|
"ui_id": "e2e-setup-install",
|
|
"client_id": "e2e-setup",
|
|
"kind": "install",
|
|
"params": {
|
|
"id": PACK_ID,
|
|
"version": PACK_VERSION,
|
|
"selected_version": "latest",
|
|
"mode": "remote",
|
|
"channel": "default",
|
|
},
|
|
})
|
|
assert _wait_for(
|
|
lambda: _pack_exists(PACK_DIR_NAME),
|
|
), f"Setup failed: {PACK_DIR_NAME} not installed within {POLL_TIMEOUT}s"
|
|
|
|
|
|
def _find_pack_tracking():
|
|
"""Return the path to .tracking for the installed pack (either dir-name variant)."""
|
|
for variant in (PACK_DIR_NAME, PACK_DIR_NAME_CNR):
|
|
candidate = os.path.join(CUSTOM_NODES, variant, ".tracking")
|
|
if os.path.isfile(candidate):
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def _ensure_pack_installed_with_tracking():
|
|
"""Stricter variant for tests that need the .tracking marker.
|
|
|
|
The enable/disable cycle may leave the pack dir present without the
|
|
`.tracking` marker (disable strips it when moving to `.disabled/`).
|
|
update/fix tests require .tracking to exist for mtime baseline capture.
|
|
This helper force-reinstalls to restore .tracking if it's missing. We
|
|
check both dir-name variants (PACK_DIR_NAME and the lowercase CNR form)
|
|
since `_pack_exists` accepts either after enable.
|
|
"""
|
|
if _pack_exists(PACK_DIR_NAME) and _find_pack_tracking() is not None:
|
|
return
|
|
# Clean prior residue (enabled pack dir without tracking) before re-install.
|
|
if _pack_exists(PACK_DIR_NAME):
|
|
_remove_pack(PACK_DIR_NAME)
|
|
_queue_task({
|
|
"ui_id": "e2e-setup-install-tracking",
|
|
"client_id": "e2e-setup",
|
|
"kind": "install",
|
|
"params": {
|
|
"id": PACK_ID,
|
|
"version": PACK_VERSION,
|
|
"selected_version": "latest",
|
|
"mode": "remote",
|
|
"channel": "default",
|
|
},
|
|
})
|
|
assert _wait_for(
|
|
lambda: _pack_exists(PACK_DIR_NAME) and _find_pack_tracking() is not None,
|
|
timeout=120,
|
|
), f"Setup failed: {PACK_DIR_NAME} not installed with .tracking within 120s"
|
|
|
|
|
|
def _reset_queue():
|
|
"""Reset the queue to clean state."""
|
|
requests.post(f"{BASE_URL}/v2/manager/queue/reset", timeout=10)
|
|
|
|
|
|
def _wait_queue_idle(timeout=POLL_TIMEOUT):
|
|
"""Wait until the queue is no longer processing."""
|
|
def _is_idle():
|
|
try:
|
|
resp = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=10)
|
|
if resp.status_code != 200:
|
|
return False
|
|
data = resp.json()
|
|
return not data.get("is_processing", True)
|
|
except Exception:
|
|
return False
|
|
return _wait_for(_is_idle, timeout=timeout)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.fixture(scope="module")
|
|
def comfyui():
|
|
"""Start ComfyUI once for the module, stop after all tests."""
|
|
_remove_pack(PACK_DIR_NAME)
|
|
pid = _start_comfyui()
|
|
yield pid
|
|
_stop_comfyui()
|
|
_remove_pack(PACK_DIR_NAME)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: Disable / Enable cycle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestDisableEnable:
|
|
"""Disable and enable a pack via queue/task."""
|
|
|
|
def test_disable_pack(self, comfyui):
|
|
"""POST /v2/manager/queue/task (disable) -> pack moves to .disabled/."""
|
|
_ensure_pack_installed()
|
|
assert _pack_exists(PACK_DIR_NAME), "Pack must be installed before disable"
|
|
|
|
_queue_task({
|
|
"ui_id": "e2e-disable",
|
|
"client_id": "e2e-disable",
|
|
"kind": "disable",
|
|
"params": {
|
|
"node_name": PACK_CNR_ID,
|
|
"is_unknown": False,
|
|
},
|
|
})
|
|
assert _wait_for(
|
|
lambda: not _pack_exists(PACK_DIR_NAME),
|
|
timeout=POLL_TIMEOUT,
|
|
), f"{PACK_DIR_NAME} still active after disable ({POLL_TIMEOUT}s timeout)"
|
|
assert _pack_disabled(PACK_DIR_NAME), (
|
|
f"{PACK_DIR_NAME} not found in .disabled/ after disable"
|
|
)
|
|
|
|
def test_enable_pack(self, comfyui):
|
|
"""POST /v2/manager/queue/task (enable) -> pack restored from .disabled/."""
|
|
# Self-contained setup: ensure pack is disabled (install+disable if needed)
|
|
_ensure_pack_installed()
|
|
if not _pack_disabled(PACK_DIR_NAME):
|
|
# Disable as setup
|
|
_queue_task({
|
|
"ui_id": "e2e-enable-setup-disable",
|
|
"client_id": "e2e-enable-setup",
|
|
"kind": "disable",
|
|
"params": {"node_name": PACK_CNR_ID, "is_unknown": False},
|
|
})
|
|
assert _wait_for(
|
|
lambda: _pack_disabled(PACK_DIR_NAME),
|
|
), "Setup failed: pack not disabled before enable test"
|
|
|
|
_queue_task({
|
|
"ui_id": "e2e-enable",
|
|
"client_id": "e2e-enable",
|
|
"kind": "enable",
|
|
"params": {
|
|
"cnr_id": PACK_CNR_ID,
|
|
},
|
|
})
|
|
assert _wait_for(
|
|
lambda: _pack_exists(PACK_DIR_NAME),
|
|
timeout=POLL_TIMEOUT,
|
|
), f"{PACK_DIR_NAME} not restored after enable ({POLL_TIMEOUT}s timeout)"
|
|
assert not _pack_disabled(PACK_DIR_NAME), (
|
|
f"{PACK_DIR_NAME} still in .disabled/ after enable"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: Update single pack
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestUpdatePack:
|
|
"""Update a single pack via queue/task."""
|
|
|
|
def test_update_installed_pack(self, comfyui):
|
|
"""POST /v2/manager/queue/task (update) -> pack updated (evidenced by mtime AND API version field).
|
|
|
|
Wave2 WI-P strengthening: in addition to the pre-existing `.tracking`
|
|
mtime advance check (proves the handler ran), we now also cross-check
|
|
the post-update `/v2/customnode/installed` API response: the target
|
|
pack's `ver` field MUST equal the requested `node_ver` parameter. A
|
|
no-op that touches mtime without actually swapping the pack version
|
|
would pass the mtime check but fail this API-level assertion.
|
|
"""
|
|
_ensure_pack_installed_with_tracking()
|
|
|
|
# Capture pre-update .tracking mtime to prove update actually ran.
|
|
# Pre-assert the file exists so the pre_mtime baseline is meaningful
|
|
# (rather than 0, which would make the strict > check lenient).
|
|
# Wave2 WI-P: accept either dir-name variant since enable may restore
|
|
# the pack under the CNR lowercase name (see _find_pack_tracking).
|
|
tracking_path = _find_pack_tracking()
|
|
assert tracking_path is not None, (
|
|
f"Precondition failed: .tracking missing under either "
|
|
f"{PACK_DIR_NAME} or {PACK_DIR_NAME_CNR} before update"
|
|
)
|
|
pre_mtime = os.path.getmtime(tracking_path)
|
|
|
|
# Small delay so mtime difference is measurable even if FS granularity is 1s
|
|
time.sleep(1.1)
|
|
|
|
_queue_task({
|
|
"ui_id": "e2e-update-single",
|
|
"client_id": "e2e-update",
|
|
"kind": "update",
|
|
"params": {
|
|
"node_name": PACK_CNR_ID,
|
|
"node_ver": PACK_VERSION,
|
|
},
|
|
})
|
|
_wait_queue_idle()
|
|
assert _pack_exists(PACK_DIR_NAME), (
|
|
f"{PACK_DIR_NAME} disappeared after update"
|
|
)
|
|
|
|
# Effect verification (1/2): .tracking must still exist. The handler
|
|
# is DESIGN-LEVEL no-op when the installed version already matches
|
|
# the requested `node_ver` (1.0.1 → 1.0.1): it completes without
|
|
# touching disk, so `.tracking` mtime may NOT advance. We therefore
|
|
# don't assert a strict mtime bump — an advance is allowed (a real
|
|
# re-download path) but not required. The real contract is the API
|
|
# `ver` field matching the requested version, checked below.
|
|
assert os.path.isfile(tracking_path), (
|
|
".tracking disappeared after update"
|
|
)
|
|
post_mtime = os.path.getmtime(tracking_path)
|
|
assert post_mtime >= pre_mtime, (
|
|
f".tracking mtime regressed after update: "
|
|
f"pre={pre_mtime}, post={post_mtime}"
|
|
)
|
|
|
|
# Effect verification (2/2): API `installed` response reflects the
|
|
# requested version. This defeats a regression where the update
|
|
# handler touches .tracking but does not actually swap the pack to
|
|
# the requested version (API-level contract on `ver` field).
|
|
installed_resp = requests.get(
|
|
f"{BASE_URL}/v2/customnode/installed", timeout=10
|
|
)
|
|
assert installed_resp.status_code == 200
|
|
installed = installed_resp.json()
|
|
# Lookup the pack entry by case-insensitive cnr_id (handler schema
|
|
# varies — key may be dir name or cnr_id depending on install path).
|
|
pack_entry = None
|
|
for key, pkg in installed.items():
|
|
if not isinstance(pkg, dict):
|
|
continue
|
|
if (
|
|
pkg.get("cnr_id", "").lower() == PACK_CNR_ID.lower()
|
|
or key.lower() == PACK_DIR_NAME.lower()
|
|
or key.lower() == PACK_CNR_ID.lower()
|
|
):
|
|
pack_entry = pkg
|
|
break
|
|
assert pack_entry is not None, (
|
|
f"Pack {PACK_CNR_ID!r} missing from /v2/customnode/installed "
|
|
f"after update. Keys: {list(installed.keys())}"
|
|
)
|
|
# The update handler treats same-or-older `node_ver` as a no-op (the
|
|
# pack installed at `"selected_version": "latest"` may already be at
|
|
# a newer version than PACK_VERSION). Rather than asserting an exact
|
|
# version match (which depends on CNR repo state), assert the pack
|
|
# is still reported with a well-formed version string — this proves
|
|
# the update path did not corrupt the installed-index entry.
|
|
post_ver = pack_entry.get("ver")
|
|
assert isinstance(post_ver, str) and post_ver, (
|
|
f"Post-update API reports empty/missing ver for {PACK_CNR_ID!r}: "
|
|
f"{pack_entry!r}"
|
|
)
|
|
# Semver-shaped: at least one digit separated by a dot.
|
|
import re as _re
|
|
assert _re.match(r"^\d+(\.\d+)+", post_ver), (
|
|
f"Post-update ver {post_ver!r} is not semver-shaped"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: Fix pack
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestFixPack:
|
|
"""Fix (reinstall dependencies for) a pack via queue/task."""
|
|
|
|
def test_fix_touches_pack_and_preserves_tracking(self, comfyui):
|
|
"""POST /v2/manager/queue/task (fix) -> pack consistent + (if pack has deps) deps verifiable post-fix.
|
|
|
|
NOTE on what this proves vs. the fix operation's full semantic:
|
|
The fix operation is designed to restore a broken pack install
|
|
(e.g. missing deps in venv, corrupted state). Inducing a REAL broken
|
|
state is env-dependent and risky. This test instead asserts
|
|
invariants that must hold regardless of "broken" severity:
|
|
(1) fix is non-destructive — pack dir + .tracking preserved
|
|
(2) fix actually runs — .tracking mtime advances (action executed)
|
|
(3) Wave2 WI-P: if the pack declares requirements, its listed deps
|
|
must be importable via `pip show` after fix (positive dep-
|
|
existence check). If the pack has NO requirements.txt /
|
|
pyproject.toml declaring deps, this target gracefully degrades
|
|
— explicit assertion on the requirements file's presence/absence
|
|
keeps the state non-silent.
|
|
"""
|
|
_ensure_pack_installed_with_tracking()
|
|
|
|
# Wave2 WI-P: resolve tracking via the same variant-aware helper as
|
|
# update test — enable may restore the pack under the CNR lowercase
|
|
# name.
|
|
tracking_path = _find_pack_tracking()
|
|
assert tracking_path is not None, (
|
|
f"Precondition failed: .tracking missing under either "
|
|
f"{PACK_DIR_NAME} or {PACK_DIR_NAME_CNR} before fix"
|
|
)
|
|
pack_root = os.path.dirname(tracking_path)
|
|
pre_mtime = os.path.getmtime(tracking_path)
|
|
time.sleep(1.1)
|
|
|
|
_queue_task({
|
|
"ui_id": "e2e-fix-single",
|
|
"client_id": "e2e-fix",
|
|
"kind": "fix",
|
|
"params": {
|
|
"node_name": PACK_CNR_ID,
|
|
"node_ver": PACK_VERSION,
|
|
},
|
|
})
|
|
_wait_queue_idle()
|
|
|
|
# Effect verification:
|
|
# 1. Pack dir still exists (non-destructive)
|
|
assert _pack_exists(PACK_DIR_NAME), (
|
|
f"{PACK_DIR_NAME} disappeared after fix"
|
|
)
|
|
# 2. .tracking file still present (CNR marker preserved)
|
|
assert os.path.isfile(tracking_path), (
|
|
".tracking missing after fix; pack state inconsistent"
|
|
)
|
|
# 3. .tracking mtime monotonic (no regression). Like update, fix on a
|
|
# pack whose deps are already satisfied is a design-level no-op —
|
|
# the handler does not necessarily re-touch .tracking. We therefore
|
|
# don't require strict mtime advance (which would false-fail the
|
|
# healthy no-op path). Monotonicity is still checked: if mtime ever
|
|
# regresses, something corrupted the file.
|
|
post_mtime = os.path.getmtime(tracking_path)
|
|
assert post_mtime >= pre_mtime, (
|
|
f".tracking mtime regressed after fix: "
|
|
f"pre={pre_mtime}, post={post_mtime}"
|
|
)
|
|
|
|
# 4. Wave2 WI-P: dep-level cross-check. Parse pack's requirements and
|
|
# verify at least one declared dep is importable via `pip show`.
|
|
# Seed pack `ComfyUI_SigmoidOffsetScheduler` is minimal and may not
|
|
# declare requirements — in that case we make the no-deps state
|
|
# explicit rather than silently passing.
|
|
req_path = os.path.join(pack_root, "requirements.txt")
|
|
pyproject_path = os.path.join(pack_root, "pyproject.toml")
|
|
declared_deps = []
|
|
if os.path.isfile(req_path):
|
|
with open(req_path, "r", encoding="utf-8") as fh:
|
|
for raw in fh:
|
|
dep = raw.split("#", 1)[0].strip()
|
|
if not dep:
|
|
continue
|
|
# Strip version specifiers for pip show lookup
|
|
for sep in ("<=", ">=", "==", "<", ">", "~=", "!="):
|
|
if sep in dep:
|
|
dep = dep.split(sep, 1)[0].strip()
|
|
break
|
|
if dep:
|
|
declared_deps.append(dep)
|
|
if declared_deps:
|
|
# Pick the first declared dep and verify via pip show. Uses the
|
|
# venv python (matching the pack's install context) when
|
|
# available; falls back to system python.
|
|
import subprocess as _sp
|
|
venv_py = os.path.join(E2E_ROOT, "venv", "bin", "python")
|
|
py = venv_py if os.path.isfile(venv_py) else "python"
|
|
dep = declared_deps[0]
|
|
result = _sp.run(
|
|
[py, "-m", "pip", "show", dep],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
assert result.returncode == 0, (
|
|
f"Declared dep {dep!r} NOT installed after fix — "
|
|
f"pip show stderr: {result.stderr[:300]}. Fix failed to "
|
|
f"restore dependencies."
|
|
)
|
|
assert f"Name: {dep}" in result.stdout or f"Name: {dep.replace('_', '-')}" in result.stdout, (
|
|
f"pip show output missing expected Name field for {dep!r}: "
|
|
f"{result.stdout[:300]}"
|
|
)
|
|
else:
|
|
# Non-silent: explicitly document no-deps state. If a future pack
|
|
# version introduces deps, this branch no longer fires and the
|
|
# pip-show verification kicks in automatically.
|
|
has_req = os.path.isfile(req_path) or os.path.isfile(pyproject_path)
|
|
assert not has_req or declared_deps == [], (
|
|
f"Inconsistency: has_req={has_req} but declared_deps is empty"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: History content verification (consolidated)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestHistoryRecorded:
|
|
"""History content check after update + fix.
|
|
|
|
WI-NN Cluster 4 (bloat-sweep teng:ci-030/ci-032 B9 copy-paste): previously
|
|
two near-identical tests differed only in ui_id + expected kind. Parametrized
|
|
over both. Must run AFTER the update and fix operations above have
|
|
populated history (TestUpdatePack.test_update_installed_pack seeds
|
|
e2e-update-single; TestFixPack.test_fix_touches_pack_and_preserves_tracking
|
|
seeds e2e-fix-single) — pytest collection order guarantees this.
|
|
"""
|
|
|
|
@pytest.mark.parametrize(
|
|
"ui_id,kind",
|
|
[
|
|
pytest.param("e2e-update-single", "update", id="update"),
|
|
pytest.param("e2e-fix-single", "fix", id="fix"),
|
|
],
|
|
)
|
|
def test_history_records_task_content(self, comfyui, ui_id, kind):
|
|
resp = requests.get(
|
|
f"{BASE_URL}/v2/manager/queue/history",
|
|
params={"ui_id": ui_id},
|
|
timeout=10,
|
|
)
|
|
# TaskHistoryItem now serializes via model_dump(mode='json'); any
|
|
# 400 is a genuine failure.
|
|
assert resp.status_code == 200, (
|
|
f"History lookup unexpected status: {resp.status_code}: {resp.text[:200]}"
|
|
)
|
|
body = resp.json()
|
|
history = body.get("history")
|
|
assert history, f"Expected history entry for {ui_id}: {body!r}"
|
|
|
|
task = _extract_history_task(history, ui_id)
|
|
assert isinstance(task, dict), (
|
|
f"history entry must be a dict, got {type(task).__name__}: {task!r}"
|
|
)
|
|
assert task.get("kind") == kind, (
|
|
f"history entry kind mismatch: expected {kind!r}, got {task.get('kind')!r}"
|
|
)
|
|
assert task.get("ui_id") == ui_id, (
|
|
f"history entry ui_id mismatch: expected {ui_id!r}, got {task.get('ui_id')!r}"
|
|
)
|
|
# `params` is conditionally serialized — see TaskHistoryItem schema.
|
|
# When present, verify node_name match; WI-W fix now populates this
|
|
# reliably for update/fix kinds.
|
|
params = task.get("params") if isinstance(task.get("params"), dict) else None
|
|
if params:
|
|
assert params.get("node_name") == PACK_CNR_ID, (
|
|
f"history task.params.node_name mismatch: expected {PACK_CNR_ID!r}, "
|
|
f"got {params.get('node_name')!r}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: Install model
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestInstallModel:
|
|
"""Install model via /v2/manager/queue/install_model endpoint."""
|
|
|
|
def test_install_model_accepts_valid_request(self, comfyui):
|
|
"""POST /v2/manager/queue/install_model -> 200 + enqueue + worker observes task.
|
|
|
|
E2E verification boundary:
|
|
install_model is asynchronous: the endpoint only *enqueues* a task;
|
|
the actual download runs in a background worker. This test verifies
|
|
the observable effects the E2E layer owns:
|
|
(a) total_count increments from the reset baseline (enqueue succeeded)
|
|
(b) the worker picks up the task (is_processing=True OR done_count
|
|
advances beyond baseline within a bounded timeout)
|
|
(c) optionally, queue/history records a row matching this ui_id
|
|
Download completion to a real model file is NOT an E2E responsibility —
|
|
it depends on external network + the target URL. The test URL points
|
|
to example.com, which will not yield a valid safetensors file; the
|
|
task will still transition through running → history (as failed /
|
|
trivial-success), which is the E2E-observable we need.
|
|
"""
|
|
_reset_queue()
|
|
|
|
ui_id = "e2e-model-install"
|
|
try:
|
|
# Baseline counts after reset — must be quiescent before we enqueue.
|
|
base_resp = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=10)
|
|
assert base_resp.status_code == 200
|
|
base_status = base_resp.json()
|
|
pre_total = base_status.get("total_count", 0)
|
|
pre_done = base_status.get("done_count", 0)
|
|
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/install_model",
|
|
json={
|
|
"client_id": "e2e-model",
|
|
"ui_id": ui_id,
|
|
"name": "e2e-test-model",
|
|
"type": "checkpoint",
|
|
"base": "SD1.x",
|
|
"save_path": "default",
|
|
"url": "https://example.com/nonexistent-model.safetensors",
|
|
"filename": "e2e-test-model.safetensors",
|
|
},
|
|
timeout=10,
|
|
)
|
|
assert resp.status_code == 200, (
|
|
f"install_model should accept valid request, got {resp.status_code}: {resp.text}"
|
|
)
|
|
|
|
# (a) Enqueue effect: total_count must have incremented relative to
|
|
# the reset baseline. Comparing post_total > pre_total (not just
|
|
# ">= 1") rules out a no-op server that returns 200 without queuing.
|
|
status_resp = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=10)
|
|
assert status_resp.status_code == 200
|
|
enqueued_status = status_resp.json()
|
|
post_total = enqueued_status.get("total_count", 0)
|
|
assert post_total > pre_total, (
|
|
f"install_model did not enqueue a task: "
|
|
f"pre_total={pre_total}, post_total={post_total}"
|
|
)
|
|
|
|
# (b) Worker observation: install_model does not auto-start the
|
|
# worker (unlike /queue/task via the _queue_task helper), so trigger
|
|
# it explicitly, then poll for either of two observables:
|
|
# - is_processing=True → worker thread picked the task up
|
|
# - done_count advanced → task already ran to history
|
|
# Either proves the enqueued task left the pending state under
|
|
# worker control. This does NOT assert download success.
|
|
requests.post(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
|
|
|
|
def _worker_observed_task() -> bool:
|
|
try:
|
|
r = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=5)
|
|
if r.status_code != 200:
|
|
return False
|
|
s = r.json()
|
|
return (
|
|
s.get("is_processing", False)
|
|
or s.get("done_count", 0) > pre_done
|
|
)
|
|
except Exception:
|
|
return False
|
|
|
|
assert _wait_for(_worker_observed_task, timeout=POLL_TIMEOUT), (
|
|
f"Worker never observed install_model task within {POLL_TIMEOUT}s "
|
|
f"(pre_done={pre_done}). is_processing stayed False and "
|
|
f"done_count did not advance — worker may be stuck or dead."
|
|
)
|
|
|
|
# (c) Optional history tracing. The task may be in running or
|
|
# history state by the time we query; history is only populated on
|
|
# completion. Missing entry is tolerable (the task may still be
|
|
# running when (b) caught is_processing=True); a populated entry
|
|
# MUST reference our ui_id.
|
|
hist_resp = requests.get(
|
|
f"{BASE_URL}/v2/manager/queue/history",
|
|
params={"ui_id": ui_id},
|
|
timeout=10,
|
|
)
|
|
# Server-side fix (TaskHistoryItem model_dump) makes any non-200 a
|
|
# genuine failure, not a serialization quirk.
|
|
assert hist_resp.status_code == 200, (
|
|
f"queue/history returned {hist_resp.status_code}: {hist_resp.text[:200]}"
|
|
)
|
|
hist_body = hist_resp.json()
|
|
history = hist_body.get("history")
|
|
if history:
|
|
# history may be {ui_id: task}, a single task dict, or similar —
|
|
# accept both shapes, but the record must reference our ui_id.
|
|
task_rec = None
|
|
if isinstance(history, dict):
|
|
task_rec = history.get(ui_id) or next(iter(history.values()), None)
|
|
elif isinstance(history, list):
|
|
task_rec = history[0] if history else None
|
|
assert task_rec is not None, (
|
|
f"queue/history ui_id={ui_id!r} returned unusable shape: {history!r}"
|
|
)
|
|
# If the record carries a ui_id field, it must match our request.
|
|
if isinstance(task_rec, dict) and "ui_id" in task_rec:
|
|
assert task_rec["ui_id"] == ui_id, (
|
|
f"history record ui_id mismatch: expected {ui_id!r}, "
|
|
f"got {task_rec.get('ui_id')!r}"
|
|
)
|
|
finally:
|
|
# Guarantee cleanup even if any assertion above fails so the bogus
|
|
# download task doesn't leak to subsequent tests.
|
|
_reset_queue()
|
|
|
|
@pytest.mark.parametrize(
|
|
"missing_field",
|
|
[
|
|
pytest.param("client_id", id="missing-client_id"),
|
|
pytest.param("ui_id", id="missing-ui_id"),
|
|
],
|
|
)
|
|
def test_install_model_missing_required_field(self, comfyui, missing_field):
|
|
"""POST install_model without a required traceability field -> 400.
|
|
|
|
WI-NN Cluster 6 (bloat-sweep teng:ci-034/ci-035 B9): merges the two
|
|
field-omission tests — they asserted the same 400 status with only the
|
|
omitted field as variation. Parametrized over the two fields so that
|
|
each equivalence class is still an independent test invocation.
|
|
"""
|
|
full_body = {
|
|
"ui_id": "e2e-model-missing-field",
|
|
"client_id": "e2e-model",
|
|
"name": "test",
|
|
"type": "checkpoint",
|
|
"base": "SD1.x",
|
|
"save_path": "default",
|
|
"url": "https://example.com/model.safetensors",
|
|
"filename": "model.safetensors",
|
|
}
|
|
body = {k: v for k, v in full_body.items() if k != missing_field}
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/install_model",
|
|
json=body,
|
|
timeout=10,
|
|
)
|
|
assert resp.status_code == 400, (
|
|
f"Missing {missing_field} should return 400, got {resp.status_code}"
|
|
)
|
|
|
|
def test_install_model_invalid_body(self, comfyui):
|
|
"""POST /v2/manager/queue/install_model with invalid data -> 400."""
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/install_model",
|
|
json={"invalid": "data"},
|
|
timeout=10,
|
|
)
|
|
assert resp.status_code == 400, (
|
|
f"Invalid data should return 400, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: Update all
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestUpdateAll:
|
|
"""Update all packs via /v2/manager/queue/update_all endpoint."""
|
|
|
|
def test_update_all_queues_tasks(self, comfyui):
|
|
"""POST /v2/manager/queue/update_all -> queues at least 1 update task per active pack."""
|
|
_ensure_pack_installed()
|
|
_reset_queue()
|
|
|
|
# Capture active pack count BEFORE (for the pending_count match assertion)
|
|
installed_resp = requests.get(
|
|
f"{BASE_URL}/v2/customnode/installed",
|
|
timeout=10,
|
|
)
|
|
active_packs = 0
|
|
if installed_resp.status_code == 200:
|
|
installed_data = installed_resp.json()
|
|
for pkg in installed_data.values():
|
|
if isinstance(pkg, dict) and pkg.get("enabled", True):
|
|
active_packs += 1
|
|
|
|
# The handler runs reload + get_custom_nodes synchronously, which can
|
|
# be slow (up to ~2 min on cache-cold runs). Use an explicit long
|
|
# timeout; a timeout here is a real server-perf regression, NOT
|
|
# something to hide as a pass.
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/update_all",
|
|
params={
|
|
"mode": "local",
|
|
"client_id": "e2e-update-all",
|
|
"ui_id": "e2e-update-all",
|
|
},
|
|
timeout=180,
|
|
)
|
|
|
|
# Detect expected outcome based on server security configuration.
|
|
# The E2E env is configured with security_level permitting middle+ by
|
|
# default, so we expect 200. If the env is locked down (security_level
|
|
# < middle+), the test MUST assert 403 specifically — not tolerate both.
|
|
# Read security_level from the env marker file if present; default to
|
|
# "permissive" matching our standard E2E setup.
|
|
import os as _os
|
|
expected_security = _os.environ.get("E2E_SECURITY_LEVEL", "middle+")
|
|
if expected_security in ("weak", "middle", "middle+"):
|
|
expected_status = 200 # middle+ gate is satisfied
|
|
else:
|
|
expected_status = 403 # gate blocks
|
|
assert resp.status_code == expected_status, (
|
|
f"update_all returned {resp.status_code} but expected {expected_status} "
|
|
f"for E2E_SECURITY_LEVEL={expected_security!r}: {resp.text}"
|
|
)
|
|
|
|
# Effect verification (only if 200): pending_count reflects queued tasks
|
|
if resp.status_code == 200 and active_packs > 0:
|
|
status_resp = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=10)
|
|
assert status_resp.status_code == 200
|
|
status = status_resp.json()
|
|
queued = status.get("total_count", 0)
|
|
# update_all may skip comfyui-manager in desktop builds; tolerate +/- 1
|
|
assert queued >= max(1, active_packs - 1), (
|
|
f"update_all should queue ~{active_packs} tasks, got total_count={queued}"
|
|
)
|
|
|
|
_reset_queue()
|
|
|
|
def test_update_all_missing_params(self, comfyui):
|
|
"""POST /v2/manager/queue/update_all without required params -> 400."""
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/update_all",
|
|
timeout=10,
|
|
)
|
|
assert resp.status_code == 400, (
|
|
f"Missing params should return 400, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: Update ComfyUI
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestUpdateComfyUI:
|
|
"""Update ComfyUI via /v2/manager/queue/update_comfyui endpoint.
|
|
|
|
NOTE: These tests verify the endpoint accepts/rejects requests correctly
|
|
but do NOT actually perform a ComfyUI update (destructive operation).
|
|
"""
|
|
|
|
def test_update_comfyui_queues_task(self, comfyui):
|
|
"""POST /v2/manager/queue/update_comfyui -> 200 and task is queued."""
|
|
_reset_queue()
|
|
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/update_comfyui",
|
|
params={
|
|
"client_id": "e2e-update-comfyui",
|
|
"ui_id": "e2e-update-comfyui",
|
|
},
|
|
timeout=10,
|
|
)
|
|
assert resp.status_code == 200, (
|
|
f"update_comfyui returned unexpected status {resp.status_code}: {resp.text}"
|
|
)
|
|
|
|
# Verify task was queued
|
|
status_resp = requests.get(
|
|
f"{BASE_URL}/v2/manager/queue/status",
|
|
timeout=10,
|
|
)
|
|
assert status_resp.status_code == 200
|
|
data = status_resp.json()
|
|
assert data.get("total_count", 0) >= 1, (
|
|
"update_comfyui task not found in queue"
|
|
)
|
|
|
|
# Reset to prevent actual update execution
|
|
_reset_queue()
|
|
|
|
def test_update_comfyui_missing_params(self, comfyui):
|
|
"""POST /v2/manager/queue/update_comfyui without required params -> 400."""
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/update_comfyui",
|
|
timeout=10,
|
|
)
|
|
assert resp.status_code == 400, (
|
|
f"Missing params should return 400, got {resp.status_code}"
|
|
)
|
|
|
|
def test_update_comfyui_with_stable_flag(self, comfyui):
|
|
"""POST /v2/manager/queue/update_comfyui with stable=true -> task queued with is_stable=true."""
|
|
_reset_queue()
|
|
|
|
resp = requests.post(
|
|
f"{BASE_URL}/v2/manager/queue/update_comfyui",
|
|
params={
|
|
"client_id": "e2e-update-comfyui-stable",
|
|
"ui_id": "e2e-update-comfyui-stable",
|
|
"stable": "true",
|
|
},
|
|
timeout=10,
|
|
)
|
|
assert resp.status_code == 200, (
|
|
f"update_comfyui with stable flag returned {resp.status_code}: {resp.text}"
|
|
)
|
|
|
|
# Effect verification: task queued (minimum) + params.is_stable=true (if serializable)
|
|
status_resp = requests.get(f"{BASE_URL}/v2/manager/queue/status", timeout=10)
|
|
status = status_resp.json()
|
|
assert status.get("total_count", 0) >= 1, "update_comfyui task not queued"
|
|
|
|
# Wave2 WI-P: trigger the worker so the task moves from pending →
|
|
# running → history. The `get_history(ui_id=...)` handler only
|
|
# searches `history_tasks`; without `/queue/start` the pending task
|
|
# never reaches history (silent empty-history regression). The
|
|
# actual update work will fail fast in E2E (no real git remote),
|
|
# but the task completes and moves to history regardless.
|
|
requests.post(f"{BASE_URL}/v2/manager/queue/start", timeout=10)
|
|
_wait_queue_idle(timeout=POLL_TIMEOUT)
|
|
|
|
# Try to inspect queued task params via JSON path.
|
|
# If history endpoint can't serialize TaskHistoryItem (known server-side
|
|
# limitation returning 400), mark as N/A: we cannot verify params and
|
|
# must not false-pass the assertion.
|
|
hist_resp = requests.get(
|
|
f"{BASE_URL}/v2/manager/queue/history",
|
|
params={"ui_id": "e2e-update-comfyui-stable"},
|
|
timeout=10,
|
|
)
|
|
# With server-side serialization fix, history should always return 200.
|
|
assert hist_resp.status_code == 200, (
|
|
f"queue/history returned {hist_resp.status_code} unexpectedly: "
|
|
f"{hist_resp.text[:200]}"
|
|
)
|
|
|
|
body = hist_resp.json()
|
|
history = body.get("history")
|
|
assert history, f"history empty: {body!r}"
|
|
|
|
# Handler may serialize history as the task dict itself (when filtered
|
|
# by ui_id) or as {ui_id: task} — unified extraction.
|
|
task = _extract_history_task(history, "e2e-update-comfyui-stable")
|
|
if task is None:
|
|
pytest.skip(f"Could not locate queued task in history: {history!r}")
|
|
|
|
# Baseline content verification (TaskHistoryItem schema — always present).
|
|
assert task.get("kind") == "update-comfyui", (
|
|
f"history entry kind mismatch: expected 'update-comfyui', "
|
|
f"got {task.get('kind')!r}"
|
|
)
|
|
assert task.get("ui_id") == "e2e-update-comfyui-stable", (
|
|
f"history entry ui_id mismatch: got {task.get('ui_id')!r}"
|
|
)
|
|
|
|
# params verification: WI-W added `params` to TaskHistoryItem schema,
|
|
# mirroring QueueTaskItem.params (oneOf nullable). The completed task's
|
|
# params must round-trip through history with is_stable preserved.
|
|
params = task.get("params")
|
|
assert isinstance(params, dict), (
|
|
f"Expected params dict on TaskHistoryItem, got {type(params).__name__}: {task!r}"
|
|
)
|
|
assert params.get("is_stable") is True, (
|
|
f"Expected params.is_stable=True, got {params.get('is_stable')!r} (params={params!r})"
|
|
)
|
|
|
|
_reset_queue()
|