Compare commits

..

6 Commits

Author SHA1 Message Date
Simon Pinfold
6deeae99e7 feat(api): expose effective extension filter on /experiment/models
Some checks failed
Python Linting / Run Ruff (push) Has been cancelled
Python Linting / Run Pylint (push) Has been cancelled
Each folder in the listing now carries its effective display filter:
the registered extension set, or the global supported_pt_extensions for
match-all folders (registered with an empty set, e.g. LLM). Resolved per
request so runtime registrations by custom nodes are reflected.

Lets the asset-backed model sidebar filter match-all folder contents
(hiding README/config noise) without hardcoding the global set client
side, while registered folders stay governed by the scanner's
extension-filtered model_type tags.
2026-07-03 14:47:51 +12:00
Simon Pinfold
529abc991f feat(assets): drop logical_path from the asset response
Ratified in the loader-key sync: the namespaced locator buys nothing
today (hash/ID-based locating is the long-term direction), and keeping
it adds a third path-shaped field for clients to confuse. loader_path
and display_name carry the loader and display concerns; the storage-root
matching stays internal, still powering display_name.

Response-surface only; no column or migration involved.
2026-07-03 13:53:35 +12:00
Simon Pinfold
bd7d95bdb4 test(assets): lock loader_path matrix (asymmetry, null, persist/read)
Cover the behaviour that has no production change but is easy to regress:
the extra-path asymmetry (loadable but no storage namespace), null
loader_path persistence for orphan files, and the response reading the
stored column with a compute fallback for un-backfilled rows.
2026-07-03 08:52:21 +12:00
Simon Pinfold
c9693374df feat(assets): rename response field to loader_path and persist it
Rename the in-root loader path response field from `file_path` to
`loader_path` (matching compute_loader_path), and persist it on
asset_references so the API reads it directly instead of re-resolving
against every registered model-folder base per request.

- add loader_path column (migration 0006) populated at scan/ingest from
  the already-computed loader path
- response prefers the stored value, falling back to compute for rows
  written before the column existed
2026-07-03 08:51:20 +12:00
Simon Pinfold
e807d60e45 feat(assets): add in-root loader file_path, rename storage locator to logical_path
Split the Asset response path fields so model-loader consumers get a
category-relative path. The namespaced storage locator moves to
`logical_path`; the new `file_path` is the in-root loader path (model
category dropped), e.g. models/checkpoints/foo/bar.safetensors -> foo/bar.safetensors.
2026-07-03 08:50:27 +12:00
Simon Pinfold
06b2d4c1d0 fix(assets): filter model_type tags by bucket extension sets
Buckets sharing a base directory (e.g. diffusion_models and a custom
unet_gguf) tagged every file in the directory regardless of whether the
bucket could load it, so .safetensors files were tagged
model_type:unet_gguf and vice versa. Carry each bucket's registered
extension set through get_comfy_models_folders and only emit a
model_type tag when the file extension matches, keeping the empty-set
match-all convention from folder_paths.filter_files_extensions.

Files under a model base matching no bucket now keep only the models
tag instead of every directory-matching model_type tag.
2026-07-03 08:50:13 +12:00
13 changed files with 69 additions and 135 deletions

View File

@ -39,7 +39,10 @@ from app.assets.services import (
upload_from_temp_path,
)
from app.assets.services.cursor import InvalidCursorError
from app.assets.services.path_utils import compute_display_name
from app.assets.services.path_utils import (
compute_display_name,
compute_loader_path,
)
from app.assets.services.tagging import list_tag_histogram
ROUTES = web.RouteTableDef()
@ -164,7 +167,11 @@ def _build_asset_response(result: schemas.AssetDetailResult | schemas.UploadResu
if result.ref.file_path:
display_name = compute_display_name(result.ref.file_path)
# In-root loader path (model category dropped): what model loaders consume.
# Persisted at scan/ingest; fall back to computing for rows written
# before the column existed.
loader_path = result.ref.loader_path
if loader_path is None:
loader_path = compute_loader_path(result.ref.file_path)
else:
display_name, loader_path = None, None
asset_content_hash = result.asset.hash if result.asset else None

View File

@ -17,11 +17,11 @@ class Asset(BaseModel):
hash: str | None = None
loader_path: str | None = Field(
default=None,
description="The value a loader consumes to load this asset. `None` when no loader can resolve the file.",
description="In-root loader path for filesystem-backed assets: the path relative to its storage root with the top-level model category dropped (e.g. `models/checkpoints/foo/bar.safetensors` -> `foo/bar.safetensors`). This is the value model loaders consume. `None` when the file is not within a recognized root or model category.",
)
display_name: str | None = Field(
default=None,
description="Human-facing label for the asset. Not unique.",
description="Human-facing label for filesystem-backed assets: the path below the top-level storage namespace (e.g. `checkpoints/foo/bar.safetensors` under `models/`). Not unique.",
)
asset_hash: str | None = None
size: int | None = None

View File

@ -76,7 +76,9 @@ class AssetReference(Base):
# Cache state fields (from former AssetCacheState)
file_path: Mapped[str | None] = mapped_column(Text, nullable=True)
# In-root loader path derived from file_path at scan/ingest time.
# In-root loader path derived from file_path at scan/ingest time (model
# category dropped). Persisted so responses read it directly instead of
# re-resolving against every registered model-folder base per request.
loader_path: Mapped[str | None] = mapped_column(Text, nullable=True)
mtime_ns: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
needs_verify: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)

View File

@ -688,14 +688,13 @@ def upsert_reference(
AssetReference.asset_id != asset_id,
AssetReference.mtime_ns.is_(None),
AssetReference.mtime_ns != int(mtime_ns),
AssetReference.loader_path.is_distinct_from(loader_path),
AssetReference.is_missing == True, # noqa: E712
AssetReference.deleted_at.isnot(None),
)
)
.values(
asset_id=asset_id, mtime_ns=int(mtime_ns), loader_path=loader_path,
is_missing=False, deleted_at=None, updated_at=now,
asset_id=asset_id, mtime_ns=int(mtime_ns), is_missing=False,
deleted_at=None, updated_at=now,
)
)
res2 = session.execute(upd)

View File

@ -245,7 +245,7 @@ def ingest_existing_file(
"mtime_ns": mtime_ns,
"info_name": name,
"tags": tags,
"fname": compute_loader_path(abs_path),
"fname": os.path.basename(abs_path),
"metadata": None,
"hash": None,
"mime_type": mime_type,

View File

@ -209,14 +209,8 @@ def get_asset_category_and_relative_path(
return "temp", _compute_relative(fp_abs, temp_base)
# 4) models (check deepest matching base to avoid ambiguity)
ext = os.path.splitext(fp_abs)[1].lower()
best: tuple[int, str, str] | None = None # (base_len, bucket, rel_inside_bucket)
for bucket, bases, extensions in get_comfy_models_folders():
# A bucket only lists files within its extension set (empty set
# accepts any extension), so a bucket that cannot load the file
# must not contribute a loader path.
if extensions and ext not in extensions:
continue
for bucket, bases, _exts in get_comfy_models_folders():
for b in bases:
base_abs = os.path.abspath(b)
if not _check_is_within(fp_abs, base_abs):
@ -245,9 +239,10 @@ def get_backend_system_tags_from_path(path: str) -> list[str]:
A ``model_type:<folder_name>`` tag is only emitted when the file's
extension is accepted by that folder's registered extension set, so
categories sharing a base directory tag only the files they can
actually load. Files under a model base whose extension matches no
category still get the ``models`` tag.
categories sharing a base directory (e.g. ``diffusion_models`` and a
custom ``unet_gguf``) tag only the files they can actually load. Files
under a model base whose extension matches no category still get the
``models`` tag.
"""
fp_abs = os.path.abspath(path)
fp_path = Path(fp_abs)

View File

@ -35,7 +35,17 @@ class ModelFileManager:
for folder in model_types:
if folder in folder_black_list:
continue
output_folders.append({"name": folder, "folders": folder_paths.get_folder_paths(folder)})
# Effective display filter: the folder's registered extension
# set, or the global supported_pt_extensions for match-all
# folders (empty set), resolved live so runtime registrations
# by custom nodes are reflected.
registered = folder_paths.folder_names_and_paths[folder][1]
effective = set(registered) if registered else set(folder_paths.supported_pt_extensions)
output_folders.append({
"name": folder,
"folders": folder_paths.get_folder_paths(folder),
"extensions": sorted(effective),
})
return web.json_response(output_folders)
# NOTE: This is an experiment to replace `/models/{folder}`

View File

@ -12,11 +12,11 @@ components:
pattern: ^blake3:[a-f0-9]{64}$
type: string
loader_path:
description: The value a loader consumes to load this asset. Null when no loader can resolve the file.
description: 'In-root loader path for filesystem-backed assets: the path relative to its storage root with the top-level model category dropped (e.g. `models/checkpoints/foo/bar.safetensors` -> `foo/bar.safetensors`). This is the value model loaders consume. `None` when the file is not within a recognized root or model category.'
nullable: true
type: string
display_name:
description: Human-facing label for the asset. Not unique.
description: 'Human-facing label for filesystem-backed assets: the path below the top-level storage namespace (e.g. `checkpoints/foo/bar.safetensors` under `models/`). Not unique.'
nullable: true
type: string
id:
@ -775,6 +775,14 @@ components:
ModelFolder:
description: Represents a folder containing models
properties:
extensions:
description: 'Effective file-extension display filter for this folder: the registered extension set, or the global supported model extensions for folders registered without one (match-all). Resolved live, so runtime registrations by custom nodes are reflected.'
example:
- .ckpt
- .safetensors
items:
type: string
type: array
folders:
description: List of paths where models of this type are stored
example:

View File

@ -24,6 +24,28 @@ def app(model_manager):
app.add_routes(routes)
return app
async def test_get_model_folders_includes_effective_extensions(aiohttp_client, app, tmp_path):
"""Folders expose their effective display filter: the registered extension
set, or the global supported_pt_extensions for match-all (empty) folders."""
with patch('folder_paths.folder_names_and_paths', {
'test_checkpoints': ([str(tmp_path)], {'.safetensors', '.ckpt'}),
'test_configs': ([str(tmp_path)], ['.yaml']),
'test_match_all': ([str(tmp_path)], set()),
'configs': ([str(tmp_path)], ['.yaml']),
}), patch('folder_paths.supported_pt_extensions', {'.safetensors', '.bin'}):
client = await aiohttp_client(app)
response = await client.get('/experiment/models')
assert response.status == 200
folders = {f['name']: f for f in await response.json()}
assert 'configs' not in folders # blocklisted
assert folders['test_checkpoints']['folders'] == [str(tmp_path)]
assert folders['test_checkpoints']['extensions'] == ['.ckpt', '.safetensors']
assert folders['test_configs']['extensions'] == ['.yaml']
# Match-all folders substitute the live global set.
assert folders['test_match_all']['extensions'] == ['.bin', '.safetensors']
async def test_get_model_preview_safetensors(aiohttp_client, app, tmp_path):
img = Image.new('RGB', (100, 100), 'white')
img_byte_arr = BytesIO()

View File

@ -176,39 +176,6 @@ class TestUpsertReference:
ref = session.query(AssetReference).filter_by(file_path=file_path).one()
assert ref.mtime_ns == final_mtime
def test_upsert_refreshes_loader_path_on_existing_reference(self, session: Session):
"""Re-ingesting an existing reference writes the loader_path computed
by that ingest, healing NULL or stale values even when nothing else
about the row changed."""
asset = _make_asset(session, "hash1")
file_path = "/models/checkpoints/sub/model.safetensors"
upsert_reference(
session, asset_id=asset.id, file_path=file_path, name="model",
mtime_ns=100, loader_path=None,
)
session.commit()
created, updated = upsert_reference(
session, asset_id=asset.id, file_path=file_path, name="model",
mtime_ns=100, loader_path="sub/model.safetensors",
)
session.commit()
assert created is False
assert updated is True
ref = session.query(AssetReference).filter_by(file_path=file_path).one()
assert ref.loader_path == "sub/model.safetensors"
# Identical loader_path is a no-op, not a spurious update.
created, updated = upsert_reference(
session, asset_id=asset.id, file_path=file_path, name="model",
mtime_ns=100, loader_path="sub/model.safetensors",
)
session.commit()
assert created is False
assert updated is False
def test_upsert_restores_missing_reference(self, session: Session):
"""Upserting a reference that was marked missing should restore it."""
asset = _make_asset(session, "hash1")

View File

@ -1,8 +1,8 @@
"""Tests for how _build_asset_response derives the response `loader_path`.
Guards the persist-and-read contract: the response reads the stored
`loader_path` verbatim, with no read-time recomputation. Like tags, the
value is a seed-time derivative healed by the scan lifecycle.
`loader_path` directly, and only recomputes when the column is NULL (rows
written before the column existed).
"""
from datetime import datetime
@ -48,9 +48,8 @@ def test_uses_persisted_loader_path_without_recomputing():
assert resp.loader_path == "SENTINEL/stored.safetensors"
def test_null_stored_loader_path_is_served_as_null(tmp_path: Path):
"""No read-time recomputation: a NULL column is served as null even when
the path would resolve."""
def test_falls_back_to_compute_when_stored_loader_path_is_null(tmp_path: Path):
"""A NULL column (pre-migration row) is backfilled at read time."""
models = tmp_path / "models"
ckpt = models / "checkpoints"
ckpt.mkdir(parents=True)
@ -69,7 +68,7 @@ def test_null_stored_loader_path_is_served_as_null(tmp_path: Path):
result = _make_result(file_path=str(f), loader_path=None)
resp = _build_asset_response(result)
assert resp.loader_path is None
assert resp.loader_path == "bar.safetensors"
assert resp.display_name == "checkpoints/bar.safetensors"

View File

@ -329,45 +329,6 @@ class TestIngestExistingFileTagFK:
assert "output" in ref_tag_names
class TestIngestExistingFileLoaderPath:
"""Outputs saved into a subfolder must persist the subfolder-qualified
loader path, not the bare basename (regression: spec["fname"] was
os.path.basename)."""
def test_subfoldered_output_persists_relative_loader_path(
self, mock_create_session, temp_dir: Path, session: Session
):
input_dir = temp_dir / "input"
output_dir = temp_dir / "output"
temp_root = temp_dir / "temp"
for directory in (input_dir, output_dir, temp_root):
directory.mkdir()
file_path = output_dir / "sub" / "img_00001_.png"
file_path.parent.mkdir()
file_path.write_bytes(b"image data")
with (
patch("app.assets.services.path_utils.folder_paths") as mock_fp,
patch(
"app.assets.services.path_utils.get_comfy_models_folders",
return_value=[],
),
):
mock_fp.get_input_directory.return_value = str(input_dir)
mock_fp.get_output_directory.return_value = str(output_dir)
mock_fp.get_temp_directory.return_value = str(temp_root)
assert ingest_existing_file(abs_path=str(file_path)) is True
ref = (
session.query(AssetReference)
.filter_by(file_path=str(file_path))
.one()
)
assert ref.loader_path == "sub/img_00001_.png"
assert (ref.user_metadata or {}).get("filename") == "sub/img_00001_.png"
class TestIngestImageDimensions:
"""system_metadata should carry {kind, width, height} for image assets."""

View File

@ -523,42 +523,6 @@ class TestLoaderPath:
assert compute_logical_path(str(f)) == "models/not_registered/orphan.bin"
assert compute_loader_path(str(f)) is None
def test_extension_mismatch_in_registered_bucket_has_no_loader_path(self, fake_dirs):
# Inside a registered bucket, but the bucket's extension set cannot
# load it: no model_type tag, and no loader path either.
f = fake_dirs["models"] / "notes.txt"
f.touch()
assert compute_logical_path(str(f)) == "models/checkpoints/notes.txt"
assert compute_loader_path(str(f)) is None
def test_shared_base_loader_path_uses_extension_matching_bucket(self, fake_dirs):
shared_root = fake_dirs["models"].parent / "unet"
shared_root.mkdir()
f = shared_root / "wan.gguf"
f.touch()
with patch(
"app.assets.services.path_utils.get_comfy_models_folders",
return_value=[
("diffusion_models", [str(shared_root)], {".safetensors"}),
("unet_gguf", [str(shared_root)], {".gguf"}),
],
):
assert compute_loader_path(str(f)) == "wan.gguf"
def test_match_all_bucket_provides_loader_path_for_any_extension(self, fake_dirs):
custom_root = fake_dirs["models"].parent / "custom_bucket"
custom_root.mkdir()
f = custom_root / "weights.bin"
f.touch()
with patch(
"app.assets.services.path_utils.get_comfy_models_folders",
return_value=[("custom_bucket", [str(custom_root)], set())],
):
assert compute_loader_path(str(f)) == "weights.bin"
def test_extra_path_model_has_loader_path_but_no_logical_path(self, tmp_path: Path):
"""Registered category base outside models_dir (extra_model_paths style).