Compare commits

..

6 Commits

Author SHA1 Message Date
Simon Pinfold
72d0f86a1d fix(assets): refresh loader_path when re-ingesting an existing reference
Some checks failed
Python Linting / Run Ruff (push) Has been cancelled
Python Linting / Run Pylint (push) Has been cancelled
upsert_reference only wrote loader_path on the INSERT branch, so
re-ingesting an existing reference (an output overwritten in place, or a
file re-registered after its loader_path derivation changed) kept the
stale or NULL value forever. Write it on the UPDATE branch too, with a
null-safe change guard so a loader_path difference alone is enough to
trigger the update, and identical values stay a no-op.
2026-07-07 08:59:46 +12:00
Simon Pinfold
09086138db fix(assets): only extension-matching buckets contribute a loader_path
The model-base match in get_asset_category_and_relative_path ignored
each bucket's extension set, so a file inside a registered base whose
extension the bucket cannot load (e.g. a .txt uploaded into
model_type:checkpoints) advertised a loader_path that no loader list
would ever resolve, while the tag side of the same stack already
excluded it. Apply the extension check used for backend tags (empty set
accepts any extension), keeping loader_path null exactly when no loader
can resolve the file.
2026-07-07 08:55:53 +12:00
Simon Pinfold
72f239a0e3 fix(assets): persist subfolder-qualified loader_path for ingested outputs
ingest_existing_file built its seed spec with the file's basename, so
outputs saved into a subfolder persisted loader_path (and the
user_metadata filename that preview URLs split for their subfolder
param) as just the basename: the served locator pointed at a file that
does not exist at that path. Scanner and seeder specs already derive
fname via compute_loader_path; use the same derivation here.
2026-07-07 08:55:53 +12:00
Simon Pinfold
a37b178116 test(assets): lock loader_path matrix (asymmetry, null, persist/read)
Cover the behaviour that has no production change but is easy to regress:
the extra-path asymmetry (loadable but no storage namespace), null
loader_path persistence for orphan files, and the response reading the
stored column with a compute fallback for un-backfilled rows.
2026-07-07 08:33:59 +12:00
Simon Pinfold
958a7f1f19 feat(assets): replace response file_path with persisted loader_path
The old file_path response field was a namespaced storage locator
(models/checkpoints/foo.safetensors): not an absolute path, not unique
identity, and not the value a loader consumes. Nothing needs that shape
on the wire (hash/ID-based locating is the long-term direction), so it
is dropped rather than renamed; the storage-root matching stays internal,
powering display_name.

What loaders DO need is the in-root loader path (category dropped:
models/checkpoints/foo/bar.safetensors -> foo/bar.safetensors). Serve it
as a first-class loader_path field, persisted on asset_references
(migration 0006) and written by every ingest pipeline at insert, so
responses read the column verbatim.

Like the model_type tags, loader_path is a seed-time derivative of the
model folder registry, maintained by the same scan lifecycle (new files seed
fresh values, pruning retires rows whose bucket disappeared). Rows
predating the column serve a null loader_path; databases from before
this stack already need recreating for the base branch's tag changes.

loader_path resolves every registered base including extra_model_paths
entries; display_name only the canonical storage roots. A file can
therefore be loadable with no display name (extra-path models) or the
reverse (unregistered files under the models root), and loader_path is
null exactly when no loader can resolve the file.
2026-07-07 08:33:47 +12:00
Simon Pinfold
8130443398 fix(assets): filter model_type tags by bucket extension sets
Buckets sharing a base directory (e.g. diffusion_models and a custom
unet_gguf) tagged every file in the directory regardless of whether the
bucket could load it, so .safetensors files were tagged
model_type:unet_gguf and vice versa. Carry each bucket's registered
extension set through get_comfy_models_folders and only emit a
model_type tag when the file extension matches, keeping the empty-set
match-all convention from folder_paths.filter_files_extensions.

Files under a model base matching no bucket now keep only the models
tag instead of every directory-matching model_type tag.
2026-07-07 08:23:24 +12:00
13 changed files with 135 additions and 69 deletions

View File

@ -39,10 +39,7 @@ from app.assets.services import (
upload_from_temp_path,
)
from app.assets.services.cursor import InvalidCursorError
from app.assets.services.path_utils import (
compute_display_name,
compute_loader_path,
)
from app.assets.services.path_utils import compute_display_name
from app.assets.services.tagging import list_tag_histogram
ROUTES = web.RouteTableDef()
@ -167,11 +164,7 @@ def _build_asset_response(result: schemas.AssetDetailResult | schemas.UploadResu
if result.ref.file_path:
display_name = compute_display_name(result.ref.file_path)
# In-root loader path (model category dropped): what model loaders consume.
# Persisted at scan/ingest; fall back to computing for rows written
# before the column existed.
loader_path = result.ref.loader_path
if loader_path is None:
loader_path = compute_loader_path(result.ref.file_path)
else:
display_name, loader_path = None, None
asset_content_hash = result.asset.hash if result.asset else None

View File

@ -17,11 +17,11 @@ class Asset(BaseModel):
hash: str | None = None
loader_path: str | None = Field(
default=None,
description="In-root loader path for filesystem-backed assets: the path relative to its storage root with the top-level model category dropped (e.g. `models/checkpoints/foo/bar.safetensors` -> `foo/bar.safetensors`). This is the value model loaders consume. `None` when the file is not within a recognized root or model category.",
description="The value a loader consumes to load this asset. `None` when no loader can resolve the file.",
)
display_name: str | None = Field(
default=None,
description="Human-facing label for filesystem-backed assets: the path below the top-level storage namespace (e.g. `checkpoints/foo/bar.safetensors` under `models/`). Not unique.",
description="Human-facing label for the asset. Not unique.",
)
asset_hash: str | None = None
size: int | None = None

View File

@ -76,9 +76,7 @@ class AssetReference(Base):
# Cache state fields (from former AssetCacheState)
file_path: Mapped[str | None] = mapped_column(Text, nullable=True)
# In-root loader path derived from file_path at scan/ingest time (model
# category dropped). Persisted so responses read it directly instead of
# re-resolving against every registered model-folder base per request.
# In-root loader path derived from file_path at scan/ingest time.
loader_path: Mapped[str | None] = mapped_column(Text, nullable=True)
mtime_ns: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
needs_verify: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)

View File

@ -688,13 +688,14 @@ def upsert_reference(
AssetReference.asset_id != asset_id,
AssetReference.mtime_ns.is_(None),
AssetReference.mtime_ns != int(mtime_ns),
AssetReference.loader_path.is_distinct_from(loader_path),
AssetReference.is_missing == True, # noqa: E712
AssetReference.deleted_at.isnot(None),
)
)
.values(
asset_id=asset_id, mtime_ns=int(mtime_ns), is_missing=False,
deleted_at=None, updated_at=now,
asset_id=asset_id, mtime_ns=int(mtime_ns), loader_path=loader_path,
is_missing=False, deleted_at=None, updated_at=now,
)
)
res2 = session.execute(upd)

View File

@ -245,7 +245,7 @@ def ingest_existing_file(
"mtime_ns": mtime_ns,
"info_name": name,
"tags": tags,
"fname": os.path.basename(abs_path),
"fname": compute_loader_path(abs_path),
"metadata": None,
"hash": None,
"mime_type": mime_type,

View File

@ -209,8 +209,14 @@ def get_asset_category_and_relative_path(
return "temp", _compute_relative(fp_abs, temp_base)
# 4) models (check deepest matching base to avoid ambiguity)
ext = os.path.splitext(fp_abs)[1].lower()
best: tuple[int, str, str] | None = None # (base_len, bucket, rel_inside_bucket)
for bucket, bases, _exts in get_comfy_models_folders():
for bucket, bases, extensions in get_comfy_models_folders():
# A bucket only lists files within its extension set (empty set
# accepts any extension), so a bucket that cannot load the file
# must not contribute a loader path.
if extensions and ext not in extensions:
continue
for b in bases:
base_abs = os.path.abspath(b)
if not _check_is_within(fp_abs, base_abs):
@ -239,10 +245,9 @@ def get_backend_system_tags_from_path(path: str) -> list[str]:
A ``model_type:<folder_name>`` tag is only emitted when the file's
extension is accepted by that folder's registered extension set, so
categories sharing a base directory (e.g. ``diffusion_models`` and a
custom ``unet_gguf``) tag only the files they can actually load. Files
under a model base whose extension matches no category still get the
``models`` tag.
categories sharing a base directory tag only the files they can
actually load. Files under a model base whose extension matches no
category still get the ``models`` tag.
"""
fp_abs = os.path.abspath(path)
fp_path = Path(fp_abs)

View File

@ -35,17 +35,7 @@ class ModelFileManager:
for folder in model_types:
if folder in folder_black_list:
continue
# Effective display filter: the folder's registered extension
# set, or the global supported_pt_extensions for match-all
# folders (empty set), resolved live so runtime registrations
# by custom nodes are reflected.
registered = folder_paths.folder_names_and_paths[folder][1]
effective = set(registered) if registered else set(folder_paths.supported_pt_extensions)
output_folders.append({
"name": folder,
"folders": folder_paths.get_folder_paths(folder),
"extensions": sorted(effective),
})
output_folders.append({"name": folder, "folders": folder_paths.get_folder_paths(folder)})
return web.json_response(output_folders)
# NOTE: This is an experiment to replace `/models/{folder}`

View File

@ -12,11 +12,11 @@ components:
pattern: ^blake3:[a-f0-9]{64}$
type: string
loader_path:
description: 'In-root loader path for filesystem-backed assets: the path relative to its storage root with the top-level model category dropped (e.g. `models/checkpoints/foo/bar.safetensors` -> `foo/bar.safetensors`). This is the value model loaders consume. `None` when the file is not within a recognized root or model category.'
description: The value a loader consumes to load this asset. Null when no loader can resolve the file.
nullable: true
type: string
display_name:
description: 'Human-facing label for filesystem-backed assets: the path below the top-level storage namespace (e.g. `checkpoints/foo/bar.safetensors` under `models/`). Not unique.'
description: Human-facing label for the asset. Not unique.
nullable: true
type: string
id:
@ -775,14 +775,6 @@ components:
ModelFolder:
description: Represents a folder containing models
properties:
extensions:
description: 'Effective file-extension display filter for this folder: the registered extension set, or the global supported model extensions for folders registered without one (match-all). Resolved live, so runtime registrations by custom nodes are reflected.'
example:
- .ckpt
- .safetensors
items:
type: string
type: array
folders:
description: List of paths where models of this type are stored
example:

View File

@ -24,28 +24,6 @@ def app(model_manager):
app.add_routes(routes)
return app
async def test_get_model_folders_includes_effective_extensions(aiohttp_client, app, tmp_path):
"""Folders expose their effective display filter: the registered extension
set, or the global supported_pt_extensions for match-all (empty) folders."""
with patch('folder_paths.folder_names_and_paths', {
'test_checkpoints': ([str(tmp_path)], {'.safetensors', '.ckpt'}),
'test_configs': ([str(tmp_path)], ['.yaml']),
'test_match_all': ([str(tmp_path)], set()),
'configs': ([str(tmp_path)], ['.yaml']),
}), patch('folder_paths.supported_pt_extensions', {'.safetensors', '.bin'}):
client = await aiohttp_client(app)
response = await client.get('/experiment/models')
assert response.status == 200
folders = {f['name']: f for f in await response.json()}
assert 'configs' not in folders # blocklisted
assert folders['test_checkpoints']['folders'] == [str(tmp_path)]
assert folders['test_checkpoints']['extensions'] == ['.ckpt', '.safetensors']
assert folders['test_configs']['extensions'] == ['.yaml']
# Match-all folders substitute the live global set.
assert folders['test_match_all']['extensions'] == ['.bin', '.safetensors']
async def test_get_model_preview_safetensors(aiohttp_client, app, tmp_path):
img = Image.new('RGB', (100, 100), 'white')
img_byte_arr = BytesIO()

View File

@ -176,6 +176,39 @@ class TestUpsertReference:
ref = session.query(AssetReference).filter_by(file_path=file_path).one()
assert ref.mtime_ns == final_mtime
def test_upsert_refreshes_loader_path_on_existing_reference(self, session: Session):
"""Re-ingesting an existing reference writes the loader_path computed
by that ingest, healing NULL or stale values even when nothing else
about the row changed."""
asset = _make_asset(session, "hash1")
file_path = "/models/checkpoints/sub/model.safetensors"
upsert_reference(
session, asset_id=asset.id, file_path=file_path, name="model",
mtime_ns=100, loader_path=None,
)
session.commit()
created, updated = upsert_reference(
session, asset_id=asset.id, file_path=file_path, name="model",
mtime_ns=100, loader_path="sub/model.safetensors",
)
session.commit()
assert created is False
assert updated is True
ref = session.query(AssetReference).filter_by(file_path=file_path).one()
assert ref.loader_path == "sub/model.safetensors"
# Identical loader_path is a no-op, not a spurious update.
created, updated = upsert_reference(
session, asset_id=asset.id, file_path=file_path, name="model",
mtime_ns=100, loader_path="sub/model.safetensors",
)
session.commit()
assert created is False
assert updated is False
def test_upsert_restores_missing_reference(self, session: Session):
"""Upserting a reference that was marked missing should restore it."""
asset = _make_asset(session, "hash1")

View File

@ -1,8 +1,8 @@
"""Tests for how _build_asset_response derives the response `loader_path`.
Guards the persist-and-read contract: the response reads the stored
`loader_path` directly, and only recomputes when the column is NULL (rows
written before the column existed).
`loader_path` verbatim, with no read-time recomputation. Like tags, the
value is a seed-time derivative healed by the scan lifecycle.
"""
from datetime import datetime
@ -48,8 +48,9 @@ def test_uses_persisted_loader_path_without_recomputing():
assert resp.loader_path == "SENTINEL/stored.safetensors"
def test_falls_back_to_compute_when_stored_loader_path_is_null(tmp_path: Path):
"""A NULL column (pre-migration row) is backfilled at read time."""
def test_null_stored_loader_path_is_served_as_null(tmp_path: Path):
"""No read-time recomputation: a NULL column is served as null even when
the path would resolve."""
models = tmp_path / "models"
ckpt = models / "checkpoints"
ckpt.mkdir(parents=True)
@ -68,7 +69,7 @@ def test_falls_back_to_compute_when_stored_loader_path_is_null(tmp_path: Path):
result = _make_result(file_path=str(f), loader_path=None)
resp = _build_asset_response(result)
assert resp.loader_path == "bar.safetensors"
assert resp.loader_path is None
assert resp.display_name == "checkpoints/bar.safetensors"

View File

@ -329,6 +329,45 @@ class TestIngestExistingFileTagFK:
assert "output" in ref_tag_names
class TestIngestExistingFileLoaderPath:
"""Outputs saved into a subfolder must persist the subfolder-qualified
loader path, not the bare basename (regression: spec["fname"] was
os.path.basename)."""
def test_subfoldered_output_persists_relative_loader_path(
self, mock_create_session, temp_dir: Path, session: Session
):
input_dir = temp_dir / "input"
output_dir = temp_dir / "output"
temp_root = temp_dir / "temp"
for directory in (input_dir, output_dir, temp_root):
directory.mkdir()
file_path = output_dir / "sub" / "img_00001_.png"
file_path.parent.mkdir()
file_path.write_bytes(b"image data")
with (
patch("app.assets.services.path_utils.folder_paths") as mock_fp,
patch(
"app.assets.services.path_utils.get_comfy_models_folders",
return_value=[],
),
):
mock_fp.get_input_directory.return_value = str(input_dir)
mock_fp.get_output_directory.return_value = str(output_dir)
mock_fp.get_temp_directory.return_value = str(temp_root)
assert ingest_existing_file(abs_path=str(file_path)) is True
ref = (
session.query(AssetReference)
.filter_by(file_path=str(file_path))
.one()
)
assert ref.loader_path == "sub/img_00001_.png"
assert (ref.user_metadata or {}).get("filename") == "sub/img_00001_.png"
class TestIngestImageDimensions:
"""system_metadata should carry {kind, width, height} for image assets."""

View File

@ -523,6 +523,42 @@ class TestLoaderPath:
assert compute_logical_path(str(f)) == "models/not_registered/orphan.bin"
assert compute_loader_path(str(f)) is None
def test_extension_mismatch_in_registered_bucket_has_no_loader_path(self, fake_dirs):
# Inside a registered bucket, but the bucket's extension set cannot
# load it: no model_type tag, and no loader path either.
f = fake_dirs["models"] / "notes.txt"
f.touch()
assert compute_logical_path(str(f)) == "models/checkpoints/notes.txt"
assert compute_loader_path(str(f)) is None
def test_shared_base_loader_path_uses_extension_matching_bucket(self, fake_dirs):
shared_root = fake_dirs["models"].parent / "unet"
shared_root.mkdir()
f = shared_root / "wan.gguf"
f.touch()
with patch(
"app.assets.services.path_utils.get_comfy_models_folders",
return_value=[
("diffusion_models", [str(shared_root)], {".safetensors"}),
("unet_gguf", [str(shared_root)], {".gguf"}),
],
):
assert compute_loader_path(str(f)) == "wan.gguf"
def test_match_all_bucket_provides_loader_path_for_any_extension(self, fake_dirs):
custom_root = fake_dirs["models"].parent / "custom_bucket"
custom_root.mkdir()
f = custom_root / "weights.bin"
f.touch()
with patch(
"app.assets.services.path_utils.get_comfy_models_folders",
return_value=[("custom_bucket", [str(custom_root)], set())],
):
assert compute_loader_path(str(f)) == "weights.bin"
def test_extra_path_model_has_loader_path_but_no_logical_path(self, tmp_path: Path):
"""Registered category base outside models_dir (extra_model_paths style).