diff --git a/alembic_db/versions/0006_add_loader_path.py b/alembic_db/versions/0006_add_loader_path.py new file mode 100644 index 000000000..afa65312d --- /dev/null +++ b/alembic_db/versions/0006_add_loader_path.py @@ -0,0 +1,30 @@ +""" +Add loader_path column to asset_references. + +Stores the in-root loader path (path relative to the storage root with the +top-level model category dropped) derived from file_path at scan/ingest time, +so the assets API can return it without re-resolving against every registered +model-folder base on every request. + +Revision ID: 0006_add_loader_path +Revises: 0005_allow_case_sensitive_tags +Create Date: 2026-07-02 +""" + +from alembic import op +import sqlalchemy as sa + +revision = "0006_add_loader_path" +down_revision = "0005_allow_case_sensitive_tags" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("asset_references") as batch_op: + batch_op.add_column(sa.Column("loader_path", sa.Text(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("asset_references") as batch_op: + batch_op.drop_column("loader_path") diff --git a/app/assets/api/routes.py b/app/assets/api/routes.py index c0a5228d8..f1d9c118c 100644 --- a/app/assets/api/routes.py +++ b/app/assets/api/routes.py @@ -168,7 +168,11 @@ def _build_asset_response(result: schemas.AssetDetailResult | schemas.UploadResu paths = compute_asset_response_paths(result.ref.file_path) logical_path, display_name = paths if paths else (None, None) # In-root loader path (model category dropped): what model loaders consume. - loader_path = compute_loader_path(result.ref.file_path) + # Persisted at scan/ingest; fall back to computing for rows written + # before the column existed. + loader_path = result.ref.loader_path + if loader_path is None: + loader_path = compute_loader_path(result.ref.file_path) else: logical_path, display_name, loader_path = None, None, None asset_content_hash = result.asset.hash if result.asset else None @@ -176,7 +180,7 @@ def _build_asset_response(result: schemas.AssetDetailResult | schemas.UploadResu id=result.ref.id, name=result.ref.name, hash=asset_content_hash, - file_path=loader_path, + loader_path=loader_path, logical_path=logical_path, display_name=display_name, asset_hash=asset_content_hash, diff --git a/app/assets/api/schemas_out.py b/app/assets/api/schemas_out.py index 7143ce77a..4bede72d7 100644 --- a/app/assets/api/schemas_out.py +++ b/app/assets/api/schemas_out.py @@ -12,10 +12,10 @@ class Asset(BaseModel): name: str = Field( ..., deprecated=True, - description="Reference label, often caller-provided or derived from the filename. Deprecated for storage path/display semantics; use `file_path`, `logical_path`, and `display_name` when present.", + description="Reference label, often caller-provided or derived from the filename. Deprecated for storage path/display semantics; use `loader_path`, `logical_path`, and `display_name` when present.", ) hash: str | None = None - file_path: str | None = Field( + loader_path: str | None = Field( default=None, description="In-root loader path for filesystem-backed assets: the path relative to its storage root with the top-level model category dropped (e.g. `models/checkpoints/foo/bar.safetensors` -> `foo/bar.safetensors`). This is the value model loaders consume. `None` when the file is not within a recognized root or model category.", ) diff --git a/app/assets/database/models.py b/app/assets/database/models.py index 9b61d309a..6cdd903c3 100644 --- a/app/assets/database/models.py +++ b/app/assets/database/models.py @@ -76,6 +76,10 @@ class AssetReference(Base): # Cache state fields (from former AssetCacheState) file_path: Mapped[str | None] = mapped_column(Text, nullable=True) + # In-root loader path derived from file_path at scan/ingest time (model + # category dropped). Persisted so responses read it directly instead of + # re-resolving against every registered model-folder base per request. + loader_path: Mapped[str | None] = mapped_column(Text, nullable=True) mtime_ns: Mapped[int | None] = mapped_column(BigInteger, nullable=True) needs_verify: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) is_missing: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) diff --git a/app/assets/database/queries/asset_reference.py b/app/assets/database/queries/asset_reference.py index 792411800..26c75ad93 100644 --- a/app/assets/database/queries/asset_reference.py +++ b/app/assets/database/queries/asset_reference.py @@ -650,6 +650,7 @@ def upsert_reference( name: str, mtime_ns: int, owner_id: str = "", + loader_path: str | None = None, ) -> tuple[bool, bool]: """Upsert a reference by file_path. Returns (created, updated). @@ -659,6 +660,7 @@ def upsert_reference( vals = { "asset_id": asset_id, "file_path": file_path, + "loader_path": loader_path, "name": name, "owner_id": owner_id, "mtime_ns": int(mtime_ns), diff --git a/app/assets/services/bulk_ingest.py b/app/assets/services/bulk_ingest.py index 4036530e6..c98658bf1 100644 --- a/app/assets/services/bulk_ingest.py +++ b/app/assets/services/bulk_ingest.py @@ -56,6 +56,7 @@ class ReferenceRow(TypedDict): id: str asset_id: str file_path: str + loader_path: str | None mtime_ns: int owner_id: str name: str @@ -172,6 +173,8 @@ def batch_insert_seed_assets( "id": reference_id, "asset_id": asset_id, "file_path": absolute_path, + # spec["fname"] is compute_loader_path(abs_path) from build_asset_specs. + "loader_path": spec["fname"], "mtime_ns": spec["mtime_ns"], "owner_id": owner_id, "name": spec["info_name"], diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index e6cc42fe9..6f6508207 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -92,6 +92,7 @@ def _ingest_file_from_path( name=info_name or os.path.basename(locator), mtime_ns=mtime_ns, owner_id=owner_id, + loader_path=compute_loader_path(locator), ) # Get the reference we just created/updated diff --git a/app/assets/services/schemas.py b/app/assets/services/schemas.py index 4d2af8a02..0fda6871d 100644 --- a/app/assets/services/schemas.py +++ b/app/assets/services/schemas.py @@ -25,6 +25,7 @@ class ReferenceData: preview_id: str | None created_at: datetime updated_at: datetime + loader_path: str | None = None system_metadata: dict[str, Any] | None = None job_id: str | None = None last_access_time: datetime | None = None @@ -93,6 +94,7 @@ def extract_reference_data(ref: AssetReference) -> ReferenceData: id=ref.id, name=ref.name, file_path=ref.file_path, + loader_path=ref.loader_path, user_metadata=ref.user_metadata, preview_id=ref.preview_id, system_metadata=ref.system_metadata, diff --git a/openapi.yaml b/openapi.yaml index 094b2277d..68f59dd62 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -11,7 +11,7 @@ components: description: Blake3 hash of the asset content. pattern: ^blake3:[a-f0-9]{64}$ type: string - file_path: + loader_path: description: 'In-root loader path for filesystem-backed assets: the path relative to its storage root with the top-level model category dropped (e.g. `models/checkpoints/foo/bar.safetensors` -> `foo/bar.safetensors`). This is the value model loaders consume. `None` when the file is not within a recognized root or model category.' nullable: true type: string diff --git a/tests-unit/assets_test/services/test_bulk_ingest.py b/tests-unit/assets_test/services/test_bulk_ingest.py index 07cd5510d..c174fce44 100644 --- a/tests-unit/assets_test/services/test_bulk_ingest.py +++ b/tests-unit/assets_test/services/test_bulk_ingest.py @@ -191,6 +191,8 @@ class TestBatchInsertSeedAssets: refs = session.query(AssetReference).all() assert len(refs) == 1 assert refs[0].file_path == absolute_path + # loader_path is persisted from the spec's fname (compute_loader_path). + assert refs[0].loader_path == "same-file.safetensors" assert set(get_reference_tags(session, reference_id=refs[0].id)) == { "models", "model_type:checkpoints", diff --git a/tests-unit/assets_test/test_uploads.py b/tests-unit/assets_test/test_uploads.py index d428e74ca..f27172f4a 100644 --- a/tests-unit/assets_test/test_uploads.py +++ b/tests-unit/assets_test/test_uploads.py @@ -55,7 +55,7 @@ def test_upload_ok_duplicate_reference(http: requests.Session, api_base: str, ma assert a2["asset_hash"] == a1["asset_hash"] assert a2["hash"] == a1["hash"] assert a2["id"] != a1["id"] # new reference with same content - assert a2.get("file_path") is None + assert a2.get("loader_path") is None assert a2.get("display_name") is None # Third upload with the same data but different name also creates new AssetReference @@ -67,7 +67,7 @@ def test_upload_ok_duplicate_reference(http: requests.Session, api_base: str, ma assert a3["asset_hash"] == a1["asset_hash"] assert a3["id"] != a1["id"] assert a3["id"] != a2["id"] - assert a3.get("file_path") is None + assert a3.get("loader_path") is None assert a3.get("display_name") is None @@ -102,17 +102,17 @@ def test_upload_fastpath_from_existing_hash_no_file(http: requests.Session, api_ assert "checkpoints" in b2["tags"] assert "uploaded" not in b2["tags"] assert not any(tag.startswith("model_type:") for tag in b2["tags"]) - assert b2.get("file_path") is None + assert b2.get("loader_path") is None assert b2.get("display_name") is None rg = http.get(f"{api_base}/api/assets/{b2['id']}", timeout=120) detail = rg.json() assert rg.status_code == 200, detail - assert detail.get("file_path") is None + assert detail.get("loader_path") is None assert detail.get("display_name") is None -def test_create_from_hash_with_model_tags_does_not_synthesize_file_path( +def test_create_from_hash_with_model_tags_does_not_synthesize_loader_path( http: requests.Session, api_base: str ): seed_name = "from_hash_seed.safetensors" @@ -137,13 +137,13 @@ def test_create_from_hash_with_model_tags_does_not_synthesize_file_path( assert created_r.status_code == 201, created assert created["created_new"] is False assert created["asset_hash"] == seed["asset_hash"] - assert created.get("file_path") is None + assert created.get("loader_path") is None assert created.get("display_name") is None detail_r = http.get(f"{api_base}/api/assets/{created['id']}", timeout=120) detail = detail_r.json() assert detail_r.status_code == 200, detail - assert detail.get("file_path") is None + assert detail.get("loader_path") is None assert detail.get("display_name") is None @@ -204,7 +204,7 @@ def test_duplicate_byte_upload_is_reference_only_and_does_not_need_destination( assert "not-a-destination" in duplicate["tags"] assert "uploaded" not in duplicate["tags"] assert "input" not in duplicate["tags"] - assert duplicate.get("file_path") is None + assert duplicate.get("loader_path") is None assert duplicate.get("display_name") is None @@ -246,7 +246,7 @@ def test_upload_multiple_tags_fields_are_merged(http: requests.Session, api_base ), ], ) -def test_upload_response_includes_file_path_and_display_name( +def test_upload_response_includes_loader_path_and_display_name( tags: list[str], extension: str, expected_prefix: str, @@ -273,16 +273,16 @@ def test_upload_response_includes_file_path_and_display_name( expected_logical_path = f"{expected_prefix}/{expected_suffix}" expected_display_name = f"{expected_display_prefix}{expected_suffix}" # In-root loader path: model category dropped, no subfolders here -> just the filename. - expected_file_path = expected_suffix + expected_loader_path = expected_suffix - assert created["file_path"] == expected_file_path + assert created["loader_path"] == expected_loader_path assert created["logical_path"] == expected_logical_path assert created["display_name"] == expected_display_name detail_r = http.get(f"{api_base}/api/assets/{created['id']}", timeout=120) detail = detail_r.json() assert detail_r.status_code == 200, detail - assert detail["file_path"] == expected_file_path + assert detail["loader_path"] == expected_loader_path assert detail["logical_path"] == expected_logical_path assert detail["display_name"] == expected_display_name @@ -294,7 +294,7 @@ def test_upload_response_includes_file_path_and_display_name( listed = list_r.json() assert list_r.status_code == 200, listed match = next(a for a in listed["assets"] if a["id"] == created["id"]) - assert match["file_path"] == expected_file_path + assert match["loader_path"] == expected_loader_path assert match["logical_path"] == expected_logical_path assert match["display_name"] == expected_display_name