diff --git a/app/assets/api/routes.py b/app/assets/api/routes.py index d156daa79..2f845b228 100644 --- a/app/assets/api/routes.py +++ b/app/assets/api/routes.py @@ -1,3 +1,4 @@ +import json import logging import os import urllib.parse @@ -80,8 +81,6 @@ def _build_error_response( def _build_validation_error_response(code: str, ve: ValidationError) -> web.Response: - import json - errors = json.loads(ve.json()) return _build_error_response(400, code, "Validation failed.", {"errors": errors}) @@ -489,8 +488,6 @@ async def get_tags(request: web.Request) -> web.Response: try: query = schemas_in.TagsListQuery.model_validate(query_map) except ValidationError as e: - import json - return _build_error_response( 400, "INVALID_QUERY", @@ -631,12 +628,22 @@ async def seed_assets(request: web.Request) -> web.Response: wait_param = request.query.get("wait", "").lower() should_wait = wait_param in ("true", "1", "yes") + # Temporarily enable seeder for explicit API calls (--disable-assets-autoscan + # only prevents the automatic startup scan, not manual triggers) + was_disabled = asset_seeder.is_disabled() + if was_disabled: + asset_seeder.enable() + started = asset_seeder.start(roots=valid_roots) if not started: + if was_disabled: + asset_seeder.disable() return web.json_response({"status": "already_running"}, status=409) if should_wait: asset_seeder.wait() + if was_disabled: + asset_seeder.disable() status = asset_seeder.get_status() return web.json_response( { diff --git a/app/assets/database/queries/asset.py b/app/assets/database/queries/asset.py index ba711ed86..24aac350e 100644 --- a/app/assets/database/queries/asset.py +++ b/app/assets/database/queries/asset.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.dialects import sqlite from sqlalchemy.orm import Session -from app.assets.database.models import Asset +from app.assets.database.models import Asset, AssetReference from app.assets.database.queries.common import calculate_rows_per_statement, iter_chunks @@ -130,8 +130,6 @@ def reassign_asset_references( Used when merging a stub asset into an existing asset with the same hash. """ - from app.assets.database.models import AssetReference - ref = session.get(AssetReference, reference_id) if ref: ref.asset_id = to_asset_id diff --git a/app/assets/database/queries/asset_reference.py b/app/assets/database/queries/asset_reference.py index 0a0510fda..55e087b6e 100644 --- a/app/assets/database/queries/asset_reference.py +++ b/app/assets/database/queries/asset_reference.py @@ -32,11 +32,6 @@ from app.assets.database.queries.common import ( from app.assets.helpers import escape_sql_like_string, get_utc_now, normalize_tags -# ============================================================================= -# Metadata conversion helpers (from former asset_info.py) -# ============================================================================= - - def _check_is_scalar(v): if v is None: return True @@ -84,11 +79,6 @@ def convert_metadata_to_rows(key: str, value) -> list[dict]: return [{"key": key, "ordinal": 0, "val_json": value}] -# ============================================================================= -# Filter helpers -# ============================================================================= - - def _apply_tag_filters( stmt: sa.sql.Select, include_tags: Sequence[str] | None = None, @@ -168,11 +158,6 @@ def _apply_metadata_filter( return stmt -# ============================================================================= -# Basic CRUD operations -# ============================================================================= - - def get_reference_by_id( session: Session, reference_id: str, @@ -302,11 +287,6 @@ def update_reference_timestamps( reference.updated_at = now -# ============================================================================= -# Listing and pagination -# ============================================================================= - - def list_references_page( session: Session, owner_id: str = "", @@ -440,11 +420,6 @@ def fetch_reference_and_asset( return pair[0], pair[1] -# ============================================================================= -# Timestamp updates -# ============================================================================= - - def update_reference_access_time( session: Session, reference_id: str, @@ -491,11 +466,6 @@ def update_reference_updated_at( ) -# ============================================================================= -# Metadata operations -# ============================================================================= - - def set_reference_metadata( session: Session, reference_id: str, @@ -538,11 +508,6 @@ def set_reference_metadata( session.flush() -# ============================================================================= -# Delete operations -# ============================================================================= - - def delete_reference_by_id( session: Session, reference_id: str, @@ -576,11 +541,6 @@ def set_reference_preview( session.flush() -# ============================================================================= -# Cache state operations (from former cache_state.py) -# ============================================================================= - - class CacheStateRow(NamedTuple): """Row from reference query with cache state data.""" @@ -867,11 +827,6 @@ def delete_orphaned_seed_asset(session: Session, asset_id: str) -> bool: return False -# ============================================================================= -# Enrichment operations -# ============================================================================= - - class UnenrichedReferenceRow(NamedTuple): """Row for references needing enrichment.""" @@ -968,11 +923,6 @@ def bulk_update_enrichment_level( return result.rowcount -# ============================================================================= -# Bulk operations -# ============================================================================= - - def bulk_insert_references_ignore_conflicts( session: Session, rows: list[dict], diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 4003c8cd0..7ded8107b 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -247,7 +247,9 @@ class AssetSeeder: return True def resume(self) -> bool: - """Resume a paused scan. + """Resume a paused scan. + + This is a noop if the scan is not in the PAUSED state Returns: True if resumed, False if not paused diff --git a/app/assets/services/hashing.py b/app/assets/services/hashing.py index c77f2f916..38aeae4d7 100644 --- a/app/assets/services/hashing.py +++ b/app/assets/services/hashing.py @@ -2,23 +2,10 @@ import asyncio import os from typing import IO +from blake3 import blake3 + DEFAULT_CHUNK = 8 * 1024 * 1024 -_blake3 = None - - -def _get_blake3(): - global _blake3 - if _blake3 is None: - try: - from blake3 import blake3 as _b3 - _blake3 = _b3 - except ImportError: - raise ImportError( - "blake3 is required for asset hashing. Install with: pip install blake3" - ) - return _blake3 - def compute_blake3_hash( fp: str | IO[bytes], @@ -55,7 +42,7 @@ def _hash_file_obj(file_obj: IO, chunk_size: int = DEFAULT_CHUNK) -> str: if orig_pos != 0: file_obj.seek(0) - h = _get_blake3()() + h = blake3() while True: chunk = file_obj.read(chunk_size) if not chunk: diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index 4af4ee7f0..38c286fa1 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -13,6 +13,7 @@ from app.assets.database.queries import ( add_tags_to_reference, fetch_reference_and_asset, get_asset_by_hash, + get_reference_by_file_path, get_reference_tags, get_or_create_reference, remove_missing_tag_for_asset_id, @@ -83,7 +84,6 @@ def _ingest_file_from_path( ) # Get the reference we just created/updated - from app.assets.database.queries import get_reference_by_file_path ref = get_reference_by_file_path(session, locator) if ref: reference_id = ref.id diff --git a/app/assets/services/path_utils.py b/app/assets/services/path_utils.py index 0bc1a62d0..8fdd59c23 100644 --- a/app/assets/services/path_utils.py +++ b/app/assets/services/path_utils.py @@ -3,7 +3,8 @@ from pathlib import Path from typing import Literal import folder_paths -from app.assets.helpers import normalize_tags +from app.assets.database.queries import list_references_by_asset_id +from app.assets.helpers import normalize_tags, select_best_live_path def get_comfy_models_folders() -> list[tuple[str, list[str]]]: @@ -157,9 +158,6 @@ def compute_filename_for_reference(session, ref) -> str | None: def compute_filename_for_asset(session, asset_id: str) -> str | None: """Compute the relative filename for an asset from its best live reference path.""" - from app.assets.database.queries import list_references_by_asset_id - from app.assets.helpers import select_best_live_path - primary_path = select_best_live_path( list_references_by_asset_id(session, asset_id=asset_id) ) diff --git a/main.py b/main.py index 0b3eb3d56..0249da689 100644 --- a/main.py +++ b/main.py @@ -259,10 +259,10 @@ def prompt_worker(q, server_instance): extra_data[k] = sensitive[k] asset_seeder.pause() - try: - e.execute(item[2], prompt_id, extra_data, item[4]) - finally: - asset_seeder.resume() + + e.execute(item[2], prompt_id, extra_data, item[4]) + + asset_seeder.resume() need_gc = True remove_sensitive = lambda prompt: prompt[:5] + prompt[6:] diff --git a/tests-unit/assets_test/helpers.py b/tests-unit/assets_test/helpers.py index 72d875bea..770e011f4 100644 --- a/tests-unit/assets_test/helpers.py +++ b/tests-unit/assets_test/helpers.py @@ -1,14 +1,27 @@ """Helper functions for assets integration tests.""" +import time + import requests def trigger_sync_seed_assets(session: requests.Session, base_url: str) -> None: - """Force a synchronous sync/seed pass by calling the seed endpoint with wait=true.""" - session.post( - base_url + "/api/assets/seed?wait=true", - json={"roots": ["models", "input", "output"]}, - timeout=60, - ) + """Force a synchronous sync/seed pass by calling the seed endpoint with wait=true. + + Retries on 409 (already running) until the previous scan finishes. + """ + deadline = time.monotonic() + 60 + while True: + r = session.post( + base_url + "/api/assets/seed?wait=true", + json={"roots": ["models", "input", "output"]}, + timeout=60, + ) + if r.status_code != 409: + assert r.status_code == 200, f"seed endpoint returned {r.status_code}: {r.text}" + return + if time.monotonic() > deadline: + raise TimeoutError("seed endpoint stuck in 409 (already running)") + time.sleep(0.25) def get_asset_filename(asset_hash: str, extension: str) -> str: diff --git a/tests-unit/assets_test/services/test_enrich.py b/tests-unit/assets_test/services/test_enrich.py index cb3c8920f..54bab642a 100644 --- a/tests-unit/assets_test/services/test_enrich.py +++ b/tests-unit/assets_test/services/test_enrich.py @@ -2,7 +2,6 @@ from pathlib import Path from unittest.mock import patch -import pytest from sqlalchemy.orm import Session from app.assets.database.models import Asset, AssetReference