optimization: fast scan: commit to the DB in chunks

app/assets_scanner.py

@@ -1,4 +1,5 @@
 import asyncio
+import contextlib
 import logging
 import os
 import time
@@ -95,6 +96,8 @@ async def schedule_scans(roots: list[schemas_in.RootType]) -> schemas_out.AssetS
 
 
 async def sync_seed_assets(roots: list[schemas_in.RootType]) -> None:
+    t_total = time.perf_counter()
+    try:
         for r in roots:
             try:
                 await _fast_db_consistency_pass(r)
@@ -109,6 +112,8 @@ async def sync_seed_assets(roots: list[schemas_in.RootType]) -> None:
         if "output" in roots:
             paths.extend(list_tree(folder_paths.get_output_directory()))
 
+        processed = 0
+        async with await create_session() as sess:
             for p in paths:
                 try:
                     st = os.stat(p, follow_symlinks=True)
@@ -117,13 +122,7 @@ async def sync_seed_assets(roots: list[schemas_in.RootType]) -> None:
                     size_bytes = int(st.st_size)
                     mtime_ns = getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000))
                     name, tags = get_name_and_tags_from_asset_path(p)
-            await _seed_one_async(p, size_bytes, mtime_ns, name, tags)
-        except OSError:
-            continue
 
-
-async def _seed_one_async(p: str, size_bytes: int, mtime_ns: int, name: str, tags: list[str]) -> None:
-    async with await create_session() as sess:
                     await ensure_seed_for_path(
                         sess,
                         abs_path=p,
@@ -133,7 +132,19 @@ async def _seed_one_async(p: str, size_bytes: int, mtime_ns: int, name: str, tag
                         tags=tags,
                         owner_id="",
                     )
+
+                    processed += 1
+                    if processed % 500 == 0:
                         await sess.commit()
+                except OSError:
+                    continue
+            await sess.commit()
+    finally:
+        LOGGER.info(
+            "Assets scan(roots=%s) completed in %.3f s",
+            roots,
+            time.perf_counter() - t_total,
+        )
 
 
 def _status_response_for(progresses: list[ScanProgress]) -> schemas_out.AssetScanStatusResponse:
@@ -482,20 +493,13 @@ async def _fast_db_consistency_pass(root: schemas_in.RootType) -> None:
                 if any_fast_ok:
                     # Remove 'missing' and delete just the stale state rows
                     for st in missing_states:
-                        try:
+                        with contextlib.suppress(Exception):
                             await sess.delete(await sess.get(AssetCacheState, st.id))
-                        except Exception:
-                            pass
-                    try:
+                    with contextlib.suppress(Exception):
                         await remove_missing_tag_for_asset_id(sess, asset_id=aid)
-                    except Exception:
-                        pass
                 else:
-                    # No fast-ok path: mark as missing
-                    try:
+                    with contextlib.suppress(Exception):
                         await add_missing_tag_for_asset_id(sess, asset_id=aid, origin="automatic")
-                    except Exception:
-                        pass
 
         await sess.flush()
         await sess.commit()

app/database/services/content.py

@@ -87,6 +87,7 @@ async def ensure_seed_for_path(
             state_row.needs_verify = True
             if asset_row.size_bytes == 0 and size_bytes > 0:
                 asset_row.size_bytes = int(size_bytes)
+            await session.flush()
         return asset_row.id
 
     asset = Asset(hash=None, size_bytes=int(size_bytes), mime_type=None, created_at=now)