From 37d8d9c15c982f2da17b2fad4ea1a90613b560a6 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Wed, 26 Aug 2026 00:40:20 +0200 Subject: feat(node): share the (path,size,mtime)->hash index cache across every group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An operator routinely shares the same physical folder into more than one group (a music library, a Séries drive) — IndexCache used to be opened once per group (data_dir/{group_id}/index_cache.db), so the second group to reference an already-fully-hashed multi-terabyte folder paid the same full content read the first one did. IndexCache itself carried no group_id in its schema; only daemon.py's wiring did. Now one instance, opened once at startup (data_dir/index_cache.db), shared by every group's DirectoryIndexer. Confirmed against a real deployment (2026-08-25/26): a group sharing an already-indexed folder with an existing group indexes it instantly, with zero rehashing. Also fixes a related cross-group correctness gap found during this work: media_cache.db (thumbnails, TMDB/MusicBrainz metadata — already node-wide, untouched by this change) was pruned for a file the moment it left *one* group's index, even if another group's index still held the same content hash — forcing a redundant re-fetch/re-probe/re-thumbnail for a group that never actually lost anything. Prune now runs only once no group's index references the file_id any more. Adds a node admin UI action ("Maintenance" card, prune-index-cache) to drop cache rows that no longer belong to any group's roots — skips anything under a root that is merely temporarily unavailable (indexer.py's "a root that goes away freezes, never empties" rule extends to this cache too, or a reconnected drive would pay a full rehash for no reason). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013XSohfUQQiaE77qyFLgSv3 --- .../meshbay-node/src/meshbay_node/indexer/cache.py | 42 ++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) (limited to 'packages/meshbay-node/src/meshbay_node/indexer') diff --git a/packages/meshbay-node/src/meshbay_node/indexer/cache.py b/packages/meshbay-node/src/meshbay_node/indexer/cache.py index c26ddf1..31b9b15 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/cache.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/cache.py @@ -1,5 +1,5 @@ """ -MeshBay Node — persistent (path, size, mtime) -> hash cache, one per group. +MeshBay Node — persistent (path, size, mtime) -> hash cache, one per node. Without this, every node restart re-reads and re-hashes every file in every root, even when nothing changed — measured at 23 minutes for a 114 GB library @@ -10,6 +10,16 @@ It is a path-keyed accelerator only. The GroupIndex itself stays keyed by content hash (see indexer.py's note on why two identical files are one entry) — this cache never changes that, it only avoids recomputing a hash that has not changed. + +**Shared by every group's DirectoryIndexer, one instance, one open +connection** (2026-08-25) — an operator very often shares the same physical +folder (a music library, a Séries drive) into more than one group, and a +cache keyed purely by absolute path has no reason to care which group asked. +It used to be opened once per group (`data_dir/{group_id}/index_cache.db`), +which meant the second group to reference an already-fully-hashed multi- +terabyte folder paid the same full read the first one did — exactly the cost +this cache exists to avoid. The schema carries no group_id and never has; +only daemon.py's wiring changed. """ import logging @@ -40,7 +50,7 @@ class CachedEntry: class IndexCache: - """Async SQLite (path, size, mtime) -> hash cache for one group.""" + """Async SQLite (path, size, mtime) -> hash cache, shared node-wide.""" def __init__(self, db_path: Path): self._db_path = db_path @@ -93,3 +103,31 @@ class IndexCache: "type = excluded.type, added_at = excluded.added_at", (path, mtime, size, hash, type, added_at)) await self._db.commit() + + # ── Maintenance (node admin UI "prune index cache") ────────────────────── + + async def count(self) -> int: + """Cheap — used for the dashboard stat, never for the prune decision + itself (that needs the actual paths, see all_paths).""" + async with self._db.execute("SELECT COUNT(*) FROM files") as cur: + row = await cur.fetchone() + return row[0] if row else 0 + + async def all_paths(self) -> list[str]: + """Every cached path, for a caller that decides staleness itself — + this cache has no notion of which paths are still claimed by a + group's roots, on purpose (see ops.prune_index_cache).""" + async with self._db.execute("SELECT path FROM files") as cur: + rows = await cur.fetchall() + return [row[0] for row in rows] + + async def remove_many(self, paths: list[str]) -> None: + """Drop rows outright — used only for paths a caller has already + decided are gone for good. Losing one costs a rehash next time that + path is scanned, never a wrong answer (lookup() always re-validates + against a live stat()).""" + if not paths: + return + await self._db.executemany( + "DELETE FROM files WHERE path = ?", [(p,) for p in paths]) + await self._db.commit() -- cgit v1.2.3