From 37d8d9c15c982f2da17b2fad4ea1a90613b560a6 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Wed, 26 Aug 2026 00:40:20 +0200 Subject: feat(node): share the (path,size,mtime)->hash index cache across every group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An operator routinely shares the same physical folder into more than one group (a music library, a Séries drive) — IndexCache used to be opened once per group (data_dir/{group_id}/index_cache.db), so the second group to reference an already-fully-hashed multi-terabyte folder paid the same full content read the first one did. IndexCache itself carried no group_id in its schema; only daemon.py's wiring did. Now one instance, opened once at startup (data_dir/index_cache.db), shared by every group's DirectoryIndexer. Confirmed against a real deployment (2026-08-25/26): a group sharing an already-indexed folder with an existing group indexes it instantly, with zero rehashing. Also fixes a related cross-group correctness gap found during this work: media_cache.db (thumbnails, TMDB/MusicBrainz metadata — already node-wide, untouched by this change) was pruned for a file the moment it left *one* group's index, even if another group's index still held the same content hash — forcing a redundant re-fetch/re-probe/re-thumbnail for a group that never actually lost anything. Prune now runs only once no group's index references the file_id any more. Adds a node admin UI action ("Maintenance" card, prune-index-cache) to drop cache rows that no longer belong to any group's roots — skips anything under a root that is merely temporarily unavailable (indexer.py's "a root that goes away freezes, never empties" rule extends to this cache too, or a reconnected drive would pay a full rehash for no reason). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013XSohfUQQiaE77qyFLgSv3 --- packages/meshbay-node/src/meshbay_node/ops.py | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) (limited to 'packages/meshbay-node/src/meshbay_node/ops.py') diff --git a/packages/meshbay-node/src/meshbay_node/ops.py b/packages/meshbay-node/src/meshbay_node/ops.py index c9375b3..c880c75 100644 --- a/packages/meshbay-node/src/meshbay_node/ops.py +++ b/packages/meshbay-node/src/meshbay_node/ops.py @@ -922,6 +922,66 @@ async def set_scan_settings(state: dict, group_id: str, reconcile_interval_secs: "debounce_secs": debounce_secs, "group_id": group_id} +# ── Index cache maintenance ─────────────────────────────────────────────────── +# +# The (path, size, mtime) -> hash accelerator (indexer/cache.py) is node-wide +# and grows for as long as a path was ever seen — a folder an operator later +# stops sharing (root removed, or every group hosting it is deleted) leaves +# its rows behind forever otherwise. Nothing about correctness needs this: +# a stale row just sits unused (lookup() keys on the live path string, so a +# path nothing scans any more is never looked up). This is disk space +# hygiene the operator can run when they want it, not a background job. + +async def index_cache_stats(state: dict) -> dict: + """Row count only — cheap, safe to call on every dashboard render. + The actual staleness check (prune_index_cache) is not this cheap and + must never run implicitly.""" + cache = state.get("index_cache") + return {"count": await cache.count() if cache else 0} + + +async def prune_index_cache(state: dict) -> dict: + """ + Drop cache rows that cannot be right for anything any more: the path is + not under any group's root at all, or it is under a root that is + available right now and the file is genuinely gone from disk. + + Deliberately leaves alone anything under a root that is currently + *unavailable* (a disconnected drive) — indexer.py's own rule is that + such a root freezes rather than empties, precisely so it does not pay a + full rehash the moment it comes back. Pruning through an unavailable + root here would reintroduce exactly that cost via a different door, so + an owning-but-unavailable root wins over "the file isn't there right + now" every time, unconditionally. + + A row lost here costs one rehash the next time that path is scanned, + never a wrong answer: lookup() (cache.py) always re-validates size and + mtime against a live stat() before trusting a cached hash. + """ + cache = state.get("index_cache") + if cache is None: + raise OpError("No index cache in this process", status=503) + + indexers = list((state.get("indexers") or {}).values()) + roots = [root for indexer in indexers for root in indexer.roots] + + def _is_stale(path_str: str) -> bool: + path = Path(path_str) + owning = [r for r in roots if r.path in path.parents] + if not owning: + return True + if any(not r.available for r in owning): + return False + return not path.exists() + + paths = await cache.all_paths() + stale = await asyncio.to_thread(lambda: [p for p in paths if _is_stale(p)]) + await cache.remove_many(stale) + log.info("Index cache pruned: %d stale row(s) removed, %d kept", + len(stale), len(paths) - len(stale)) + return {"status": "pruned", "removed": len(stale), "kept": len(paths) - len(stale)} + + # ── Reload ────────────────────────────────────────────────────────────────── async def reload_config(state: dict) -> dict: -- cgit v1.2.3