From 37d8d9c15c982f2da17b2fad4ea1a90613b560a6 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Wed, 26 Aug 2026 00:40:20 +0200 Subject: feat(node): share the (path,size,mtime)->hash index cache across every group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An operator routinely shares the same physical folder into more than one group (a music library, a Séries drive) — IndexCache used to be opened once per group (data_dir/{group_id}/index_cache.db), so the second group to reference an already-fully-hashed multi-terabyte folder paid the same full content read the first one did. IndexCache itself carried no group_id in its schema; only daemon.py's wiring did. Now one instance, opened once at startup (data_dir/index_cache.db), shared by every group's DirectoryIndexer. Confirmed against a real deployment (2026-08-25/26): a group sharing an already-indexed folder with an existing group indexes it instantly, with zero rehashing. Also fixes a related cross-group correctness gap found during this work: media_cache.db (thumbnails, TMDB/MusicBrainz metadata — already node-wide, untouched by this change) was pruned for a file the moment it left *one* group's index, even if another group's index still held the same content hash — forcing a redundant re-fetch/re-probe/re-thumbnail for a group that never actually lost anything. Prune now runs only once no group's index references the file_id any more. Adds a node admin UI action ("Maintenance" card, prune-index-cache) to drop cache rows that no longer belong to any group's roots — skips anything under a root that is merely temporarily unavailable (indexer.py's "a root that goes away freezes, never empties" rule extends to this cache too, or a reconnected drive would pay a full rehash for no reason). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013XSohfUQQiaE77qyFLgSv3 --- packages/meshbay-node/tests/test_indexer.py | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'packages/meshbay-node/tests/test_indexer.py') diff --git a/packages/meshbay-node/tests/test_indexer.py b/packages/meshbay-node/tests/test_indexer.py index 9aa9fb9..729dade 100644 --- a/packages/meshbay-node/tests/test_indexer.py +++ b/packages/meshbay-node/tests/test_indexer.py @@ -283,6 +283,44 @@ async def test_second_scan_with_same_cache_hashes_nothing( assert {e.id for e in second.index.entries} == {e.id for e in first.index.entries} +@pytest.mark.asyncio +async def test_second_group_sharing_the_same_folder_hashes_nothing( + shared_dir, sk_node, gek, index_cache): + """ + The scenario the cache was made node-wide for (2026-08-25): an operator + shares the same physical folder into a second group. `IndexCache` is + keyed purely by absolute path, with no notion of group_id at all — a + *different* group_id scanning the same folder through the same shared + cache instance must hit exactly as hard as a same-group restart does + (the test right above this one). Before this cache was shared node-wide, + each group got its own on-disk cache file and this scan would have + rehashed every byte again. + """ + first = DirectoryIndexer(roots=one_root(shared_dir), group_id="group-a", + sk_node=sk_node, gek=gek, cache=index_cache) + await first.initial_scan() + assert first.index.count == 4 + + calls = [] + real_scan_file = indexer_mod._scan_file + + def spy(root, path): + calls.append(path) + return real_scan_file(root, path) + + indexer_mod._scan_file = spy + try: + second = DirectoryIndexer(roots=one_root(shared_dir), group_id="group-b", + sk_node=sk_node, gek=gek, cache=index_cache) + await second.initial_scan() + finally: + indexer_mod._scan_file = real_scan_file + + assert calls == [], ( + f"a second group scanning the same folder must not rehash it, got {calls}") + assert {e.id for e in second.index.entries} == {e.id for e in first.index.entries} + + @pytest.mark.asyncio async def test_modified_file_is_rehashed(tmp_path, sk_node, gek, index_cache): d = tmp_path / "shared" -- cgit v1.2.3