From 37d8d9c15c982f2da17b2fad4ea1a90613b560a6 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Wed, 26 Aug 2026 00:40:20 +0200 Subject: feat(node): share the (path,size,mtime)->hash index cache across every group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An operator routinely shares the same physical folder into more than one group (a music library, a Séries drive) — IndexCache used to be opened once per group (data_dir/{group_id}/index_cache.db), so the second group to reference an already-fully-hashed multi-terabyte folder paid the same full content read the first one did. IndexCache itself carried no group_id in its schema; only daemon.py's wiring did. Now one instance, opened once at startup (data_dir/index_cache.db), shared by every group's DirectoryIndexer. Confirmed against a real deployment (2026-08-25/26): a group sharing an already-indexed folder with an existing group indexes it instantly, with zero rehashing. Also fixes a related cross-group correctness gap found during this work: media_cache.db (thumbnails, TMDB/MusicBrainz metadata — already node-wide, untouched by this change) was pruned for a file the moment it left *one* group's index, even if another group's index still held the same content hash — forcing a redundant re-fetch/re-probe/re-thumbnail for a group that never actually lost anything. Prune now runs only once no group's index references the file_id any more. Adds a node admin UI action ("Maintenance" card, prune-index-cache) to drop cache rows that no longer belong to any group's roots — skips anything under a root that is merely temporarily unavailable (indexer.py's "a root that goes away freezes, never empties" rule extends to this cache too, or a reconnected drive would pay a full rehash for no reason). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013XSohfUQQiaE77qyFLgSv3 --- packages/meshbay-node/tests/test_daemon.py | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) (limited to 'packages/meshbay-node/tests/test_daemon.py') diff --git a/packages/meshbay-node/tests/test_daemon.py b/packages/meshbay-node/tests/test_daemon.py index b367a20..71aae78 100644 --- a/packages/meshbay-node/tests/test_daemon.py +++ b/packages/meshbay-node/tests/test_daemon.py @@ -439,6 +439,71 @@ async def test_delta_reflects_additions_and_deletions(tmp_path, shared_dir, gek) assert [a["id"] for a in delta_msg["additions"]] == ["new-file-id"] +@pytest.mark.asyncio +async def test_media_cache_not_pruned_when_another_group_still_has_the_content( + tmp_path, shared_dir, gek): + """ + media_cache.db is node-wide, keyed by content hash — a file shared into + two groups is one row there. Removing it from ONE group's index (root + unshared, group left) must not wipe the thumbnail/tmdb/mbid mapping the + OTHER group's copy still needs, or that surviving group pays for a + redundant re-fetch/re-probe/re-thumbnail for content it never lost. + """ + daemon = _new_daemon_for_group(tmp_path, shared_dir, gek, group_id="a" * 32) + daemon._media_cache = AsyncMock() + daemon._webrtc = MagicMock() + daemon._webrtc._sessions = {} + + indexer_a = DirectoryIndexer(roots=one_root(shared_dir), group_id="a" * 32, + sk_node=Ed25519PrivateKey.generate(), gek=gek) + await indexer_a.initial_scan() + shared_id = indexer_a.index.entries[0].id + + indexer_b = DirectoryIndexer(roots=one_root(shared_dir), group_id="b" * 32, + sk_node=Ed25519PrivateKey.generate(), gek=gek) + await indexer_b.initial_scan() + assert indexer_b.index.get_entry(shared_id) is not None + + daemon._indexers = [indexer_a, indexer_b] + + await daemon._on_index_change(indexer_a) # establishes the snapshot + await asyncio.sleep(0.05) + + indexer_a.index.remove_entry(shared_id) + await daemon._on_index_change(indexer_a) + await asyncio.sleep(0.05) + + daemon._media_cache.prune_file.assert_not_called() + + +@pytest.mark.asyncio +async def test_media_cache_pruned_once_no_group_has_the_content_left( + tmp_path, shared_dir, gek): + """Counterpart of the test above: with only one group ever having held + the content, its removal must still prune media_cache as before — the + fix only withholds pruning when the content genuinely survives + elsewhere, it must not make pruning stop happening altogether.""" + daemon = _new_daemon_for_group(tmp_path, shared_dir, gek, group_id="a" * 32) + daemon._media_cache = AsyncMock() + daemon._webrtc = MagicMock() + daemon._webrtc._sessions = {} + + indexer = DirectoryIndexer(roots=one_root(shared_dir), group_id="a" * 32, + sk_node=Ed25519PrivateKey.generate(), gek=gek) + await indexer.initial_scan() + removed_id = indexer.index.entries[0].id + daemon._indexers = [indexer] + + await daemon._on_index_change(indexer) + await asyncio.sleep(0.05) + + indexer.index.remove_entry(removed_id) + await daemon._on_index_change(indexer) + await asyncio.sleep(0.05) + + daemon._media_cache.prune_file.assert_called_once_with(removed_id) + + @pytest.mark.asyncio async def test_a_burst_of_changes_produces_one_broadcast(tmp_path, shared_dir, gek): """Coalescing: several _on_index_change calls in quick succession (one -- cgit v1.2.3