From 37d8d9c15c982f2da17b2fad4ea1a90613b560a6 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Wed, 26 Aug 2026 00:40:20 +0200 Subject: feat(node): share the (path,size,mtime)->hash index cache across every group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An operator routinely shares the same physical folder into more than one group (a music library, a Séries drive) — IndexCache used to be opened once per group (data_dir/{group_id}/index_cache.db), so the second group to reference an already-fully-hashed multi-terabyte folder paid the same full content read the first one did. IndexCache itself carried no group_id in its schema; only daemon.py's wiring did. Now one instance, opened once at startup (data_dir/index_cache.db), shared by every group's DirectoryIndexer. Confirmed against a real deployment (2026-08-25/26): a group sharing an already-indexed folder with an existing group indexes it instantly, with zero rehashing. Also fixes a related cross-group correctness gap found during this work: media_cache.db (thumbnails, TMDB/MusicBrainz metadata — already node-wide, untouched by this change) was pruned for a file the moment it left *one* group's index, even if another group's index still held the same content hash — forcing a redundant re-fetch/re-probe/re-thumbnail for a group that never actually lost anything. Prune now runs only once no group's index references the file_id any more. Adds a node admin UI action ("Maintenance" card, prune-index-cache) to drop cache rows that no longer belong to any group's roots — skips anything under a root that is merely temporarily unavailable (indexer.py's "a root that goes away freezes, never empties" rule extends to this cache too, or a reconnected drive would pay a full rehash for no reason). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013XSohfUQQiaE77qyFLgSv3 --- packages/meshbay-node/tests/test_index_cache.py | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'packages/meshbay-node/tests/test_index_cache.py') diff --git a/packages/meshbay-node/tests/test_index_cache.py b/packages/meshbay-node/tests/test_index_cache.py index db0e37e..24e2f76 100644 --- a/packages/meshbay-node/tests/test_index_cache.py +++ b/packages/meshbay-node/tests/test_index_cache.py @@ -59,6 +59,52 @@ async def test_put_overwrites_previous_row_for_same_path(cache): assert hit.hash == "new" +@pytest.mark.asyncio +async def test_count_reflects_number_of_rows(cache): + assert await cache.count() == 0 + + await cache.put("/lib/a.mkv", size=1000, mtime=111.0, hash="a", + type="video", added_at=1) + await cache.put("/lib/b.mkv", size=2000, mtime=222.0, hash="b", + type="video", added_at=2) + + assert await cache.count() == 2 + + +@pytest.mark.asyncio +async def test_all_paths_returns_every_row(cache): + await cache.put("/lib/a.mkv", size=1000, mtime=111.0, hash="a", + type="video", added_at=1) + await cache.put("/lib/b.mkv", size=2000, mtime=222.0, hash="b", + type="video", added_at=2) + + assert set(await cache.all_paths()) == {"/lib/a.mkv", "/lib/b.mkv"} + + +@pytest.mark.asyncio +async def test_remove_many_drops_only_the_given_paths(cache): + await cache.put("/lib/a.mkv", size=1000, mtime=111.0, hash="a", + type="video", added_at=1) + await cache.put("/lib/b.mkv", size=2000, mtime=222.0, hash="b", + type="video", added_at=2) + + await cache.remove_many(["/lib/a.mkv"]) + + assert await cache.lookup("/lib/a.mkv", size=1000, mtime=111.0) is None + assert await cache.lookup("/lib/b.mkv", size=2000, mtime=222.0) is not None + assert await cache.count() == 1 + + +@pytest.mark.asyncio +async def test_remove_many_with_empty_list_is_a_no_op(cache): + await cache.put("/lib/a.mkv", size=1000, mtime=111.0, hash="a", + type="video", added_at=1) + + await cache.remove_many([]) + + assert await cache.count() == 1 + + @pytest.mark.asyncio async def test_cache_survives_reopen(tmp_path): db_path = tmp_path / "index_cache.db" -- cgit v1.2.3