From 37d8d9c15c982f2da17b2fad4ea1a90613b560a6 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Wed, 26 Aug 2026 00:40:20 +0200 Subject: feat(node): share the (path,size,mtime)->hash index cache across every group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An operator routinely shares the same physical folder into more than one group (a music library, a Séries drive) — IndexCache used to be opened once per group (data_dir/{group_id}/index_cache.db), so the second group to reference an already-fully-hashed multi-terabyte folder paid the same full content read the first one did. IndexCache itself carried no group_id in its schema; only daemon.py's wiring did. Now one instance, opened once at startup (data_dir/index_cache.db), shared by every group's DirectoryIndexer. Confirmed against a real deployment (2026-08-25/26): a group sharing an already-indexed folder with an existing group indexes it instantly, with zero rehashing. Also fixes a related cross-group correctness gap found during this work: media_cache.db (thumbnails, TMDB/MusicBrainz metadata — already node-wide, untouched by this change) was pruned for a file the moment it left *one* group's index, even if another group's index still held the same content hash — forcing a redundant re-fetch/re-probe/re-thumbnail for a group that never actually lost anything. Prune now runs only once no group's index references the file_id any more. Adds a node admin UI action ("Maintenance" card, prune-index-cache) to drop cache rows that no longer belong to any group's roots — skips anything under a root that is merely temporarily unavailable (indexer.py's "a root that goes away freezes, never empties" rule extends to this cache too, or a reconnected drive would pay a full rehash for no reason). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013XSohfUQQiaE77qyFLgSv3 --- packages/meshbay-node/src/meshbay_node/ui/app.py | 59 +++++++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) (limited to 'packages/meshbay-node/src/meshbay_node/ui/app.py') diff --git a/packages/meshbay-node/src/meshbay_node/ui/app.py b/packages/meshbay-node/src/meshbay_node/ui/app.py index 18764e8..3dc320b 100644 --- a/packages/meshbay-node/src/meshbay_node/ui/app.py +++ b/packages/meshbay-node/src/meshbay_node/ui/app.py @@ -169,6 +169,14 @@ def create_ui_app(state: dict) -> FastAPI: async def api_denylist_clear(subject: str = ""): return await _op(lambda: ops.clear_denylist(state, subject=subject)) + @app.get("/api/index-cache") + async def api_index_cache_stats(): + return await _op(lambda: ops.index_cache_stats(state)) + + @app.post("/api/index-cache/prune") + async def api_index_cache_prune(): + return await _op(lambda: ops.prune_index_cache(state)) + @app.get("/api/groups/{group_id}/files") async def api_group_files(group_id: str): groups_ctx = state.get("groups_ctx", {}) @@ -458,7 +466,9 @@ def create_ui_app(state: dict) -> FastAPI: "members": await roster.list_members(), "invites": await roster.list_invites(), } - return _render_page(state, roster_view) + index_cache = state.get("index_cache") + cache_count = await index_cache.count() if index_cache else None + return _render_page(state, roster_view, cache_count) @app.get("/audit", response_class=HTMLResponse) async def audit_page(): @@ -540,7 +550,8 @@ def _render_roster(roster_view: dict | None) -> str:

""" -def _render_page(state: dict, roster_view: dict | None = None) -> str: +def _render_page(state: dict, roster_view: dict | None = None, + index_cache_count: int | None = None) -> str: token_js = json.dumps(state.get("ui_token", "")) status = state.get("status", "starting") indexes = state.get("indexes", {}) @@ -723,6 +734,26 @@ def _render_page(state: dict, roster_view: dict | None = None) -> str:

Node ID: {state.get("endpoint_hint") or "—"}

+

Maintenance

+
+

Index cache: { + index_cache_count if index_cache_count is not None else "—" + } path(s) remembered (size/mtime → hash, shared by every group)

+

Removes rows whose path no longer belongs to any group's + root, or whose file is genuinely gone from a root that is currently + reachable. Never touches a root that is temporarily unavailable + (unplugged drive) — that one still needs its full cache back the + moment it returns.

+
+ + +
+
+

Link Node to Hub Account

To connect to your group from a browser, link this node to your hub account. @@ -774,6 +805,30 @@ async function initGEK(groupId) {{ if (btn) btn.disabled = false; }} }} +async function pruneIndexCache() {{ + const btn = document.getElementById('prune-cache-btn'); + const status = document.getElementById('prune-cache-status'); + if (btn) btn.disabled = true; + if (status) {{ status.textContent = 'Pruning...'; status.style.color = ''; }} + try {{ + const resp = await fetch('/api/index-cache/prune?t=' + TOKEN, {{ method: 'POST' }}); + const data = await resp.json(); + if (resp.ok) {{ + if (status) status.textContent = 'Removed ' + data.removed + ', kept ' + data.kept; + if (status) status.style.color = '#22c55e'; + const count = document.getElementById('cacheCount'); + if (count) count.textContent = data.kept; + }} else {{ + if (status) status.textContent = data.error || 'Failed'; + if (status) status.style.color = '#ef4444'; + }} + }} catch (e) {{ + if (status) status.textContent = 'Error: ' + e.message; + if (status) status.style.color = '#ef4444'; + }} finally {{ + if (btn) btn.disabled = false; + }} +}} setTimeout(()=>location.reload(), 10000); -- cgit v1.2.3