From 3c44f55f6b0aba77c7ad57d0a5ebe3e55b409473 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Tue, 25 Aug 2026 18:18:48 +0200 Subject: fix(hub,node): Create Group wizard silently skipped apps, and lost track of scanning progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real-world bugs found together while testing multi-root group creation: - CreateGroupWizard only sent the enabled-apps PUT when the operator had *unchecked* something, assuming "every box left checked" already matched the node's own default (Roster.DEFAULT_APPS = chat, files). It doesn't — so leaving every app checked, the common case, silently left Videos/Music/Photos disabled on the node. Now sent unconditionally. - The wizard's "add extra roots" step never polled index-status, so once step 3 (which only watches the first/upload root) finished, the progress bar froze while the node kept scanning the remaining roots for minutes, unwatched. Added waitForRootsIndexed (platform.js), mirroring waitForGroupHosted's own race handling. That fix exposed a deeper one: indexer.py's _scan_root() only flipped `progress.scanning` on *after* walking the directory and stat()-ing every file — both off-loop, but slow enough on a large root that a poller's grace period (waitForRootsIndexed's 5s) could expire before ever observing `scanning: true` (confirmed against production logs: a GEK-init step fired 5.058s after a root started scanning, matching the grace period almost exactly). The stat() pass was also a synchronous loop directly on the asyncio event loop — blocking the whole daemon (WebRTC, chat, admin UI) for as long as it took on a root with many files. Both fixed: `scanning` now flips on before the walk starts, and stat()-ing is now off-loop too (_size_files). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013XSohfUQQiaE77qyFLgSv3 --- .../src/meshbay_node/indexer/indexer.py | 65 ++++++++++++++++------ 1 file changed, 49 insertions(+), 16 deletions(-) (limited to 'packages/meshbay-node/src/meshbay_node') diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index 10981c0..f78465b 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -139,6 +139,27 @@ def _walk_root(root: Root) -> list[Path]: return [p for p in root.path.rglob("*") if p.is_file()] +def _size_files(files: list[Path]) -> list[tuple[Path, int]]: + """ + Blocking: stat() every file from an already-completed walk — run via an + executor for the same reason `_walk_root` is (its own docstring above). + Previously a plain loop straight on the asyncio event loop thread: for a + root with many thousands of files (a real personal library, not a + hypothetical — docs/musicbay.md's own "several thousand files" example) + that blocked the entire daemon, every WebRTC session and the admin UI + included, for as long as the stat() calls took — and did so *before* + `_scan_root` had even set `progress.scanning`, so a consumer polling it + saw "not scanning" the whole time real, blocking work was happening. + """ + sized: list[tuple[Path, int]] = [] + for p in files: + try: + sized.append((p, p.stat().st_size)) + except OSError: + continue + return sized + + def _scan_file(root: Root, file_path: Path) -> IndexEntry | None: """Compute IndexEntry for a file. Blocking — run in executor. Uses streaming blake3 so arbitrarily large files (ISOs, VM images, etc.) @@ -304,26 +325,38 @@ class DirectoryIndexer: log.info("Scanning %s (root %r) ...", root.path, root.name) count = 0 loop = asyncio.get_event_loop() - try: - files = await loop.run_in_executor(self._executor, _walk_root, root) - except OSError as e: - log.warning("Cannot scan root %r: %s", root.name, e) - return 0 - - # Sizes up front, off the same listing that already walked the tree — - # the progress bar's denominator, not a second pass over the disk. - sized: list[tuple[Path, int]] = [] - for p in files: - try: - sized.append((p, p.stat().st_size)) - except OSError: - continue + # `scanning` flips on here, before the walk — not after it and the + # sizing pass, which for a large or slow (network/USB) root can + # themselves take a long time despite running off-loop. A consumer + # polling IndexProgress (index-status; the Create Group wizard's + # own progress bar) must see "scanning" the moment real work starts, + # not only once the file list and its total size are known. Found + # live (2026-08-25): the wizard's post-creation "add extra roots" + # step gave up waiting after a short grace period because the flag + # had not yet turned on, even though the node was already several + # seconds into walking and sizing a large root — total_bytes is + # unknown at this point, so it starts at 0 and is corrected below. self.progress.scanning = True self.progress.scanned_bytes = 0 - self.progress.total_bytes = sum(size for _, size in sized) - self.progress.current_dir = "" + self.progress.total_bytes = 0 + self.progress.current_dir = root.name try: + try: + files = await loop.run_in_executor(self._executor, _walk_root, root) + except OSError as e: + log.warning("Cannot scan root %r: %s", root.name, e) + return 0 + + # Sizes up front, off the same listing that already walked the + # tree — the progress bar's denominator, not a second pass over + # the disk. Off-loop for the same reason the walk itself is + # (_size_files's own docstring: this used to be a synchronous + # loop right here, blocking the whole daemon for a large root). + sized = await loop.run_in_executor(self._executor, _size_files, files) + + self.progress.total_bytes = sum(size for _, size in sized) + self.progress.current_dir = "" for file_path, size in sized: self.progress.current_dir = file_path.parent.name entry = await self._hash_or_cached(root, file_path) -- cgit v1.2.3