diff options
Diffstat (limited to 'packages/meshbay-node')
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/indexer/indexer.py | 65 | ||||
| -rw-r--r-- | packages/meshbay-node/tests/test_indexer.py | 85 |
2 files changed, 134 insertions, 16 deletions
diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index 10981c0..f78465b 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -139,6 +139,27 @@ def _walk_root(root: Root) -> list[Path]: return [p for p in root.path.rglob("*") if p.is_file()] +def _size_files(files: list[Path]) -> list[tuple[Path, int]]: + """ + Blocking: stat() every file from an already-completed walk — run via an + executor for the same reason `_walk_root` is (its own docstring above). + Previously a plain loop straight on the asyncio event loop thread: for a + root with many thousands of files (a real personal library, not a + hypothetical — docs/musicbay.md's own "several thousand files" example) + that blocked the entire daemon, every WebRTC session and the admin UI + included, for as long as the stat() calls took — and did so *before* + `_scan_root` had even set `progress.scanning`, so a consumer polling it + saw "not scanning" the whole time real, blocking work was happening. + """ + sized: list[tuple[Path, int]] = [] + for p in files: + try: + sized.append((p, p.stat().st_size)) + except OSError: + continue + return sized + + def _scan_file(root: Root, file_path: Path) -> IndexEntry | None: """Compute IndexEntry for a file. Blocking — run in executor. Uses streaming blake3 so arbitrarily large files (ISOs, VM images, etc.) @@ -304,26 +325,38 @@ class DirectoryIndexer: log.info("Scanning %s (root %r) ...", root.path, root.name) count = 0 loop = asyncio.get_event_loop() - try: - files = await loop.run_in_executor(self._executor, _walk_root, root) - except OSError as e: - log.warning("Cannot scan root %r: %s", root.name, e) - return 0 - - # Sizes up front, off the same listing that already walked the tree — - # the progress bar's denominator, not a second pass over the disk. - sized: list[tuple[Path, int]] = [] - for p in files: - try: - sized.append((p, p.stat().st_size)) - except OSError: - continue + # `scanning` flips on here, before the walk — not after it and the + # sizing pass, which for a large or slow (network/USB) root can + # themselves take a long time despite running off-loop. A consumer + # polling IndexProgress (index-status; the Create Group wizard's + # own progress bar) must see "scanning" the moment real work starts, + # not only once the file list and its total size are known. Found + # live (2026-08-25): the wizard's post-creation "add extra roots" + # step gave up waiting after a short grace period because the flag + # had not yet turned on, even though the node was already several + # seconds into walking and sizing a large root — total_bytes is + # unknown at this point, so it starts at 0 and is corrected below. self.progress.scanning = True self.progress.scanned_bytes = 0 - self.progress.total_bytes = sum(size for _, size in sized) - self.progress.current_dir = "" + self.progress.total_bytes = 0 + self.progress.current_dir = root.name try: + try: + files = await loop.run_in_executor(self._executor, _walk_root, root) + except OSError as e: + log.warning("Cannot scan root %r: %s", root.name, e) + return 0 + + # Sizes up front, off the same listing that already walked the + # tree — the progress bar's denominator, not a second pass over + # the disk. Off-loop for the same reason the walk itself is + # (_size_files's own docstring: this used to be a synchronous + # loop right here, blocking the whole daemon for a large root). + sized = await loop.run_in_executor(self._executor, _size_files, files) + + self.progress.total_bytes = sum(size for _, size in sized) + self.progress.current_dir = "" for file_path, size in sized: self.progress.current_dir = file_path.parent.name entry = await self._hash_or_cached(root, file_path) diff --git a/packages/meshbay-node/tests/test_indexer.py b/packages/meshbay-node/tests/test_indexer.py index 4bae620..9aa9fb9 100644 --- a/packages/meshbay-node/tests/test_indexer.py +++ b/packages/meshbay-node/tests/test_indexer.py @@ -2,6 +2,7 @@ import asyncio import os +import threading import time import pytest from pathlib import Path @@ -521,6 +522,90 @@ async def test_walk_root_does_not_stall_the_event_loop(tmp_path, sk_node, gek): @pytest.mark.asyncio +async def test_scanning_flag_is_true_while_the_walk_is_still_running(tmp_path, sk_node, gek): + """ + Regression for the actual bug this was found by (2026-08-25): `scanning` + used to flip on only *after* the walk finished, so a consumer polling + IndexProgress — index-status; the Create Group wizard's own progress + poll, which gives up after a short grace period if it never observes + `scanning: true` — read "not scanning" for however long a large/slow + root's discovery phase took, even though the node was already doing real + work. Confirmed against production logs: a wizard step waiting on this + flag gave up exactly at its grace-period deadline for a root whose walk + was still running. + """ + d = tmp_path / "shared" + d.mkdir() + (d / "f.bin").write_bytes(b"x") + + real_walk = indexer_mod._walk_root + walk_started = threading.Event() + release_walk = threading.Event() + + def slow_walk(root): + walk_started.set() + release_walk.wait(timeout=5) + return real_walk(root) + + indexer_mod._walk_root = slow_walk + indexer = DirectoryIndexer(roots=one_root(d), group_id="g", sk_node=sk_node, gek=gek) + try: + scan_task = asyncio.create_task(indexer.initial_scan()) + # Off-loop wait for the walk to actually start — busy-polling the + # event loop itself here would defeat the point of the test. + await asyncio.get_event_loop().run_in_executor(None, walk_started.wait, 5) + + assert indexer.progress.scanning is True, ( + "scanning must be True the moment the walk starts, not only " + "once it (and the sizing pass) finish") + finally: + release_walk.set() + indexer_mod._walk_root = real_walk + await scan_task + + assert indexer.progress.scanning is False + + +@pytest.mark.asyncio +async def test_scanning_flag_is_true_while_sizing_files_is_still_running( + tmp_path, sk_node, gek): + """ + Same regression, one phase later: sizing (stat()-ing every walked file) + used to be a synchronous loop straight on the asyncio event loop thread — + for a root with many thousands of files (a real personal library, not a + hypothetical) that blocked the entire daemon, and did so before + `scanning` was ever set. Now off-loop (_size_files) and `scanning` is + already true throughout, same principle as the walk phase above. + """ + d = tmp_path / "shared" + d.mkdir() + (d / "f.bin").write_bytes(b"x") + + real_size = indexer_mod._size_files + sizing_started = threading.Event() + release_sizing = threading.Event() + + def slow_size(files): + sizing_started.set() + release_sizing.wait(timeout=5) + return real_size(files) + + indexer_mod._size_files = slow_size + indexer = DirectoryIndexer(roots=one_root(d), group_id="g", sk_node=sk_node, gek=gek) + try: + scan_task = asyncio.create_task(indexer.initial_scan()) + await asyncio.get_event_loop().run_in_executor(None, sizing_started.wait, 5) + + assert indexer.progress.scanning is True + finally: + release_sizing.set() + indexer_mod._size_files = real_size + await scan_task + + assert indexer.progress.scanning is False + + +@pytest.mark.asyncio async def test_reconcile_backoff_grows_with_no_changes_then_caps(shared_dir, sk_node, gek): indexer = DirectoryIndexer(roots=one_root(shared_dir), group_id="g", sk_node=sk_node, gek=gek, reconcile_secs=0.01) |