From d4b37774118a2689c67b2ad802b9763f2ef448fd Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Mon, 14 Sep 2026 09:40:23 +0200 Subject: fix(node): a reload and a plug rescan outlive the session that asked A root added from the client arrives over MNP, and _retarget_indexer started the daemon's reload with the session's own _spawn. When that session closed - a client reconnecting 47 s into the scan of a 900 GB root - shutdown_tasks() cancelled the reload mid-scan, and the reload queued behind it, without a line in the log. The new root was in node.toml and in the indexer's set but never in the group's context; the lock was free and nothing retried, so the node served the old roots table for hours while reconcile hashed the whole drive as missed events. One loopback reload fixed the live node in 9 ms. _reload_config now runs the work in a node-owned task and awaits it through asyncio.shield, so a caller that goes away only stops waiting; a cancelled reload is logged. plug_root does the same for its rescan, which drops the root's entries before walking the disk and so left the root empty when its admin op's session closed. The existing MNP test replaced _spawn with a list and could not cancel anything; the new tests close the session for real and fail without this. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01T6jPTeocXA1BePekdsgPya --- packages/meshbay-node/src/meshbay_node/daemon.py | 22 ++++++++++++++++++++-- .../src/meshbay_node/indexer/indexer.py | 18 ++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) (limited to 'packages/meshbay-node/src/meshbay_node') diff --git a/packages/meshbay-node/src/meshbay_node/daemon.py b/packages/meshbay-node/src/meshbay_node/daemon.py index c9c362a..932a90d 100644 --- a/packages/meshbay-node/src/meshbay_node/daemon.py +++ b/packages/meshbay-node/src/meshbay_node/daemon.py @@ -764,9 +764,27 @@ class NodeDaemon: Serialised by _reload_lock: fire-and-forget reloads from config-mutating endpoints can overlap with the wizard's explicit /api/reload call, and two concurrent hot-loads of the same group corrupt the runtime state. + + The reload belongs to the node, never to whoever asked for it. A root + added from a browser reaches here through the operator's WebRTC session, + whose tasks are all cancelled when that session closes — and on + 2026-09-14 one closed 47 s into the scan of a 900 GB root. The reload + died without a line in the log, the new root was in node.toml and in + the indexer but never in the group's context, and nothing ever tried + again: the lock was free, and nobody was waiting on it. So the work runs + in a task of its own, and a caller that goes away only stops waiting. """ - async with self._reload_lock: - await self._reload_config_inner() + await asyncio.shield(spawn(self._reload_config_locked(), what="config reload")) + + async def _reload_config_locked(self) -> None: + try: + async with self._reload_lock: + await self._reload_config_inner() + except asyncio.CancelledError: + log.warning("Config reload cancelled before it finished — the node may " + "be serving part of the previous configuration until the " + "next reload") + raise async def _reload_config_inner(self) -> None: log.info("Reloading config from %s", self._config_path) diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index 2b68550..c20902e 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -953,8 +953,22 @@ class DirectoryIndexer: return root.ejected = False root.available = root.is_live() - if root.available: - log.info("Root %r plugged — rescanning", root.name) + if not root.available: + await self._finish_plug(None) + return + log.info("Root %r plugged — rescanning", root.name) + # `_rescan_root` drops the root's entries before it walks the disk, and + # this is called from an admin op running in the operator's WebRTC + # session — which cancels everything it started when it closes. So the + # rescan runs in a task this indexer owns, and a caller that goes away + # only stops waiting for it instead of leaving the root emptied. + task = asyncio.create_task(self._finish_plug(root)) + self._scan_tasks.add(task) + task.add_done_callback(self._scan_tasks.discard) + await asyncio.shield(task) + + async def _finish_plug(self, root: Root | None) -> None: + if root is not None: await self._rescan_root(root) self._restart_observer() self._index.roots = self.roots.describe() -- cgit v1.2.3