""" Wire shapes shared by every node transport. `file_chunk` lives in `meshbay_common.protocol` — it is pure crypto and shape, so a client can use the same encoder. This module is for the messages that also need the node's own view of its disk, which `meshbay-common` cannot see. Why it exists at all: `index_sync` was built twice, and the two copies did not agree. WebRTC sent `{group_id, version, entries, dirs, roots}` — the shape the shipping client reads — while QUIC sent `{index_b64}`, a signed, compressed, GEK-encrypted envelope produced by `GroupIndex.serialize()`. Same message type, two encodings, one consumer each and nothing asserting they matched. Same failure mode as the two `file_chunk` encoders, and the same fix: one builder, used by both. That envelope, and `GroupIndex.serialize()`/`deserialize()` which produced it, are gone: once both transports built `index_sync` here, nothing stored or exchanged it. It was never a candidate for reuse below either — it compressed with zstd, which no browser can decompress (`DecompressionStream` offers gzip and deflate only). Since MNP 1.0 both messages carry their payload **sealed under a GEK-derived subkey** (`meshbay_common.groupbox`). Only the routing fields — `type`, `v`, `group_id` — stay in clear: a receiver must route and version-check before it can decrypt, and `group_id` is the AAD and selects the key besides. `version`/`base_version` moved *inside* the payload; there is no reason to act on a version number carried by a message we have not yet authenticated. """ from __future__ import annotations import os from pathlib import Path from meshbay_common import MNP_VERSION from meshbay_common.groupbox import PURPOSE_INDEX, seal from meshbay_common.protocol import MNP, index_entry_wire from meshbay_node.roots import RootSet, off_disk # A group with a deep tree can hold more directories than anyone will navigate in one # sitting, and the whole list rides on one message. MAX_DIRS = 2000 def list_dirs(roots: RootSet | None) -> list[str]: """ Every directory in the group, as members address them, sorted. Each root appears as a directory in its own right, so a root holding no files yet is still somewhere a member can navigate to and upload into. An unavailable root is listed too — its content is frozen, not gone, and hiding it would look exactly like deletion. """ return _walk_dirs(_dir_roots(roots)) if roots else [] async def list_dirs_off_loop(roots: RootSet | None) -> list[str]: """ `list_dirs`, on the thread that serves `roots`. The walk is a syscall per directory over every root of the group, and it ran on the event loop for each full index: on a library of several large roots that was seconds in which the node answered nobody, not its members and not the desktop client asking whether it was there. Found live: a tester adding a sixth directory saw "Add a directory" vanish because the node missed a 3 s check while it indexed the other five. """ if not roots: return [] return await off_disk(roots, _walk_dirs, _dir_roots(roots)) def _dir_roots(roots: RootSet) -> list[tuple[str, Path, bool]]: """What the walk needs from each root, read on the loop that owns them.""" return [(root.name, root.path, root.available) for root in roots] def _walk_dirs(dir_roots: list[tuple[str, Path, bool]]) -> list[str]: """Blocking. Directories only: `os.walk` reads each one once and never stats a file, which `rglob("*")` and an `is_dir()` per entry did.""" out: list[str] = [] for name, path, available in dir_roots: out.append(name) if not available: continue for top, subdirs, _files in os.walk(path): # Pruned in place: nothing under a hidden directory is listed either. subdirs[:] = [d for d in subdirs if not d.startswith(".")] rel = Path(top).relative_to(path) for d in subdirs: out.append(f"{name}/{(rel / d).as_posix()}") return sorted(out)[:MAX_DIRS] def index_sync_message(index, roots: RootSet | None, hidden=(), dirs: list[str] | None = None) -> dict: """ The full `index_sync` message for one group. `dirs` and `roots` are in the payload because directories are not index entries: without them a folder someone just created, or one they emptied, does not exist as far as a client is concerned, and a member cannot tell "the drive is unplugged" from "it is all still there". `hidden` is the content blocklist in a public group (`meshbay_node.blocklist`): those entries are not sent at all. `dirs` is `list_dirs_off_loop(roots)`, awaited by a caller on the event loop; left out, the walk runs here, where the caller is. """ payload = { "version": index.version, "entries": [index_entry_wire(e) for e in index.entries if e.id not in hidden], "dirs": list_dirs(roots) if dirs is None else dirs, "roots": roots.describe() if roots else [], } return { "type": MNP.INDEX_SYNC, "v": MNP_VERSION, "group_id": index.group_id, **seal(index.gek, PURPOSE_INDEX, MNP.INDEX_SYNC, index.group_id, payload), } def index_delta_message(index, delta, roots=None, hidden=()) -> dict: """ One `index_delta` — what changed since the last thing this node broadcast. Built here rather than inline in the daemon, which is where it lived and which made it the third place an index message was constructed: precisely the drift that produced two `index_sync` encodings and two `file_chunk` encodings before it. `roots` rides along (MNP 1.1, additive — a 1.0 client ignores it). It used to travel on `index_sync` alone, which is a *full* index and therefore only ever sent on request. So a root added, removed, ejected or plugged left every connected client's directory table stale until somebody reloaded the page: the delta that told them something had changed was the one message that could not say what. It is a handful of dicts, bounded by the number of directories a group has, and it is sealed with the rest. `hidden`, as for `index_sync_message`: a blocked entry is never added or updated; its deletion still goes out. """ payload = { "base_version": delta.base_version, "version": delta.version, "additions": [index_entry_wire(e) for e in delta.additions if e.id not in hidden], "deletions": list(delta.deletions), "updates": [index_entry_wire(e) for e in delta.updates if e.id not in hidden], } if roots is not None: payload["roots"] = roots.describe() return { "type": MNP.INDEX_DELTA, "v": MNP_VERSION, "group_id": index.group_id, **seal(index.gek, PURPOSE_INDEX, MNP.INDEX_DELTA, index.group_id, payload), }