From e9d5e979fdab9a1cc3c729d602e6f27207b9480c Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Tue, 18 Aug 2026 02:15:02 +0200 Subject: feat(node): several named roots per group, and one implementation per operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage A — a group's content is a set of named roots --------------------------------------------------- `shared_dir` becomes a list of {name, path, kind}. The name is the directory's basename, derived once at add time and *stored*: recomputing it would re-identify a whole library the day someone renames a folder on disk. Duplicate names are refused case-insensitively and no root may contain another — both compared with NFC folding, because most of these directories live on exFAT or NTFS where `Films` and `films` are one directory. Every index path carries its root name, in a one-root group as much as in a five-root one. One path shape has to be got right once; two have to be kept right for ever. **A root that goes away freezes; it never empties.** Unmounting a volume makes watchdog report every file under it as deleted, or presents an empty directory to the next scan. Acting on either propagates deletions for a whole library to every member, as though the owner had erased it. So a deletion is acted on only once its root is confirmed readable, and availability is tracked per root — one unplugged drive leaves the others serving. 12 tests, verified to fail against an indexer without the check. Events are not trusted to be complete either: ReadDirectoryChangesW drops them under load and inotify on a FUSE mount misses changes made outside it. A periodic reconciliation sweep is the only thing that recovers a missed event. MNP 0.2 → 0.3 (additive). The hub needs no change: SwarmSource carries a content hash, a node id and an endpoint — no paths, no filenames — and private groups register nothing (H7). Stage B — one implementation behind every front door ---------------------------------------------------- C1 and C6 were both "a second path into the node with its own weaker handshake". Two implementations of `revoke` with two authorization checks is that shape one size down. `meshbay_node/ops.py` holds each operation once, takes the daemon state, and knows nothing about HTTP, argv or MNP. The loopback API is one `_op(...)` line per endpoint; the MNP handlers call the same functions. test_ops.py asserts the shape rather than trusting it. Phase 14 is finished on top of it — `group list`, `gek init|rotate`, `reload` (SIGHUP), `denylist show|clear`, `file list|rm`. **No operator action requires a browser any more.** Plus `gek_rotate` and `member_unpin` as operator-signed MNP operations: rotation is the half of revocation that revocation cannot do, since the ex-member holds the current key, and the node generates the replacement with its own CSPRNG — no key material crosses the wire, which is what the C5b rule is actually about. Two bugs found by running it rather than by testing it ------------------------------------------------------ GroupIndex is keyed by **content hash**, so the same bytes at two paths are one entry — which is also why a scan reports ten files and indexes nine. Reconciliation compared paths, so it decided the second path was a missed event every 60 s, rewrote the entry and pushed an index update to every connected peer. Seen in a live node's log. `meshbay-node reload` crashed on first use with `subprocess` unimported: the module compiles fine, which is the "syntax, not names" trap already recorded for the SPA. test_cli_dispatch.py now walks every verb and refuses to let one be added to the parser without an entry there. Also corrected: protocol.py declared a second MNP_VERSION of "0.1" while the wire carried "0.2" — harmless only because nothing imported it. And _do_dir_create/_do_dir_delete referenced an undefined `filename` on their error path. 740 tests pass; QE/deploy/e2e.py passes end to end against the live deployment. Co-Authored-By: Claude Opus 5 --- .../meshbay-common/src/meshbay_common/__init__.py | 2 +- .../meshbay-common/src/meshbay_common/adminop.py | 8 ++ .../meshbay-common/src/meshbay_common/paths.py | 150 +++++++++++++++++++++ .../meshbay-common/src/meshbay_common/protocol.py | 12 +- packages/meshbay-common/tests/test_paths.py | 123 +++++++++++++++++ 5 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 packages/meshbay-common/src/meshbay_common/paths.py create mode 100644 packages/meshbay-common/tests/test_paths.py (limited to 'packages/meshbay-common') diff --git a/packages/meshbay-common/src/meshbay_common/__init__.py b/packages/meshbay-common/src/meshbay_common/__init__.py index 60a9dc7..814c606 100644 --- a/packages/meshbay-common/src/meshbay_common/__init__.py +++ b/packages/meshbay-common/src/meshbay_common/__init__.py @@ -4,5 +4,5 @@ __version__ = "0.5.0" # 0.2: added PING/PONG, and `before`/`has_more` on chat history. Both are # additive — an 0.1 peer sends no `before` and gets the newest page, which is # what it wanted — so this is a MINOR bump, not a MAJOR one. -MNP_VERSION = "0.2" +MNP_VERSION = "0.3" MHP_VERSION = "0.1" diff --git a/packages/meshbay-common/src/meshbay_common/adminop.py b/packages/meshbay-common/src/meshbay_common/adminop.py index fe4be83..4e28f65 100644 --- a/packages/meshbay-common/src/meshbay_common/adminop.py +++ b/packages/meshbay-common/src/meshbay_common/adminop.py @@ -37,6 +37,14 @@ OP_FILE_DELETE = "file_delete" OP_DIR_DELETE = "dir_delete" OP_INVITE_CREATE = "invite_create" OP_MEMBER_REVOKE = "member_revoke" +# Rotating the group key is what actually takes it away from a revoked member: +# revocation stops the node serving the *next* key, and they still hold the +# current one. The node generates the new key itself with its own CSPRNG, so +# nothing arriving over MNP contributes key material — the C5b rule is about +# key material from outside, not about the instruction. +OP_GEK_ROTATE = "gek_rotate" +# Forgetting a pinned identity, so someone can pair again after losing a device. +OP_MEMBER_UNPIN = "member_unpin" # OP_GEK_BUNDLE_STORE is gone. Members no longer hand the node key material at # all: the node holds the GEK and wraps it itself, for a key the recipient proved # they hold (see `join.py` and docs/invite-pairing-v1.md). The operation existed diff --git a/packages/meshbay-common/src/meshbay_common/paths.py b/packages/meshbay-common/src/meshbay_common/paths.py new file mode 100644 index 0000000..45bf34e --- /dev/null +++ b/packages/meshbay-common/src/meshbay_common/paths.py @@ -0,0 +1,150 @@ +""" +Name rules that have to give the same answer everywhere. + +Most people share from an external drive formatted exFAT or NTFS, on Windows. +Those filesystems are case-insensitive and case-preserving, store no POSIX +permissions and no symlinks, and Windows refuses a set of names outright. So two +names that are plainly different on ext4 can be the same file somewhere else, or +no file at all — and an index built on one machine is read on another. + +"Are these two names the same?" therefore has one answer, and it lives here. + +What this module deliberately does **not** do: rewrite names. A name is stored as +the filesystem gave it, because that is the string that opens the file. Folding +and normalization exist for *comparison*, never for storage. +""" + +from __future__ import annotations + +import os +import re +import unicodedata +from pathlib import Path, PurePath + +# Windows refuses these as a basename, with or without an extension: `AUX.txt` +# is as impossible as `AUX`. A group indexed on Linux can hold them, and a +# Windows client then cannot write the file it just downloaded. +WINDOWS_RESERVED = frozenset({ + "CON", "PRN", "AUX", "NUL", + *(f"COM{i}" for i in range(1, 10)), + *(f"LPT{i}" for i in range(1, 10)), +}) + +# Reserved on Windows; `/` is reserved everywhere. Control characters go too. +_RESERVED_CHARS = set('<>:"/\\|?*') | {chr(c) for c in range(32)} + +# Windows without long-path support. A deep media library reaches this. +MAX_PATH_WINDOWS = 260 + + +def nfc(text: str) -> str: + """ + Canonical form for comparison. + + `Café.mkv` written on macOS (NFD: `e` + combining acute) and on Windows + (NFC: precomposed `é`) are different byte strings that name the same file to + a human, and to most filesystems. For French names this is routine. + """ + return unicodedata.normalize("NFC", text) + + +def fold(text: str) -> str: + """ + The form in which two names are "the same file". + + NFC first, then `casefold` — which is not `lower()`: it handles the cases + `lower()` misses, and those are the ones that show up as a bug report rather + than a test failure. + """ + return nfc(text).casefold() + + +def fold_path(rel: str) -> str: + """`fold` applied per segment, so separators survive.""" + return "/".join(fold(seg) for seg in rel.split("/")) + + +def portable_name_problem(name: str) -> str | None: + """ + Why `name` cannot be written on some supported platform, or None. + + Used to warn, not to refuse: a file already on the operator's disk is a fact, + and the answer is to tell whoever downloads it that it was renamed — not to + pretend it is not there. + """ + if not name: + return "empty name" + if name in (".", ".."): + return "reserved name" + bad = sorted(set(name) & _RESERVED_CHARS) + if bad: + printable = "".join(c if c.isprintable() else "?" for c in bad) + return f"contains reserved characters: {printable}" + # Windows strips these silently, so `file .` and `file` become the same + # thing after a round trip. + if name[-1] in " .": + return "ends with a space or a dot" + stem = name.split(".", 1)[0] + if stem.upper() in WINDOWS_RESERVED: + return f"reserved on Windows ({stem.upper()})" + return None + + +def is_portable_name(name: str) -> bool: + return portable_name_problem(name) is None + + +def long_path(path: Path | str) -> str: + """ + A path string safe to hand to the OS. + + On Windows, prefix with `\\\\?\\` so `MAX_PATH` does not truncate a deep + library. The prefix requires an absolute, already-normalized path, and it is + a no-op everywhere else. + """ + text = str(path) + if os.name != "nt" or text.startswith("\\\\?\\"): + return text + resolved = str(PurePath(text)) + if len(resolved) < MAX_PATH_WINDOWS and not text.startswith("\\\\"): + return text + if text.startswith("\\\\"): # UNC: \\server\share → \\?\UNC\server\share + return "\\\\?\\UNC" + text[1:] + return "\\\\?\\" + text + + +def find_fold_collisions(names: list[str]) -> dict[str, list[str]]: + """ + Names that collide once folded, keyed by the folded form. + + Two entries that fold together cannot both exist on a case-insensitive + filesystem. On ext4 they can, which is how an index becomes unrepresentable + for the person who downloads it — so this is reported to the operator rather + than resolved silently: only they know which file they meant. + """ + seen: dict[str, list[str]] = {} + for name in names: + seen.setdefault(fold(name), []).append(name) + return {k: v for k, v in seen.items() if len(v) > 1} + + +_TRAILING = re.compile(r"[ .]+$") + + +def sanitize_for_download(name: str, *, replacement: str = "_") -> str: + """ + A name that can be written on the running platform, from one that may not be. + + For the client saving a file, never for the node storing one. Returns the + name unchanged when it is already portable, so the common case is identity + and the caller can tell whether it renamed anything by comparing. + """ + if is_portable_name(name): + return name + out = "".join(replacement if c in _RESERVED_CHARS else c for c in name) + out = _TRAILING.sub("", out) + stem, dot, ext = out.partition(".") + if stem.upper() in WINDOWS_RESERVED: + stem = f"{stem}{replacement}" + out = f"{stem}{dot}{ext}" + return out or "unnamed" diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index 1aadafe..a1c971f 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -11,8 +11,10 @@ Every message carries a "v" field for protocol version. from dataclasses import dataclass, field from typing import Any -MNP_VERSION = "0.1" -MHP_VERSION = "0.1" +# The wire versions live in meshbay_common/__init__.py — one source, because a +# second copy here said "0.1" while every message on the wire carried "0.2". +# Nothing imported it, which is the only reason it was harmless. +from meshbay_common import MNP_VERSION, MHP_VERSION # noqa: F401 (re-export) # ── MNP message types ───────────────────────────────────────────────────────── @@ -74,6 +76,12 @@ class MNP: INVITE_CREATE = "invite_create" # operator → node: issue a pairing code MEMBER_REVOKE = "member_revoke" # operator → node: stop serving the key MEMBER_REVOKE_ACK = "member_revoke_ack" + MEMBER_UNPIN = "member_unpin" # operator → node: forget an identity + MEMBER_UNPIN_ACK = "member_unpin_ack" + # Rotation is the half of revocation that revocation cannot do: the node + # generates a fresh key itself, so no key material crosses the wire. + GEK_ROTATE = "gek_rotate" # operator → node: new group key + GEK_ROTATE_ACK = "gek_rotate_ack" INVITE_RESULT = "invite_result" # node → operator: the code, once diff --git a/packages/meshbay-common/tests/test_paths.py b/packages/meshbay-common/tests/test_paths.py new file mode 100644 index 0000000..b9052e3 --- /dev/null +++ b/packages/meshbay-common/tests/test_paths.py @@ -0,0 +1,123 @@ +""" +Name rules that have to give the same answer on every platform. + +Most people share from an external drive formatted exFAT or NTFS, on Windows, so +these are not compatibility niceties — they decide whether two index entries are +one file, and whether a member can save what they downloaded. +""" + +import pytest + +from meshbay_common.paths import ( + find_fold_collisions, + fold, + fold_path, + is_portable_name, + nfc, + portable_name_problem, + sanitize_for_download, +) + + +# ── Same file or not ───────────────────────────────────────────────────────── + +def test_case_differences_fold_together(): + assert fold("README.TXT") == fold("readme.txt") + + +def test_folding_is_not_lowercasing(): + """`casefold` handles what `lower()` misses, and those are the pairs that + arrive as a bug report rather than a test failure.""" + assert fold("STRASSE") == fold("straße") + assert "STRASSE".lower() != "straße".lower() + + +def test_the_two_unicode_spellings_of_an_accent_are_one_name(): + """ + `Café.mkv` written on macOS (NFD) and on Windows (NFC) are different byte + strings naming the same file. For French filenames this is routine. + """ + nfc_form = "Café.mkv" # é precomposed + nfd_form = "Café.mkv" # e + combining acute + assert nfc_form != nfd_form + assert nfc(nfd_form) == nfc_form + assert fold(nfd_form) == fold(nfc_form) + + +def test_distinct_names_stay_distinct(): + assert fold("film.mkv") != fold("film2.mkv") + + +def test_folding_a_path_keeps_its_separators(): + assert fold_path("Films/2024/A.MKV") == "films/2024/a.mkv" + + +def test_collisions_are_found_and_grouped(): + found = find_fold_collisions( + ["README.txt", "readme.TXT", "notes.md", "Café.mkv", "Café.mkv"]) + groups = {frozenset(v) for v in found.values()} + assert len(groups) == 2 + assert frozenset({"README.txt", "readme.TXT"}) in groups + # The other pair is the two Unicode spellings of the same accented name, + # which are different byte strings — so compare by size, not by literal. + assert any(len(g) == 2 and all(f.endswith('.mkv') for f in g) + for g in groups) + assert all("notes.md" not in v for v in found.values()) + + +def test_no_collision_means_no_report(): + assert find_fold_collisions(["a.txt", "b.txt"]) == {} + + +# ── What Windows refuses ───────────────────────────────────────────────────── + +@pytest.mark.parametrize("name", ["CON", "PRN", "AUX", "NUL", "COM1", "LPT9", + "aux", "Aux.txt", "COM3.tar.gz"]) +def test_windows_reserved_names_are_flagged(name): + """`AUX.txt` is as impossible as `AUX` — the extension does not help.""" + assert portable_name_problem(name) is not None + + +@pytest.mark.parametrize("name", ['ab', 'a:b', 'a"b', "a/b", "a\\b", + "a|b", "a?b", "a*b", "a\x00b", "a\tb"]) +def test_reserved_characters_are_flagged(name): + assert portable_name_problem(name) is not None + + +@pytest.mark.parametrize("name", ["trailing ", "trailing.", " "]) +def test_a_trailing_space_or_dot_is_flagged(name): + """Windows strips them silently, so `file .` and `file` come back the same.""" + assert portable_name_problem(name) is not None + + +@pytest.mark.parametrize("name", ["film.mkv", "Café.mkv", "rapport (1).pdf", + "été.txt", "COMET.txt", "AUXILIARY.doc", + "日本語.txt"]) +def test_ordinary_names_are_portable(name): + assert is_portable_name(name), portable_name_problem(name) + + +def test_empty_and_dot_names_are_refused(): + assert portable_name_problem("") is not None + assert portable_name_problem(".") is not None + assert portable_name_problem("..") is not None + + +# ── Saving a file whose name the platform refuses ──────────────────────────── + +def test_a_portable_name_is_returned_unchanged(): + """Identity in the common case, so a caller can tell whether it renamed + anything by comparing.""" + assert sanitize_for_download("film.mkv") == "film.mkv" + assert sanitize_for_download("Café (2024).mkv") == "Café (2024).mkv" + + +def test_sanitizing_produces_something_writable(): + for hostile in ["ac.txt", "AUX", "trailing .", "with/slash.txt", "COM1.log"]: + cleaned = sanitize_for_download(hostile) + assert is_portable_name(cleaned), f"{hostile!r} → {cleaned!r} still refused" + + +def test_sanitizing_never_returns_nothing(): + assert sanitize_for_download("...") not in ("", None) + assert sanitize_for_download("???") not in ("", None) -- cgit v1.2.3