From c1be7571973c3d0b671ed4db2da41266ae3099d8 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Thu, 3 Sep 2026 15:20:40 +0200 Subject: refactor!: one file_chunk and index_sync encoder for every transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `file_chunk` and `index_sync` were each built twice, once per transport, and the two copies did not agree. WebRTC sent binary, unsigned chunks carrying a `file_id`; QUIC sent base64 fields, two BLAKE3 hashes, a per-chunk Ed25519 signature and no `file_id`. `index_sync` was plain entries on one transport and a `GroupIndex.serialize()` envelope on the other. One message type, two shapes, one consumer each, and nothing that failed when they drifted — finding C6 one size down, in the two places the handshake unification did not reach. Phase 9.15 moved WebRTC to the binary format and dropped the per-chunk signature; the QUIC encoder was never brought along. It is dropped here rather than reintroduced: the AES-GCM tag authenticates the ciphertext under a GEK-derived key, and since C3 the node authenticates itself once in the handshake instead of once per megabyte. `meshbay_common.protocol` now owns the chunk codec (`chunk_ciphertext`, `file_chunk_wire`, `file_chunk_plaintext`) and `meshbay_node/transport/wire.py` the index builder, which also absorbs the delta the daemon used to hand-build. `test_transport_wire_parity.py` fails if either server grows its own copy back. `ChunkRequest`/`ChunkResponse` are deleted. `ChunkResponse` described the QUIC half while reading like the contract for both, which is what made the fork hard to see at all. BREAKING CHANGE: MNP 0.15 changes the encoding of `file_chunk` and `index_sync` on the QUIC transport. The WebRTC shapes are byte for byte unchanged and no QUIC client ships, which is why this is a MINOR bump; a deployed QUIC peer would have made it MAJOR. Also fixes a test fixture that put a `Path` where the daemon puts a `RootSet`. Nothing caught it: the old QUIC index handler never touched `roots`, and `entry_abs_path` fell through `Path.resolve(strict=...)`, reading the virtual path as a truthy flag and returning the right file by accident. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AsoWC3GmhNdwVFomW3QjH3 --- .../meshbay-common/src/meshbay_common/__init__.py | 11 ++- .../meshbay-common/src/meshbay_common/crypto.py | 9 +- .../meshbay-common/src/meshbay_common/protocol.py | 100 +++++++++++++++++---- 3 files changed, 102 insertions(+), 18 deletions(-) (limited to 'packages/meshbay-common') diff --git a/packages/meshbay-common/src/meshbay_common/__init__.py b/packages/meshbay-common/src/meshbay_common/__init__.py index fdf6d86..61502c9 100644 --- a/packages/meshbay-common/src/meshbay_common/__init__.py +++ b/packages/meshbay-common/src/meshbay_common/__init__.py @@ -61,5 +61,14 @@ __version__ = "0.10.0" # under the account's recovery key, so a forgotten passphrase does not strand # the identity (docs/auth-confirm.md §4.3). Additive: an older node ignores the # field on store and never returns one; an older client never sends it. -MNP_VERSION = "0.14" +# 0.15: `file_chunk` and `index_sync` had forked between the transports — WebRTC +# sent binary, unsigned chunks and plain index entries, QUIC sent base64 chunks +# with a per-chunk Ed25519 signature and a `GroupIndex.serialize()` envelope. One +# type, two shapes, a single consumer each and no test that they agreed. Both now +# come from one encoder (`protocol.file_chunk_wire`, `transport/wire.py`), which is +# QUIC adopting what WebRTC already sent. **Breaking on the QUIC wire**, and only +# there: the WebRTC shape — the one every deployed client speaks — is byte for byte +# what it was, and no QUIC client ships. Recorded as a MINOR bump for that reason; +# a deployed QUIC peer would have made it a MAJOR one. +MNP_VERSION = "0.15" MHP_VERSION = "0.1" diff --git a/packages/meshbay-common/src/meshbay_common/crypto.py b/packages/meshbay-common/src/meshbay_common/crypto.py index b2ff3c0..eadb42b 100644 --- a/packages/meshbay-common/src/meshbay_common/crypto.py +++ b/packages/meshbay-common/src/meshbay_common/crypto.py @@ -225,7 +225,14 @@ def decrypt_keystore(iv: bytes, ciphertext: bytes, tag: bytes, key: bytes) -> by def sign_chunk(sk_node: Ed25519PrivateKey, chunk_index: int, nonce: bytes, ct_hash: bytes) -> bytes: - """Sign chunk metadata. Payload: chunk_index || nonce || ct_hash.""" + """ + Sign chunk metadata. Payload: chunk_index || nonce || ct_hash. + + Despite the name, this no longer signs file chunks — Phase 9.15 dropped per-chunk + signatures on the WebRTC path and 2026-09-03 dropped the QUIC copy that had been + left behind. Its one caller is `GroupIndex.serialize()`, which signs a whole index + envelope under the pseudo-index `INDEX_CHUNK`. + """ payload = chunk_index.to_bytes(4, "big") + nonce + ct_hash return sk_node.sign(payload) diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index c789287..4d2dd9d 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -15,6 +15,11 @@ from typing import Any # second copy here said "0.1" while every message on the wire carried "0.2". # Nothing imported it, which is the only reason it was harmless. from meshbay_common import MNP_VERSION, MHP_VERSION # noqa: F401 (re-export) +from meshbay_common.webcrypto import ( + chunk_key_aes, + decrypt_chunk_aes, + encrypt_chunk_aes, +) # ── MNP message types ───────────────────────────────────────────────────────── @@ -236,21 +241,84 @@ class IndexDelta: updates: list[IndexEntry] = field(default_factory=list) -# ── Chunk request/response ──────────────────────────────────────────────────── +# ── File chunk ──────────────────────────────────────────────────────────────── +# +# One encoder and one decoder, for every transport. +# +# There used to be two. WebRTC moved to a binary wire format in Phase 9.15 (base64 +# costs 33%) and dropped the per-chunk Ed25519 signature with it; the QUIC encoder +# was not brought along, so `file_chunk` meant two different messages depending on +# which transport carried it — base64 fields, hashes and a signature on one, raw +# bytes and a `file_id` on the other. Nothing in the type name said which, and the +# dataclass that used to sit here described only the QUIC half while reading like +# the contract for both. That is finding C6 one size down: two implementations of +# one message, free to drift, with a test for neither. +# +# Why no per-chunk signature: the AES-GCM tag already authenticates the ciphertext +# under a key derived from the GEK, which only group members hold, and since C3 the +# node authenticates itself in the handshake and is pinned by the client. A +# signature per chunk re-proved, once per megabyte, what the session established +# once. (`sign_chunk` still exists in `crypto.py` — it signs the serialized index +# envelope, which is a different artifact; see `indexer/group_index.py`.) -@dataclass -class ChunkRequest: - file_id: str # blake3 hash of file (hex) - chunk_index: int -@dataclass -class ChunkResponse: - chunk_index: int - plaintext_size: int - nonce_b64: str - ct_b64: str - ct_hash_b64: str - pt_hash_b64: str - sig_b64: str - pk_node_b64: str - file_hash_b64: str +def chunk_ciphertext( + gek: bytes, + plaintext: bytes, + chunk_index: int, + file_hash: bytes, +) -> tuple[bytes, bytes]: + """ + `(nonce, ciphertext)` for one chunk. + + Split out because `stream_data` encrypts exactly like `file_chunk` — same key + derivation, indexed by segment instead of by chunk — and differs only in the + message it lands in. It used to do so through its own copy of these two lines. + """ + ckey = chunk_key_aes(gek, file_hash, chunk_index) + return encrypt_chunk_aes(ckey, plaintext) + + +def file_chunk_wire( + gek: bytes, + plaintext: bytes, + chunk_index: int, + file_hash: bytes, + file_id: str = "", +) -> dict: + """ + Encrypt one chunk and build the `file_chunk` message. + + `file_hash` is the raw content hash the chunk key is derived from; `file_id` is + the same value hex-encoded, echoed so a client running several downloads at once + can tell whose reply arrived. Serving a thumbnail or a cached transcode passes + the cache blob's own hash for both. + """ + nonce, ct = chunk_ciphertext(gek, plaintext, chunk_index, file_hash) + return { + "type": MNP.FILE_CHUNK, + "v": MNP_VERSION, + "file_id": file_id, + "chunk_index": chunk_index, + "plaintext_size": len(plaintext), + "nonce": nonce, + "ct": ct, + } + + +def file_chunk_plaintext( + gek: bytes, + msg: dict, + file_hash: bytes | None = None, +) -> bytes: + """ + Decrypt a `file_chunk`. Raises `InvalidTag` if the ciphertext was tampered with. + + `file_hash` defaults to the message's own `file_id`, which is what a client that + asked for a whole file already has. Pass it explicitly only when the caller knows + better than the peer does. + """ + if file_hash is None: + file_hash = bytes.fromhex(msg["file_id"]) + ckey = chunk_key_aes(gek, file_hash, msg["chunk_index"]) + return decrypt_chunk_aes(ckey, msg["nonce"], msg["ct"]) -- cgit v1.2.3