From 8980a8e42d94ab7c0bc9739283d39f938f8402b0 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Mon, 7 Sep 2026 17:46:33 +0200 Subject: feat(mnp)!: seal the upload under the group key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Downloads have been encrypted under a GEK-derived key since the beginning: `file_chunk` and `stream_data` both go through `chunk_ciphertext`. Uploads never were. `file_upload` carried the filename and the raw bytes in plain msgpack, and `file_upload_ack` carried the name the node stored them under — so the same file was ciphertext leaving a node and plaintext arriving at one. There was no threat model behind that asymmetry. Both halves now travel sealed under a third groupbox purpose, HKDF(GEK, info="meshbay:upload:v1"). The filename, the destination folder and the bytes are all inside the seal; only `upload_id` and `chunk_index` stay in clear, because the node routes and orders on them before it can decrypt. This direction seals *towards* the node — it holds the GEK for its own group — and it opens the payload before it picks a destination or touches the disk. What that forced, and why none of it is optional: - `filename` was the correlation key on both sides. It cannot be: matching an ack to its request by name would hand back exactly what the seal hides. `upload_id` replaces it — client-drawn, opaque to the node, unique within a connection, never an authorization input. The property it guarded (one refusal fails one upload, not every upload in flight) is unchanged. - Refusals can no longer quote what they refused. `No directory named 'X'` becomes `No such directory in this group` plus the `code` that was already there; the client knows what it sent. - No plaintext fallback. A path that still accepts plaintext is not a sealed path, so an unsealed `file_upload` is refused with `upload_not_sealed`. Hardened while here, because what comes out of a seal is authenticated but not validated — a member can seal anything: `filename` and `data` have their types checked before any upload state is created, and `chunk_index`/`total_chunks`, which are outside the seal by necessity, can no longer raise where a refusal was meant. Tests. `test_upload_sealed.py` pins the node half: nothing identifying on the wire, tamper/wrong-key/wrong-group all refused with nothing written, and multi-chunk reassembly unchanged. `test_upload_seal_client.py` drives the shipped `uploadFile` over the shipped `crypto.js` under node and feeds its real frames to the real `_do_file_upload` — the file lands intact, and the ack the node actually produced comes back with the name it chose for a collision, which is the half a source-reading test cannot see. Both upload purposes join the JS/Python groupbox parity vectors. BREAKING CHANGE: MNP 2.0. `file_upload`/`file_upload_ack` change shape on the wire every deployed client speaks, which is MAJOR by the same rule 1.0 was — but the break is confined to uploads. `MNP_MIN_SUPPORTED` stays at "1.0", so a 1.x peer still connects, browses, downloads, streams and chats; only its uploads are refused, with a message saying which side is old. The client checks the node's version before sending a chunk, so neither side meets this as a timeout. This is the version negotiation shipped in 1.0 earning its keep: 1.0 cost a flag day, 2.0 costs a refusal code. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AsoWC3GmhNdwVFomW3QjH3 --- .../meshbay-common/src/meshbay_common/__init__.py | 35 ++++++- .../meshbay-common/src/meshbay_common/groupbox.py | 17 ++++ .../meshbay-common/src/meshbay_common/protocol.py | 109 +++++++++++++++++++++ 3 files changed, 160 insertions(+), 1 deletion(-) (limited to 'packages/meshbay-common/src') diff --git a/packages/meshbay-common/src/meshbay_common/__init__.py b/packages/meshbay-common/src/meshbay_common/__init__.py index b3e24e3..ac69e18 100644 --- a/packages/meshbay-common/src/meshbay_common/__init__.py +++ b/packages/meshbay-common/src/meshbay_common/__init__.py @@ -93,5 +93,38 @@ __version__ = "0.11.0" # The index at rest, `index_progress` (counters only, never a path — see # `groupbox.py` and daemon.py `_push_index_progress`), chat, and file content # on the operator's disk are all deliberately unchanged. -MNP_VERSION = "1.1" +# 1.1: `index_delta` carries `roots` (additive — a 1.0 client ignores it), and +# the per-root/per-app operations: `root_update`/`root_eject`/`root_plug`, +# `app_directories`, `chat_directory`, `chat_link_preview`. +# 2.0: the **write** path is sealed, and the last unencrypted content message +# is gone. +# +# - `file_upload` and `file_upload_ack` travel sealed under a GEK-derived +# subkey (`groupbox.PURPOSE_UPLOAD`). The filename, the destination folder +# and the bytes all ride inside the seal; `upload_id` — a fresh +# client-chosen correlation id — and `chunk_index` stay in clear because +# the node routes and orders on them. `filename` used to be the +# correlation key and cannot be any more, which is what forced `upload_id`. +# - `stream_seg` is **removed**. It answered with an MPEG-TS segment as +# base64 with no encryption at all, on both transports, to any +# authenticated member — the one content-plane message that never went +# through a GEK-derived key. `stream_data` has done the job properly since +# Phase 12, and `fetchStreamSegment`, its only browser caller, was defined +# and never once invoked. +# +# **Breaking, on the wire every deployed client speaks**, and MAJOR by the same +# rule 1.0 was: a 1.x client's upload reaches a 2.0 node with no `filename` and +# no `data` outside the seal. That is why the break is confined rather than +# total — `MNP_MIN_SUPPORTED` stays at "1.0", so a 1.x peer still connects, +# browses, downloads, streams and chats; only its uploads are refused, with +# `upload_not_sealed` and a message saying which side is old. A client refuses +# symmetrically before sending (`supportsSealedUpload` in transport.js), so +# neither side discovers this as a timeout. +# +# Still deliberately in clear, and none of it is content: the handshake itself, +# `index_progress` (counters only — see daemon.py `_push_index_progress`), the +# admin and configuration acks, and the media-metadata replies. Chat is next, +# under sender keys. The index at rest and file content on the operator's disk +# are unchanged. +MNP_VERSION = "2.0" MHP_VERSION = "0.1" diff --git a/packages/meshbay-common/src/meshbay_common/groupbox.py b/packages/meshbay-common/src/meshbay_common/groupbox.py index f1091c3..eb2f630 100644 --- a/packages/meshbay-common/src/meshbay_common/groupbox.py +++ b/packages/meshbay-common/src/meshbay_common/groupbox.py @@ -24,6 +24,14 @@ and group configuration. It buys nothing against a network observer (DTLS/TLS already covers that), nothing against the hub (it never sees channel traffic), and nothing against a member — who holds the GEK. That is the whole claim. +**Both, for the upload (MNP 2.0).** `file_upload` carried the filename and the raw +bytes in clear, and `file_upload_ack` carried the name it was stored under. The +download path had been sealed end to end since the beginning — so the same file was +ciphertext coming out of a node and plaintext going in, which is not a threat model, +it is an oversight. The node holds the GEK for its own group, so unlike the index +this direction seals *towards* the node: it opens the payload before it writes +anything to disk, and refuses a chunk that does not open rather than guessing. + Purpose separation is deliberate. `GroupIndex.serialize()` reuses `chunk_key_aes(gek, file_hash, chunk_index)` with a pseudo-file ("the index as chunk 0 of a virtual index file"), which borrows a file's key space for something that is @@ -41,6 +49,7 @@ from cryptography.hazmat.primitives.kdf.hkdf import HKDF PURPOSE_INDEX = "index" PURPOSE_ACK = "ack" +PURPOSE_UPLOAD = "upload" # `salt=None` here and `salt: new Uint8Array(0)` in crypto.js agree — RFC 5869 # extracts with a zero key either way. Already proven in production by @@ -48,8 +57,16 @@ PURPOSE_ACK = "ack" _INFO = { PURPOSE_INDEX: b"meshbay:index:v1", PURPOSE_ACK: b"meshbay:ack:v1", + PURPOSE_UPLOAD: b"meshbay:upload:v1", } +# One subkey per purpose, and `seal` draws a fresh 96-bit nonce per message, so +# the bound that matters is birthday collision under `PURPOSE_UPLOAD` — the only +# purpose with real volume, one message per 48 KiB chunk. 2**32 chunks is 200 TB +# uploaded under a single GEK before the collision probability reaches 2**-32, +# and `gek_rotate` exists. Deriving the nonce from the payload instead would be +# worse, not better: two chunks of identical bytes are ordinary in a file. + NONCE_LEN = 12 # 96-bit, the WebCrypto AES-GCM standard diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index bb52a51..876f465 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -30,6 +30,7 @@ number — it is a label chosen by the peer, and the only thing it decides is which local promise a reply belongs to. """ +import os from dataclasses import dataclass, field from typing import Any @@ -37,6 +38,7 @@ from typing import Any # second copy here said "0.1" while every message on the wire carried "0.2". # Nothing imported it, which is the only reason it was harmless. from meshbay_common import MNP_VERSION, MHP_VERSION # noqa: F401 (re-export) +from meshbay_common.groupbox import PURPOSE_UPLOAD, seal, unseal from meshbay_common.webcrypto import ( chunk_key_aes, decrypt_chunk_aes, @@ -370,3 +372,110 @@ def file_chunk_plaintext( file_hash = bytes.fromhex(msg["file_id"]) ckey = chunk_key_aes(gek, file_hash, msg["chunk_index"]) return decrypt_chunk_aes(ckey, msg["nonce"], msg["ct"]) + + +# ── Uploads (MNP 2.0) ───────────────────────────────────────────────────────── +# +# The write path, sealed under the group key the way the read path always was. +# One encoder for both directions, here rather than in the client, for the reason +# `file_chunk` has one: two copies of a wire shape with a single consumer each is +# how `index_sync` and `file_chunk` forked (finding C6), and nothing noticed +# until someone went looking. +# +# What stays in clear, and why each has to: +# `type`, `v` — routed and version-checked before anything can be decrypted +# `upload_id` — the correlation key. It replaces `filename`, which used to +# play that role and cannot any more: naming the file in clear +# to match an ack against a request would give back exactly +# what the seal is for. Client-chosen, opaque to the node, +# unique within one connection; never an authorization input. +# `chunk_index` — ordering, which the node enforces before it opens anything +# `total_chunks` — how many to expect +# +# `group_id` is *not* on the message: the session already decided which group it +# is on, and the node uses that as the AAD. A client naming its own group here +# would be choosing which key its bytes are checked against. + +UPLOAD_ID_LEN = 16 # 128 bits of client-chosen correlation, hex on the wire + + +def new_upload_id() -> str: + """A fresh correlation id for one upload.""" + return os.urandom(UPLOAD_ID_LEN).hex() + + +def file_upload_wire( + gek: bytes, + group_id: str, + *, + upload_id: str, + chunk_index: int, + total_chunks: int, + filename: str, + data: bytes, + dir: str = "", + root: str = "", +) -> dict: + """ + One sealed `file_upload` chunk. + + `filename`, `dir` and `root` ride inside the seal with the bytes: sealing the + content and announcing the name beside it would be theatre. They are repeated + on every chunk rather than sent once — a hundred bytes against a 48 KiB chunk + — because a header that arrives once is state the node has to carry, and + upload state that can disagree with the chunk in hand is what `_free_name` and + the chunk-ordering rule exist to prevent. + """ + payload = {"filename": filename, "data": data, "dir": dir, "root": root} + return { + "type": MNP.FILE_UPLOAD, + "v": MNP_VERSION, + "upload_id": upload_id, + "chunk_index": chunk_index, + "total_chunks": total_chunks, + **seal(gek, PURPOSE_UPLOAD, MNP.FILE_UPLOAD, group_id, payload), + } + + +def file_upload_payload(gek: bytes, group_id: str, msg: dict) -> dict: + """ + Open a `file_upload`. Raises on anything that does not open. + + Never a partial result and never a default: a chunk that does not open is not + an empty file with an empty name, it is a peer we cannot talk to. `unseal` + says why at length. + """ + return unseal(gek, PURPOSE_UPLOAD, MNP.FILE_UPLOAD, group_id, msg) + + +def file_upload_ack_wire( + gek: bytes, + group_id: str, + *, + upload_id: str, + chunk_index: int, + filename: str, + stored_as: str, + dir: str = "", +) -> dict: + """ + The node's answer to one chunk, sealed the same way. + + `stored_as` is the name the node settled on — it finds a free one rather than + replacing anything — and `dir` is where it landed. Both name the operator's + content, so both belong inside the seal; only `upload_id` and `chunk_index` + stay out, because the client matches on them. + """ + payload = {"filename": filename, "stored_as": stored_as, "dir": dir} + return { + "type": MNP.FILE_UPLOAD_ACK, + "v": MNP_VERSION, + "upload_id": upload_id, + "chunk_index": chunk_index, + **seal(gek, PURPOSE_UPLOAD, MNP.FILE_UPLOAD_ACK, group_id, payload), + } + + +def file_upload_ack_payload(gek: bytes, group_id: str, msg: dict) -> dict: + """Open a `file_upload_ack`. Raises on anything that does not open.""" + return unseal(gek, PURPOSE_UPLOAD, MNP.FILE_UPLOAD_ACK, group_id, msg) -- cgit v1.2.3