diff options
Diffstat (limited to 'packages/meshbay-common/src')
8 files changed, 339 insertions, 37 deletions
diff --git a/packages/meshbay-common/src/meshbay_common/__init__.py b/packages/meshbay-common/src/meshbay_common/__init__.py index ac69e18..cb6ad15 100644 --- a/packages/meshbay-common/src/meshbay_common/__init__.py +++ b/packages/meshbay-common/src/meshbay_common/__init__.py @@ -90,14 +90,9 @@ __version__ = "0.11.0" # makes the *next* breaking change cost a refusal message instead of a second # flag day. `MNP_MIN_SUPPORTED` in `handshake.py` is the other half. # -# The index at rest, `index_progress` (counters only, never a path — see -# `groupbox.py` and daemon.py `_push_index_progress`), chat, and file content -# on the operator's disk are all deliberately unchanged. -# 1.1: `index_delta` carries `roots` (additive — a 1.0 client ignores it), and -# the per-root/per-app operations: `root_update`/`root_eject`/`root_plug`, -# `app_directories`, `chat_directory`, `chat_link_preview`. -# 2.0: the **write** path is sealed, and the last unencrypted content message -# is gone. +# **2.0 (2026-09-07): the write path is sealed, chat is encrypted, and the last +# unencrypted content message is gone.** Three changes that landed together and +# share one version, because they share one flag day. # # - `file_upload` and `file_upload_ack` travel sealed under a GEK-derived # subkey (`groupbox.PURPOSE_UPLOAD`). The filename, the destination folder @@ -105,6 +100,12 @@ __version__ = "0.11.0" # client-chosen correlation id — and `chunk_index` stay in clear because # the node routes and orders on them. `filename` used to be the # correlation key and cannot be any more, which is what forced `upload_id`. +# - `chat_msg` is sealed under a per-device subkey of the group's chat epoch +# key (`meshbay_common.chatbox`) and signed over the ciphertext with the +# device key the node pinned. There is deliberately **no per-group switch**: +# every node in existence is a test node, so an opt-in flag would buy +# nothing and leave a plaintext branch reachable. Existing node data is +# migrated by `QE/migration/migrate_chat_encryption.py`. # - `stream_seg` is **removed**. It answered with an MPEG-TS segment as # base64 with no encryption at all, on both transports, to any # authenticated member — the one content-plane message that never went @@ -113,18 +114,24 @@ __version__ = "0.11.0" # and never once invoked. # # **Breaking, on the wire every deployed client speaks**, and MAJOR by the same -# rule 1.0 was: a 1.x client's upload reaches a 2.0 node with no `filename` and -# no `data` outside the seal. That is why the break is confined rather than -# total — `MNP_MIN_SUPPORTED` stays at "1.0", so a 1.x peer still connects, -# browses, downloads, streams and chats; only its uploads are refused, with -# `upload_not_sealed` and a message saying which side is old. A client refuses -# symmetrically before sending (`supportsSealedUpload` in transport.js), so -# neither side discovers this as a timeout. +# rule 1.0 was. +# +# **`MNP_MIN_SUPPORTED` moves to "2.0" with it, and that is a change of plan +# worth reading.** The sealed upload alone was a *confined* break: a 1.x peer +# could still connect, browse, download, stream and chat, and only its uploads +# were refused — so the floor stayed at "1.0" and the refusal was per message +# (`upload_not_sealed`). Mandatory chat encryption ends that confinement. A 1.x +# peer can neither produce a sealed chat message nor read one, so it would +# connect, appear to work, and then be unable to say anything or read anything +# anyone else said. Refusing it at the handshake with `version_too_old` and a +# sentence saying so is the honest form: a stated refusal is a bug report, a +# chat that quietly does not work is a support case. The per-message +# `upload_not_sealed` path stays, unchanged — it is still the right answer if +# the floor is ever lowered again. # # Still deliberately in clear, and none of it is content: the handshake itself, # `index_progress` (counters only — see daemon.py `_push_index_progress`), the -# admin and configuration acks, and the media-metadata replies. Chat is next, -# under sender keys. The index at rest and file content on the operator's disk -# are unchanged. +# admin and configuration acks, and the media-metadata replies. The index at +# rest and file content on the operator's disk are unchanged. MNP_VERSION = "2.0" MHP_VERSION = "0.1" diff --git a/packages/meshbay-common/src/meshbay_common/adminop.py b/packages/meshbay-common/src/meshbay_common/adminop.py index 9a56934..ed6e940 100644 --- a/packages/meshbay-common/src/meshbay_common/adminop.py +++ b/packages/meshbay-common/src/meshbay_common/adminop.py @@ -112,6 +112,14 @@ OP_ROOT_REMOVE = "root_remove" OP_APP_DIRECTORIES = "app_directories" OP_CHAT_DIRECTORY = "chat_directory" OP_CHAT_LINK_PREVIEW = "chat_link_preview" +# Open a new chat epoch for a group, by hand. The removals that matter open one +# by themselves (member revoke/unpin, device revoke, gek_rotate); this is the +# operator saying "do it anyway", which is the same shape as `gek_rotate` and +# signed for the same reason. +# +# There is no op for *enabling* chat encryption. It is not a setting — MNP 2.0 +# has no plaintext chat to fall back to. +OP_CHAT_EPOCH = "chat_epoch" OP_ROOT_UPDATE = "root_update" OP_ROOT_EJECT = "root_eject" OP_ROOT_PLUG = "root_plug" diff --git a/packages/meshbay-common/src/meshbay_common/chatbox.py b/packages/meshbay-common/src/meshbay_common/chatbox.py new file mode 100644 index 0000000..e04bd9b --- /dev/null +++ b/packages/meshbay-common/src/meshbay_common/chatbox.py @@ -0,0 +1,181 @@ +""" +Chat message encryption and sender authentication. + +Design A of `docs/chat-sender-keys.md`, decided 2026-09-07. What it is, and what +it deliberately is not, in the order the decisions were made: + +**Not a ratchet.** `senderkeys.py` implements Signal-style sender keys and is +unused by production. With sender keys distributed under the group key and a +node that serves history to devices which were not present, the node must retain +and hand out each chain's *earliest* key — and a chain key at iteration *i* +yields every message key from *i* onward by pure HKDF. Forward secrecy is then +zero, and the ratchet is computing HKDF over a value every member already holds. +The property is given up on the record rather than inherited by accident. + +**A key per group, per epoch, per device.** The node generates an epoch key and +delivers it to members wrapped under the current group key. Each device derives +its *own* subkey from it, by name, so: + +* two devices never share an AES key, and nonce reuse across devices is + impossible without any coordination — the property per-device ratchet chains + were wanted for, obtained by derivation instead of by mutable state (which is + C1 one level down, and is exactly what `GroupSenderKeyStore` got wrong); +* a receiver derives any sender's subkey from the epoch key it already has, so + nothing is distributed per device and there is no per-device state to + persist, migrate or lose; +* history keeps working, for new members and new devices alike, because the + epoch key does not move as messages are sent. + +**Epochs, not rotation of the archive.** The epoch key is wrapped under the +group key *at delivery*, never stored under it — so rotating the group key +(which is the documented step after removing a member) is a re-wrap and costs +nothing. A new epoch is opened when the set of devices that may read *future* +messages shrinks; old epochs are kept and still delivered to current members, +which is what keeps the history readable to the people who could already read it. + +**Signing is separate from encryption**, and is what actually establishes who +said something. The signature is over the *ciphertext*, so it can be checked +before decryption and by anyone holding the roster, and it names the device key +the node pinned — not a fresh key the sender invented, which is what made the +sender-key distribution format forgeable by any member. + +What none of this protects against, stated per the v5/v6 convention: the node +operator and every current group member hold the group key and therefore the +epoch keys. This is the same boundary as file access, by design. It protects +against someone who obtains the node's storage without the keystore password. +""" + +from __future__ import annotations + +import os + +import msgpack +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, + Ed25519PublicKey, +) +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +EPOCH_KEY_LEN = 32 +# 96-bit, the WebCrypto AES-GCM standard — and the same reasoning as +# `groupbox.py`: one key per device per epoch puts the NIST SP 800-38D ceiling +# of 2**32 random nonces far out of reach of a person typing. +NONCE_LEN = 12 +SIG_LEN = 64 + +_DEVICE_KEY_INFO = "meshbay:chat:dev:v1" +_SIG_PREFIX = b"meshbay:chat:v1" + + +def new_epoch_key() -> bytes: + """A fresh epoch key. Generated by the node — never by a member (C5b).""" + return os.urandom(EPOCH_KEY_LEN) + + +def device_key(epoch_key: bytes, group_id: str, device_b64: str) -> bytes: + """ + The AES-256 subkey one device encrypts under, in one group, in one epoch. + + Derived by name, so every member can compute every device's subkey from the + epoch key and nobody has to distribute anything per device. `salt=None` here + matches `salt: new Uint8Array(0)` in crypto.js — RFC 5869 extracts with a + zero key either way, which `deriveChunkKey` already relies on in production + and `test_js_python_parity` holds. + """ + if not epoch_key: + raise ValueError("no epoch key") + if not device_b64: + raise ValueError("no device") + info = f"{_DEVICE_KEY_INFO}|{group_id}|{device_b64}".encode() + return HKDF( + algorithm=hashes.SHA256(), length=32, salt=None, info=info, + ).derive(epoch_key) + + +def associated_data(group_id: str, epoch: int) -> bytes: + """ + What a chat ciphertext is bound to. + + The group stops a message being moved between two groups on one node; the + epoch stops one being re-presented as belonging to a later key, which would + otherwise let a member who kept an old epoch key make an old message look + current. Same reasoning as `groupbox.associated_data`, one field longer. + """ + return f"chat_msg|{group_id}|{epoch}".encode() + + +def signing_transcript(group_id: str, epoch: int, device: bytes, nonce: bytes, + ct: bytes) -> bytes: + """ + What the sending device signs — the ciphertext, never the plaintext. + + Over the ciphertext so a receiver can establish authorship *before* it + decrypts, and so anyone holding the roster can verify a stored message + without the epoch key. Length-prefixed and domain-separated, per L4, like + every other transcript in this codebase: without the lengths, a message + could be re-cut into a different one with the same bytes. + + Note what is *not* in here: this connection's nonce. Every other transcript + binds one, and this one cannot — a receiver reading history has no access to + the connection the message arrived on. Replay is therefore refused by + storage instead, on the unique `(device, nonce)` pair. + """ + out = bytearray(_SIG_PREFIX) + for field in (group_id.encode(), str(epoch).encode(), device, nonce, ct): + out += len(field).to_bytes(4, "big") + out += field + return bytes(out) + + +def seal(epoch_key: bytes, group_id: str, epoch: int, device_b64: str, + device_raw: bytes, sk: Ed25519PrivateKey, payload: dict) -> dict: + """ + One message, sealed and signed: `{nonce, ct, sig}` for the caller to merge. + + The routing and authentication fields — `format`, `epoch`, `device` — stay + in clear, because a receiver has to select a key and check a signature + before it can decrypt, and because the node routes on them without being + able to read anything. + """ + key = device_key(epoch_key, group_id, device_b64) + # Random per message. Never derived from the payload: two identical messages + # under one device's key would then reuse it, and AES-GCM's failure under + # nonce reuse is not graceful. + nonce = os.urandom(NONCE_LEN) + ct = AESGCM(key).encrypt( + nonce, msgpack.packb(payload, use_bin_type=True), + associated_data(group_id, epoch)) + sig = sk.sign(signing_transcript(group_id, epoch, device_raw, nonce, ct)) + return {"nonce": nonce, "ct": ct, "sig": sig} + + +def verify(group_id: str, epoch: int, device_raw: bytes, nonce: bytes, + ct: bytes, sig: bytes) -> bool: + """Whether `sig` is this device's signature over this ciphertext.""" + try: + pk = Ed25519PublicKey.from_public_bytes(device_raw) + pk.verify(sig, signing_transcript(group_id, epoch, device_raw, nonce, ct)) + return True + except Exception: + return False + + +def open_message(epoch_key: bytes, group_id: str, epoch: int, device_b64: str, + nonce: bytes, ct: bytes) -> dict: + """ + Open a sealed message. Raises on anything that does not open. + + Never a partial result and never a default — an unopenable message is not an + empty one, and rendering it as blank would make a message nobody can read + indistinguishable from a message nobody wrote. The caller marks it as + unreadable and shows the gap, which is the honest thing for a reader to see. + """ + key = device_key(epoch_key, group_id, device_b64) + plain = AESGCM(key).decrypt(bytes(nonce), bytes(ct), + associated_data(group_id, epoch)) + payload = msgpack.unpackb(plain, raw=False) + if not isinstance(payload, dict): + raise ValueError("chat: sealed payload is not a map") + return payload diff --git a/packages/meshbay-common/src/meshbay_common/device.py b/packages/meshbay-common/src/meshbay_common/device.py index 8951dbb..cfa8dd6 100644 --- a/packages/meshbay-common/src/meshbay_common/device.py +++ b/packages/meshbay-common/src/meshbay_common/device.py @@ -36,6 +36,7 @@ import hashlib DEVICE_REQUEST_PREFIX = b"meshbay:device_req:v1" DEVICE_ADD_PREFIX = b"meshbay:device_add:v1" +DEVICE_HELLO_PREFIX = b"meshbay:device_hello:v1" # Same as the join and admin transcripts: interactive exchanges that complete in # milliseconds, so anything older is a replay. @@ -123,3 +124,41 @@ def device_add_transcript( nonce_node, str(ts).encode(), ]) + + +def device_hello_transcript( + node_pk_b64: str, + group_id: str, + user_id: str, + pk_ed25519_b64: str, + nonce_node: bytes, + ts: int, +) -> bytes: + """ + Signed by the device on an already-authenticated connection, saying **which + of the account's devices this connection is**. + + The handshake proves membership of a group (a GEK-HMAC) and carries an + account from the hub's token; it proves nothing about *which* device is + talking. The node needed that the moment one account could hold several: + `_load_pinned_pk` was resolving "this account's oldest live device" and + recording it as the uploader of every file, so a phone's uploads were + attributed to a laptop. + + Additive and optional. A client that does not send it leaves the node where + it was, which is why this could ship without a breaking protocol change — + but a node that *has* been told refuses a later claim to be a different + device on the same connection. + + `nonce_node` is this connection's handshake nonce, so the signature cannot + be lifted onto another connection, and `node_pk` binds it to one node — + the same rule as every other transcript here. + """ + return _pack(DEVICE_HELLO_PREFIX, [ + node_pk_b64.encode(), + group_id.encode(), + user_id.encode(), + pk_ed25519_b64.encode(), + nonce_node, + str(ts).encode(), + ]) diff --git a/packages/meshbay-common/src/meshbay_common/groupbox.py b/packages/meshbay-common/src/meshbay_common/groupbox.py index eb2f630..e020ed8 100644 --- a/packages/meshbay-common/src/meshbay_common/groupbox.py +++ b/packages/meshbay-common/src/meshbay_common/groupbox.py @@ -50,6 +50,10 @@ from cryptography.hazmat.primitives.kdf.hkdf import HKDF PURPOSE_INDEX = "index" PURPOSE_ACK = "ack" PURPOSE_UPLOAD = "upload" +# The chat epoch keys themselves, on their way to a member. The keys are what +# the chat archive is encrypted under; this is only how they travel, which is +# why rotating the group key costs a re-wrap and not a re-encryption. +PURPOSE_CHAT_KEYS = "chat_keys" # `salt=None` here and `salt: new Uint8Array(0)` in crypto.js agree — RFC 5869 # extracts with a zero key either way. Already proven in production by @@ -58,6 +62,7 @@ _INFO = { PURPOSE_INDEX: b"meshbay:index:v1", PURPOSE_ACK: b"meshbay:ack:v1", PURPOSE_UPLOAD: b"meshbay:upload:v1", + PURPOSE_CHAT_KEYS: b"meshbay:chat_keys:v1", } # One subkey per purpose, and `seal` draws a fresh 96-bit nonce per message, so diff --git a/packages/meshbay-common/src/meshbay_common/handshake.py b/packages/meshbay-common/src/meshbay_common/handshake.py index f7d4911..188a8aa 100644 --- a/packages/meshbay-common/src/meshbay_common/handshake.py +++ b/packages/meshbay-common/src/meshbay_common/handshake.py @@ -66,11 +66,19 @@ from meshbay_common import MNP_VERSION HANDSHAKE_PREFIX = b"meshbay:mnp:handshake:v1" -# The oldest peer this build will talk to. MNP 1.0 sealed `index_sync`, -# `index_delta` and the `handshake_ack` payload under the group key, which no -# 0.x peer can open and which a 0.x peer's own messages do not carry — there is -# nothing to be compatible with, which is what makes it a MAJOR bump. -MNP_MIN_SUPPORTED = "1.0" +# The oldest peer this build will talk to. +# +# 2.0 (2026-09-07): chat messages are sealed under a per-device subkey of the +# group's chat epoch key, and the node refuses a plaintext one. A 1.x peer can +# neither produce nor read that, so there is nothing to be compatible with — +# the same reasoning that made 1.0 a MAJOR bump for the sealed index. +# +# Moving the floor with the version is the point: a 1.x client is refused here, +# with `version_too_old` and a sentence saying so, instead of completing a +# handshake and then discovering that every message it sends is rejected and +# every message it receives is unreadable. A stated refusal is a bug report; a +# chat that quietly does not work is a support case. +MNP_MIN_SUPPORTED = "2.0" ROLE_CLIENT = "client" ROLE_NODE = "node" diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index 876f465..cc5324a 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -68,7 +68,14 @@ class MNP: # per segment), and its browser caller `fetchStreamSegment` was defined and # never once invoked. A live handler on both transports, plaintext media, # and no client: removed rather than repaired. - CHAT_MESSAGE = "chat_msg" # Double Ratchet message + # + # Not a Double Ratchet message, and never was — `first-review.md` C1 + # rejected exactly that for groups. Since MNP 2.0 it is AES-256-GCM under a + # per-device subkey of the group's chat epoch key, signed over the + # ciphertext with the sending device's pinned Ed25519 key. There is no + # plaintext form on the wire (`chatbox.py`, docs/chat-sender-keys.md); + # `format` distinguishes a *stored* pre-2.0 row, which is still served. + CHAT_MESSAGE = "chat_msg" # one chat message, sealed and signed CHAT_ATTACHMENT = "chat_attach" # attachment metadata CHAT_HISTORY = "chat_hist" # request message history (newest, or before a cursor) CHAT_HISTORY_RESPONSE = "chat_hist_resp" # history response with messages @@ -183,6 +190,14 @@ class MNP: DEVICE_LIST = "device_list" # anyone → node: my devices DEVICE_LIST_RESULT = "device_list_result" DEVICE_REVOKE = "device_revoke" # a device retires another + # "Which of this account's devices am I?" — signed, on an + # already-authenticated connection. The handshake proves the account and the + # group; it never proved the device, so the node attributed uploads to the + # account's oldest key and could not tell one device's chat from another's. + # Additive (MNP 1.2): a client that stays silent leaves the node exactly + # where it was. + DEVICE_HELLO = "device_hello" # device → node: this is me + DEVICE_HELLO_ACK = "device_hello_ack" # Rotation is the half of revocation that revocation cannot do: the node # generates a fresh key itself, so no key material crosses the wire. GEK_ROTATE = "gek_rotate" # operator → node: new group key @@ -206,6 +221,20 @@ class MNP: CHAT_DIRECTORY_ACK = "chat_directory_ack" CHAT_LINK_PREVIEW = "chat_link_preview" CHAT_LINK_PREVIEW_ACK = "chat_link_preview_ack" + # The keys a group's chat archive is encrypted under, on their way to a + # member. Sealed under a group-derived subkey, so the payload carries an + # authentication tag from a key the hub does not hold — and a member who has + # not completed the handshake is served a ciphertext rather than the keys. + # Requested rather than pushed on the ack: a group with no chat should not + # pay for this on every connection. + CHAT_KEYS_REQ = "chat_keys_req" + CHAT_KEYS_RESP = "chat_keys_resp" + # Node → this group: a new chat epoch was opened, because somebody was + # removed. Not a setting — there is no switch; chat is always encrypted + # (MNP 2.0). Pushed so a connected client stops sealing under the retired + # key without having to reconnect. + CHAT_EPOCH = "chat_epoch" + CHAT_EPOCH_ACK = "chat_epoch_ack" ROOT_UPDATE = "root_update" # operator → node: change writable/removable on a root ROOT_UPDATE_ACK = "root_update_ack" ROOT_EJECT = "root_eject" # operator → node: mark removable root as ejected diff --git a/packages/meshbay-common/src/meshbay_common/senderkeys.py b/packages/meshbay-common/src/meshbay_common/senderkeys.py index 932e2e6..9ad5107 100644 --- a/packages/meshbay-common/src/meshbay_common/senderkeys.py +++ b/packages/meshbay-common/src/meshbay_common/senderkeys.py @@ -1,21 +1,40 @@ """ -MeshBay — Sender Keys protocol for group messaging. +MeshBay — Sender Keys protocol. **Not used by group chat. Not used at all.** -Signal Groups approach: each member maintains their own sending chain. -Advantages over shared Double Ratchet: - - O(N) state per group (one chain per member) vs O(N^2) pairwise - - Single encrypt per message (not N encryptions) - - No key/nonce reuse — each sender has an independent chain +Kept the way `ratchet.py` is kept: a working implementation of a protocol that +may earn a place in a future 1:1 DM, where there is no server-side history to +contradict it. Group chat is `chatbox.py`, and the decision to build that +instead is `docs/chat-sender-keys.md` §4 (operator, 2026-09-07). Do not read a +green test run here as evidence that group chat is encrypted; nothing in +production imports this module. -Key components: - - Chain key ratchet: HKDF per message, provides forward secrecy +**Why it is not what group chat uses.** With sender keys distributed under the +group key, and a node that serves history to devices which were not present when +a message was sent, the node must retain and hand out each chain's *earliest* +key — and a chain key at iteration *i* yields every message key from *i* onward +by pure HKDF. Forward secrecy is then zero, and what is left is a large amount +of stateful client code whose failure modes are silent. Three of them are real +and reproduced in the design document: + + * `GroupSenderKeyStore.add_sender` accepts any distribution for any + `sender_id` and overwrites what is there, and `SenderKeyRecord.create` + invents a signing key bound to nothing — so under group-key distribution any + member can replace another member's chain and sign as them (F1); + * a second device registering under one `sender_id` drops the first device's + chain, and its messages then fail signature verification rather than failing + visibly at registration (F2); + * `SenderKeyState.advance_to` caches every skipped message key and nothing + trims `_skipped_keys` (F3). + +They are findings about a module nothing calls, and are deliberately not fixed +here. Anyone bringing this back for 1:1 DM must fix all three first — and must +bind the distribution to a key the node pinned, which is what F1 is really about. + +Key components, as implemented: + - Chain key ratchet: HKDF per message - Message key derivation: separate HKDF from chain key - Ed25519 signing: each sender signs their ciphertext - AES-256-GCM encryption: browser-compatible symmetric cipher - -Key distribution: - - On join: admin wraps each sender's SenderKeyDistribution with GEK - - On leave: all remaining members rotate their chain keys """ import os @@ -169,7 +188,13 @@ class SenderKeyRecord: # ── Group store ────────────────────────────────────────────────────────────── class GroupSenderKeyStore: - """All sender key states for one group, held by one member.""" + """All sender key states for one group, held by one member. + + One chain per **device**, were this ever used: a shared per-person chain + advanced by two devices produces key and nonce reuse, which is `first- + review.md` C1 one level down. `add_sender` does not enforce that — see the + module docstring, F2. + """ def __init__(self, group_id: str): self.group_id = group_id |