diff options
| -rw-r--r-- | docs/MESHBAY_DESIGN.md | 17 | ||||
| -rw-r--r-- | docs/MESHBAY_NODE_PROTOCOL.md | 34 | ||||
| -rw-r--r-- | packages/meshbay-common/pyproject.toml | 1 | ||||
| -rw-r--r-- | packages/meshbay-common/src/meshbay_common/crypto.py | 92 | ||||
| -rw-r--r-- | packages/meshbay-common/src/meshbay_common/groupbox.py | 8 | ||||
| -rw-r--r-- | packages/meshbay-common/src/meshbay_common/protocol.py | 8 | ||||
| -rw-r--r-- | packages/meshbay-common/src/meshbay_common/webcrypto.py | 27 | ||||
| -rw-r--r-- | packages/meshbay-common/tests/test_groupbox.py | 4 | ||||
| -rw-r--r-- | packages/meshbay-common/tests/test_webcrypto.py | 27 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/indexer/group_index.py | 151 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/transport/wire.py | 9 | ||||
| -rw-r--r-- | packages/meshbay-node/tests/test_indexer.py | 96 | ||||
| -rw-r--r-- | packages/meshbay-node/tests/test_webrtc_transport.py | 6 | ||||
| -rw-r--r-- | packaging/win/README.md | 2 | ||||
| -rw-r--r-- | packaging/win/meshbay-node.spec | 1 |
15 files changed, 63 insertions, 420 deletions
diff --git a/docs/MESHBAY_DESIGN.md b/docs/MESHBAY_DESIGN.md index 74c1050..23c587e 100644 --- a/docs/MESHBAY_DESIGN.md +++ b/docs/MESHBAY_DESIGN.md @@ -638,17 +638,17 @@ Every private key lives in an encrypted keystore on the machine that owns it. Th hub never sees one. **Domain separation is consistent and mandatory.** Every derivation uses a -distinct `info` string, and the AES variant adds an `:aes` suffix so two ciphers -can never derive the same key from one group key. This is a small detail that -prevents cross-protocol key reuse, and it is checked rather than assumed. +distinct `info` string, so no two purposes can derive the same key from one group +key. The chunk and wrap strings end in `:aes`, left from a second cipher that no +longer exists; it stays because it is part of every key already derived. ### 4.2 Group key wrapping (ECIES) ``` wrap: sk_eph, pk_eph = X25519.generate() # fresh per bundle shared = X25519(sk_eph, pk_recipient) - wrap_key = HKDF(shared, salt=pk_eph, info="meshbay:gek_wrap:v1", len=32) - wrapped = AEAD(wrap_key).encrypt(nonce, gek, aad=pk_recipient) + wrap_key = HKDF(shared, salt=pk_eph, info="meshbay:gek_wrap:v1:aes", len=32) + wrapped = AES-256-GCM(wrap_key).encrypt(nonce, gek, aad=pk_recipient) bundle = pk_eph ‖ nonce ‖ wrapped unwrap: shared = X25519(sk_recipient, pk_eph) # same derivation @@ -678,20 +678,19 @@ time. This avoids double storage and makes key rotation feasible without re-encrypting terabytes. ``` -disk (plaintext) → compress → per-chunk AEAD under a group-derived key → transport → client +disk (plaintext) → per-chunk AES-256-GCM under a group-derived key → transport → client ``` - Chunk size 1 MB: amortises AEAD overhead and enables seeking, because each chunk is independently decryptable. -- `chunk_key = HKDF(GEK, salt=None, info="file:" ‖ blake3(file) ‖ ":chunk:" ‖ index)`. +- `chunk_key = HKDF(GEK, salt=None, info="file:" ‖ blake3(file) ‖ ":chunk:" ‖ index ‖ ":aes")`. The salt is omitted deliberately: the group key is CSPRNG output and already uniform, so the file and chunk context belongs in `info`, which is the correct HKDF usage (**M5**, first review). - **Chunk authentication is the AEAD tag**, not a per-chunk signature. The tag authenticates the ciphertext under a key only members hold, which is what the signature was for. -- Compression precedes encryption, because compression is ineffective on - ciphertext. +- **Chunks are not compressed**: a chunk is encrypted and sent as it was read. - Upload chunk size is 48 KB, which is what fits the SCTP limit after msgpack overhead. diff --git a/docs/MESHBAY_NODE_PROTOCOL.md b/docs/MESHBAY_NODE_PROTOCOL.md index 642e966..07b179c 100644 --- a/docs/MESHBAY_NODE_PROTOCOL.md +++ b/docs/MESHBAY_NODE_PROTOCOL.md @@ -728,9 +728,9 @@ wrap_key = HKDF-SHA256(shared, salt = pk_eph, info = "meshbay:gek_wrap:v wrapped = AES-256-GCM(wrap_key).encrypt(nonce_96, GEK, aad = pk_recipient) ``` -AES-GCM because WebCrypto has no ChaCha20-Poly1305; a `chacha20-poly1305` variant with -`info = "meshbay:gek_wrap:v1"` exists for native clients. The recipient's public key is -the AEAD's associated data, so a bundle cannot be re-addressed. +AES-GCM because WebCrypto has no ChaCha20-Poly1305, and one cipher serves every +client. The recipient's public key is the AEAD's associated data, so a bundle cannot be +re-addressed. `found: false` is the normal answer: per-member bundles are not stored, and the key is produced on demand by the join path (§8). The node keeps one stored bundle of its own @@ -1290,9 +1290,9 @@ ciphertext. That is the same statement the index makes, one step stronger. `salt = <none>` is Python's `salt=None` and WebCrypto's `salt: new Uint8Array(0)`; RFC 5869 extracts with a zero key either way. The subkeys are purpose-separated -rather than borrowed from a file's key space — `GroupIndex.serialize()` reuses -`chunk_key_aes` with a pseudo-file ("the index as chunk 0 of a virtual index file"), -which is a hack this deliberately does not repeat. +rather than borrowed from a file's key space — reusing `chunk_key_aes` with a +pseudo-file ("the index as chunk 0 of a virtual index file") is a hack this +deliberately does not repeat. **What stays in clear, and why each one has to:** @@ -1337,10 +1337,6 @@ purpose and a fresh 96-bit random nonce per message: at one message per 48 KiB c reaches 2⁻³², and `gek_rotate` exists. Deriving the nonce from the payload instead would be worse, not better — two chunks of identical bytes are ordinary in a file. -`GroupIndex.serialize()` is not a candidate for reuse here: it compresses with zstd, -which no browser can decompress (`DecompressionStream` offers gzip and deflate only), -so reusing it would mean shipping a WASM decoder to every client for no gain. - **Failure is fatal, never degraded** (I8). A client that cannot open an index message ends the session naming the message type; it never reports an empty index, because "the group has no files" is a state a real group can be in. @@ -1531,8 +1527,9 @@ nonce = 12 random bytes ct = AES-256-GCM(chunk_key).encrypt(nonce, plaintext) no AAD ``` -* The `:aes` suffix keeps AES keys distinct from the ChaCha20 variant - (`chunk_key`/`encrypt_chunk`, `info` without the suffix) derived from the same GEK. +* The `:aes` suffix is part of every chunk key: it once kept these distinct from a + ChaCha20 variant derived from the same GEK, which no longer exists, and it stays + because removing it would change every key. * `nonce` and `ct` are msgpack **binary**, not base64. Every message that carries content carries it this way, and none carries it outside an AEAD. * The key is a pure function of (GEK, file hash, index), so chunks are cacheable, @@ -2094,7 +2091,7 @@ speak, and every message above is available on it. **QUIC is in development** (§5.2): a partial message set, no client, and not a shipped feature. Nothing about it is a compatibility commitment yet. -Two rules hold across transports, and both are about there being exactly one of each +One rule holds across transports, and it is about there being exactly one of each message: * **One encoder per message type, shared by every transport.** `file_chunk` comes from @@ -2103,11 +2100,6 @@ message: its own. Two encoders for one type is a type free to drift, with a name that no longer says which shape will arrive — and, when one of them is a sealed envelope, a second construction site that goes on sending cleartext. -* **`GroupIndex.serialize()` / `deserialize()` describes no MNP message.** It is a - signed, compressed, encrypted at-rest and interchange format, and reading it as a wire - contract is a mistake worth naming: the sealed envelope of §11.1a is what index - messages travel under, and it is deliberately not this, because zstd decompresses in - no browser. --- @@ -2371,7 +2363,7 @@ LP(x) = uint32be(len(x)) || x every field, no exceptions | `MAX_CHAT_CIPHERTEXT` / chat rate | 64 KiB / 60 per 60 s per account per group | `webrtc/chat.py` | | GEK | 256-bit, node CSPRNG | `crypto.py` | | Chat epoch key | 256-bit, node CSPRNG, one per group per epoch | `chatbox.py` | -| Chunk cipher | AES-256-GCM, 96-bit nonce (ChaCha20-Poly1305 variant for native) | `webcrypto.py`, `crypto.py` | +| Chunk cipher | AES-256-GCM, 96-bit nonce | `webcrypto.py` | | GEK wrap | X25519 + HKDF-SHA256 + AES-256-GCM, AAD = recipient public key | `crypto.py` | | Sealed message envelope | HKDF-SHA256 subkey per purpose, AES-256-GCM, 96-bit random nonce | `groupbox.py`, `static/crypto.js` | | Chat envelope | HKDF-SHA256 subkey per device per epoch, AES-256-GCM, 96-bit random nonce, Ed25519 over the ciphertext | `chatbox.py` | @@ -2391,8 +2383,8 @@ meshbay-common/ protocol.py message types, chunk and upload codecs adminop.py the admin transcript and the operation catalogue join.py join transcript and pairing codes device.py device request / add / hello transcripts - crypto.py GEK, chunk keys, ECIES wrap, BLAKE3 ids - webcrypto.py the AES variants the browser can also compute + crypto.py GEK, ECIES wrap, keystore, BLAKE3 ids + webcrypto.py the content cipher: per-chunk AES-GCM keys tokens.py the two token audiences (node vs hub API) meshbay-node/ transport/webrtc_server.py the reference implementation of MNP, diff --git a/packages/meshbay-common/pyproject.toml b/packages/meshbay-common/pyproject.toml index a454521..9f43cf2 100644 --- a/packages/meshbay-common/pyproject.toml +++ b/packages/meshbay-common/pyproject.toml @@ -12,7 +12,6 @@ dependencies = [ "PyJWT>=2.9", "blake3>=1.0", "msgpack>=1.1", - "zstandard>=0.23", ] [project.optional-dependencies] diff --git a/packages/meshbay-common/src/meshbay_common/crypto.py b/packages/meshbay-common/src/meshbay_common/crypto.py index e537500..8fb80f8 100644 --- a/packages/meshbay-common/src/meshbay_common/crypto.py +++ b/packages/meshbay-common/src/meshbay_common/crypto.py @@ -41,25 +41,6 @@ def generate_gek() -> bytes: """Generate a fresh 256-bit Group Encryption Key.""" return ChaCha20Poly1305.generate_key() -def chunk_key(gek: bytes, file_hash: bytes, chunk_index: int) -> bytes: - """Derive a per-chunk encryption key from the GEK (deterministic).""" - return HKDF( - algorithm=hashes.SHA256(), - length=32, - salt=None, - info=b"file:" + file_hash + b":chunk:" + chunk_index.to_bytes(4, "big"), - ).derive(gek) - -def encrypt_chunk(key: bytes, plaintext: bytes) -> tuple[bytes, bytes]: - """Encrypt plaintext with ChaCha20-Poly1305. Returns (nonce, ciphertext).""" - nonce = os.urandom(12) - ct = ChaCha20Poly1305(key).encrypt(nonce, plaintext, None) - return nonce, ct - -def decrypt_chunk(key: bytes, nonce: bytes, ciphertext: bytes) -> bytes: - """Decrypt ciphertext. Raises InvalidTag on authentication failure.""" - return ChaCha20Poly1305(key).decrypt(nonce, ciphertext, None) - def file_hash(path_or_bytes) -> bytes: """Compute blake3 hash of a file (bytes or path-like).""" if isinstance(path_or_bytes, (str, bytes)) and not isinstance(path_or_bytes, bytes): @@ -73,58 +54,6 @@ def file_hash(path_or_bytes) -> bytes: # ── GEK wrapping (ECIES-like) ───────────────────────────────────────────────── -GEK_WRAP_INFO = b"meshbay:gek_wrap:v1" - -def wrap_gek(gek: bytes, pk_recipient: bytes) -> dict: - """ - Wrap a GEK for a recipient using ephemeral X25519 + HKDF + ChaCha20-Poly1305. - - Protocol: - 1. Generate ephemeral (sk_eph, pk_eph) - 2. shared = X25519(sk_eph, pk_recipient) - 3. wrap_key = HKDF(shared, salt=pk_eph, info=GEK_WRAP_INFO) - 4. wrapped = ChaCha20-Poly1305(wrap_key).encrypt(nonce, gek, aad=pk_recipient) - - The hub stores {pk_eph, nonce, wrapped} — opaque, cannot decrypt. - """ - sk_eph = X25519PrivateKey.generate() - pk_eph_raw = pk_to_raw(sk_eph.public_key()) - - shared = sk_eph.exchange(X25519PublicKey.from_public_bytes(pk_recipient)) - wrap_key = HKDF( - algorithm=hashes.SHA256(), length=32, - salt=pk_eph_raw, info=GEK_WRAP_INFO, - ).derive(shared) - - nonce = os.urandom(12) - wrapped = ChaCha20Poly1305(wrap_key).encrypt(nonce, gek, pk_recipient) - - return { - "pk_eph_b64": base64.b64encode(pk_eph_raw).decode(), - "nonce_b64": base64.b64encode(nonce).decode(), - "wrapped_b64": base64.b64encode(wrapped).decode(), - } - -def unwrap_gek(bundle: dict, sk_recipient: bytes, pk_recipient: bytes) -> bytes: - """ - Unwrap a GEK bundle using the recipient's X25519 private key. - Raises InvalidTag if the key is wrong or the bundle was tampered. - """ - pk_eph_raw = base64.b64decode(bundle["pk_eph_b64"]) - nonce = base64.b64decode(bundle["nonce_b64"]) - wrapped = base64.b64decode(bundle["wrapped_b64"]) - - shared = X25519PrivateKey.from_private_bytes(sk_recipient).exchange( - X25519PublicKey.from_public_bytes(pk_eph_raw) - ) - wrap_key = HKDF( - algorithm=hashes.SHA256(), length=32, - salt=pk_eph_raw, info=GEK_WRAP_INFO, - ).derive(shared) - - return ChaCha20Poly1305(wrap_key).decrypt(nonce, wrapped, pk_recipient) - - GEK_WRAP_INFO_AES = b"meshbay:gek_wrap:v1:aes" def wrap_gek_aes(gek: bytes, pk_recipient: bytes) -> dict: @@ -220,24 +149,3 @@ def decrypt_keystore(iv: bytes, ciphertext: bytes, tag: bytes, key: bytes) -> by """Decrypt keystore blob. Raises on authentication failure.""" dec = Cipher(algorithms.AES(key), modes.GCM(iv, tag)).decryptor() return dec.update(ciphertext) + dec.finalize() - -# ── Chunk signing ───────────────────────────────────────────────────────────── - -def sign_chunk(sk_node: Ed25519PrivateKey, chunk_index: int, - nonce: bytes, ct_hash: bytes) -> bytes: - """ - Sign chunk metadata. Payload: chunk_index || nonce || ct_hash. - - Despite the name, this no longer signs file chunks — Phase 9.15 dropped per-chunk - signatures on the WebRTC path and 2026-09-03 dropped the QUIC copy that had been - left behind. Its one caller is `GroupIndex.serialize()`, which signs a whole index - envelope under the pseudo-index `INDEX_CHUNK`. - """ - payload = chunk_index.to_bytes(4, "big") + nonce + ct_hash - return sk_node.sign(payload) - -def verify_chunk_signature(pk_node: Ed25519PublicKey, chunk_index: int, - nonce: bytes, ct_hash: bytes, signature: bytes) -> None: - """Verify chunk signature. Raises InvalidSignature on failure.""" - payload = chunk_index.to_bytes(4, "big") + nonce + ct_hash - pk_node.verify(signature, payload) diff --git a/packages/meshbay-common/src/meshbay_common/groupbox.py b/packages/meshbay-common/src/meshbay_common/groupbox.py index 3b45dba..260e362 100644 --- a/packages/meshbay-common/src/meshbay_common/groupbox.py +++ b/packages/meshbay-common/src/meshbay_common/groupbox.py @@ -32,10 +32,10 @@ it is an oversight. The node holds the GEK for its own group, so unlike the inde this direction seals *towards* the node: it opens the payload before it writes anything to disk, and refuses a chunk that does not open rather than guessing. -Purpose separation is deliberate. `GroupIndex.serialize()` reuses -`chunk_key_aes(gek, file_hash, chunk_index)` with a pseudo-file ("the index as chunk -0 of a virtual index file"), which borrows a file's key space for something that is -not a file. Each purpose here derives its own subkey instead. +Purpose separation is deliberate. The alternative — reusing +`chunk_key_aes(gek, file_hash, chunk_index)` with a pseudo-file, "the index as chunk +0 of a virtual index file" — borrows a file's key space for something that is not a +file. Each purpose here derives its own subkey instead. """ from __future__ import annotations diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index 6b929cd..5e666e9 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -318,9 +318,8 @@ class IndexEntry: def index_entry_wire(e: IndexEntry) -> dict: """ - The wire-dict shape used by INDEX_SYNC/INDEX_DELTA hand-built messages - (as opposed to GroupIndex.serialize()'s asdict() encoding of the whole - index). Centralized so the three call sites that build these + The wire-dict shape used by INDEX_SYNC/INDEX_DELTA hand-built messages. + Centralized so the three call sites that build these (webrtc/files.py's _do_index_sync, daemon._broadcast_index_change's two branches) can't drift from each other as fields are added. """ @@ -369,8 +368,7 @@ class IndexDelta: # under a key derived from the GEK, which only group members hold, and since C3 the # node authenticates itself in the handshake and is pinned by the client. A # signature per chunk re-proved, once per megabyte, what the session established -# once. (`sign_chunk` still exists in `crypto.py` — it signs the serialized index -# envelope, which is a different artifact; see `indexer/group_index.py`.) +# once. def chunk_ciphertext( diff --git a/packages/meshbay-common/src/meshbay_common/webcrypto.py b/packages/meshbay-common/src/meshbay_common/webcrypto.py index 3bd5ae6..7119e2e 100644 --- a/packages/meshbay-common/src/meshbay_common/webcrypto.py +++ b/packages/meshbay-common/src/meshbay_common/webcrypto.py @@ -1,28 +1,21 @@ """ -MeshBay — AES-256-GCM cipher variant for browser-accessible groups. +MeshBay — the content cipher: AES-256-GCM, per-chunk keys derived from the GEK. -The ChaCha20-Poly1305 GEK used in MNP (TCP+TLS and QUIC transport) -is NOT available in the WebCrypto API. For groups whose content must -be decryptable by a web browser (using SubtleCrypto), an AES-256-GCM -variant is used instead. - -The GEK wrapping (X25519 + HKDF) is identical — only the content -cipher changes. The hub stores and distributes GEK bundles the same way. - -Cipher selection is declared per-group in the hub registry: - "cipher": "chacha20-poly1305" (default, native clients) - "cipher": "aes-256-gcm" (browser-compatible groups) +AES-GCM because it is what WebCrypto offers, and one cipher serves every client: +the browser, the desktop client (the same engine) and the Python side here. Python side (this module): - encrypt_chunk_aes / decrypt_chunk_aes + chunk_key_aes / encrypt_chunk_aes / decrypt_chunk_aes JavaScript side (in static/crypto.js): - Uses SubtleCrypto.importKey + SubtleCrypto.decrypt with AES-GCM. + SubtleCrypto.importKey + SubtleCrypto.decrypt with AES-GCM. -Key derivation for AES variant — same HKDF info string with suffix: +Key derivation: info = b"file:" + file_hash + b":chunk:" + chunk_index + b":aes" -This ensures AES and ChaCha20 keys are always distinct even from the same GEK. +The `:aes` suffix dates from a ChaCha20-Poly1305 variant derived from the same +GEK without it, which nothing used and which is gone. It stays: it is part of +every chunk key in existence, and changing it would change them all. """ import os @@ -33,7 +26,7 @@ from cryptography.hazmat.primitives.kdf.hkdf import HKDF def chunk_key_aes(gek: bytes, file_hash: bytes, chunk_index: int) -> bytes: - """Derive a per-chunk AES-256 key. Distinct from ChaCha20 key.""" + """Derive a per-chunk AES-256 key from the GEK.""" return HKDF( algorithm=hashes.SHA256(), length=32, salt=None, info=b"file:" + file_hash + b":chunk:" + chunk_index.to_bytes(4, "big") + b":aes", diff --git a/packages/meshbay-common/tests/test_groupbox.py b/packages/meshbay-common/tests/test_groupbox.py index 65d8ce6..6e687ea 100644 --- a/packages/meshbay-common/tests/test_groupbox.py +++ b/packages/meshbay-common/tests/test_groupbox.py @@ -45,8 +45,8 @@ def test_purposes_are_separate_key_spaces(gek): The reason there are two info strings rather than one key reused. An ack sealed under the index subkey would otherwise be openable by anything - holding the index subkey, which is the confusion `GroupIndex.serialize()`'s - "the index as chunk 0 of a virtual index file" creates for chunk keys. + holding the index subkey — the confusion that treating "the index as chunk 0 of + a virtual index file" would create for chunk keys. """ assert group_key(gek, PURPOSE_INDEX) != group_key(gek, PURPOSE_ACK) sealed = seal(gek, PURPOSE_INDEX, "index_sync", "g1", PAYLOAD) diff --git a/packages/meshbay-common/tests/test_webcrypto.py b/packages/meshbay-common/tests/test_webcrypto.py index fbffdc1..ffe3f36 100644 --- a/packages/meshbay-common/tests/test_webcrypto.py +++ b/packages/meshbay-common/tests/test_webcrypto.py @@ -17,17 +17,6 @@ def test_aes_roundtrip(): assert decrypt_chunk_aes(key, nonce, ct) == data -def test_aes_key_distinct_from_chacha_key(): - """AES and ChaCha20 keys for the same chunk must differ.""" - from meshbay_common.crypto import chunk_key as chacha_key - gek = generate_gek() - data = os.urandom(100) - fh = blake3.blake3(data).digest() - aes_k = chunk_key_aes(gek, fh, 0) - chacha_k = chacha_key(gek, fh, 0) - assert aes_k != chacha_k - - def test_aes_wrong_key_rejected(): gek = generate_gek() data = b"private content" @@ -76,19 +65,3 @@ def test_aes_gek_wrap_wrong_key_rejected(): bundle = wrap_gek_aes(gek, pk_to_raw(sk_a.public_key())) with pytest.raises(Exception): unwrap_gek_aes(bundle, sk_to_raw(sk_b), pk_to_raw(sk_b.public_key())) - - -def test_aes_gek_wrap_differs_from_chacha_wrap(): - from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey - from meshbay_common.crypto import ( - pk_to_raw, - wrap_gek, - wrap_gek_aes, - ) - gek = generate_gek() - sk = X25519PrivateKey.generate() - pk_raw = pk_to_raw(sk.public_key()) - - bundle_aes = wrap_gek_aes(gek, pk_raw) - bundle_chacha = wrap_gek(gek, pk_raw) - assert bundle_aes["wrapped_b64"] != bundle_chacha["wrapped_b64"] diff --git a/packages/meshbay-node/src/meshbay_node/indexer/group_index.py b/packages/meshbay-node/src/meshbay_node/indexer/group_index.py index 9e4e780..85bc13e 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/group_index.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/group_index.py @@ -1,67 +1,41 @@ """ -Mesh Group Index — encrypted file listing for a group. +Mesh Group Index — the file listing for one group, and the deltas between versions. -Wire format (private group): - msgpack({entries, version, group_id}) → zstd compress → GEK ChaCha20 encrypt → sign - -Wire format (public group): - msgpack({entries, version, group_id}) → sign (no encryption) +What travels on the wire is built from it by `transport/wire.py` and sealed under +the group key (`meshbay_common.groupbox`); this module holds no encoding of its own. Delta format: {base_version, version, additions: [...], deletions: [id, ...]} """ -import base64 import logging -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field -import blake3 -import msgpack -import zstandard as zstd from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from meshbay_common.crypto import ( - pk_to_b64, - sign_chunk, - verify_chunk_signature, -) from meshbay_common.protocol import IndexDelta, IndexEntry -from meshbay_common.webcrypto import ( - chunk_key_aes as derive_chunk_key, -) -from meshbay_common.webcrypto import ( - decrypt_chunk_aes as decrypt_chunk, -) -from meshbay_common.webcrypto import ( - encrypt_chunk_aes as encrypt_chunk, -) log = logging.getLogger(__name__) -ZSTD_LEVEL = 3 # fast compression -INDEX_CHUNK = 0 # the index itself is treated as chunk 0 of a virtual "index file" - @dataclass class GroupIndex: """ - Encrypted, signed Mesh Group Index for one group. + Mesh Group Index for one group. Usage: idx = GroupIndex(group_id="...", sk_node=sk, gek=gek_bytes) idx.add_entry(entry) - wire_bytes = idx.serialize() # for sending to members - recovered = GroupIndex.deserialize(wire_bytes, sk_node=sk, gek=gek_bytes) """ group_id: str sk_node: Ed25519PrivateKey gek: bytes | None = None # None → public group (no encryption) version: int = 1 - # The group's roots and whether each is readable right now. Travels inside - # the encrypted payload because it names the operator's directories, and a - # member needs it to tell "temporarily unavailable" from "deleted" — a - # distinction the entries alone cannot carry, since an unavailable root's - # files are still listed. Absent in an index written before roots existed. + # The group's roots and whether each is readable right now, as the indexer + # last described them. A member needs this to tell "temporarily unavailable" + # from "deleted" — a distinction the entries alone cannot carry, since an + # unavailable root's files are still listed. What members receive is built + # from the RootSet by `transport/wire.py`, not from this copy. roots: list = field(default_factory=list) _entries: dict = field(default_factory=dict, repr=False) # id → IndexEntry @@ -101,111 +75,6 @@ class GroupIndex: idx._entries = dict(entries_by_id) return idx - # ── Serialisation ───────────────────────────────────────────────────────── - - def serialize(self) -> bytes: - """ - Produce a signed index envelope: - msgpack → zstd → [GEK encrypt if private] → sign → length-prefixed envelope - - **This is not an MNP message.** It was the payload of `index_sync` on the QUIC - transport, while WebRTC sent plain entries under the same type — one message - type, two encodings (2026-09-03). Both transports now build `index_sync` from - `transport/wire.py`. This stays as a correct at-rest/interchange format, and - as the only thing that signs and encrypts a whole index; read it as that, not - as a wire contract. - """ - payload = msgpack.packb({ - "group_id": self.group_id, - "version": self.version, - "roots": list(self.roots), - "entries": [asdict(e) for e in self.entries], - }, use_bin_type=True) - - compressed = zstd.compress(payload, level=ZSTD_LEVEL) - - if self.gek is not None: - # Private group: encrypt with GEK-derived key - idx_hash = blake3.blake3(compressed).digest() - ckey = derive_chunk_key(self.gek, idx_hash, INDEX_CHUNK) - nonce, ct = encrypt_chunk(ckey, compressed) - sig = sign_chunk(self.sk_node, INDEX_CHUNK, nonce, blake3.blake3(ct).digest()) - envelope = msgpack.packb({ - "type": "index", - "encrypted": True, - "version": self.version, - "group_id": self.group_id, - "idx_hash_b64": base64.b64encode(idx_hash).decode(), - "nonce_b64": base64.b64encode(nonce).decode(), - "ct_b64": base64.b64encode(ct).decode(), - "sig_b64": base64.b64encode(sig).decode(), - "pk_node_b64": pk_to_b64(self.sk_node.public_key()), - }, use_bin_type=True) - else: - # Public group: just sign the compressed payload - payload_hash = blake3.blake3(compressed).digest() - sig = sign_chunk(self.sk_node, INDEX_CHUNK, - bytes(12), # zero nonce for plaintext - payload_hash) - envelope = msgpack.packb({ - "type": "index", - "encrypted": False, - "version": self.version, - "group_id": self.group_id, - "data_b64": base64.b64encode(compressed).decode(), - "hash_b64": base64.b64encode(payload_hash).decode(), - "sig_b64": base64.b64encode(sig).decode(), - "pk_node_b64": pk_to_b64(self.sk_node.public_key()), - }, use_bin_type=True) - - return envelope - - @classmethod - def deserialize( - cls, - data: bytes, - sk_node: Ed25519PrivateKey, - gek: bytes | None = None, - ) -> "GroupIndex": - """Deserialize, verify signature, and decrypt (if private).""" - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey - - envelope = msgpack.unpackb(data, raw=False) - - pk_node_raw = base64.b64decode(envelope["pk_node_b64"]) - pk_node = Ed25519PublicKey.from_public_bytes(pk_node_raw) - sig = base64.b64decode(envelope["sig_b64"]) - - if envelope["encrypted"]: - if gek is None: - raise ValueError("GEK required to decrypt private group index") - ct = base64.b64decode(envelope["ct_b64"]) - nonce = base64.b64decode(envelope["nonce_b64"]) - ct_hash = blake3.blake3(ct).digest() - verify_chunk_signature(pk_node, INDEX_CHUNK, nonce, ct_hash, sig) - - idx_hash = base64.b64decode(envelope["idx_hash_b64"]) - ckey = derive_chunk_key(gek, idx_hash, INDEX_CHUNK) - compressed = decrypt_chunk(ckey, nonce, ct) - else: - compressed = base64.b64decode(envelope["data_b64"]) - payload_hash = base64.b64decode(envelope["hash_b64"]) - verify_chunk_signature(pk_node, INDEX_CHUNK, bytes(12), payload_hash, sig) - - payload = msgpack.unpackb(zstd.decompress(compressed), raw=False) - idx = cls( - group_id=payload["group_id"], - sk_node=sk_node, - gek=gek, - version=payload["version"], - # Absent from an index written before roots existed; an empty list - # reads as "nothing known about availability", not "no roots". - roots=payload.get("roots") or [], - ) - for e in payload["entries"]: - idx.add_entry(IndexEntry(**e)) - return idx - # ── Delta ───────────────────────────────────────────────────────────────── def diff(self, previous: "GroupIndex") -> IndexDelta: diff --git a/packages/meshbay-node/src/meshbay_node/transport/wire.py b/packages/meshbay-node/src/meshbay_node/transport/wire.py index 6986b01..1f9c925 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/wire.py +++ b/packages/meshbay-node/src/meshbay_node/transport/wire.py @@ -12,11 +12,10 @@ envelope produced by `GroupIndex.serialize()`. Same message type, two encodings, consumer each and nothing asserting they matched. Same failure mode as the two `file_chunk` encoders, and the same fix: one builder, used by both. -`GroupIndex.serialize()`/`deserialize()` are unchanged and still tested — they remain -a correct signed index envelope — but they no longer describe any MNP message. Read -them as an at-rest/interchange format, not as a wire contract. It is also not a -candidate for reuse below: it compresses with zstd, which no browser can decompress -(`DecompressionStream` offers gzip and deflate only). +That envelope, and `GroupIndex.serialize()`/`deserialize()` which produced it, are +gone: once both transports built `index_sync` here, nothing stored or exchanged it. +It was never a candidate for reuse below either — it compressed with zstd, which no +browser can decompress (`DecompressionStream` offers gzip and deflate only). Since MNP 1.0 both messages carry their payload **sealed under a GEK-derived subkey** (`meshbay_common.groupbox`). Only the routing fields — `type`, `v`, `group_id` — stay diff --git a/packages/meshbay-node/tests/test_indexer.py b/packages/meshbay-node/tests/test_indexer.py index e97e2ef..c60600f 100644 --- a/packages/meshbay-node/tests/test_indexer.py +++ b/packages/meshbay-node/tests/test_indexer.py @@ -42,58 +42,6 @@ def shared_dir(tmp_path): # ── GroupIndex tests ────────────────────────────────────────────────────────── -def test_group_index_serialize_deserialize_private(sk_node, gek, shared_dir): - idx = GroupIndex(group_id="grp-001", sk_node=sk_node, gek=gek) - from meshbay_common.protocol import IndexEntry - idx.add_entry(IndexEntry( - id="abc123", name="video.mkv", path="", size=1024, - type="video", added_at=int(time.time()), duration=120)) - - wire = idx.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=gek) - - assert recovered.group_id == "grp-001" - assert recovered.count == 1 - assert recovered.entries[0].name == "video.mkv" - assert recovered.entries[0].type == "video" - - -def test_group_index_serialize_deserialize_public(sk_node): - idx = GroupIndex(group_id="pub-001", sk_node=sk_node, gek=None) - from meshbay_common.protocol import IndexEntry - idx.add_entry(IndexEntry( - id="xyz789", name="readme.txt", path="", size=42, - type="document", added_at=int(time.time()))) - - wire = idx.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=None) - assert recovered.count == 1 - assert recovered.entries[0].id == "xyz789" - - -def test_group_index_wrong_gek_rejected(sk_node, gek): - idx = GroupIndex(group_id="grp-002", sk_node=sk_node, gek=gek) - from meshbay_common.protocol import IndexEntry - idx.add_entry(IndexEntry(id="a", name="f.mp3", path="", size=1, - type="audio", added_at=0)) - wire = idx.serialize() - - wrong_gek = generate_gek() - with pytest.raises(Exception): # InvalidTag from AEAD - GroupIndex.deserialize(wire, sk_node=sk_node, gek=wrong_gek) - - -def test_group_index_tampered_rejected(sk_node, gek): - idx = GroupIndex(group_id="grp-003", sk_node=sk_node, gek=gek) - from meshbay_common.protocol import IndexEntry - idx.add_entry(IndexEntry(id="b", name="f.mp4", path="", size=1, - type="video", added_at=0)) - wire = bytearray(idx.serialize()) - wire[-5] ^= 0xFF # flip bytes at the end - with pytest.raises(Exception): - GroupIndex.deserialize(bytes(wire), sk_node=sk_node, gek=gek) - - def test_group_index_diff(sk_node, gek): from meshbay_common.protocol import IndexEntry v1 = GroupIndex(group_id="g", sk_node=sk_node, gek=gek, version=1) @@ -229,16 +177,6 @@ async def test_on_change_callback(shared_dir, sk_node, gek): assert len(changes) >= 1, "on_change should have been called" -@pytest.mark.asyncio -async def test_index_roundtrip_after_scan(shared_dir, sk_node, gek): - indexer = DirectoryIndexer(roots=one_root(shared_dir), group_id="g", sk_node=sk_node, gek=gek) - await indexer.initial_scan() - - wire = indexer.index.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=gek) - assert recovered.count == indexer.index.count - - # ── Cache-aware scanning ─────────────────────────────────────────────────────── @pytest.fixture @@ -796,35 +734,13 @@ def test_partial_hash_is_deterministic(tmp_path): assert e1.id == e2.id -def test_group_index_roundtrip_preserves_hash_version(sk_node, gek): - from meshbay_common.protocol import IndexEntry - idx = GroupIndex(group_id="hv-test", sk_node=sk_node, gek=gek) - idx.add_entry(IndexEntry( - id="aaa", name="small.mp4", path="root", size=1024, - type="video", added_at=100, hash_version=1)) - idx.add_entry(IndexEntry( - id="bbb", name="big.mkv", path="root", size=50_000_000, - type="video", added_at=200, hash_version=2)) - - wire = idx.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=gek) - - by_id = {e.id: e for e in recovered.entries} - assert by_id["aaa"].hash_version == 1 - assert by_id["bbb"].hash_version == 2 - - -def test_deserialize_without_hash_version_defaults_to_1(sk_node, gek): - """Entries serialized by old code (no hash_version field) must deserialize - as hash_version=1.""" +def test_an_entry_without_hash_version_defaults_to_1(): + """An entry built without the field (written before it existed) reads as a + full-read hash.""" from meshbay_common.protocol import IndexEntry - idx = GroupIndex(group_id="compat", sk_node=sk_node, gek=gek) - idx.add_entry(IndexEntry( - id="old", name="f.mp4", path="root", size=1024, - type="video", added_at=100)) - wire = idx.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=gek) - assert recovered.entries[0].hash_version == 1 + e = IndexEntry(id="old", name="f.mp4", path="root", size=1024, + type="video", added_at=100) + assert e.hash_version == 1 def test_index_entry_wire_includes_hash_version(): diff --git a/packages/meshbay-node/tests/test_webrtc_transport.py b/packages/meshbay-node/tests/test_webrtc_transport.py index 990b1da..4542817 100644 --- a/packages/meshbay-node/tests/test_webrtc_transport.py +++ b/packages/meshbay-node/tests/test_webrtc_transport.py @@ -38,9 +38,7 @@ from meshbay_common.adminop import ( from meshbay_common.crypto import ( generate_gek, pk_to_b64, - unwrap_gek, unwrap_gek_aes, - wrap_gek, wrap_gek_aes, ) from meshbay_common.groupbox import PURPOSE_ACK, PURPOSE_INDEX, unseal @@ -1570,7 +1568,7 @@ async def test_gek_bundle_fetch_during_handshake(sk_node, sk_hub, gek, shared_di await bundle_store.open() # Pre-populate a bundle for user-001 in group "g" - bundle = wrap_gek(gek, pk_x_raw) + bundle = wrap_gek_aes(gek, pk_x_raw) await bundle_store.store("g", "user-001", bundle["pk_eph_b64"], bundle["nonce_b64"], bundle["wrapped_b64"]) @@ -1631,7 +1629,7 @@ async def test_gek_bundle_fetch_during_handshake(sk_node, sk_hub, gek, shared_di assert bundle_resp["found"] is True # Step 3: Unwrap GEK and compute HMAC proof - recovered_gek = unwrap_gek(bundle_resp, sk_x_raw, pk_x_raw) + recovered_gek = unwrap_gek_aes(bundle_resp, sk_x_raw, pk_x_raw) assert recovered_gek == gek nonce_s = base64.b64decode(msg["nonce"]) diff --git a/packaging/win/README.md b/packaging/win/README.md index 3c141d1..b84ba98 100644 --- a/packaging/win/README.md +++ b/packaging/win/README.md @@ -278,7 +278,7 @@ the journal. `meshbay-node.spec` pulls the awkward packages in whole (`collect_all`) because they have C/Rust extensions or do dynamic imports: `aiortc`, `av` (bundles FFmpeg DLLs), `aioquic`, `pydantic_core`, `uvicorn`, `watchdog`, `guessit`, -`blake3`, `zstandard`, `msgpack`. If a frozen run raises `ModuleNotFoundError`, +`blake3`, `msgpack`. If a frozen run raises `ModuleNotFoundError`, add the package to `COLLECT_ALL` / `HIDDEN` / `COPY_META` in the spec — that is the expected way the list grows. diff --git a/packaging/win/meshbay-node.spec b/packaging/win/meshbay-node.spec index 324d8c1..3261778 100644 --- a/packaging/win/meshbay-node.spec +++ b/packaging/win/meshbay-node.spec @@ -82,7 +82,6 @@ COLLECT_ALL = [ "PIL", "blake3", "msgpack", - "zstandard", "aiosqlite", "jwt", "dns", |