From a4b36e5fe31cb671a4cbbdaad75746cd68b601fe Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Mon, 28 Sep 2026 16:24:12 +0200 Subject: refactor: remove the unused GroupIndex.serialize chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serialize/deserialize had no production caller, and took with them the per-chunk signature, the ChaCha20 cipher variant and the zstandard dependency. Key derivations are unchanged. Docs corrected, including design §4.3's claim that chunks are compressed. Co-Authored-By: Claude Opus 5.5 --- .../src/meshbay_node/indexer/group_index.py | 151 ++------------------- .../src/meshbay_node/transport/wire.py | 9 +- packages/meshbay-node/tests/test_indexer.py | 96 +------------ .../meshbay-node/tests/test_webrtc_transport.py | 6 +- 4 files changed, 22 insertions(+), 240 deletions(-) (limited to 'packages/meshbay-node') diff --git a/packages/meshbay-node/src/meshbay_node/indexer/group_index.py b/packages/meshbay-node/src/meshbay_node/indexer/group_index.py index 9e4e780..85bc13e 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/group_index.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/group_index.py @@ -1,67 +1,41 @@ """ -Mesh Group Index — encrypted file listing for a group. +Mesh Group Index — the file listing for one group, and the deltas between versions. -Wire format (private group): - msgpack({entries, version, group_id}) → zstd compress → GEK ChaCha20 encrypt → sign - -Wire format (public group): - msgpack({entries, version, group_id}) → sign (no encryption) +What travels on the wire is built from it by `transport/wire.py` and sealed under +the group key (`meshbay_common.groupbox`); this module holds no encoding of its own. Delta format: {base_version, version, additions: [...], deletions: [id, ...]} """ -import base64 import logging -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field -import blake3 -import msgpack -import zstandard as zstd from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from meshbay_common.crypto import ( - pk_to_b64, - sign_chunk, - verify_chunk_signature, -) from meshbay_common.protocol import IndexDelta, IndexEntry -from meshbay_common.webcrypto import ( - chunk_key_aes as derive_chunk_key, -) -from meshbay_common.webcrypto import ( - decrypt_chunk_aes as decrypt_chunk, -) -from meshbay_common.webcrypto import ( - encrypt_chunk_aes as encrypt_chunk, -) log = logging.getLogger(__name__) -ZSTD_LEVEL = 3 # fast compression -INDEX_CHUNK = 0 # the index itself is treated as chunk 0 of a virtual "index file" - @dataclass class GroupIndex: """ - Encrypted, signed Mesh Group Index for one group. + Mesh Group Index for one group. Usage: idx = GroupIndex(group_id="...", sk_node=sk, gek=gek_bytes) idx.add_entry(entry) - wire_bytes = idx.serialize() # for sending to members - recovered = GroupIndex.deserialize(wire_bytes, sk_node=sk, gek=gek_bytes) """ group_id: str sk_node: Ed25519PrivateKey gek: bytes | None = None # None → public group (no encryption) version: int = 1 - # The group's roots and whether each is readable right now. Travels inside - # the encrypted payload because it names the operator's directories, and a - # member needs it to tell "temporarily unavailable" from "deleted" — a - # distinction the entries alone cannot carry, since an unavailable root's - # files are still listed. Absent in an index written before roots existed. + # The group's roots and whether each is readable right now, as the indexer + # last described them. A member needs this to tell "temporarily unavailable" + # from "deleted" — a distinction the entries alone cannot carry, since an + # unavailable root's files are still listed. What members receive is built + # from the RootSet by `transport/wire.py`, not from this copy. roots: list = field(default_factory=list) _entries: dict = field(default_factory=dict, repr=False) # id → IndexEntry @@ -101,111 +75,6 @@ class GroupIndex: idx._entries = dict(entries_by_id) return idx - # ── Serialisation ───────────────────────────────────────────────────────── - - def serialize(self) -> bytes: - """ - Produce a signed index envelope: - msgpack → zstd → [GEK encrypt if private] → sign → length-prefixed envelope - - **This is not an MNP message.** It was the payload of `index_sync` on the QUIC - transport, while WebRTC sent plain entries under the same type — one message - type, two encodings (2026-09-03). Both transports now build `index_sync` from - `transport/wire.py`. This stays as a correct at-rest/interchange format, and - as the only thing that signs and encrypts a whole index; read it as that, not - as a wire contract. - """ - payload = msgpack.packb({ - "group_id": self.group_id, - "version": self.version, - "roots": list(self.roots), - "entries": [asdict(e) for e in self.entries], - }, use_bin_type=True) - - compressed = zstd.compress(payload, level=ZSTD_LEVEL) - - if self.gek is not None: - # Private group: encrypt with GEK-derived key - idx_hash = blake3.blake3(compressed).digest() - ckey = derive_chunk_key(self.gek, idx_hash, INDEX_CHUNK) - nonce, ct = encrypt_chunk(ckey, compressed) - sig = sign_chunk(self.sk_node, INDEX_CHUNK, nonce, blake3.blake3(ct).digest()) - envelope = msgpack.packb({ - "type": "index", - "encrypted": True, - "version": self.version, - "group_id": self.group_id, - "idx_hash_b64": base64.b64encode(idx_hash).decode(), - "nonce_b64": base64.b64encode(nonce).decode(), - "ct_b64": base64.b64encode(ct).decode(), - "sig_b64": base64.b64encode(sig).decode(), - "pk_node_b64": pk_to_b64(self.sk_node.public_key()), - }, use_bin_type=True) - else: - # Public group: just sign the compressed payload - payload_hash = blake3.blake3(compressed).digest() - sig = sign_chunk(self.sk_node, INDEX_CHUNK, - bytes(12), # zero nonce for plaintext - payload_hash) - envelope = msgpack.packb({ - "type": "index", - "encrypted": False, - "version": self.version, - "group_id": self.group_id, - "data_b64": base64.b64encode(compressed).decode(), - "hash_b64": base64.b64encode(payload_hash).decode(), - "sig_b64": base64.b64encode(sig).decode(), - "pk_node_b64": pk_to_b64(self.sk_node.public_key()), - }, use_bin_type=True) - - return envelope - - @classmethod - def deserialize( - cls, - data: bytes, - sk_node: Ed25519PrivateKey, - gek: bytes | None = None, - ) -> "GroupIndex": - """Deserialize, verify signature, and decrypt (if private).""" - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey - - envelope = msgpack.unpackb(data, raw=False) - - pk_node_raw = base64.b64decode(envelope["pk_node_b64"]) - pk_node = Ed25519PublicKey.from_public_bytes(pk_node_raw) - sig = base64.b64decode(envelope["sig_b64"]) - - if envelope["encrypted"]: - if gek is None: - raise ValueError("GEK required to decrypt private group index") - ct = base64.b64decode(envelope["ct_b64"]) - nonce = base64.b64decode(envelope["nonce_b64"]) - ct_hash = blake3.blake3(ct).digest() - verify_chunk_signature(pk_node, INDEX_CHUNK, nonce, ct_hash, sig) - - idx_hash = base64.b64decode(envelope["idx_hash_b64"]) - ckey = derive_chunk_key(gek, idx_hash, INDEX_CHUNK) - compressed = decrypt_chunk(ckey, nonce, ct) - else: - compressed = base64.b64decode(envelope["data_b64"]) - payload_hash = base64.b64decode(envelope["hash_b64"]) - verify_chunk_signature(pk_node, INDEX_CHUNK, bytes(12), payload_hash, sig) - - payload = msgpack.unpackb(zstd.decompress(compressed), raw=False) - idx = cls( - group_id=payload["group_id"], - sk_node=sk_node, - gek=gek, - version=payload["version"], - # Absent from an index written before roots existed; an empty list - # reads as "nothing known about availability", not "no roots". - roots=payload.get("roots") or [], - ) - for e in payload["entries"]: - idx.add_entry(IndexEntry(**e)) - return idx - # ── Delta ───────────────────────────────────────────────────────────────── def diff(self, previous: "GroupIndex") -> IndexDelta: diff --git a/packages/meshbay-node/src/meshbay_node/transport/wire.py b/packages/meshbay-node/src/meshbay_node/transport/wire.py index 6986b01..1f9c925 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/wire.py +++ b/packages/meshbay-node/src/meshbay_node/transport/wire.py @@ -12,11 +12,10 @@ envelope produced by `GroupIndex.serialize()`. Same message type, two encodings, consumer each and nothing asserting they matched. Same failure mode as the two `file_chunk` encoders, and the same fix: one builder, used by both. -`GroupIndex.serialize()`/`deserialize()` are unchanged and still tested — they remain -a correct signed index envelope — but they no longer describe any MNP message. Read -them as an at-rest/interchange format, not as a wire contract. It is also not a -candidate for reuse below: it compresses with zstd, which no browser can decompress -(`DecompressionStream` offers gzip and deflate only). +That envelope, and `GroupIndex.serialize()`/`deserialize()` which produced it, are +gone: once both transports built `index_sync` here, nothing stored or exchanged it. +It was never a candidate for reuse below either — it compressed with zstd, which no +browser can decompress (`DecompressionStream` offers gzip and deflate only). Since MNP 1.0 both messages carry their payload **sealed under a GEK-derived subkey** (`meshbay_common.groupbox`). Only the routing fields — `type`, `v`, `group_id` — stay diff --git a/packages/meshbay-node/tests/test_indexer.py b/packages/meshbay-node/tests/test_indexer.py index e97e2ef..c60600f 100644 --- a/packages/meshbay-node/tests/test_indexer.py +++ b/packages/meshbay-node/tests/test_indexer.py @@ -42,58 +42,6 @@ def shared_dir(tmp_path): # ── GroupIndex tests ────────────────────────────────────────────────────────── -def test_group_index_serialize_deserialize_private(sk_node, gek, shared_dir): - idx = GroupIndex(group_id="grp-001", sk_node=sk_node, gek=gek) - from meshbay_common.protocol import IndexEntry - idx.add_entry(IndexEntry( - id="abc123", name="video.mkv", path="", size=1024, - type="video", added_at=int(time.time()), duration=120)) - - wire = idx.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=gek) - - assert recovered.group_id == "grp-001" - assert recovered.count == 1 - assert recovered.entries[0].name == "video.mkv" - assert recovered.entries[0].type == "video" - - -def test_group_index_serialize_deserialize_public(sk_node): - idx = GroupIndex(group_id="pub-001", sk_node=sk_node, gek=None) - from meshbay_common.protocol import IndexEntry - idx.add_entry(IndexEntry( - id="xyz789", name="readme.txt", path="", size=42, - type="document", added_at=int(time.time()))) - - wire = idx.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=None) - assert recovered.count == 1 - assert recovered.entries[0].id == "xyz789" - - -def test_group_index_wrong_gek_rejected(sk_node, gek): - idx = GroupIndex(group_id="grp-002", sk_node=sk_node, gek=gek) - from meshbay_common.protocol import IndexEntry - idx.add_entry(IndexEntry(id="a", name="f.mp3", path="", size=1, - type="audio", added_at=0)) - wire = idx.serialize() - - wrong_gek = generate_gek() - with pytest.raises(Exception): # InvalidTag from AEAD - GroupIndex.deserialize(wire, sk_node=sk_node, gek=wrong_gek) - - -def test_group_index_tampered_rejected(sk_node, gek): - idx = GroupIndex(group_id="grp-003", sk_node=sk_node, gek=gek) - from meshbay_common.protocol import IndexEntry - idx.add_entry(IndexEntry(id="b", name="f.mp4", path="", size=1, - type="video", added_at=0)) - wire = bytearray(idx.serialize()) - wire[-5] ^= 0xFF # flip bytes at the end - with pytest.raises(Exception): - GroupIndex.deserialize(bytes(wire), sk_node=sk_node, gek=gek) - - def test_group_index_diff(sk_node, gek): from meshbay_common.protocol import IndexEntry v1 = GroupIndex(group_id="g", sk_node=sk_node, gek=gek, version=1) @@ -229,16 +177,6 @@ async def test_on_change_callback(shared_dir, sk_node, gek): assert len(changes) >= 1, "on_change should have been called" -@pytest.mark.asyncio -async def test_index_roundtrip_after_scan(shared_dir, sk_node, gek): - indexer = DirectoryIndexer(roots=one_root(shared_dir), group_id="g", sk_node=sk_node, gek=gek) - await indexer.initial_scan() - - wire = indexer.index.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=gek) - assert recovered.count == indexer.index.count - - # ── Cache-aware scanning ─────────────────────────────────────────────────────── @pytest.fixture @@ -796,35 +734,13 @@ def test_partial_hash_is_deterministic(tmp_path): assert e1.id == e2.id -def test_group_index_roundtrip_preserves_hash_version(sk_node, gek): - from meshbay_common.protocol import IndexEntry - idx = GroupIndex(group_id="hv-test", sk_node=sk_node, gek=gek) - idx.add_entry(IndexEntry( - id="aaa", name="small.mp4", path="root", size=1024, - type="video", added_at=100, hash_version=1)) - idx.add_entry(IndexEntry( - id="bbb", name="big.mkv", path="root", size=50_000_000, - type="video", added_at=200, hash_version=2)) - - wire = idx.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=gek) - - by_id = {e.id: e for e in recovered.entries} - assert by_id["aaa"].hash_version == 1 - assert by_id["bbb"].hash_version == 2 - - -def test_deserialize_without_hash_version_defaults_to_1(sk_node, gek): - """Entries serialized by old code (no hash_version field) must deserialize - as hash_version=1.""" +def test_an_entry_without_hash_version_defaults_to_1(): + """An entry built without the field (written before it existed) reads as a + full-read hash.""" from meshbay_common.protocol import IndexEntry - idx = GroupIndex(group_id="compat", sk_node=sk_node, gek=gek) - idx.add_entry(IndexEntry( - id="old", name="f.mp4", path="root", size=1024, - type="video", added_at=100)) - wire = idx.serialize() - recovered = GroupIndex.deserialize(wire, sk_node=sk_node, gek=gek) - assert recovered.entries[0].hash_version == 1 + e = IndexEntry(id="old", name="f.mp4", path="root", size=1024, + type="video", added_at=100) + assert e.hash_version == 1 def test_index_entry_wire_includes_hash_version(): diff --git a/packages/meshbay-node/tests/test_webrtc_transport.py b/packages/meshbay-node/tests/test_webrtc_transport.py index 990b1da..4542817 100644 --- a/packages/meshbay-node/tests/test_webrtc_transport.py +++ b/packages/meshbay-node/tests/test_webrtc_transport.py @@ -38,9 +38,7 @@ from meshbay_common.adminop import ( from meshbay_common.crypto import ( generate_gek, pk_to_b64, - unwrap_gek, unwrap_gek_aes, - wrap_gek, wrap_gek_aes, ) from meshbay_common.groupbox import PURPOSE_ACK, PURPOSE_INDEX, unseal @@ -1570,7 +1568,7 @@ async def test_gek_bundle_fetch_during_handshake(sk_node, sk_hub, gek, shared_di await bundle_store.open() # Pre-populate a bundle for user-001 in group "g" - bundle = wrap_gek(gek, pk_x_raw) + bundle = wrap_gek_aes(gek, pk_x_raw) await bundle_store.store("g", "user-001", bundle["pk_eph_b64"], bundle["nonce_b64"], bundle["wrapped_b64"]) @@ -1631,7 +1629,7 @@ async def test_gek_bundle_fetch_during_handshake(sk_node, sk_hub, gek, shared_di assert bundle_resp["found"] is True # Step 3: Unwrap GEK and compute HMAC proof - recovered_gek = unwrap_gek(bundle_resp, sk_x_raw, pk_x_raw) + recovered_gek = unwrap_gek_aes(bundle_resp, sk_x_raw, pk_x_raw) assert recovered_gek == gek nonce_s = base64.b64decode(msg["nonce"]) -- cgit v1.2.3