summaryrefslogtreecommitdiffstats
path: root/packages/meshbay-node/src/meshbay_node/indexer
diff options
context:
space:
mode:
authorChristophe Besson <cbesson@gmail.com>2026-08-09 22:56:18 +0200
committerChristophe Besson <cbesson@gmail.com>2026-08-09 22:56:18 +0200
commit217b61ff34390fd24d0a2ad338f6094183debbe3 (patch)
treed0e51d3138dfe4df6581c3e69b1c3536267ee2e5 /packages/meshbay-node/src/meshbay_node/indexer
parent82ce18ef9f45f817505b12e24c958beda45465f0 (diff)
downloadmeshbay-217b61ff34390fd24d0a2ad338f6094183debbe3.tar.gz
fix: 4 corrections — streaming hash, watchdog bug, cipher doc, deps
1. indexer.py: streaming blake3 (8MB chunks) instead of read_bytes(). Large files (initrd.img, ISOs, VM images) no longer load into RAM. 2. QE/demo-v1/run_node.py: call indexer.start() not initial_scan(). initial_scan() alone never starts the watchdog observer — files added after startup were silently ignored. Added indexer.stop() on shutdown. 3. USERGUIDE.md §8: clarify symmetric vs asymmetric. Ed25519/X25519 = asymmetric (key pairs). ChaCha20-Poly1305 and AES-256-GCM = symmetric AEAD 256-bit (content encryption). ChaCha20 is PRIMARY; AES-GCM is optional browser-compat variant only. 4. pyproject.toml: aioquic, websockets, aiosqlite, slowapi added to proper package deps (were installed manually, now declared). Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'packages/meshbay-node/src/meshbay_node/indexer')
-rw-r--r--packages/meshbay-node/src/meshbay_node/indexer/indexer.py14
1 files changed, 11 insertions, 3 deletions
diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py
index 5cd7205..4bb1527 100644
--- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py
+++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py
@@ -56,14 +56,22 @@ def _is_indexable(path: Path) -> bool:
)
+_HASH_CHUNK = 8 * 1024 * 1024 # 8 MB streaming hash chunks
+
+
def _scan_file(root: Path, file_path: Path) -> IndexEntry | None:
- """Compute IndexEntry for a file. Blocking — run in executor."""
+ """Compute IndexEntry for a file. Blocking — run in executor.
+ Uses streaming blake3 so arbitrarily large files (ISOs, VM images, etc.)
+ don't require loading the whole file into memory."""
if not _is_indexable(file_path):
return None
try:
stat = file_path.stat()
- data = file_path.read_bytes()
- file_id = blake3.blake3(data).hexdigest()
+ hasher = blake3.blake3()
+ with open(file_path, "rb") as f:
+ while chunk := f.read(_HASH_CHUNK):
+ hasher.update(chunk)
+ file_id = hasher.hexdigest()
rel_path = str(file_path.parent.relative_to(root))
if rel_path == ".":
rel_path = ""