diff options
Diffstat (limited to 'packages')
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/indexer/indexer.py | 14 |
1 files changed, 11 insertions, 3 deletions
diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index 5cd7205..4bb1527 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -56,14 +56,22 @@ def _is_indexable(path: Path) -> bool: ) +_HASH_CHUNK = 8 * 1024 * 1024 # 8 MB streaming hash chunks + + def _scan_file(root: Path, file_path: Path) -> IndexEntry | None: - """Compute IndexEntry for a file. Blocking — run in executor.""" + """Compute IndexEntry for a file. Blocking — run in executor. + Uses streaming blake3 so arbitrarily large files (ISOs, VM images, etc.) + don't require loading the whole file into memory.""" if not _is_indexable(file_path): return None try: stat = file_path.stat() - data = file_path.read_bytes() - file_id = blake3.blake3(data).hexdigest() + hasher = blake3.blake3() + with open(file_path, "rb") as f: + while chunk := f.read(_HASH_CHUNK): + hasher.update(chunk) + file_id = hasher.hexdigest() rel_path = str(file_path.parent.relative_to(root)) if rel_path == ".": rel_path = "" |