From c27e04c88557716bbe8e42b9174ba9b07f90facf Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Thu, 8 Oct 2026 01:27:58 +0200 Subject: perf(node): sample 9 MB with the size above 9 MB, keep known ids hash_version 3: size + first 4 MB + last 4 MB + 1 MB at the middle, 5.5x faster cold on a USB disk than the 45 MB sample. The cache now serves a hit under whatever version it holds, so no existing id moves. Co-Authored-By: Claude Opus 5.5 --- packages/meshbay-node/tests/test_indexer.py | 51 ++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 8 deletions(-) (limited to 'packages/meshbay-node/tests/test_indexer.py') diff --git a/packages/meshbay-node/tests/test_indexer.py b/packages/meshbay-node/tests/test_indexer.py index c60600f..fda40c8 100644 --- a/packages/meshbay-node/tests/test_indexer.py +++ b/packages/meshbay-node/tests/test_indexer.py @@ -678,7 +678,7 @@ def test_small_file_gets_hash_version_1(tmp_path): assert entry.hash_version == 1 -def test_large_file_gets_hash_version_2(tmp_path): +def test_large_file_gets_hash_version_3(tmp_path): from meshbay_node.indexer.indexer import _PARTIAL_THRESHOLD, _scan_file d = tmp_path / "root" d.mkdir() @@ -688,7 +688,7 @@ def test_large_file_gets_hash_version_2(tmp_path): entry = _scan_file(one_root(d).roots[0], f) assert entry is not None - assert entry.hash_version == 2 + assert entry.hash_version == 3 assert entry.size == size @@ -719,7 +719,7 @@ def test_partial_hash_differs_from_full_hash(tmp_path): full_hash = b3.blake3(content).hexdigest() assert entry.id != full_hash - assert entry.hash_version == 2 + assert entry.hash_version == 3 def test_partial_hash_is_deterministic(tmp_path): @@ -771,16 +771,51 @@ async def test_cache_aware_scan_uses_hash_version(tmp_path, sk_node, gek): by_name = {e.name: e for e in indexer.index.entries} assert by_name["small.mp4"].hash_version == 1 - assert by_name["big.mkv"].hash_version == 2 + assert by_name["big.mkv"].hash_version == 3 hit_small = await cache.lookup( - str(small), small.stat().st_size, small.stat().st_mtime, - hash_version=1) + str(small), small.stat().st_size, small.stat().st_mtime) assert hit_small is not None hit_big = await cache.lookup( - str(big), big.stat().st_size, big.stat().st_mtime, - hash_version=2) + str(big), big.stat().st_size, big.stat().st_mtime) assert hit_big is not None await cache.close() + + +def test_the_size_is_part_of_a_sampled_hash(tmp_path): + """Two preallocated downloads, all zeros, sample the same bytes.""" + from meshbay_node.indexer.indexer import _PARTIAL_THRESHOLD, _scan_file + d = tmp_path / "root" + d.mkdir() + for name, extra in (("a.mkv", 1), ("b.mkv", 2)): + with open(d / name, "wb") as f: + f.truncate(_PARTIAL_THRESHOLD + extra) + + root = one_root(d).roots[0] + assert _scan_file(root, d / "a.mkv").id != _scan_file(root, d / "b.mkv").id + + +@pytest.mark.asyncio +async def test_a_file_hashed_under_an_older_scheme_keeps_its_id(tmp_path, sk_node, gek): + """The id is what TMDB matches, thumbnails and playlists are keyed by.""" + from meshbay_node.indexer.indexer import _PARTIAL_THRESHOLD + d = tmp_path / "root" + d.mkdir() + big = d / "big.mkv" + big.write_bytes(os.urandom(_PARTIAL_THRESHOLD + 1)) + st = big.stat() + + cache = IndexCache(db_path=tmp_path / "cache.db") + await cache.open() + await cache.put(str(big), st.st_size, st.st_mtime, "v2_id", "video", 0, + hash_version=2) + try: + indexer = DirectoryIndexer(roots=one_root(d), group_id="g", + sk_node=sk_node, gek=gek, cache=cache) + await indexer.initial_scan() + [entry] = indexer.index.entries + assert (entry.id, entry.hash_version) == ("v2_id", 2) + finally: + await cache.close() -- cgit v1.2.3