From f2da33a648f86e34ddbbb5f6bec124828ef2a847 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Sun, 6 Sep 2026 01:33:16 +0200 Subject: feat(node): indexing v2 — partial-read hashing for files above 40 MB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files above 40 MB are no longer read in full. Instead, blake3 hashes 45 MB of samples (first 20 MB + last 20 MB + 5 MB at 50% offset). Files at or below 40 MB are unchanged (full read, hash_version 1). A new `hash_version` field on IndexEntry (default 1) travels on the wire and through the cache so both versions coexist without breaking existing nodes or clients. The IndexCache auto-migrates its schema on open (ALTER TABLE), so no manual step is required on upgrade. A standalone migration script is available in QE/migration/ for operators who want to preview or force a full re-hash. Co-Authored-By: Claude Opus 4.6 --- packages/meshbay-node/tests/test_index_cache.py | 77 +++++++++++++++++++++++++ 1 file changed, 77 insertions(+) (limited to 'packages/meshbay-node/tests/test_index_cache.py') diff --git a/packages/meshbay-node/tests/test_index_cache.py b/packages/meshbay-node/tests/test_index_cache.py index 24e2f76..036b4d7 100644 --- a/packages/meshbay-node/tests/test_index_cache.py +++ b/packages/meshbay-node/tests/test_index_cache.py @@ -122,3 +122,80 @@ async def test_cache_survives_reopen(tmp_path): assert hit is not None assert hit.hash == "abc123" + + +# ── hash_version support (indexing v2) ───────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_put_v2_then_lookup_hits(cache): + await cache.put("/lib/big.mkv", size=50_000_000, mtime=111.0, + hash="partial_abc", type="video", added_at=42, + hash_version=2) + + hit = await cache.lookup("/lib/big.mkv", size=50_000_000, mtime=111.0, + hash_version=2) + assert hit is not None + assert hit.hash == "partial_abc" + assert hit.hash_version == 2 + + +@pytest.mark.asyncio +async def test_lookup_misses_on_wrong_hash_version(cache): + await cache.put("/lib/big.mkv", size=50_000_000, mtime=111.0, + hash="full_hash", type="video", added_at=42, + hash_version=1) + + assert await cache.lookup("/lib/big.mkv", size=50_000_000, mtime=111.0, + hash_version=2) is None + + +@pytest.mark.asyncio +async def test_put_v2_overwrites_v1_for_same_path(cache): + await cache.put("/lib/big.mkv", size=50_000_000, mtime=111.0, + hash="full_hash", type="video", added_at=42, + hash_version=1) + await cache.put("/lib/big.mkv", size=50_000_000, mtime=111.0, + hash="partial_hash", type="video", added_at=42, + hash_version=2) + + assert await cache.lookup("/lib/big.mkv", size=50_000_000, mtime=111.0, + hash_version=1) is None + hit = await cache.lookup("/lib/big.mkv", size=50_000_000, mtime=111.0, + hash_version=2) + assert hit is not None + assert hit.hash == "partial_hash" + + +@pytest.mark.asyncio +async def test_v1_schema_auto_migrates(tmp_path): + """An index_cache.db created by old code (no hash_version column) gains + the column on the next open(), and existing rows default to hash_version=1.""" + import aiosqlite + db_path = tmp_path / "old_cache.db" + async with aiosqlite.connect(str(db_path)) as db: + await db.executescript(""" + CREATE TABLE files ( + path TEXT PRIMARY KEY, + mtime REAL NOT NULL, + size INTEGER NOT NULL, + hash TEXT NOT NULL, + type TEXT NOT NULL, + added_at INTEGER NOT NULL + ); + """) + await db.execute( + "INSERT INTO files (path, mtime, size, hash, type, added_at) " + "VALUES (?, ?, ?, ?, ?, ?)", + ("/lib/old.mkv", 111.0, 1000, "oldhash", "video", 42)) + await db.commit() + + cache = IndexCache(db_path=db_path) + await cache.open() + hit = await cache.lookup("/lib/old.mkv", size=1000, mtime=111.0, + hash_version=1) + await cache.close() + + assert hit is not None + assert hit.hash == "oldhash" + assert hit.hash_version == 1 -- cgit v1.2.3