diff options
3 files changed, 261 insertions, 0 deletions
diff --git a/packages/meshbay-node/src/meshbay_node/daemon.py b/packages/meshbay-node/src/meshbay_node/daemon.py index 74fff4c..f9e992c 100644 --- a/packages/meshbay-node/src/meshbay_node/daemon.py +++ b/packages/meshbay-node/src/meshbay_node/daemon.py @@ -1172,6 +1172,24 @@ class NodeDaemon: # this broadcast — enrichment fields arrive later as their own # INDEX_DELTA update (_on_enriched below). new_entries = delta.additions if delta is not None else list(idx.entries) + + # A root that was ejected and plugged back in, or that fell off and + # re-mounted, has had its entries thrown away and rebuilt from disk + # (`indexer._drop_root_entries`). The rebuilt entry has the same + # content-hash id and none of the enrichment fields, so the diff above + # reports neither an addition nor a deletion — and `_enriched_attempted` + # still says "done" for a file whose album and cover no longer exist. + # Found live: a Music library came back with its files and without its + # albums, and stayed that way, because only a restart (which starts + # with no snapshot, making every entry an addition) could clear either + # gate. Treated here as what it is — those entries are new again. + rebuilt_ids = indexer.drain_rescanned_ids() + if rebuilt_ids: + rebuilt = [e for e in idx.entries if e.id in rebuilt_ids] + for entry in rebuilt: + self._enriched_attempted.discard((group_id, entry.id)) + seen = {e.id for e in new_entries} + new_entries = new_entries + [e for e in rebuilt if e.id not in seen] asyncio.ensure_future(self._enrich_new_video_entries(indexer, new_entries)) # Music app (docs/musicbay.md §6): same shape, gated on audio_root # exactly like video_root above (added later — musicbay.md's diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index f7ffdca..eea5b4f 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -314,6 +314,14 @@ class DirectoryIndexer: # stay up for exactly as long as the slow part (hashing) is running. self._burst_inflight = 0 self._burst_sizes: dict[str, int] = {} + # Ids whose entry this indexer threw away and rebuilt from disk, since + # the last time a consumer drained this. A rebuilt entry carries only + # what `_hash_or_cached` fills in — every enrichment field the Videos, + # Music and Photos apps put there is gone — but its id is the file's + # content hash, so a diff against the last broadcast sees no addition + # and no deletion and nothing downstream can tell the fields were + # wiped. See `_drop_root_entries`. + self.rescanned_ids: set[str] = set() @property def index(self) -> GroupIndex: @@ -704,9 +712,26 @@ class DirectoryIndexer: if fold(e.path).split("/", 1)[0] == prefix] def _drop_root_entries(self, root: Root) -> None: + """ + Throw away a root's entries, always in order to rescan it. + + Both callers — `reconcile` when a root reappears, `plug_root` when the + operator plugs one back in — rebuild immediately, so nothing outside + ever observes the gap: no deletion is broadcast, and the entries that + come back have the same content-hash ids they had before. What they do + not have is anything enrichment put on them, which is why the ids are + recorded for `daemon._broadcast_index_change` to re-enrich rather than + simply forgotten. + """ for entry in self._entries_under(root): + self.rescanned_ids.add(entry.id) self._index.remove_entry(entry.id) + def drain_rescanned_ids(self) -> set[str]: + """Take the ids rebuilt since the last call; leave the set empty.""" + drained, self.rescanned_ids = self.rescanned_ids, set() + return drained + @staticmethod def _entry_path(root: Root, entry: IndexEntry) -> Path | None: _, _, tail = entry.path.partition("/") diff --git a/packages/meshbay-node/tests/test_replug_restores_enrichment.py b/packages/meshbay-node/tests/test_replug_restores_enrichment.py new file mode 100644 index 0000000..23fc97e --- /dev/null +++ b/packages/meshbay-node/tests/test_replug_restores_enrichment.py @@ -0,0 +1,218 @@ +""" +A root that comes back keeps its Videos/Music/Photos metadata. + +Reported live against a real library: a removable root ejected from the +Files app and plugged back in came back with its files but *without* its +albums — Music showed "no music found" and stayed that way through a force +reload, because the loss was on the node, not in the client. + +`DirectoryIndexer.plug_root` drops the root's entries and rescans, which is +right: the drive may have changed while it was away. What it produces is +bare `IndexEntry` objects — `_hash_or_cached` fills id/name/path/size/type +and nothing else. Every enrichment field (`artist`, `album`, `track_no`, +`duration`, `thumb_hash`, `display_title`, `taken_at`, ...) is gone, and +unlike the video/photo probe results those tag fields are cached nowhere: +`enrich_audio.py` deliberately re-reads them each time so a rename can +re-derive the filename fallback. Re-enrichment is the only way back. + +Two independent gates then made sure it never ran: + +* `_broadcast_index_change` schedules enrichment for `delta.additions`. + Ejecting never broadcasts, so `_last_broadcast_snapshot` still held those + ids — the re-added entries diffed as *updates*, not additions. +* `_enrich_new_*_entries` skips anything already in `_enriched_attempted`, + which is only discarded for `delta.deletions`. Dropping and rescanning + inside one call means no deletion is ever broadcast, so the mark survived + a wipe of the very fields it was standing for. + +A restart cleared both (an empty snapshot makes every entry an addition), +which is why this looked like it might fix itself and never did. + +The same drop-and-rescan runs in `reconcile()` — "Root %r is back" — so a +USB drive that falls off and returns on its own hits this without anybody +touching the UI. +""" + +import asyncio +import os + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from meshbay_common.crypto import generate_gek +from meshbay_node.config import (Config, GroupConfig, HubConfig, KeystoreConfig, + NodeConfig) +from meshbay_node.daemon import NodeDaemon +from meshbay_node.indexer import DirectoryIndexer +from meshbay_node.media_cache import MediaCache +from meshbay_node.roots import RootSet +from meshbay_node.roster import Roster + +pytestmark = pytest.mark.asyncio + +# Above indexer.py's MIN_AUDIO_SIZE_BYTES, or nothing would be indexed. +_AUDIO_BYTES = os.urandom(60 * 1024) + + +def _free_port() -> int: + import socket + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class _CountingEnricher: + """Stands in for AudioEnricher: records who it was asked to enrich.""" + + def __init__(self): + self.spawned: list[str] = [] + + def spawn(self, entry, file_path, on_done, boundary=None): + self.spawned.append(entry.name) + + +async def _daemon(tmp_path, shared, group_id): + config = Config( + hub=HubConfig(url="http://localhost:9999", username="testuser"), + node=NodeConfig(quic_port=_free_port(), ui_port=_free_port()), + groups=[GroupConfig( + id=group_id, name="test-group", shared_dir=str(shared), + visibility="private", quic_port=_free_port(), + )], + keystore=KeystoreConfig(path=tmp_path / "keystore.enc"), + data_dir=tmp_path / "data", + ) + daemon = NodeDaemon(config) + daemon._broadcast_coalesce_secs = 0.01 + daemon._media_cache = MediaCache(db_path=tmp_path / "media_cache.db") + await daemon._media_cache.open() + daemon._roster = Roster(db_path=tmp_path / "roster.db") + await daemon._roster.open() + return daemon + + +async def _settled(daemon, indexer): + await daemon._on_index_change(indexer) + await asyncio.sleep(0.05) + + +async def test_a_replugged_root_gets_its_music_metadata_back(tmp_path): + group_id = "a" * 32 + library = tmp_path / "music" + (library / "an album").mkdir(parents=True) + (library / "an album" / "track.mp3").write_bytes(_AUDIO_BYTES) + + daemon = await _daemon(tmp_path, library, group_id) + enricher = _CountingEnricher() + daemon._audio_enricher = enricher + try: + await daemon._roster.set_app_directories( + group_id, "music", ["music/an album"], set_by="op") + + roots = RootSet.build([{"path": str(library), "removable": True}]) + indexer = DirectoryIndexer( + roots=roots, group_id=group_id, + sk_node=Ed25519PrivateKey.generate(), gek=generate_gek()) + await indexer.initial_scan() + await _settled(daemon, indexer) + assert enricher.spawned == ["track.mp3"], "the first pass never ran" + + # The operator ejects the drive from Files, then plugs it back in. + indexer.eject_root("music") + await indexer.plug_root("music") + await _settled(daemon, indexer) + + entry = next(iter(indexer.index.entries)) + assert entry.artist is None and entry.album is None, ( + "the rescan is supposed to produce a bare entry — if this ever " + "stops being true the rest of this test is measuring nothing") + assert enricher.spawned == ["track.mp3", "track.mp3"], ( + "a replugged root came back with its files and without its " + "albums, and nothing was ever going to fill them in again") + finally: + await daemon._media_cache.close() + await daemon._roster.close() + + +async def test_a_root_that_returns_on_its_own_is_treated_the_same(tmp_path): + """ + `reconcile()` rescans a root that reappears without anyone asking — a USB + drive re-mounting. It goes through the same drop-and-rescan, so it loses + the same fields, with no click anywhere to blame it on. + """ + group_id = "b" * 32 + library = tmp_path / "music" + (library / "an album").mkdir(parents=True) + (library / "an album" / "track.mp3").write_bytes(_AUDIO_BYTES) + + daemon = await _daemon(tmp_path, library, group_id) + enricher = _CountingEnricher() + daemon._audio_enricher = enricher + try: + await daemon._roster.set_app_directories( + group_id, "music", ["music/an album"], set_by="op") + + roots = RootSet.build([{"path": str(library), "removable": True}]) + indexer = DirectoryIndexer( + roots=roots, group_id=group_id, + sk_node=Ed25519PrivateKey.generate(), gek=generate_gek()) + await indexer.initial_scan() + await _settled(daemon, indexer) + assert enricher.spawned == ["track.mp3"] + + # Gone, then back — availability is what reconcile() watches. + roots.roots[0].available = False + await indexer.reconcile() + await _settled(daemon, indexer) + await indexer.reconcile() + await _settled(daemon, indexer) + + assert enricher.spawned.count("track.mp3") >= 2, ( + "a drive that fell off and came back left the library with no " + "metadata until the next daemon restart") + finally: + await daemon._media_cache.close() + await daemon._roster.close() + + +async def test_videos_and_photos_lost_the_same_fields(tmp_path): + """ + Nothing here is specific to Music — Videos and Photos hang off the same + `new_entries` list in `_broadcast_index_change`, so a replug took their + durations, titles and thumbnails with it too. Music is simply where it + shows up loudest: a track with no tags has no album to file it under, so + the app goes empty rather than merely plain. + """ + group_id = "c" * 32 + library = tmp_path / "media" + (library / "films").mkdir(parents=True) + (library / "films" / "clip.mkv").write_bytes(os.urandom(60 * 1024)) + (library / "album").mkdir(parents=True) + (library / "album" / "shot.jpg").write_bytes(os.urandom(60 * 1024)) + + daemon = await _daemon(tmp_path, library, group_id) + video, photo = _CountingEnricher(), _CountingEnricher() + daemon._enricher, daemon._photo_enricher = video, photo + try: + await daemon._roster.set_app_directories( + group_id, "video", ["media/films"], set_by="op") + await daemon._roster.set_app_directories( + group_id, "photo", ["media/album"], set_by="op") + + roots = RootSet.build([{"path": str(library), "removable": True}]) + indexer = DirectoryIndexer( + roots=roots, group_id=group_id, + sk_node=Ed25519PrivateKey.generate(), gek=generate_gek()) + await indexer.initial_scan() + await _settled(daemon, indexer) + assert video.spawned == ["clip.mkv"] and photo.spawned == ["shot.jpg"] + + indexer.eject_root("media") + await indexer.plug_root("media") + await _settled(daemon, indexer) + + assert video.spawned == ["clip.mkv"] * 2, "the film lost its probe" + assert photo.spawned == ["shot.jpg"] * 2, "the photo lost its thumbnail" + finally: + await daemon._media_cache.close() + await daemon._roster.close() |