From daec4bc6a2a9f9e038e5269586b06018ff327a27 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Wed, 14 Oct 2026 11:17:38 +0200 Subject: fix(node): index, serve and write files past Windows' MAX_PATH With the system's default setting (LongPathsEnabled off) every call given a plain path of 260 characters or more refuses it, so a deep library was partly invisible: the walk could not list such a directory and its files never reached the members. The node now hands each call that touches a group's disk the extended form (paths.long_path), and keeps the plain path in the index, the hash cache and the log. A directory is listed under its long form and its entries named from the plain one (indexer.walk_files, wire._walk_dirs). Covered: the walk, stats and hashing, reconcile and the watcher, the folder list members see, serving over WebRTC and QUIC, deletion, uploads and their recovery, folder creation and removal, audio tags and the cover beside a track, photos, and a show's episode numbering. ffmpeg extends a long path itself and keeps the plain one. long_path now normalises before prefixing (the extended form skips the system's normalisation) and starts at 248 characters, where creating a directory is already refused. test_long_paths.py runs with the setting off: 15 of its 17 tests fail without the change. Checked by hand in the installed app with files at 312 to 380 characters. Co-Authored-By: Claude Opus 5.5 --- .../src/meshbay_node/indexer/enrich.py | 11 +++--- .../src/meshbay_node/indexer/enrich_audio.py | 18 ++++++---- .../src/meshbay_node/indexer/enrich_photo.py | 3 +- .../src/meshbay_node/indexer/indexer.py | 39 ++++++++++++++-------- 4 files changed, 46 insertions(+), 25 deletions(-) (limited to 'packages/meshbay-node/src/meshbay_node/indexer') diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich.py index 9881298..53f1b27 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich.py @@ -13,14 +13,16 @@ the rest asynchronously and hands the result back via a callback. import asyncio import logging +import os from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from meshbay_node.indexer import title_parse -from meshbay_node.indexer.indexer import MEDIA_EXTENSIONS +from meshbay_node.indexer.indexer import MEDIA_EXTENSIONS, walk_files from meshbay_node.media_cache import MediaCache from meshbay_node.media_probe import probe_video @@ -102,7 +104,8 @@ def _title_from_siblings(file_path: Path) -> str | None: from another Special just like this one. """ try: - names = sorted(p.name for p in file_path.parent.iterdir() if p.is_file()) + with os.scandir(long_path(file_path.parent)) as listing: + names = sorted(e.name for e in listing if e.is_file()) except OSError: return None checked = 0 @@ -142,8 +145,8 @@ def _synthetic_episode_number(file_path: Path, show_root: Path, season: int) -> """ try: candidates = sorted( - (p for p in show_root.rglob("*") - if p.is_file() and p.suffix.lower() in MEDIA_EXTENSIONS["video"]), + (p for p in walk_files(show_root) + if p.suffix.lower() in MEDIA_EXTENSIONS["video"]), key=lambda p: (str(p.parent), p.name), ) except OSError: diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py index 397f760..ecc3b5e 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py @@ -60,11 +60,13 @@ image-file fallback for cover art, same as everything else. import asyncio import logging +import os import re from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from mutagen import File as MutagenFile from mutagen.asf import ASF @@ -155,8 +157,9 @@ _WMP_ALBUMART_GUID_RE = re.compile(r"^albumart_\{([0-9a-f-]{36})\}_(large|small) def _find_sibling_cover(folder: Path) -> Path | None: try: - candidates = [p for p in folder.iterdir() - if p.is_file() and p.suffix.lower() in _COVER_EXTS] + with os.scandir(long_path(folder)) as listing: + candidates = [folder / e.name for e in listing + if e.is_file() and Path(e.name).suffix.lower() in _COVER_EXTS] except OSError: return None if not candidates: @@ -240,12 +243,12 @@ def _read_format_specific_tags(path: Path) -> tuple[dict, float | None] | None: suffix = path.suffix.lower() if suffix == ".wma": try: - return _asf_tags_from_object(ASF(str(path))) + return _asf_tags_from_object(ASF(long_path(path))) except Exception: return {}, None if suffix == ".mpc": try: - return _musepack_tags_from_object(Musepack(str(path))) + return _musepack_tags_from_object(Musepack(long_path(path))) except Exception: return {}, None return None @@ -257,7 +260,7 @@ def _read_generic_tags_and_cover( tags: dict = {} duration: float | None = None try: - easy = MutagenFile(str(path), easy=True) + easy = MutagenFile(long_path(path), easy=True) except Exception: easy = None if easy is not None: @@ -277,7 +280,7 @@ def _read_generic_tags_and_cover( cover: bytes | None = None if not skip_cover: try: - raw = MutagenFile(str(path)) + raw = MutagenFile(long_path(path)) except Exception: raw = None if raw is not None: @@ -339,7 +342,8 @@ def _read_sibling_cover(path: Path) -> bytes | None: if sibling is None: return None try: - return sibling.read_bytes() + with open(long_path(sibling), "rb") as f: + return f.read() except OSError: return None diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py index 5cacef9..a9f85d0 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py @@ -26,6 +26,7 @@ from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from PIL import ExifTags, Image, ImageOps @@ -74,7 +75,7 @@ def _read_image(file_path: Path) -> tuple[bytes, int, int, int | None, str | Non *original* image's own dimensions — the thumbnail is a separate, resized copy, never what width/height describe. """ - with Image.open(file_path) as img: + with Image.open(long_path(file_path)) as img: taken_at = None camera = None try: diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index e4679d4..55ec56a 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -82,7 +82,7 @@ def _is_indexable_name(name: str) -> bool: def _is_indexable(path: Path) -> bool: - return path.is_file() and _is_indexable_name(path.name) + return os.path.isfile(long_path(path)) and _is_indexable_name(path.name) # ── A disk that stops answering ─────────────────────────────────────────────── @@ -348,13 +348,25 @@ def _walk_root(root: Root) -> list[Path]: every other thing the daemon is doing — WebRTC sessions, chat, the admin UI — for as long as it takes. """ + return walk_files(root.path) + + +def walk_files(top: Path) -> list[Path]: + """ + Every file below `top`, links to files included, links to directories not + entered. Raises if `top` itself cannot be listed. Blocking. + + Each directory is listed under its long form and the paths returned are + built from the plain one, so a directory past Windows' MAX_PATH is listed + and everything downstream still sees, stores and compares plain paths. + """ files: list[Path] = [] - pending = [root.path] + pending = [top] while pending: directory = pending.pop() _beat(directory) try: - with os.scandir(directory) as listing: + with os.scandir(long_path(directory)) as listing: below: list[Path] = [] for entry in listing: _beat() @@ -363,13 +375,13 @@ def _walk_root(root: Root) -> list[Path]: # a file counts whether or not it is one. try: if entry.is_dir(follow_symlinks=False): - below.append(Path(entry.path)) + below.append(directory / entry.name) elif entry.is_file(): - files.append(Path(entry.path)) + files.append(directory / entry.name) except OSError: continue except OSError as e: - if directory == root.path: + if directory == top: raise # One directory that cannot be listed costs that directory. It # used to be silent for a refusal and to end the whole walk for @@ -406,7 +418,7 @@ def _size_files(files: list[Path]) -> list[tuple[Path, os.stat_result]]: for p in files: _beat(p.parent) try: - sized.append((p, p.stat())) + sized.append((p, os.stat(long_path(p)))) except OSError: continue return sized @@ -420,7 +432,7 @@ def _scan_file(root: Root, file_path: Path) -> IndexEntry | None: if not _is_indexable(file_path): return None try: - stat = file_path.stat() + stat = os.stat(long_path(file_path)) if not _is_indexable_size(file_path, stat.st_size): return None if stat.st_size > _PARTIAL_THRESHOLD: @@ -903,7 +915,7 @@ class DirectoryIndexer: if not _is_indexable(file_path): return None try: - st = file_path.stat() + st = os.stat(long_path(file_path)) except OSError: return None elif not _is_indexable_name(file_path.name): @@ -988,7 +1000,7 @@ class DirectoryIndexer: if self._cache is None or not user_id: return try: - st = file_path.stat() + st = os.stat(long_path(file_path)) except OSError: # Gone between the rename and here. Nothing to attribute, and # nothing for anyone to delete either. @@ -1565,7 +1577,7 @@ class DirectoryIndexer: return True for path, size, mtime in known: try: - st = Path(path).stat() + st = os.stat(long_path(path)) except OSError: continue if st.st_size == size and (mtime is None or st.st_mtime == mtime): @@ -1735,7 +1747,7 @@ class DirectoryIndexer: # bookkeeping still has to reach the `fire()` that actually runs. if not deleted and key not in self._burst_sizes: try: - size = file_path.stat().st_size + size = os.stat(long_path(file_path)).st_size except OSError: size = 0 if self._burst_inflight <= 0: @@ -1790,7 +1802,8 @@ class DirectoryIndexer: covers this directory too. """ try: - return not dir_path.exists() and dir_path.parent.is_dir() + return (not os.path.exists(long_path(dir_path)) + and os.path.isdir(long_path(dir_path.parent))) except OSError: return False -- cgit v1.2.3