From daec4bc6a2a9f9e038e5269586b06018ff327a27 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Wed, 14 Oct 2026 11:17:38 +0200 Subject: fix(node): index, serve and write files past Windows' MAX_PATH With the system's default setting (LongPathsEnabled off) every call given a plain path of 260 characters or more refuses it, so a deep library was partly invisible: the walk could not list such a directory and its files never reached the members. The node now hands each call that touches a group's disk the extended form (paths.long_path), and keeps the plain path in the index, the hash cache and the log. A directory is listed under its long form and its entries named from the plain one (indexer.walk_files, wire._walk_dirs). Covered: the walk, stats and hashing, reconcile and the watcher, the folder list members see, serving over WebRTC and QUIC, deletion, uploads and their recovery, folder creation and removal, audio tags and the cover beside a track, photos, and a show's episode numbering. ffmpeg extends a long path itself and keeps the plain one. long_path now normalises before prefixing (the extended form skips the system's normalisation) and starts at 248 characters, where creating a directory is already refused. test_long_paths.py runs with the setting off: 15 of its 17 tests fail without the change. Checked by hand in the installed app with files at 312 to 380 characters. Co-Authored-By: Claude Opus 5.5 --- packages/meshbay-node/src/meshbay_node/daemon.py | 6 +-- .../meshbay-node/src/meshbay_node/enrichment.py | 9 +++-- .../src/meshbay_node/indexer/enrich.py | 11 ++++-- .../src/meshbay_node/indexer/enrich_audio.py | 18 +++++---- .../src/meshbay_node/indexer/enrich_photo.py | 3 +- .../src/meshbay_node/indexer/indexer.py | 39 ++++++++++++------- .../meshbay-node/src/meshbay_node/ops/files.py | 7 +++- packages/meshbay-node/src/meshbay_node/roots.py | 14 +++---- .../src/meshbay_node/transport/quic_server.py | 5 ++- .../src/meshbay_node/transport/webrtc/disk.py | 45 ++++++++++++++++------ .../src/meshbay_node/transport/webrtc/files.py | 3 +- .../transport/webrtc/upload_handlers.py | 8 ++-- .../src/meshbay_node/transport/wire.py | 35 +++++++++++++---- packages/meshbay-node/src/meshbay_node/uploads.py | 13 +++++-- 14 files changed, 146 insertions(+), 70 deletions(-) (limited to 'packages/meshbay-node/src') diff --git a/packages/meshbay-node/src/meshbay_node/daemon.py b/packages/meshbay-node/src/meshbay_node/daemon.py index 022553e..557b523 100644 --- a/packages/meshbay-node/src/meshbay_node/daemon.py +++ b/packages/meshbay-node/src/meshbay_node/daemon.py @@ -39,7 +39,7 @@ from pathlib import Path import uvicorn from meshbay_common import MNP_VERSION from meshbay_common.background import spawn -from meshbay_common.paths import fold +from meshbay_common.paths import fold, long_path from meshbay_common.protocol import MNP from meshbay_node import uploads as uploads_mod @@ -1209,8 +1209,8 @@ class NodeDaemon(EnrichmentMixin): for path in uploads_mod.orphaned_parts( uploads_mod.find_parts(roots.roots), live, when): try: - size = path.stat().st_size - path.unlink() + size = os.stat(long_path(path)).st_size + os.unlink(long_path(path)) except OSError as exc: log.warning("Could not remove abandoned upload %s: %s", path.name, exc) diff --git a/packages/meshbay-node/src/meshbay_node/enrichment.py b/packages/meshbay-node/src/meshbay_node/enrichment.py index 19aafba..c09c5e5 100644 --- a/packages/meshbay-node/src/meshbay_node/enrichment.py +++ b/packages/meshbay-node/src/meshbay_node/enrichment.py @@ -2,8 +2,11 @@ thumbnail for Videos, tags and cover for Music, thumbnail and EXIF for Photos. NodeDaemon runs it; the enrichers themselves are in indexer/.""" +import os from dataclasses import replace +from meshbay_common.paths import long_path + from meshbay_node.indexer import DirectoryIndexer, GroupIndex from meshbay_node.roots import entry_abs_path @@ -67,7 +70,7 @@ class EnrichmentMixin: if not _under_any_directory(entry.path, video_dirs): continue file_path = entry_abs_path(indexer.roots, entry) - if not file_path or not file_path.exists(): + if not file_path or not os.path.exists(long_path(file_path)): continue self._enriched_attempted.add((indexer.group_id, entry.id)) @@ -153,7 +156,7 @@ class EnrichmentMixin: if owner is None: continue file_path = entry_abs_path(indexer.roots, entry) - if not file_path or not file_path.exists(): + if not file_path or not os.path.exists(long_path(file_path)): continue self._enriched_attempted.add((indexer.group_id, entry.id)) @@ -236,7 +239,7 @@ class EnrichmentMixin: if not _under_any_directory(entry.path, photo_dirs): continue file_path = entry_abs_path(indexer.roots, entry) - if not file_path or not file_path.exists(): + if not file_path or not os.path.exists(long_path(file_path)): continue self._enriched_attempted.add((indexer.group_id, entry.id)) diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich.py index 9881298..53f1b27 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich.py @@ -13,14 +13,16 @@ the rest asynchronously and hands the result back via a callback. import asyncio import logging +import os from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from meshbay_node.indexer import title_parse -from meshbay_node.indexer.indexer import MEDIA_EXTENSIONS +from meshbay_node.indexer.indexer import MEDIA_EXTENSIONS, walk_files from meshbay_node.media_cache import MediaCache from meshbay_node.media_probe import probe_video @@ -102,7 +104,8 @@ def _title_from_siblings(file_path: Path) -> str | None: from another Special just like this one. """ try: - names = sorted(p.name for p in file_path.parent.iterdir() if p.is_file()) + with os.scandir(long_path(file_path.parent)) as listing: + names = sorted(e.name for e in listing if e.is_file()) except OSError: return None checked = 0 @@ -142,8 +145,8 @@ def _synthetic_episode_number(file_path: Path, show_root: Path, season: int) -> """ try: candidates = sorted( - (p for p in show_root.rglob("*") - if p.is_file() and p.suffix.lower() in MEDIA_EXTENSIONS["video"]), + (p for p in walk_files(show_root) + if p.suffix.lower() in MEDIA_EXTENSIONS["video"]), key=lambda p: (str(p.parent), p.name), ) except OSError: diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py index 397f760..ecc3b5e 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py @@ -60,11 +60,13 @@ image-file fallback for cover art, same as everything else. import asyncio import logging +import os import re from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from mutagen import File as MutagenFile from mutagen.asf import ASF @@ -155,8 +157,9 @@ _WMP_ALBUMART_GUID_RE = re.compile(r"^albumart_\{([0-9a-f-]{36})\}_(large|small) def _find_sibling_cover(folder: Path) -> Path | None: try: - candidates = [p for p in folder.iterdir() - if p.is_file() and p.suffix.lower() in _COVER_EXTS] + with os.scandir(long_path(folder)) as listing: + candidates = [folder / e.name for e in listing + if e.is_file() and Path(e.name).suffix.lower() in _COVER_EXTS] except OSError: return None if not candidates: @@ -240,12 +243,12 @@ def _read_format_specific_tags(path: Path) -> tuple[dict, float | None] | None: suffix = path.suffix.lower() if suffix == ".wma": try: - return _asf_tags_from_object(ASF(str(path))) + return _asf_tags_from_object(ASF(long_path(path))) except Exception: return {}, None if suffix == ".mpc": try: - return _musepack_tags_from_object(Musepack(str(path))) + return _musepack_tags_from_object(Musepack(long_path(path))) except Exception: return {}, None return None @@ -257,7 +260,7 @@ def _read_generic_tags_and_cover( tags: dict = {} duration: float | None = None try: - easy = MutagenFile(str(path), easy=True) + easy = MutagenFile(long_path(path), easy=True) except Exception: easy = None if easy is not None: @@ -277,7 +280,7 @@ def _read_generic_tags_and_cover( cover: bytes | None = None if not skip_cover: try: - raw = MutagenFile(str(path)) + raw = MutagenFile(long_path(path)) except Exception: raw = None if raw is not None: @@ -339,7 +342,8 @@ def _read_sibling_cover(path: Path) -> bytes | None: if sibling is None: return None try: - return sibling.read_bytes() + with open(long_path(sibling), "rb") as f: + return f.read() except OSError: return None diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py index 5cacef9..a9f85d0 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py @@ -26,6 +26,7 @@ from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from PIL import ExifTags, Image, ImageOps @@ -74,7 +75,7 @@ def _read_image(file_path: Path) -> tuple[bytes, int, int, int | None, str | Non *original* image's own dimensions — the thumbnail is a separate, resized copy, never what width/height describe. """ - with Image.open(file_path) as img: + with Image.open(long_path(file_path)) as img: taken_at = None camera = None try: diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index e4679d4..55ec56a 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -82,7 +82,7 @@ def _is_indexable_name(name: str) -> bool: def _is_indexable(path: Path) -> bool: - return path.is_file() and _is_indexable_name(path.name) + return os.path.isfile(long_path(path)) and _is_indexable_name(path.name) # ── A disk that stops answering ─────────────────────────────────────────────── @@ -348,13 +348,25 @@ def _walk_root(root: Root) -> list[Path]: every other thing the daemon is doing — WebRTC sessions, chat, the admin UI — for as long as it takes. """ + return walk_files(root.path) + + +def walk_files(top: Path) -> list[Path]: + """ + Every file below `top`, links to files included, links to directories not + entered. Raises if `top` itself cannot be listed. Blocking. + + Each directory is listed under its long form and the paths returned are + built from the plain one, so a directory past Windows' MAX_PATH is listed + and everything downstream still sees, stores and compares plain paths. + """ files: list[Path] = [] - pending = [root.path] + pending = [top] while pending: directory = pending.pop() _beat(directory) try: - with os.scandir(directory) as listing: + with os.scandir(long_path(directory)) as listing: below: list[Path] = [] for entry in listing: _beat() @@ -363,13 +375,13 @@ def _walk_root(root: Root) -> list[Path]: # a file counts whether or not it is one. try: if entry.is_dir(follow_symlinks=False): - below.append(Path(entry.path)) + below.append(directory / entry.name) elif entry.is_file(): - files.append(Path(entry.path)) + files.append(directory / entry.name) except OSError: continue except OSError as e: - if directory == root.path: + if directory == top: raise # One directory that cannot be listed costs that directory. It # used to be silent for a refusal and to end the whole walk for @@ -406,7 +418,7 @@ def _size_files(files: list[Path]) -> list[tuple[Path, os.stat_result]]: for p in files: _beat(p.parent) try: - sized.append((p, p.stat())) + sized.append((p, os.stat(long_path(p)))) except OSError: continue return sized @@ -420,7 +432,7 @@ def _scan_file(root: Root, file_path: Path) -> IndexEntry | None: if not _is_indexable(file_path): return None try: - stat = file_path.stat() + stat = os.stat(long_path(file_path)) if not _is_indexable_size(file_path, stat.st_size): return None if stat.st_size > _PARTIAL_THRESHOLD: @@ -903,7 +915,7 @@ class DirectoryIndexer: if not _is_indexable(file_path): return None try: - st = file_path.stat() + st = os.stat(long_path(file_path)) except OSError: return None elif not _is_indexable_name(file_path.name): @@ -988,7 +1000,7 @@ class DirectoryIndexer: if self._cache is None or not user_id: return try: - st = file_path.stat() + st = os.stat(long_path(file_path)) except OSError: # Gone between the rename and here. Nothing to attribute, and # nothing for anyone to delete either. @@ -1565,7 +1577,7 @@ class DirectoryIndexer: return True for path, size, mtime in known: try: - st = Path(path).stat() + st = os.stat(long_path(path)) except OSError: continue if st.st_size == size and (mtime is None or st.st_mtime == mtime): @@ -1735,7 +1747,7 @@ class DirectoryIndexer: # bookkeeping still has to reach the `fire()` that actually runs. if not deleted and key not in self._burst_sizes: try: - size = file_path.stat().st_size + size = os.stat(long_path(file_path)).st_size except OSError: size = 0 if self._burst_inflight <= 0: @@ -1790,7 +1802,8 @@ class DirectoryIndexer: covers this directory too. """ try: - return not dir_path.exists() and dir_path.parent.is_dir() + return (not os.path.exists(long_path(dir_path)) + and os.path.isdir(long_path(dir_path.parent))) except OSError: return False diff --git a/packages/meshbay-node/src/meshbay_node/ops/files.py b/packages/meshbay-node/src/meshbay_node/ops/files.py index 4609627..a448103 100644 --- a/packages/meshbay-node/src/meshbay_node/ops/files.py +++ b/packages/meshbay-node/src/meshbay_node/ops/files.py @@ -4,8 +4,11 @@ from __future__ import annotations import asyncio import logging +import os from pathlib import Path +from meshbay_common.paths import long_path + from meshbay_node.ops.core import OpError, _group_ctx log = logging.getLogger("meshbay_node.ops") @@ -41,7 +44,7 @@ async def delete_file(state: dict, group_id: str, file_id: str) -> dict: f"not readable right now — the file is frozen, not gone", status=409) try: - path.unlink() + os.unlink(long_path(path)) except FileNotFoundError: # Already gone from disk; drop the stale entry rather than refusing. log.warning("Index named a file that is not on disk: %s", path) @@ -104,7 +107,7 @@ async def prune_index_cache(state: dict) -> dict: return True if any(not r.available for r in owning): return False - return not path.exists() + return not os.path.exists(long_path(path)) paths = await cache.all_paths() stale = await asyncio.to_thread(lambda: [p for p in paths if _is_stale(p)]) diff --git a/packages/meshbay-node/src/meshbay_node/roots.py b/packages/meshbay-node/src/meshbay_node/roots.py index 61fdc9b..94560f5 100644 --- a/packages/meshbay-node/src/meshbay_node/roots.py +++ b/packages/meshbay-node/src/meshbay_node/roots.py @@ -36,7 +36,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from pathlib import Path -from meshbay_common.paths import fold, portable_name_problem +from meshbay_common.paths import fold, long_path, portable_name_problem log = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def _free_name(directory: Path, filename: str, or another upload (C5a). """ def free(name: str) -> bool: - return name not in taken and not (directory / name).exists() + return name not in taken and not os.path.exists(long_path(directory / name)) if free(filename): return filename @@ -109,17 +109,17 @@ def publish_upload(part: Path, directory: Path, stored_name: str, filename: str, for _ in range(8): target = directory / name try: - os.link(part, target) + os.link(long_path(part), long_path(target)) except FileExistsError: name = _free_name(directory, filename, taken) continue except OSError: - if target.exists(): + if os.path.exists(long_path(target)): name = _free_name(directory, filename, taken) continue - part.rename(target) + os.rename(long_path(part), long_path(target)) return name - part.unlink() + os.unlink(long_path(part)) return name raise FileExistsError(stored_name) @@ -195,7 +195,7 @@ class Root: def is_live(self) -> bool: """Readable right now. The question `available` caches.""" try: - return self.path.is_dir() + return os.path.isdir(long_path(self.path)) except OSError: return False diff --git a/packages/meshbay-node/src/meshbay_node/transport/quic_server.py b/packages/meshbay-node/src/meshbay_node/transport/quic_server.py index 715448f..3974a2f 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/quic_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/quic_server.py @@ -48,6 +48,7 @@ from meshbay_common.handshake import ( quic_binding, verify_proof, ) +from meshbay_common.paths import long_path from meshbay_common.protocol import MNP, file_chunk_wire from meshbay_node.indexer import GroupIndex @@ -434,7 +435,7 @@ class _MNPServerProtocol(QuicConnectionProtocol): if file_path is None: self._send(stream_id, {"type": "error", "detail": ROOT_NOT_SERVED}) return - if not file_path.exists(): + if not os.path.exists(long_path(file_path)): self._send(stream_id, {"type": "error", "detail": "File not on disk"}) return @@ -463,7 +464,7 @@ def _read_and_encrypt( file_id: str = "", ) -> dict: """Read one chunk off disk and encrypt it, in the one shape every transport uses.""" - with open(file_path, "rb") as f: + with open(long_path(file_path), "rb") as f: f.seek(chunk_index * CHUNK_SIZE) plaintext = f.read(CHUNK_SIZE) return file_chunk_wire(gek, plaintext, chunk_index, file_hash, file_id) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/disk.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/disk.py index e298942..386b81f 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/disk.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/disk.py @@ -1,8 +1,15 @@ -"""Blocking disk work the session hands to `off_disk`.""" +""" +Blocking disk work the session hands to `off_disk`. +Every call takes the path's long form (`long_path`): a file past Windows' +MAX_PATH is indexed, so it is served, deleted and written beside like any other. +""" + +import os import shutil from pathlib import Path +from meshbay_common.paths import long_path from meshbay_common.protocol import file_chunk_wire from meshbay_node.roots import ROOT_NOT_SERVED, RootSet, entry_abs_path @@ -22,7 +29,7 @@ def _locate(roots: RootSet, entry) -> tuple[Path | None, str | None]: path = entry_abs_path(roots, entry) if path is None: return None, ROOT_NOT_SERVED - if not path.exists(): + if not os.path.exists(long_path(path)): return None, "File not on disk" return path, None @@ -37,15 +44,16 @@ def _mkdir_if_absent(target: Path) -> str | None: `mkdir` would raise where a refusal was meant. Blocking; called through `off_disk`. """ - if target.exists(): + if os.path.exists(long_path(target)): return "Already exists" - target.mkdir(parents=False) + os.mkdir(long_path(target)) return None def _is_empty_dir(target: Path) -> bool: """Blocking; called through `off_disk`.""" - return not any(target.iterdir()) + with os.scandir(long_path(target)) as listing: + return not any(listing) def _rmdir_if_empty(target: Path) -> bool: @@ -57,9 +65,10 @@ def _rmdir_if_empty(target: Path) -> bool: operator's browser, and a file can land in between. Two awaits here would reopen the same window one size smaller. Blocking; called through `off_disk`. """ - if any(target.iterdir()): - return False - target.rmdir() + with os.scandir(long_path(target)) as listing: + if any(listing): + return False + os.rmdir(long_path(target)) return True @@ -71,7 +80,7 @@ def _read_and_encrypt( file_id: str = "", ) -> dict: """Read one chunk off disk and encrypt it. Blocking; called through `off_disk`.""" - with open(file_path, "rb") as f: + with open(long_path(file_path), "rb") as f: f.seek(chunk_index * CHUNK_SIZE) plaintext = f.read(CHUNK_SIZE) return file_chunk_wire(gek, plaintext, chunk_index, file_hash, file_id) @@ -80,10 +89,24 @@ def _read_and_encrypt( def _free_bytes(directory: Path) -> int: """Space left on the filesystem holding `directory`. Blocking; called through `off_disk`, since a `statvfs` wakes a sleeping disk like any stat.""" - return shutil.disk_usage(directory).free + return shutil.disk_usage(long_path(directory)).free def _append_chunk(tmp_path: Path, chunk_bytes: bytes, first: bool) -> None: """Add one chunk to a partial upload. Blocking; called through `off_disk`.""" - with open(tmp_path, "wb" if first else "ab") as f: + with open(long_path(tmp_path), "wb" if first else "ab") as f: f.write(chunk_bytes) + + +def _unlink(path: Path, missing_ok: bool = False) -> None: + """Remove one file. Blocking; called through `off_disk`.""" + try: + os.unlink(long_path(path)) + except FileNotFoundError: + if not missing_ok: + raise + + +def _exists(path: Path) -> bool: + """Blocking; called through `off_disk`.""" + return os.path.exists(long_path(path)) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/files.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/files.py index eafdaed..9c5fab5 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/files.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/files.py @@ -17,6 +17,7 @@ from meshbay_node.transport.webrtc.disk import ( _mkdir_if_absent, _read_and_encrypt, _rmdir_if_empty, + _unlink, ) from meshbay_node.transport.webrtc.limits import CHUNK_SIZE, LEASE_GRANTED, LEASE_QUEUED from meshbay_node.transport.wire import index_sync_message, list_dirs_off_loop @@ -439,7 +440,7 @@ class FilesMixin: self._send({"type": "error", "detail": ROOT_NOT_SERVED}) return if file_path is not None: - await off_disk(ctx["roots"], file_path.unlink) + await off_disk(ctx["roots"], _unlink, file_path) log.info("File deleted: %s", entry.name) self._audit("file_delete", entry.name) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/upload_handlers.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/upload_handlers.py index c052703..0d5bb0e 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/upload_handlers.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/upload_handlers.py @@ -17,7 +17,7 @@ from meshbay_node.roots import ( publish_upload, shell_active, ) -from meshbay_node.transport.webrtc.disk import _append_chunk, _free_bytes +from meshbay_node.transport.webrtc.disk import _append_chunk, _exists, _free_bytes, _unlink from meshbay_node.transport.webrtc.limits import LEASE_GRANTED, LEASE_NONE, LEASE_QUEUED log = logging.getLogger("meshbay_node.transport.webrtc_server") @@ -365,7 +365,7 @@ class UploadMixin: if chunk_index == 0: # Backstop: _free_name already guarantees this, and it stays because # it asserts the invariant where the write happens. - if await off_disk(roots, final_path.exists): + if await off_disk(roots, _exists, final_path): _refuse("File already exists", "already_exists") return # Asked once, before the first byte, from the size the client @@ -396,7 +396,7 @@ class UploadMixin: if state.bytes + len(chunk_bytes) > self._max_upload_bytes(): uploads.drop(user_id, rel_dir, filename) - await off_disk(roots, tmp_path.unlink, True) + await off_disk(roots, _unlink, tmp_path, True) _refuse("Upload exceeds size limit", "too_large") return @@ -410,7 +410,7 @@ class UploadMixin: # space that ran out. The client starts over from zero, which is # what it would have to do for a resume against a full disk anyway. uploads.drop(user_id, rel_dir, filename) - await off_disk(roots, tmp_path.unlink, True) + await off_disk(roots, _unlink, tmp_path, True) log.warning("Upload refused: the disk holding %s is full", rel_dir) self._audit("upload_refused", "disk_full") _refuse("The node's disk is full", "disk_full") diff --git a/packages/meshbay-node/src/meshbay_node/transport/wire.py b/packages/meshbay-node/src/meshbay_node/transport/wire.py index 77c5d62..75d27c1 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/wire.py +++ b/packages/meshbay-node/src/meshbay_node/transport/wire.py @@ -32,6 +32,7 @@ from pathlib import Path from meshbay_common import MNP_VERSION from meshbay_common.groupbox import PURPOSE_INDEX, seal +from meshbay_common.paths import long_path from meshbay_common.protocol import MNP, index_entry_wire from meshbay_node.roots import RootSet, off_disk @@ -75,22 +76,40 @@ def _dir_roots(roots: RootSet) -> list[tuple[str, Path, bool]]: def _walk_dirs(dir_roots: list[tuple[str, Path, bool]]) -> list[str]: - """Blocking. Directories only: `os.walk` reads each one once and never - stats a file, which `rglob("*")` and an `is_dir()` per entry did.""" + """Blocking. Directories only: each is read once and no file is stat'ed, + which `rglob("*")` and an `is_dir()` per entry did. What `os.walk` did, + except that each directory is listed under its long form: `os.walk` skipped + whatever lay past Windows' MAX_PATH, without a word.""" out: list[str] = [] for name, path, available in dir_roots: out.append(name) if not available: continue - for top, subdirs, _files in os.walk(path): - # Pruned in place: nothing under a hidden directory is listed either. - subdirs[:] = [d for d in subdirs if not d.startswith(".")] - rel = Path(top).relative_to(path) - for d in subdirs: - out.append(f"{name}/{(rel / d).as_posix()}") + pending = [path] + while pending: + top = pending.pop() + try: + with os.scandir(long_path(top)) as listing: + # Nothing under a hidden directory is listed either. + subdirs = [e for e in listing + if not e.name.startswith(".") and _is_dir(e)] + except OSError: + continue + rel = top.relative_to(path) + for entry in subdirs: + out.append(f"{name}/{(rel / entry.name).as_posix()}") + if not entry.is_symlink(): # listed, not entered, as os.walk did + pending.append(top / entry.name) return sorted(out)[:MAX_DIRS] +def _is_dir(entry: os.DirEntry) -> bool: + try: + return entry.is_dir() + except OSError: + return False + + def index_sync_message(index, roots: RootSet | None, hidden=(), dirs: list[str] | None = None) -> dict: """ diff --git a/packages/meshbay-node/src/meshbay_node/uploads.py b/packages/meshbay-node/src/meshbay_node/uploads.py index c485e39..a0f65dc 100644 --- a/packages/meshbay-node/src/meshbay_node/uploads.py +++ b/packages/meshbay-node/src/meshbay_node/uploads.py @@ -24,6 +24,7 @@ build first. from __future__ import annotations +import os import re import secrets import time @@ -31,6 +32,10 @@ from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path +from meshbay_common.paths import long_path + +from meshbay_node.indexer.indexer import walk_files + # What an unfinished upload is called on disk while it is being written. The # node has always used this; it is named here because the reaper below has to # recognise one, and a second spelling of it would be a bug nobody could see. @@ -208,14 +213,14 @@ def find_parts(roots: Iterable) -> list[tuple[Path, float]]: if not getattr(root, "available", False): continue try: - candidates = root.path.rglob(f"*{PART_SUFFIX}") + candidates = walk_files(root.path) except OSError: continue for path in candidates: + if not path.name.endswith(PART_SUFFIX): + continue try: - if not path.is_file(): - continue - found.append((path, path.stat().st_mtime)) + found.append((path, os.stat(long_path(path)).st_mtime)) except OSError: continue return found -- cgit v1.2.3