diff options
Diffstat (limited to 'packages/meshbay-node/src')
14 files changed, 146 insertions, 70 deletions
diff --git a/packages/meshbay-node/src/meshbay_node/daemon.py b/packages/meshbay-node/src/meshbay_node/daemon.py index 022553e..557b523 100644 --- a/packages/meshbay-node/src/meshbay_node/daemon.py +++ b/packages/meshbay-node/src/meshbay_node/daemon.py @@ -39,7 +39,7 @@ from pathlib import Path import uvicorn from meshbay_common import MNP_VERSION from meshbay_common.background import spawn -from meshbay_common.paths import fold +from meshbay_common.paths import fold, long_path from meshbay_common.protocol import MNP from meshbay_node import uploads as uploads_mod @@ -1209,8 +1209,8 @@ class NodeDaemon(EnrichmentMixin): for path in uploads_mod.orphaned_parts( uploads_mod.find_parts(roots.roots), live, when): try: - size = path.stat().st_size - path.unlink() + size = os.stat(long_path(path)).st_size + os.unlink(long_path(path)) except OSError as exc: log.warning("Could not remove abandoned upload %s: %s", path.name, exc) diff --git a/packages/meshbay-node/src/meshbay_node/enrichment.py b/packages/meshbay-node/src/meshbay_node/enrichment.py index 19aafba..c09c5e5 100644 --- a/packages/meshbay-node/src/meshbay_node/enrichment.py +++ b/packages/meshbay-node/src/meshbay_node/enrichment.py @@ -2,8 +2,11 @@ thumbnail for Videos, tags and cover for Music, thumbnail and EXIF for Photos. NodeDaemon runs it; the enrichers themselves are in indexer/.""" +import os from dataclasses import replace +from meshbay_common.paths import long_path + from meshbay_node.indexer import DirectoryIndexer, GroupIndex from meshbay_node.roots import entry_abs_path @@ -67,7 +70,7 @@ class EnrichmentMixin: if not _under_any_directory(entry.path, video_dirs): continue file_path = entry_abs_path(indexer.roots, entry) - if not file_path or not file_path.exists(): + if not file_path or not os.path.exists(long_path(file_path)): continue self._enriched_attempted.add((indexer.group_id, entry.id)) @@ -153,7 +156,7 @@ class EnrichmentMixin: if owner is None: continue file_path = entry_abs_path(indexer.roots, entry) - if not file_path or not file_path.exists(): + if not file_path or not os.path.exists(long_path(file_path)): continue self._enriched_attempted.add((indexer.group_id, entry.id)) @@ -236,7 +239,7 @@ class EnrichmentMixin: if not _under_any_directory(entry.path, photo_dirs): continue file_path = entry_abs_path(indexer.roots, entry) - if not file_path or not file_path.exists(): + if not file_path or not os.path.exists(long_path(file_path)): continue self._enriched_attempted.add((indexer.group_id, entry.id)) diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich.py index 9881298..53f1b27 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich.py @@ -13,14 +13,16 @@ the rest asynchronously and hands the result back via a callback. import asyncio import logging +import os from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from meshbay_node.indexer import title_parse -from meshbay_node.indexer.indexer import MEDIA_EXTENSIONS +from meshbay_node.indexer.indexer import MEDIA_EXTENSIONS, walk_files from meshbay_node.media_cache import MediaCache from meshbay_node.media_probe import probe_video @@ -102,7 +104,8 @@ def _title_from_siblings(file_path: Path) -> str | None: from another Special just like this one. """ try: - names = sorted(p.name for p in file_path.parent.iterdir() if p.is_file()) + with os.scandir(long_path(file_path.parent)) as listing: + names = sorted(e.name for e in listing if e.is_file()) except OSError: return None checked = 0 @@ -142,8 +145,8 @@ def _synthetic_episode_number(file_path: Path, show_root: Path, season: int) -> """ try: candidates = sorted( - (p for p in show_root.rglob("*") - if p.is_file() and p.suffix.lower() in MEDIA_EXTENSIONS["video"]), + (p for p in walk_files(show_root) + if p.suffix.lower() in MEDIA_EXTENSIONS["video"]), key=lambda p: (str(p.parent), p.name), ) except OSError: diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py index 397f760..ecc3b5e 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py @@ -60,11 +60,13 @@ image-file fallback for cover art, same as everything else. import asyncio import logging +import os import re from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from mutagen import File as MutagenFile from mutagen.asf import ASF @@ -155,8 +157,9 @@ _WMP_ALBUMART_GUID_RE = re.compile(r"^albumart_\{([0-9a-f-]{36})\}_(large|small) def _find_sibling_cover(folder: Path) -> Path | None: try: - candidates = [p for p in folder.iterdir() - if p.is_file() and p.suffix.lower() in _COVER_EXTS] + with os.scandir(long_path(folder)) as listing: + candidates = [folder / e.name for e in listing + if e.is_file() and Path(e.name).suffix.lower() in _COVER_EXTS] except OSError: return None if not candidates: @@ -240,12 +243,12 @@ def _read_format_specific_tags(path: Path) -> tuple[dict, float | None] | None: suffix = path.suffix.lower() if suffix == ".wma": try: - return _asf_tags_from_object(ASF(str(path))) + return _asf_tags_from_object(ASF(long_path(path))) except Exception: return {}, None if suffix == ".mpc": try: - return _musepack_tags_from_object(Musepack(str(path))) + return _musepack_tags_from_object(Musepack(long_path(path))) except Exception: return {}, None return None @@ -257,7 +260,7 @@ def _read_generic_tags_and_cover( tags: dict = {} duration: float | None = None try: - easy = MutagenFile(str(path), easy=True) + easy = MutagenFile(long_path(path), easy=True) except Exception: easy = None if easy is not None: @@ -277,7 +280,7 @@ def _read_generic_tags_and_cover( cover: bytes | None = None if not skip_cover: try: - raw = MutagenFile(str(path)) + raw = MutagenFile(long_path(path)) except Exception: raw = None if raw is not None: @@ -339,7 +342,8 @@ def _read_sibling_cover(path: Path) -> bytes | None: if sibling is None: return None try: - return sibling.read_bytes() + with open(long_path(sibling), "rb") as f: + return f.read() except OSError: return None diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py index 5cacef9..a9f85d0 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py @@ -26,6 +26,7 @@ from collections.abc import Awaitable, Callable from pathlib import Path import blake3 +from meshbay_common.paths import long_path from meshbay_common.protocol import IndexEntry from PIL import ExifTags, Image, ImageOps @@ -74,7 +75,7 @@ def _read_image(file_path: Path) -> tuple[bytes, int, int, int | None, str | Non *original* image's own dimensions — the thumbnail is a separate, resized copy, never what width/height describe. """ - with Image.open(file_path) as img: + with Image.open(long_path(file_path)) as img: taken_at = None camera = None try: diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index e4679d4..55ec56a 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -82,7 +82,7 @@ def _is_indexable_name(name: str) -> bool: def _is_indexable(path: Path) -> bool: - return path.is_file() and _is_indexable_name(path.name) + return os.path.isfile(long_path(path)) and _is_indexable_name(path.name) # ── A disk that stops answering ─────────────────────────────────────────────── @@ -348,13 +348,25 @@ def _walk_root(root: Root) -> list[Path]: every other thing the daemon is doing — WebRTC sessions, chat, the admin UI — for as long as it takes. """ + return walk_files(root.path) + + +def walk_files(top: Path) -> list[Path]: + """ + Every file below `top`, links to files included, links to directories not + entered. Raises if `top` itself cannot be listed. Blocking. + + Each directory is listed under its long form and the paths returned are + built from the plain one, so a directory past Windows' MAX_PATH is listed + and everything downstream still sees, stores and compares plain paths. + """ files: list[Path] = [] - pending = [root.path] + pending = [top] while pending: directory = pending.pop() _beat(directory) try: - with os.scandir(directory) as listing: + with os.scandir(long_path(directory)) as listing: below: list[Path] = [] for entry in listing: _beat() @@ -363,13 +375,13 @@ def _walk_root(root: Root) -> list[Path]: # a file counts whether or not it is one. try: if entry.is_dir(follow_symlinks=False): - below.append(Path(entry.path)) + below.append(directory / entry.name) elif entry.is_file(): - files.append(Path(entry.path)) + files.append(directory / entry.name) except OSError: continue except OSError as e: - if directory == root.path: + if directory == top: raise # One directory that cannot be listed costs that directory. It # used to be silent for a refusal and to end the whole walk for @@ -406,7 +418,7 @@ def _size_files(files: list[Path]) -> list[tuple[Path, os.stat_result]]: for p in files: _beat(p.parent) try: - sized.append((p, p.stat())) + sized.append((p, os.stat(long_path(p)))) except OSError: continue return sized @@ -420,7 +432,7 @@ def _scan_file(root: Root, file_path: Path) -> IndexEntry | None: if not _is_indexable(file_path): return None try: - stat = file_path.stat() + stat = os.stat(long_path(file_path)) if not _is_indexable_size(file_path, stat.st_size): return None if stat.st_size > _PARTIAL_THRESHOLD: @@ -903,7 +915,7 @@ class DirectoryIndexer: if not _is_indexable(file_path): return None try: - st = file_path.stat() + st = os.stat(long_path(file_path)) except OSError: return None elif not _is_indexable_name(file_path.name): @@ -988,7 +1000,7 @@ class DirectoryIndexer: if self._cache is None or not user_id: return try: - st = file_path.stat() + st = os.stat(long_path(file_path)) except OSError: # Gone between the rename and here. Nothing to attribute, and # nothing for anyone to delete either. @@ -1565,7 +1577,7 @@ class DirectoryIndexer: return True for path, size, mtime in known: try: - st = Path(path).stat() + st = os.stat(long_path(path)) except OSError: continue if st.st_size == size and (mtime is None or st.st_mtime == mtime): @@ -1735,7 +1747,7 @@ class DirectoryIndexer: # bookkeeping still has to reach the `fire()` that actually runs. if not deleted and key not in self._burst_sizes: try: - size = file_path.stat().st_size + size = os.stat(long_path(file_path)).st_size except OSError: size = 0 if self._burst_inflight <= 0: @@ -1790,7 +1802,8 @@ class DirectoryIndexer: covers this directory too. """ try: - return not dir_path.exists() and dir_path.parent.is_dir() + return (not os.path.exists(long_path(dir_path)) + and os.path.isdir(long_path(dir_path.parent))) except OSError: return False diff --git a/packages/meshbay-node/src/meshbay_node/ops/files.py b/packages/meshbay-node/src/meshbay_node/ops/files.py index 4609627..a448103 100644 --- a/packages/meshbay-node/src/meshbay_node/ops/files.py +++ b/packages/meshbay-node/src/meshbay_node/ops/files.py @@ -4,8 +4,11 @@ from __future__ import annotations import asyncio import logging +import os from pathlib import Path +from meshbay_common.paths import long_path + from meshbay_node.ops.core import OpError, _group_ctx log = logging.getLogger("meshbay_node.ops") @@ -41,7 +44,7 @@ async def delete_file(state: dict, group_id: str, file_id: str) -> dict: f"not readable right now — the file is frozen, not gone", status=409) try: - path.unlink() + os.unlink(long_path(path)) except FileNotFoundError: # Already gone from disk; drop the stale entry rather than refusing. log.warning("Index named a file that is not on disk: %s", path) @@ -104,7 +107,7 @@ async def prune_index_cache(state: dict) -> dict: return True if any(not r.available for r in owning): return False - return not path.exists() + return not os.path.exists(long_path(path)) paths = await cache.all_paths() stale = await asyncio.to_thread(lambda: [p for p in paths if _is_stale(p)]) diff --git a/packages/meshbay-node/src/meshbay_node/roots.py b/packages/meshbay-node/src/meshbay_node/roots.py index 61fdc9b..94560f5 100644 --- a/packages/meshbay-node/src/meshbay_node/roots.py +++ b/packages/meshbay-node/src/meshbay_node/roots.py @@ -36,7 +36,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from pathlib import Path -from meshbay_common.paths import fold, portable_name_problem +from meshbay_common.paths import fold, long_path, portable_name_problem log = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def _free_name(directory: Path, filename: str, or another upload (C5a). """ def free(name: str) -> bool: - return name not in taken and not (directory / name).exists() + return name not in taken and not os.path.exists(long_path(directory / name)) if free(filename): return filename @@ -109,17 +109,17 @@ def publish_upload(part: Path, directory: Path, stored_name: str, filename: str, for _ in range(8): target = directory / name try: - os.link(part, target) + os.link(long_path(part), long_path(target)) except FileExistsError: name = _free_name(directory, filename, taken) continue except OSError: - if target.exists(): + if os.path.exists(long_path(target)): name = _free_name(directory, filename, taken) continue - part.rename(target) + os.rename(long_path(part), long_path(target)) return name - part.unlink() + os.unlink(long_path(part)) return name raise FileExistsError(stored_name) @@ -195,7 +195,7 @@ class Root: def is_live(self) -> bool: """Readable right now. The question `available` caches.""" try: - return self.path.is_dir() + return os.path.isdir(long_path(self.path)) except OSError: return False diff --git a/packages/meshbay-node/src/meshbay_node/transport/quic_server.py b/packages/meshbay-node/src/meshbay_node/transport/quic_server.py index 715448f..3974a2f 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/quic_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/quic_server.py @@ -48,6 +48,7 @@ from meshbay_common.handshake import ( quic_binding, verify_proof, ) +from meshbay_common.paths import long_path from meshbay_common.protocol import MNP, file_chunk_wire from meshbay_node.indexer import GroupIndex @@ -434,7 +435,7 @@ class _MNPServerProtocol(QuicConnectionProtocol): if file_path is None: self._send(stream_id, {"type": "error", "detail": ROOT_NOT_SERVED}) return - if not file_path.exists(): + if not os.path.exists(long_path(file_path)): self._send(stream_id, {"type": "error", "detail": "File not on disk"}) return @@ -463,7 +464,7 @@ def _read_and_encrypt( file_id: str = "", ) -> dict: """Read one chunk off disk and encrypt it, in the one shape every transport uses.""" - with open(file_path, "rb") as f: + with open(long_path(file_path), "rb") as f: f.seek(chunk_index * CHUNK_SIZE) plaintext = f.read(CHUNK_SIZE) return file_chunk_wire(gek, plaintext, chunk_index, file_hash, file_id) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/disk.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/disk.py index e298942..386b81f 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/disk.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/disk.py @@ -1,8 +1,15 @@ -"""Blocking disk work the session hands to `off_disk`.""" +""" +Blocking disk work the session hands to `off_disk`. +Every call takes the path's long form (`long_path`): a file past Windows' +MAX_PATH is indexed, so it is served, deleted and written beside like any other. +""" + +import os import shutil from pathlib import Path +from meshbay_common.paths import long_path from meshbay_common.protocol import file_chunk_wire from meshbay_node.roots import ROOT_NOT_SERVED, RootSet, entry_abs_path @@ -22,7 +29,7 @@ def _locate(roots: RootSet, entry) -> tuple[Path | None, str | None]: path = entry_abs_path(roots, entry) if path is None: return None, ROOT_NOT_SERVED - if not path.exists(): + if not os.path.exists(long_path(path)): return None, "File not on disk" return path, None @@ -37,15 +44,16 @@ def _mkdir_if_absent(target: Path) -> str | None: `mkdir` would raise where a refusal was meant. Blocking; called through `off_disk`. """ - if target.exists(): + if os.path.exists(long_path(target)): return "Already exists" - target.mkdir(parents=False) + os.mkdir(long_path(target)) return None def _is_empty_dir(target: Path) -> bool: """Blocking; called through `off_disk`.""" - return not any(target.iterdir()) + with os.scandir(long_path(target)) as listing: + return not any(listing) def _rmdir_if_empty(target: Path) -> bool: @@ -57,9 +65,10 @@ def _rmdir_if_empty(target: Path) -> bool: operator's browser, and a file can land in between. Two awaits here would reopen the same window one size smaller. Blocking; called through `off_disk`. """ - if any(target.iterdir()): - return False - target.rmdir() + with os.scandir(long_path(target)) as listing: + if any(listing): + return False + os.rmdir(long_path(target)) return True @@ -71,7 +80,7 @@ def _read_and_encrypt( file_id: str = "", ) -> dict: """Read one chunk off disk and encrypt it. Blocking; called through `off_disk`.""" - with open(file_path, "rb") as f: + with open(long_path(file_path), "rb") as f: f.seek(chunk_index * CHUNK_SIZE) plaintext = f.read(CHUNK_SIZE) return file_chunk_wire(gek, plaintext, chunk_index, file_hash, file_id) @@ -80,10 +89,24 @@ def _read_and_encrypt( def _free_bytes(directory: Path) -> int: """Space left on the filesystem holding `directory`. Blocking; called through `off_disk`, since a `statvfs` wakes a sleeping disk like any stat.""" - return shutil.disk_usage(directory).free + return shutil.disk_usage(long_path(directory)).free def _append_chunk(tmp_path: Path, chunk_bytes: bytes, first: bool) -> None: """Add one chunk to a partial upload. Blocking; called through `off_disk`.""" - with open(tmp_path, "wb" if first else "ab") as f: + with open(long_path(tmp_path), "wb" if first else "ab") as f: f.write(chunk_bytes) + + +def _unlink(path: Path, missing_ok: bool = False) -> None: + """Remove one file. Blocking; called through `off_disk`.""" + try: + os.unlink(long_path(path)) + except FileNotFoundError: + if not missing_ok: + raise + + +def _exists(path: Path) -> bool: + """Blocking; called through `off_disk`.""" + return os.path.exists(long_path(path)) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/files.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/files.py index eafdaed..9c5fab5 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/files.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/files.py @@ -17,6 +17,7 @@ from meshbay_node.transport.webrtc.disk import ( _mkdir_if_absent, _read_and_encrypt, _rmdir_if_empty, + _unlink, ) from meshbay_node.transport.webrtc.limits import CHUNK_SIZE, LEASE_GRANTED, LEASE_QUEUED from meshbay_node.transport.wire import index_sync_message, list_dirs_off_loop @@ -439,7 +440,7 @@ class FilesMixin: self._send({"type": "error", "detail": ROOT_NOT_SERVED}) return if file_path is not None: - await off_disk(ctx["roots"], file_path.unlink) + await off_disk(ctx["roots"], _unlink, file_path) log.info("File deleted: %s", entry.name) self._audit("file_delete", entry.name) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/upload_handlers.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/upload_handlers.py index c052703..0d5bb0e 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/upload_handlers.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/upload_handlers.py @@ -17,7 +17,7 @@ from meshbay_node.roots import ( publish_upload, shell_active, ) -from meshbay_node.transport.webrtc.disk import _append_chunk, _free_bytes +from meshbay_node.transport.webrtc.disk import _append_chunk, _exists, _free_bytes, _unlink from meshbay_node.transport.webrtc.limits import LEASE_GRANTED, LEASE_NONE, LEASE_QUEUED log = logging.getLogger("meshbay_node.transport.webrtc_server") @@ -365,7 +365,7 @@ class UploadMixin: if chunk_index == 0: # Backstop: _free_name already guarantees this, and it stays because # it asserts the invariant where the write happens. - if await off_disk(roots, final_path.exists): + if await off_disk(roots, _exists, final_path): _refuse("File already exists", "already_exists") return # Asked once, before the first byte, from the size the client @@ -396,7 +396,7 @@ class UploadMixin: if state.bytes + len(chunk_bytes) > self._max_upload_bytes(): uploads.drop(user_id, rel_dir, filename) - await off_disk(roots, tmp_path.unlink, True) + await off_disk(roots, _unlink, tmp_path, True) _refuse("Upload exceeds size limit", "too_large") return @@ -410,7 +410,7 @@ class UploadMixin: # space that ran out. The client starts over from zero, which is # what it would have to do for a resume against a full disk anyway. uploads.drop(user_id, rel_dir, filename) - await off_disk(roots, tmp_path.unlink, True) + await off_disk(roots, _unlink, tmp_path, True) log.warning("Upload refused: the disk holding %s is full", rel_dir) self._audit("upload_refused", "disk_full") _refuse("The node's disk is full", "disk_full") diff --git a/packages/meshbay-node/src/meshbay_node/transport/wire.py b/packages/meshbay-node/src/meshbay_node/transport/wire.py index 77c5d62..75d27c1 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/wire.py +++ b/packages/meshbay-node/src/meshbay_node/transport/wire.py @@ -32,6 +32,7 @@ from pathlib import Path from meshbay_common import MNP_VERSION from meshbay_common.groupbox import PURPOSE_INDEX, seal +from meshbay_common.paths import long_path from meshbay_common.protocol import MNP, index_entry_wire from meshbay_node.roots import RootSet, off_disk @@ -75,22 +76,40 @@ def _dir_roots(roots: RootSet) -> list[tuple[str, Path, bool]]: def _walk_dirs(dir_roots: list[tuple[str, Path, bool]]) -> list[str]: - """Blocking. Directories only: `os.walk` reads each one once and never - stats a file, which `rglob("*")` and an `is_dir()` per entry did.""" + """Blocking. Directories only: each is read once and no file is stat'ed, + which `rglob("*")` and an `is_dir()` per entry did. What `os.walk` did, + except that each directory is listed under its long form: `os.walk` skipped + whatever lay past Windows' MAX_PATH, without a word.""" out: list[str] = [] for name, path, available in dir_roots: out.append(name) if not available: continue - for top, subdirs, _files in os.walk(path): - # Pruned in place: nothing under a hidden directory is listed either. - subdirs[:] = [d for d in subdirs if not d.startswith(".")] - rel = Path(top).relative_to(path) - for d in subdirs: - out.append(f"{name}/{(rel / d).as_posix()}") + pending = [path] + while pending: + top = pending.pop() + try: + with os.scandir(long_path(top)) as listing: + # Nothing under a hidden directory is listed either. + subdirs = [e for e in listing + if not e.name.startswith(".") and _is_dir(e)] + except OSError: + continue + rel = top.relative_to(path) + for entry in subdirs: + out.append(f"{name}/{(rel / entry.name).as_posix()}") + if not entry.is_symlink(): # listed, not entered, as os.walk did + pending.append(top / entry.name) return sorted(out)[:MAX_DIRS] +def _is_dir(entry: os.DirEntry) -> bool: + try: + return entry.is_dir() + except OSError: + return False + + def index_sync_message(index, roots: RootSet | None, hidden=(), dirs: list[str] | None = None) -> dict: """ diff --git a/packages/meshbay-node/src/meshbay_node/uploads.py b/packages/meshbay-node/src/meshbay_node/uploads.py index c485e39..a0f65dc 100644 --- a/packages/meshbay-node/src/meshbay_node/uploads.py +++ b/packages/meshbay-node/src/meshbay_node/uploads.py @@ -24,6 +24,7 @@ build first. from __future__ import annotations +import os import re import secrets import time @@ -31,6 +32,10 @@ from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path +from meshbay_common.paths import long_path + +from meshbay_node.indexer.indexer import walk_files + # What an unfinished upload is called on disk while it is being written. The # node has always used this; it is named here because the reaper below has to # recognise one, and a second spelling of it would be a bug nobody could see. @@ -208,14 +213,14 @@ def find_parts(roots: Iterable) -> list[tuple[Path, float]]: if not getattr(root, "available", False): continue try: - candidates = root.path.rglob(f"*{PART_SUFFIX}") + candidates = walk_files(root.path) except OSError: continue for path in candidates: + if not path.name.endswith(PART_SUFFIX): + continue try: - if not path.is_file(): - continue - found.append((path, path.stat().st_mtime)) + found.append((path, os.stat(long_path(path)).st_mtime)) except OSError: continue return found |