diff options
Diffstat (limited to 'packages/meshbay-node/src/meshbay_node/indexer')
5 files changed, 64 insertions, 58 deletions
diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich.py index 6eeb1ef..b44a246 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich.py @@ -4,7 +4,7 @@ filename parsing (title_parse), and thumbnail generation (ffmpeg) for a newly-added video IndexEntry. Runs through its own small bounded worker pool — separate from the streaming -transcode pool (docs/mediacenter.md §5.2, mirroring webrtc_server.py's +transcode pool (docs/MESHBAY_DESIGN.md §6.5, mirroring webrtc_server.py's `_transcode_semaphore`) — so indexing a large library never blocks on this, and enrichment never competes with an active viewer for CPU. The scan itself already put the entry in the index with hash/size/type only; this fills in @@ -33,26 +33,27 @@ THUMB_WIDTH = 320 # "Show/SeasonFolder/episode.mkv" is the expected shape, with a little slack # for an extra wrapper folder — not an attempt to find the exact group root. MAX_ANCESTOR_DEPTH = 4 -# Bounds the "borrow a title from a sibling episode filename" scan (§3.4) so -# a folder with thousands of files costs a fixed, small amount of work. +# Bounds the "borrow a title from a sibling episode filename" scan +# (docs/MESHBAY_DESIGN.md §9.7) so a folder with thousands of files costs a +# fixed, small amount of work. MAX_SIBLINGS_CHECKED = 20 # Bounds the whole-show scan _synthetic_episode_number uses to rank a -# season's files across more than one folder (§3.4c) — a show's total file -# count, not just one folder's, so this needs more headroom than -# MAX_SIBLINGS_CHECKED. +# season's files across more than one folder (docs/MESHBAY_DESIGN.md §9.7) — +# a show's total file count, not just one folder's, so this needs more +# headroom than MAX_SIBLINGS_CHECKED. MAX_SEASON_FILES_CHECKED = 500 def _season_and_show_from_ancestors(file_path: Path) -> tuple[int, Path] | None: """ - §3.4/§3.4c: walk up ancestor folders for a season-like one (a numbered - season, or Specials/Bonus/Extras -> season 0) — season from the first - (innermost) match, but the show's own name from *above every - consecutive season-like ancestor*, not just the first one. A - per-season Bonus folder (`Show/Season N/Bonus/file.ext`) is nested two - levels inside the show, both of them season-like on their own - ("Bonus" and "Season N") — stopping at the first would hand back - "Season N" as the show's name instead of "Show". + docs/MESHBAY_DESIGN.md §9.7: walk up ancestor folders for a season-like one + (a numbered season, or Specials/Bonus/Extras -> season 0) — season from the + first (innermost) match, but the show's own name from *above every + consecutive season-like ancestor*, not just the first one. A per-season + Bonus folder (`Show/Season N/Bonus/file.ext`) is nested two levels inside + the show, both of them season-like on their own ("Bonus" and "Season N") — + stopping at the first would hand back "Season N" as the show's name instead + of "Show". Trusted over any per-file guessit title once found: a bare episode numbering convention with no show name embedded at all @@ -87,15 +88,17 @@ def _season_and_show_from_ancestors(file_path: Path) -> tuple[int, Path] | None: def _title_from_siblings(file_path: Path) -> str | None: """ - §3.4: an episode filename with no show name in it borrows the title from - a representative sibling in the same folder, never from the folder name - alone (an acronym-named show folder is a real, observed case). + docs/MESHBAY_DESIGN.md §9.7: an episode filename with no show name in it + borrows the title from a representative sibling in the same folder, never + from the folder name alone (an acronym-named show folder is a real, + observed case). Requires the sibling to carry its own episode number too, not just a - title — a folder where every file is a one-off-named Special (§3.4b) - has plenty of `display_title`s (guessit reads *a* title off nearly - anything) but none of them name the show; requiring a real episode - number alongside is what tells apart a genuinely representative sibling + title — a folder where every file is a one-off-named Special + (docs/MESHBAY_DESIGN.md §9.7) has plenty of `display_title`s (guessit + reads *a* title off nearly anything) but none of them name the show; + requiring a real episode number alongside is what tells apart a + genuinely representative sibling from another Special just like this one. """ try: @@ -119,11 +122,11 @@ def _title_from_siblings(file_path: Path) -> str | None: def _synthetic_episode_number(file_path: Path, show_root: Path, season: int) -> int: """ - §3.4b/§3.4c: a Specials/Bonus folder's files often carry no episode - number at all — each is just named after its own one-off title. The - frontend (video-app.js's buildSeasons) sorts within a season by this - number but only needs it to provide a stable order, not to mean - anything beyond that. + docs/MESHBAY_DESIGN.md §9.7: a Specials/Bonus folder's files often carry no + episode number at all — each is just named after its own one-off title. The + frontend (video-app.js's buildSeasons) sorts within a season by this number + but only needs it to provide a stable order, not to mean anything beyond + that. Ranked across the *whole show*, not just this file's own folder: season 0 routinely spans more than one folder under the show's root — a @@ -289,10 +292,11 @@ class Enricher: and (title_parse.has_episode_marker(entry.name) or title_parse.year_in(entry.name) is None)): # No season-like ancestor at all (a flat library) but the - # filename itself carries season+episode (§3.4) — *and* it - # is a real marker, not guessit reading a bare number as - # SxxExx. A movie whose "1080p" tag was truncated to "108", - # or "1280" left in the name, otherwise parses to S01E08 / + # filename itself carries season+episode + # (docs/MESHBAY_DESIGN.md §9.7) — *and* it is a real marker, + # not guessit reading a bare number as SxxExx. A movie whose + # "1080p" tag was truncated to "108", or "1280" left in the + # name, otherwise parses to S01E08 / # S12E80 and gets shelved as a nonexistent series # (found live 2026-08-29). A genuine flat-dumped episode # has an explicit SxxExx/1x08/"Episode N" marker; a movie diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py index 4fa08ef..d484e35 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich_audio.py @@ -1,7 +1,7 @@ """ Index-time enrichment for the Music group app: embedded tag/cover extraction (mutagen) and filename-parse fallback for a newly-added audio -IndexEntry (docs/musicbay.md §2.1, §6). +IndexEntry (docs/MESHBAY_DESIGN.md §9.8). Runs through its own small bounded worker pool, the same discipline as the Videos app's `enrich.py` — separate from any other pool, never blocking a @@ -15,14 +15,13 @@ block the event loop. MusicBrainz lookups are **not** done here. Tag/cover extraction is free and local, so it runs for every audio file the Music app is enabled for, regardless of whether MusicBrainz itself is turned on for the group — the -flat view (docs/musicbay.md §5.2) needs nothing more than this. MusicBrainz +flat view (docs/MESHBAY_DESIGN.md §9.8) needs nothing more than this. MusicBrainz is a separate, lazy, per-request enrichment (`music_meta_req`, handled in webrtc_server.py), the same "fetched on demand, cached once" shape TMDB already uses. **Revised 2026-08-24** against a real ~5700-file library (folder-per-artist -mostly, but not uniformly — see musicbay.md's own "what got measured" note -if one gets added). Two findings drove this revision, both confirmed with +mostly, but not uniformly). Two findings drove this revision, both confirmed with real data before writing the fix: 1. The original ancestor walk always went up two levels (parent = album, diff --git a/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py b/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py index 93a68cc..6613f85 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/enrich_photo.py @@ -3,7 +3,7 @@ Index-time enrichment for the Photos group app: a resized thumbnail and a minimal, best-effort info set (`taken_at`, `camera`) read from the image's own EXIF block, for a newly-added image IndexEntry. -Deliberately small — docs/photos.md §2.4 is explicit that this app does not +Deliberately small — docs/MESHBAY_DESIGN.md §9.9 is explicit that this app does not build a full EXIF-viewer panel. Two fields only, both best-effort (missing EXIF is the ordinary case for a screenshot or a re-saved/edited image, not an error). GPS is never read here, on purpose: it is a location disclosure @@ -12,7 +12,8 @@ module extracts, caches, or hands it to a caller. Runs through its own small bounded worker pool, separate from the video (ffmpeg) and audio (mutagen) enrichment pools — mirrors enrich.py exactly, -per docs/photos.md §5's "never shared with either" rule, even though +per docs/MESHBAY_DESIGN.md §6.5's "its own small bounded pool, never the +streaming pool" rule, even though Pillow's own work is comparatively cheap: a burst of hundreds of newly shared photos should not peg every CPU core at once. """ @@ -100,7 +101,7 @@ def _read_image(file_path: Path) -> tuple[bytes, int, int, int | None, str | Non # width/height and resizing — otherwise a phone photo stored # "sideways" reports its raw, pre-rotation dimensions (swapped from # what it actually displays as) and produces a sideways thumbnail - # (docs/photos.md §2.4). Never reads Orientation itself as a + # (docs/MESHBAY_DESIGN.md §9.9). Never reads Orientation itself as a # client-visible field; this is display correction only, and # width/height must describe the *displayed* image, matching what # the lightbox and the info panel show. diff --git a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py index 887d435..3fe4f3e 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/indexer.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/indexer.py @@ -82,7 +82,8 @@ def _is_indexable(path: Path) -> bool: # Found live: a 1256-byte ".mp3" with no audio stream at all, just an ID3 # tag — a truncated/corrupted rip, sitting between two good tracks of the -# same album (docs/musicbay.md). A source this small claiming to be audio +# same album (docs/MESHBAY_DESIGN.md §9.8). A source this small claiming to +# be audio # is far more likely broken than real, so it is skipped before ever being # hashed rather than indexed and left to fail at playback time. Scoped to # audio only — a tiny real file of any other type is still worth indexing. @@ -186,9 +187,8 @@ def _size_files(files: list[Path]) -> list[tuple[Path, int]]: executor for the same reason `_walk_root` is (its own docstring above). Previously a plain loop straight on the asyncio event loop thread: for a root with many thousands of files (a real personal library, not a - hypothetical — docs/musicbay.md's own "several thousand files" example) - that blocked the entire daemon, every WebRTC session and the admin UI - included, for as long as the stat() calls took — and did so *before* + hypothetical) that blocked the entire daemon, every WebRTC session and + the admin UI included, for as long as the stat() calls took — and did so *before* `_scan_root` had even set `progress.scanning`, so a consumer polling it saw "not scanning" the whole time real, blocking work was happening. """ @@ -673,8 +673,9 @@ class DirectoryIndexer: Entries under a root that is gone from the config are dropped — the operator removed it deliberately, which is not the same event as a - volume disappearing, and conflating the two is what §6.9 exists to - prevent. Roots that survive keep their entries; new ones are scanned. + volume disappearing, and conflating the two is what + docs/MESHBAY_DESIGN.md §6.2 exists to prevent. Roots that survive + keep their entries; new ones are scanned. The set takes effect before anything is scanned: the roots table, the watcher and `self.roots` all move at once. With ``wait=False`` the scan diff --git a/packages/meshbay-node/src/meshbay_node/indexer/title_parse.py b/packages/meshbay-node/src/meshbay_node/indexer/title_parse.py index beb0d3f..14728ff 100644 --- a/packages/meshbay-node/src/meshbay_node/indexer/title_parse.py +++ b/packages/meshbay-node/src/meshbay_node/indexer/title_parse.py @@ -1,18 +1,18 @@ """ Filename -> title/year/season/episode parsing for the Videos group app. -Wraps `guessit` and layers the fixes from docs/mediacenter.md §3.3/§3.4 on +Wraps `guessit` and layers the fixes from docs/MESHBAY_DESIGN.md §9.7 on top of it: none of them are per-title hacks, each is a generic rule found by validating guessit's raw output against real TMDB search results over a ~1950-file library (movies, TV shows, and a small franchise set). -Scope is deliberately narrow (§3.5): title, year, season, episode. Technical +Scope is deliberately narrow: title, year, season, episode. Technical facts (resolution, codec, duration) come from ffprobe, never the filename — a mislabeled `1080p` tag is a real, observed failure mode. This module never touches the filesystem or the network. The orchestration that decides *which* file supplies a show's title (a representative episode -filename, not the folder name — §3.4) lives in the indexer, which has the +filename, not the folder name) lives in the indexer, which has the directory listing; this module only parses strings it's handed. """ @@ -91,7 +91,7 @@ _YEAR_RE = re.compile(r"(?<!\d)(?:19|20)\d{2}(?!\d)") def year_in(text: str) -> int | None: """First 19xx/20xx in `text`, or None — used to lift a year off a show - folder name ("Some.Show.2022.S01") for the search fallback (§10.1/V8).""" + folder name ("Some.Show.2022.S01") for the search fallback (V8).""" m = _YEAR_RE.search(text or "") return int(m.group(0)) if m else None @@ -102,7 +102,7 @@ def clean_query(s: str) -> str: parenthesized-year stripping `naive_title` does. `naive_title` assumes a real filename; a show's `display_title` is a folder basename ("Some.Show.Name" — `rsplit('.', 1)` would eat ".Name"), so it needs a - gentler normaliser (§10.1/V8). + gentler normaliser (V8). """ s = re.sub(r"[._-]+", " ", s or "") s = _strip_editions(s) @@ -115,7 +115,7 @@ def _strip_editions(title: str) -> str: def naive_title(filename: str) -> str: """ - The mandated fallback (§3.6, §4.1): strip the extension, replace every + The mandated fallback: strip the extension, replace every `.`/`_`/`-` with a space, drop a trailing parenthesized year, collapse whitespace. Always computable, never fails, used both as the flat-mode display name of last resort and as a second TMDB query candidate. @@ -150,14 +150,14 @@ def sequel_variants(title: str) -> list[str]: A trailing sequel index often has no exact match in the real TMDB title: the file has a digit where TMDB uses a Roman numeral (or the reverse), spells the number out, or wraps it as "Part N" / "Chapitre N" - (§3.3 row 4, §10.1/V10). Returns extra candidate titles to try — the + (V10). Returns extra candidate titles to try — the index re-rendered as digit and as Roman numeral, plus (only when there is no "Part"/"Episode"/… keyword) the bare base. The bare base is withheld for a keyword'd index — "<Saga> Chapter III" → "<Saga>" — because a franchise's bare name is very often a real, *different* film (the series' first entry), and that variant matched - every later entry to it (§10.1/V14). Without the keyword ("<Franchise> + every later entry to it (V14). Without the keyword ("<Franchise> 3") the number is decoration and the bare base is the right thing to try. """ @@ -190,7 +190,7 @@ def sequel_variants(title: str) -> list[str]: def season_from_folder_name(name: str) -> int | None: """ - §3.4: a season-like ancestor folder, vocabulary-driven rather than + A season-like ancestor folder, vocabulary-driven rather than assuming a numeric convention everywhere. A specials/bonus/extras folder maps to season 0 (matching TMDB's own `season_number: 0`). Returns None if `name` doesn't look like a season folder at all. @@ -212,7 +212,8 @@ def season_from_folder_name(name: str) -> int | None: @dataclass class ParsedName: display_title: str | None # None => caller must supply from elsewhere (e.g. a sibling file) - alt_title: str | None = None # guessit's alternative_title, a second query candidate (§3.3 row 1) + alt_title: str | None = None # guessit's alternative_title, a second + # query candidate naive_title: str = "" # always available, fully punctuation-normalized fallback year: int | None = None season: int | None = None @@ -249,7 +250,7 @@ def parse_movie_filename(filename: str) -> ParsedName: -# ── Music app (docs/musicbay.md §2.1) ──────────────────────────────────────── +# ── Music app (docs/MESHBAY_DESIGN.md §9.8) ──────────────────────────────────────── # # Filename parsing is the *fallback* here, not the primary source (unlike # Videos, where guessit does all the work): embedded ID3/Vorbis tags are read @@ -307,7 +308,7 @@ def strip_track_prefix(text: str) -> str: # "Season 1"/"Saison 1". guessit will also invent a season+episode from a # bare 3-4 digit run ("1080p" truncated to "108" -> S01E08; "1280" -> # S12E80), which is how a plain movie ends up shelved as a series -# (§10.1/V14). The indexer uses this to tell a real flat-library episode +# (V14). The indexer uses this to tell a real flat-library episode # from that hallucination. _EPISODE_MARKER_RE = re.compile( r"s\d{1,2}[\s._-]*e\d{1,3}" @@ -326,9 +327,9 @@ def has_episode_marker(filename: str) -> bool: def parse_episode_filename(filename: str) -> ParsedName: """ Parse an episode filename. `display_title` may come back None (e.g. - `S08E02.SUBFRENCH.720p.mkv` carries no show name at all, §3.2) — the + `S08E02.SUBFRENCH.720p.mkv` carries no show name at all) — the indexer then supplies the show title from a representative sibling - filename in the same folder rather than the folder name itself (§3.4). + filename in the same folder rather than the folder name itself. """ g = guessit(filename) title = g.get("title") @@ -351,7 +352,7 @@ def parse_episode_filename(filename: str) -> ParsedName: ) -# A bare leading episode number, no show name attached (§3.4c) — the same +# A bare leading episode number, no show name attached — the same # shape as music's _TRACK_PREFIX_RE, capped at 3 digits for the same reason: # a leading year ("2010 - Episode.mkv") is 4 digits and must not match. # guessit's own `episode` is not a substitute here: given exactly 3 digits it |