From ad4ca3229002997934ccb5b2eaeb553c13b8888f Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Thu, 17 Sep 2026 13:39:23 +0200 Subject: feat: embedded subtitles in the video player (MNP 3.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MSE decodes no in-band text track, so a subtitle cannot ride inside the fragmented MP4 the player is fed. The node extracts one track whole, converts it to WebVTT and caches it under its own hash; the client pulls that blob through the ordinary file_req/chunk path and hangs a on the video element — the same indirection as a TMDB poster or an audio transcode, which is what makes a film's subtitles extracted once in the life of the file rather than once per viewing. Whole-file also makes the cues absolute, so a seek and an audio-language change both leave the track untouched. **The ordinal counts every subtitle stream, including the ones never listed.** Only text codecs are offered: a bitmap track (PGS, VOBSUB — about a fifth of a real library) has no path to WebVTT without OCR, and one extracted anyway yields a header with no cues, which is a menu entry that shows nothing and reports no error. Numbering the survivors of that filter would give a PGS/SRT/SRT file the ordinals 0 and 1 for its text tracks and `-map 0:s:0` would then extract the PGS — the same trap `AudioTrack.ordinal` exists for, one level deeper. A fixture whose first subtitle stream cannot be decoded pins it, and the handler checks membership of the probed list, never a range. Additive and MINOR: the selector is drawn from `subtitle_tracks` in the node's own `stream_init` and from no version number, so `subtitle_req` is never sent to a peer that would not answer it. The floor stays at 3.0. Also here: a failed extraction never touches playback, a superseded reply cannot install its blob over a newer choice, and `_languageName` is shared with the audio labels — lifted by both label harnesses, since a lift that names one function stops covering the rule the moment logic moves out of it. Tests: 9 node (tracks told apart by the words in the extracted cues, not by tags), 10 client. Full suite green: 1545 node/common, 1252 hub. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01UGY17EPph5LsLzePPXhUVc --- .../src/meshbay_node/transport/webrtc_server.py | 165 +++++++++++++++++++++ 1 file changed, 165 insertions(+) (limited to 'packages/meshbay-node/src/meshbay_node/transport') diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py index 67ca380..f858a53 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py @@ -287,6 +287,15 @@ BROWSER_INCOMPATIBLE_AUDIO_EXTS = frozenset({".wma", ".mpc"}) # bounded generously so one slow/huge outlier can't pin a transcode slot # (shared with video, MAX_CONCURRENT_TRANSCODES above) indefinitely. AUDIO_TRANSCODE_TIMEOUT_SECS = 120 +# Extracting one subtitle track is a demux and a text conversion, not an +# encode: measured at ~1.2 s for a full film. The bound is generous against a +# pathological container rather than against the work itself, and it is short +# next to the audio one because nothing here decodes a media stream. +SUBTITLE_EXTRACT_TIMEOUT_SECS = 60 +# A subtitle file is text; a film's is ~96 KB. Anything past this is not a +# subtitle track, it is an ffmpeg that found something else to write, and it +# would sit in the media cache for ever. +SUBTITLE_MAX_BYTES = 8 * 1024 * 1024 # Bundle fetches are served in the pre-proof window (C4). Bounded and audited # until the native client removes remote keypair bundles entirely. MAX_PRE_PROOF_FETCHES = 4 @@ -675,6 +684,8 @@ class WebRTCPeerSession: self._spawn(self._do_music_meta_request(msg)) elif mtype == MNP.AUDIO_TRANSCODE_REQ: self._spawn(self._do_audio_transcode_request(msg)) + elif mtype == MNP.SUBTITLE_REQ: + self._spawn(self._do_subtitle_request(msg)) elif mtype == MNP.MEMBER_UNPIN: self._do_member_unpin(msg) elif mtype == MNP.GEK_ROTATE: @@ -4016,6 +4027,87 @@ class WebRTCPeerSession: "file_id": file_id, "hash": transcode_hash, "size": len(blob), "mime": "audio/mp4"}) + async def _do_subtitle_request(self, msg: dict) -> None: + """ + One embedded subtitle track, extracted whole-file to WebVTT and served + back through the ordinary file_req/chunk path — the same indirection + as `_do_audio_transcode_request` above, and cached the same way, so a + film's subtitles are extracted once in the life of the file rather + than once per viewing. + + The ordinal is validated against `probe_video`'s *filtered* list and + then used as the ffmpeg `-map 0:s:` argument, which is only correct + because `SubtitleTrack.ordinal` counts every subtitle stream including + the bitmap ones the list omits (see media_probe.py). Checking + membership rather than range is what makes that hold: a bitmap + ordinal is in range and is not in the list, and extracting it would + produce an empty WebVTT — a subtitle track with no subtitles in it, + which reports no error anywhere. + """ + ctx = self._group_ctx() + file_id = msg.get("file_id", "") + entry = ctx["index"].get_entry(file_id) + if not entry: + self._send({"type": "error", "detail": "File not found"}) + return + file_path = entry_abs_path(ctx["roots"], entry) + if file_path is None: + self._send({"type": "error", "detail": ROOT_NOT_SERVED}) + return + if not file_path.exists(): + self._send({"type": "error", "detail": "File not on disk"}) + return + + media_cache = self._ctx.get("media_cache") + if media_cache is None: + self._send({"type": "error", "detail": "Subtitles unavailable"}) + return + + try: + ordinal = int(msg.get("track", 0) or 0) + except (TypeError, ValueError): + ordinal = -1 + + synthetic_id = f"subtitle:{entry.id}:{ordinal}" + cached_hash = await media_cache.get_thumb_hash_by_file_id(synthetic_id) + if cached_hash is not None: + blob = await media_cache.get_thumb(cached_hash) + if blob is not None: + self._send({"type": MNP.SUBTITLE_RESP, "v": MNP_VERSION, + "file_id": file_id, "track": ordinal, + "hash": cached_hash, "size": len(blob), + "mime": "text/vtt"}) + return + # Cached hash but the blob was pruned: fall through and extract + # again, same as a cold cache. + + probe = await _probe_video(str(file_path)) + if not any(tr.ordinal == ordinal for tr in probe.subtitle_tracks): + # Not a range check — see this method's docstring. + self._send({"type": "error", "detail": "No such subtitle track"}) + return + + sem = self._transcode_semaphore() + if sem.locked() and sem._value <= 0: + self._send({"type": "error", "detail": "Server busy, retry shortly"}) + return + async with sem: + try: + blob = await _extract_subtitle_to_webvtt(file_path, ordinal) + except Exception as e: + log.warning("Subtitle extract failed for %s track %d: %s", + entry.id[:12], ordinal, e) + self._send({"type": "error", "detail": f"Subtitle extraction failed: {e}"}) + return + + subtitle_hash = blake3.blake3(blob).hexdigest() + await media_cache.put_thumb(subtitle_hash, synthetic_id, blob) + self._audit("subtitle_extract", f"{entry.name} [{ordinal}]") + self._send({"type": MNP.SUBTITLE_RESP, "v": MNP_VERSION, + "file_id": file_id, "track": ordinal, + "hash": subtitle_hash, "size": len(blob), + "mime": "text/vtt"}) + async def _do_music_meta_request(self, msg: dict) -> None: """ docs/musicbay.md §4.3: MusicBrainz metadata for one track, resolved @@ -6143,6 +6235,21 @@ class WebRTCPeerSession: for tr in probe.audio_tracks ], "audio_track": audio_track if has_audio else None, + # Same discovery-from-the-answer shape as `audio_tracks`: a node + # too old to enumerate sends no list, the client shows no selector + # and never sends `subtitle_req` to a peer that would answer + # "unknown message type". Text tracks only — a bitmap one has no + # WebVTT to offer (media_probe.py), so it is absent here rather + # than present and unplayable. + "subtitle_tracks": [ + { + "i": tr.ordinal, + "lang": tr.language, + "title": tr.title, + "codec": tr.codec_name, + } + for tr in probe.subtitle_tracks + ], }) # A client that says nothing gets the old behaviour, which is why this @@ -6360,6 +6467,64 @@ async def _transcode_audio_to_aac(file_path: Path) -> bytes: tmp_path.unlink(missing_ok=True) +async def _extract_subtitle_to_webvtt(file_path: Path, ordinal: int) -> bytes: + """ + One subtitle track out of a container, whole, as WebVTT. + + Whole-file rather than following the stream, which is what makes the + result reusable: the cues carry the source's own absolute timestamps, so + the same extraction serves every seek, every audio-language change and + every later viewing, and the `` the client attaches never has to be + rebuilt. It is also the only shape the cache makes sense in — a segment + keyed on a seek position would be a different blob every time. + + `-map 0:s:` counts subtitle streams (see media_probe.py), and + `-c:s webvtt` converts subrip/ass to text; a bitmap codec reaching here + would produce an empty file rather than an error, which is why the caller + checks membership of the probed text list first and never a range. + + Written to a temp file rather than read off a pipe: the caller wants one + complete blob to hash and cache, and there is nothing to gain from + streaming a hundred kilobytes. + """ + fd, tmp_name = tempfile.mkstemp(suffix=".vtt") + os.close(fd) + tmp_path = Path(tmp_name) + try: + proc = await asyncio.create_subprocess_exec( + platform.ffmpeg_cmd(), "-hide_banner", "-loglevel", "error", "-y", + "-i", str(file_path), + "-map", f"0:s:{ordinal}", "-c:s", "webvtt", + "-f", "webvtt", str(tmp_path), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + try: + _, stderr = await asyncio.wait_for( + proc.communicate(), timeout=SUBTITLE_EXTRACT_TIMEOUT_SECS) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise RuntimeError(f"ffmpeg timed out after {SUBTITLE_EXTRACT_TIMEOUT_SECS}s") + if proc.returncode != 0: + raise RuntimeError( + f"ffmpeg exited {proc.returncode}: {stderr.decode(errors='replace')[:300]}") + size = tmp_path.stat().st_size + if size > SUBTITLE_MAX_BYTES: + raise RuntimeError(f"subtitle track is {size} bytes, over the {SUBTITLE_MAX_BYTES} cap") + blob = tmp_path.read_bytes() + # A WebVTT file that is only its header has no cues in it. That is what + # a bitmap track extracted by mistake produces, and what a text track + # whose stream is empty produces; either way there is nothing to show, + # and an empty track attached to the player is worse than none — it + # appears in the menu and does nothing when picked. + if len(blob.strip()) <= len(b"WEBVTT"): + raise RuntimeError("extracted subtitle contains no cues") + return blob + finally: + tmp_path.unlink(missing_ok=True) + + class WebRTCTransport: """ Manages WebRTC peer connections for browser clients. -- cgit v1.2.3