diff options
| author | Christophe Besson <cbesson@gmail.com> | 2026-09-17 13:39:23 +0200 |
|---|---|---|
| committer | Christophe Besson <cbesson@gmail.com> | 2026-09-17 13:39:23 +0200 |
| commit | ad4ca3229002997934ccb5b2eaeb553c13b8888f (patch) | |
| tree | a680f9e5d3ba43a84236b84f73e0b71eaf916db4 /packages | |
| parent | 3e6d514663a5df1be3b2f0286c5f67f669d9c1d6 (diff) | |
| download | meshbay-ad4ca3229002997934ccb5b2eaeb553c13b8888f.tar.gz | |
feat: embedded subtitles in the video player (MNP 3.3)
MSE decodes no in-band text track, so a subtitle cannot ride inside the
fragmented MP4 the player is fed. The node extracts one track whole, converts
it to WebVTT and caches it under its own hash; the client pulls that blob
through the ordinary file_req/chunk path and hangs a <track> on the video
element — the same indirection as a TMDB poster or an audio transcode, which
is what makes a film's subtitles extracted once in the life of the file rather
than once per viewing. Whole-file also makes the cues absolute, so a seek and
an audio-language change both leave the track untouched.
**The ordinal counts every subtitle stream, including the ones never listed.**
Only text codecs are offered: a bitmap track (PGS, VOBSUB — about a fifth of a
real library) has no path to WebVTT without OCR, and one extracted anyway
yields a header with no cues, which is a menu entry that shows nothing and
reports no error. Numbering the survivors of that filter would give a
PGS/SRT/SRT file the ordinals 0 and 1 for its text tracks and `-map 0:s:0`
would then extract the PGS — the same trap `AudioTrack.ordinal` exists for,
one level deeper. A fixture whose first subtitle stream cannot be decoded pins
it, and the handler checks membership of the probed list, never a range.
Additive and MINOR: the selector is drawn from `subtitle_tracks` in the node's
own `stream_init` and from no version number, so `subtitle_req` is never sent
to a peer that would not answer it. The floor stays at 3.0.
Also here: a failed extraction never touches playback, a superseded reply
cannot install its blob over a newer choice, and `_languageName` is shared
with the audio labels — lifted by both label harnesses, since a lift that
names one function stops covering the rule the moment logic moves out of it.
Tests: 9 node (tracks told apart by the words in the extracted cues, not by
tags), 10 client. Full suite green: 1545 node/common, 1252 hub.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UGY17EPph5LsLzePPXhUVc
Diffstat (limited to 'packages')
20 files changed, 1067 insertions, 5 deletions
diff --git a/packages/meshbay-common/src/meshbay_common/__init__.py b/packages/meshbay-common/src/meshbay_common/__init__.py index c4c4bf4..c5b0279 100644 --- a/packages/meshbay-common/src/meshbay_common/__init__.py +++ b/packages/meshbay-common/src/meshbay_common/__init__.py @@ -201,5 +201,23 @@ __version__ = "0.14.0" # compatibility switch 3.0 refused, because there is no second branch on the # node: a 3.2 node always enumerates, always honours what it is asked for, and # always says which track it used. `MNP_MIN_SUPPORTED` does not move. -MNP_VERSION = "3.2" +# +# **3.3 (2026-09-17): the viewer picks a subtitle track.** +# +# `stream_init` carries `subtitle_tracks` (ordinal, language, title, codec); +# `subtitle_req` asks for one by ordinal and `subtitle_resp` names a cache hash +# to pull through the ordinary `file_req`/`file_chunk` path, exactly as +# `audio_transcode_resp` does. Subtitles travel beside the stream because MSE +# decodes no in-band text track, and whole-file because the extraction is then +# absolute in time: a seek re-extracts nothing and the `<track>` is untouched. +# +# **Additive, MINOR, and for once nothing can be served wrongly.** An +# `audio_track` a 3.2 node ignores is the wrong language played in silence; +# `subtitle_req` has no such failure — a node too old to know the message +# answers `unknown message type` and the client shows no subtitles, which is +# what it showed before. The selector is still drawn from `subtitle_tracks` in +# the node's own `stream_init` and from no version number, so the request is +# never sent to a peer that could not answer it. `MNP_MIN_SUPPORTED` does not +# move. +MNP_VERSION = "3.3" MHP_VERSION = "0.1" diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index 8ad36e6..c3cb099 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -191,6 +191,13 @@ class MNP: # TMDB poster or a MusicBrainz cover. AUDIO_TRANSCODE_REQ = "audio_transcode_req" # client → node: transcode this file id AUDIO_TRANSCODE_RESP = "audio_transcode_resp" # node → client: cache hash/size/mime + # One embedded subtitle track, extracted whole-file to WebVTT and cached + # under its own hash — same shape as the audio transcode above, and for + # the same reason: the result is small, identical every time, and MSE + # cannot decode an in-band text track, so it has to travel beside the + # stream rather than inside it. Whole-file, so a seek re-extracts nothing. + SUBTITLE_REQ = "subtitle_req" # client → node: file id + track ordinal + SUBTITLE_RESP = "subtitle_resp" # node → client: cache hash/size/mime # Device linking. A new device files a request bound to a code it displays; # an already-pinned device of the same account approves it. Neither the hub # nor the node can produce the countersignature. diff --git a/packages/meshbay-hub/src/meshbay_hub/static/icon.js b/packages/meshbay-hub/src/meshbay_hub/static/icon.js index 306158a..94a82fb 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/icon.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/icon.js @@ -100,6 +100,11 @@ const ICON_PATHS = { // overlapping outlines read as a tangle at 20 px. 'speech-pair': ['M11 2.5H4A1.5 1.5 0 0 0 2.5 4v4A1.5 1.5 0 0 0 4 9.5h1V12l3-2.5h3A1.5 1.5 0 0 0 12.5 8V4A1.5 1.5 0 0 0 11 2.5z', 'M13 12.5h7A1.5 1.5 0 0 1 21.5 14v4a1.5 1.5 0 0 1-1.5 1.5h-1V22l-3-2.5h-3A1.5 1.5 0 0 1 11.5 18v-4a1.5 1.5 0 0 1 1.5-1.5z'], + // A frame with two lines at its foot — the conventional subtitle glyph, and + // the reason `speech-pair` above is not a frame: the two sit side by side in + // the player's toolbar and have to be told apart at a glance. + subtitles: ['M3.5 5.5h17a1 1 0 0 1 1 1v11a1 1 0 0 1-1 1h-17a1 1 0 0 1-1-1v-11a1 1 0 0 1 1-1z', + 'M6 14.5h7', 'M15.5 14.5h2.5'], image: ['M5 3.5h14a1.5 1.5 0 0 1 1.5 1.5v14a1.5 1.5 0 0 1-1.5 1.5H5a1.5 1.5 0 0 1-1.5-1.5V5a1.5 1.5 0 0 1 1.5-1.5z', 'M7 9.5a1.5 1.5 0 1 0 3 0 1.5 1.5 0 0 0-3 0', 'M20.5 15l-5-5-9.5 9.5'], diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js index 806fa38..dd53bcf 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js @@ -251,6 +251,10 @@ export default { 'video.from_start': "Von vorn beginnen", 'video.audio_track': 'Tonspur', 'video.audio_track_n': 'Spur {n}', + 'video.subtitles': 'Untertitel', + 'video.subtitles_off': 'Aus', + 'video.subtitle_track_n': 'Spur {n}', + 'video.err_subtitle': 'Diese Untertitelspur konnte nicht geladen werden', 'video.close': 'Schließen (Esc)', 'preview.pdf_fallback': 'Dieser Browser zeigt das PDF nicht direkt an. Laden Sie es ' + 'stattdessen herunter — entschlüsselt wurde es ohnehin hier.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js index 92114c9..e9f3d8d 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js @@ -250,6 +250,10 @@ export default { 'video.from_start': "Start from the beginning", 'video.audio_track': 'Audio track', 'video.audio_track_n': 'Track {n}', + 'video.subtitles': 'Subtitles', + 'video.subtitles_off': 'Off', + 'video.subtitle_track_n': 'Track {n}', + 'video.err_subtitle': 'Could not load this subtitle track', 'video.close': 'Close (Esc)', 'preview.pdf_fallback': 'This browser will not display the PDF inline. Download it instead — it was decrypted here either way.', 'preview.too_large': 'This file is {size}, more than this page can hold in memory ({limit}). Download it instead — a download is written straight to disk.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js index d9b8f63..68fbe10 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js @@ -249,6 +249,10 @@ export default { 'video.from_start': "Empezar desde el principio", 'video.audio_track': 'Pista de audio', 'video.audio_track_n': 'Pista {n}', + 'video.subtitles': 'Subtítulos', + 'video.subtitles_off': 'Ninguno', + 'video.subtitle_track_n': 'Pista {n}', + 'video.err_subtitle': 'No se pudo cargar esta pista de subtítulos', 'video.close': 'Cerrar (Esc)', 'preview.pdf_fallback': 'Este navegador no mostrará el PDF integrado. Descárguelo ' + 'en su lugar — en cualquier caso se descifró aquí.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js index d507d28..679a458 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js @@ -250,6 +250,10 @@ export default { 'video.from_start': "Reprendre depuis le début", 'video.audio_track': 'Piste audio', 'video.audio_track_n': 'Piste {n}', + 'video.subtitles': 'Sous-titres', + 'video.subtitles_off': 'Aucun', + 'video.subtitle_track_n': 'Piste {n}', + 'video.err_subtitle': 'Impossible de charger cette piste de sous-titres', 'video.close': 'Fermer (Échap)', 'preview.pdf_fallback': 'Ce navigateur n’affichera pas le PDF directement. ' + 'Téléchargez-le plutôt — il a été déchiffré ici dans les deux cas.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js index b6c68dd..9a34c93 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js @@ -250,6 +250,10 @@ export default { 'video.from_start': "Riparti dall'inizio", 'video.audio_track': 'Traccia audio', 'video.audio_track_n': 'Traccia {n}', + 'video.subtitles': 'Sottotitoli', + 'video.subtitles_off': 'Nessuno', + 'video.subtitle_track_n': 'Traccia {n}', + 'video.err_subtitle': 'Impossibile caricare questa traccia di sottotitoli', 'video.close': 'Chiudi (Esc)', 'preview.pdf_fallback': 'Questo browser non mostrerà il PDF nella pagina. Lo scarichi ' + 'invece — in ogni caso è stato decifrato qui.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js index 99e2e1f..1d58e2c 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js @@ -247,6 +247,10 @@ export default { 'video.from_start': "最初から再生する", 'video.audio_track': '音声トラック', 'video.audio_track_n': 'トラック {n}', + 'video.subtitles': '字幕', + 'video.subtitles_off': 'なし', + 'video.subtitle_track_n': 'トラック {n}', + 'video.err_subtitle': 'この字幕トラックを読み込めませんでした', 'video.close': '閉じる(Esc)', 'preview.pdf_fallback': 'このブラウザーはページ内に PDF を表示しません。' + 'ダウンロードしてご覧ください。いずれにせよ復号はここで行われています。', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js index 269fa70..5bef61f 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js @@ -251,6 +251,10 @@ export default { 'video.from_start': "Vanaf het begin afspelen", 'video.audio_track': 'Audiospoor', 'video.audio_track_n': 'Spoor {n}', + 'video.subtitles': 'Ondertiteling', + 'video.subtitles_off': 'Geen', + 'video.subtitle_track_n': 'Spoor {n}', + 'video.err_subtitle': 'Dit ondertitelspoor kon niet worden geladen', 'video.close': 'Sluiten (Esc)', 'preview.pdf_fallback': 'Deze browser toont de PDF niet in de pagina zelf. Download ' + 'hem in plaats daarvan — ontsleuteld werd hij hoe dan ook hier.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js index 363296c..8989880 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js @@ -256,6 +256,10 @@ export default { 'video.from_start': "Odtwórz od początku", 'video.audio_track': 'Ścieżka dźwiękowa', 'video.audio_track_n': 'Ścieżka {n}', + 'video.subtitles': 'Napisy', + 'video.subtitles_off': 'Brak', + 'video.subtitle_track_n': 'Ścieżka {n}', + 'video.err_subtitle': 'Nie udało się wczytać tej ścieżki napisów', 'video.close': 'Zamknij (Esc)', 'preview.pdf_fallback': 'Ta przeglądarka nie wyświetli pliku PDF na stronie. Proszę ' + 'go pobrać — i tak został odszyfrowany tutaj.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js index 2a90956..f1b8643 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js @@ -251,6 +251,10 @@ export default { 'video.from_start': "Começar do início", 'video.audio_track': 'Faixa de áudio', 'video.audio_track_n': 'Faixa {n}', + 'video.subtitles': 'Legendas', + 'video.subtitles_off': 'Nenhuma', + 'video.subtitle_track_n': 'Faixa {n}', + 'video.err_subtitle': 'Não foi possível carregar esta faixa de legendas', 'video.close': 'Fechar (Esc)', 'preview.pdf_fallback': 'Este navegador não exibirá o PDF na própria página. Baixe ' + 'o arquivo — de todo modo ele foi descriptografado aqui.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js index e3aa440..91da464 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js @@ -245,6 +245,10 @@ export default { 'video.from_start': "从头开始播放", 'video.audio_track': '音轨', 'video.audio_track_n': '音轨 {n}', + 'video.subtitles': '字幕', + 'video.subtitles_off': '关闭', + 'video.subtitle_track_n': '字幕 {n}', + 'video.err_subtitle': '无法加载此字幕轨道', 'video.close': '关闭(Esc)', 'preview.pdf_fallback': '此浏览器不会在页面内显示该 PDF。请改为下载——无论如何它都已在本地解密。', 'preview.too_large': '该文件为 {size},超出本页面可在内存中保存的上限({limit})。请改为下载——下载会直接写入磁盘。', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/transport.js b/packages/meshbay-hub/src/meshbay_hub/static/transport.js index 262b00e..63c71ff 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/transport.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/transport.js @@ -1607,6 +1607,25 @@ class MeshBayTransport { } /** + * One embedded subtitle track, extracted node-side to WebVTT. Returns + * `{ hash, size, mime, track }` — the *cache* hash to pull through the + * ordinary file_req/chunk path, the same indirection as an audio transcode + * or a TMDB poster, and cached node-side under the file's own id so a film + * is extracted once rather than once per viewing. + * + * `track` is the ordinal the node published in `stream_init.subtitle_tracks` + * and is passed back untouched: it counts every subtitle stream in the + * container, including the bitmap ones that are never listed, so it is not + * a position in the list this client received. + */ + async requestSubtitle(fileId, track) { + const msg = await this._sendAndWait( + { type: 'subtitle_req', v: '0.9', file_id: fileId, track }, 90000); + if (msg.type === 'error') throw new Error(msg.detail); + return msg; + } + + /** * Whether MusicBrainz lookups run for this group at all — per-group from * the start (docs/musicbay.md §3.2/§6). Signed like setTmdbEnabled. */ @@ -2968,6 +2987,12 @@ class MeshBayTransport { // Same reordering hazard as media_meta_req: the player prefetches // the next track while the current one may still be transcoding. : obj.type === 'audio_transcode_req' ? `audio_transcode:${obj.file_id}` + // The track ordinal is part of the key, not just the file id: a + // viewer who opens the menu and picks a second language before the + // first extraction has answered has two of these in flight for the + // same film, and the one that arrives first is not necessarily the + // one that was asked for first. + : obj.type === 'subtitle_req' ? `subtitle:${obj.file_id}:${obj.track}` // Two-step admin-op flow (_authorizeAdminOp) — see ADMIN_OP_TYPES' // own comment for the race this closes. The initial request and // the admin_response that follows it are keyed the same way @@ -3399,6 +3424,18 @@ class MeshBayTransport { return; } + // Keyed on file *and* track — see the `subtitle_req` key above. The node + // echoes `track` back for exactly this: without it a reply could only be + // matched to the film, and the two tracks of one film are precisely the + // pair that can be in flight together. + if (msg.type === 'subtitle_resp') { + const key = `subtitle:${msg.file_id}:${msg.track}`; + for (const [, handler] of this._pending) { + if (handler._key === key) { handler.resolve(msg); return; } + } + return; + } + // Same reasoning as media_meta_resp: keyed, not arrival-order, and // "nobody's waiting any more" must not fall through either. if (msg.type === 'season_meta_resp') { diff --git a/packages/meshbay-hub/src/meshbay_hub/static/video-player.js b/packages/meshbay-hub/src/meshbay_hub/static/video-player.js index 23b0699..5725fc4 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/video-player.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/video-player.js @@ -3,7 +3,7 @@ import { } from './vendor/htm-preact.js'; import { t, getLocale } from './i18n.js'; import { Icon } from './icon.js'; -import { formatSize } from './file-utils.js'; +import { formatSize, CHUNK_SIZE, pipelinedDownload } from './file-utils.js'; import { loadAuth } from './hub-client.js'; import * as platform from './platform.js'; @@ -68,8 +68,8 @@ const _ISO639 = { * otherwise indistinguishable in the menu — which is common, since a stereo * downmix usually sits beside the surround track it came from. */ -function audioTrackLabel(track) { - const code = (track.lang || '').toLowerCase(); +function _languageName(lang) { + const code = (lang || '').toLowerCase(); let name = null; const iso = _ISO639[code] || (code.length === 2 ? code : null); if (iso) { @@ -84,6 +84,11 @@ function audioTrackLabel(track) { } catch { /* no Intl.DisplayNames, or a code it does not know */ } } if (!name && code && code !== 'und') name = code; + return name; +} + +function audioTrackLabel(track) { + let name = _languageName(track.lang); if (!name) name = t('video.audio_track_n', { n: track.i + 1 }); // Two tracks in the same language are one menu entry repeated without // this, and a library where a stereo downmix sits beside the surround @@ -96,6 +101,21 @@ function audioTrackLabel(track) { return detail ? `${name} — ${detail}` : name; } +/** + * What to call one subtitle track. + * + * Same shape as `audioTrackLabel`, minus the channel layout, which subtitles + * have no equivalent of. The container's title still wins where there is one: + * "Forced", "SDH" and "Signs & Songs" are all the same language tag as the + * ordinary track they sit beside, and picking the wrong one of those is the + * difference between a full translation and three lines in a whole film. + */ +function subtitleTrackLabel(track) { + const name = _languageName(track.lang) + || t('video.subtitle_track_n', { n: track.i + 1 }); + return track.title ? `${name} — ${track.title}` : name; +} + function _mseSupported(codec) { if (!window.MediaSource) return false; const mime = `video/mp4; codecs="${codec}"`; @@ -218,6 +238,25 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { // Read inside the effect's closures, which are built once and would // otherwise capture the first track forever. const audioTrackRef = useRef(null); + // Subtitles. The node lists only the tracks it can turn into WebVTT, so an + // empty list means "nothing showable here" whatever the container holds, + // and draws no selector — the same discovery-from-the-answer shape as the + // audio tracks above. `null` is off, and off is where a film opens. + // + // None of this is torn down by a seek or a language change: the extraction + // is whole-file, so the cues are absolute and the <track> outlives every + // restart of the MediaSource underneath it. + const [subtitleTracks, setSubtitleTracks] = useState([]); + const [subtitleTrack, setSubtitleTrack] = useState(null); + const [subtitleMenuOpen, setSubtitleMenuOpen] = useState(false); + const [subtitleUrl, setSubtitleUrl] = useState(null); + const [subtitleBusy, setSubtitleBusy] = useState(false); + const [subtitleError, setSubtitleError] = useState(false); + const subtitleUrlRef = useRef(null); + // Two extractions can be in flight when the viewer changes their mind, and + // the first one asked for is not necessarily the first one answered. Only + // the newest request may install its blob. + const subtitleGenRef = useRef(0); const [castActive, setCastActive] = useState(false); const [castUrl, setCastUrl] = useState(null); const [castPickerOpen, setCastPickerOpen] = useState(false); @@ -629,6 +668,12 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { // was drawn falls back to the first track, and the selector must show // the truth rather than the request. setAudioTracks(Array.isArray(msg.audio_tracks) ? msg.audio_tracks : []); + // Re-stated on every stream_init, including the ones a seek and an + // audio-language change produce. Deliberately does not touch + // `subtitleTrack` or the blob: the cues are absolute, so the track + // showing before the restart is still the right one after it. + setSubtitleTracks( + Array.isArray(msg.subtitle_tracks) ? msg.subtitle_tracks : []); if (Number.isInteger(msg.audio_track)) { audioTrackRef.current = msg.audio_track; setAudioTrack(msg.audio_track); @@ -1025,6 +1070,80 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { return () => window.removeEventListener('keydown', onKey); }, [onClose]); + /** + * Show one subtitle track, or none. + * + * The node extracts the whole track to WebVTT and caches it under its own + * hash; what comes back here is that hash, pulled through the ordinary + * chunk path like any other file. So this is slow exactly once per film per + * track, and instant every time after — including in a later sitting, which + * is the part a per-seek extraction could never have given. + * + * A failure here never touches playback. Subtitles are an addition to a + * film that is already running, and taking the film down because a text + * track could not be read would be a worse answer than no subtitles. + */ + const selectSubtitle = useCallback(async (track) => { + const gen = ++subtitleGenRef.current; + if (subtitleUrlRef.current) { + URL.revokeObjectURL(subtitleUrlRef.current); + subtitleUrlRef.current = null; + } + setSubtitleUrl(null); + setSubtitleError(false); + if (track === null) { + setSubtitleTrack(null); + setSubtitleBusy(false); + return; + } + const transport = transportRef.current; + if (!transport) return; + setSubtitleTrack(track.i); + setSubtitleBusy(true); + try { + const info = await transport.requestSubtitle(entry.id, track.i); + const chunks = await pipelinedDownload( + transport, gekRef.current, info.hash, Math.ceil(info.size / CHUNK_SIZE)); + const url = URL.createObjectURL( + new Blob(chunks, { type: info.mime || 'text/vtt' })); + // Someone changed their mind while this was in flight. Dropping the blob + // rather than installing it is the whole point of the generation: the + // reply that arrives last is not the choice that was made last. + if (subtitleGenRef.current !== gen) { URL.revokeObjectURL(url); return; } + subtitleUrlRef.current = url; + setSubtitleUrl(url); + } catch (err) { + if (subtitleGenRef.current !== gen) return; + console.warn('[MeshBay] subtitle track', track.i, 'failed:', err); + setSubtitleTrack(null); + setSubtitleError(true); + } finally { + if (subtitleGenRef.current === gen) setSubtitleBusy(false); + } + }, [entry, transportRef, gekRef]); + + // A <track> added to a media element after it started playing is not shown + // by the `default` attribute — that one is read when the element is first + // parsed, and by then this track did not exist. The mode has to be set on + // the live TextTrack, which only appears once the element has adopted the + // child preact just rendered. + useEffect(() => { + const v = videoRef.current; + if (!v) return; + for (let i = 0; i < v.textTracks.length; i++) { + v.textTracks[i].mode = subtitleUrl ? 'showing' : 'disabled'; + } + }, [subtitleUrl]); + + useEffect(() => { + return () => { + if (subtitleUrlRef.current) { + URL.revokeObjectURL(subtitleUrlRef.current); + subtitleUrlRef.current = null; + } + }; + }, []); + return html` <div class="video-overlay" onClick=${(e) => { if (e.target.classList.contains('video-overlay')) onClose(); @@ -1065,6 +1184,46 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { `} </div> `} + ${subtitleTracks.length > 0 && html` + <div class="cast-wrapper" style="position:relative"> + <button class="video-close ${subtitleUrl ? 'cast-active' : ''}" + onClick=${() => setSubtitleMenuOpen(!subtitleMenuOpen)} + title="${t('video.subtitles')}"> + ${subtitleBusy + ? html`<span class="spinner"></span>` + : html`<${Icon} name="subtitles" />`}</button> + ${subtitleMenuOpen && html` + <div class="cast-picker"> + <button class="cast-picker-item" onClick=${() => { + setSubtitleMenuOpen(false); + selectSubtitle(null); + }}> + ${subtitleTrack === null + ? html`<${Icon} name="check" />` + : html`<span style="display:inline-block;width:14px"></span>`} + ${' '}${t('video.subtitles_off')} + </button> + ${subtitleTracks.map((track) => html` + <button class="cast-picker-item" onClick=${() => { + setSubtitleMenuOpen(false); + if (track.i === subtitleTrack) return; + selectSubtitle(track); + }}> + ${track.i === subtitleTrack + ? html`<${Icon} name="check" />` + : html`<span style="display:inline-block;width:14px"></span>`} + ${' '}${subtitleTrackLabel(track)} + </button> + `)} + ${subtitleError && html` + <div class="cast-picker-item cast-picker-empty"> + ${t('video.err_subtitle')} + </div> + `} + </div> + `} + </div> + `} ${platform.capabilities.lanCast && html` <div class="cast-wrapper" style="position:relative"> <button class="video-close ${castActive ? 'cast-active' : ''}" @@ -1167,7 +1326,16 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { ${(phase === 'streaming' || phase === 'loading') && html` <div class="video-container"> - <video ref=${videoRef} controls autoplay /> + <video ref=${videoRef} controls autoplay> + ${subtitleUrl && html` + <track key=${subtitleUrl} kind="subtitles" src=${subtitleUrl} + srclang=${(subtitleTracks.find((s) => s.i === subtitleTrack) || {}).lang || ''} + label=${subtitleTrack === null ? '' + : subtitleTrackLabel( + subtitleTracks.find((s) => s.i === subtitleTrack) || { i: 0 })} + default /> + `} + </video> ${phase === 'loading' && html` <div class="video-loading"> <div class="video-loading-label"> diff --git a/packages/meshbay-hub/tests/test_video_audio_track.py b/packages/meshbay-hub/tests/test_video_audio_track.py index 534da6d..6eb98ba 100644 --- a/packages/meshbay-hub/tests/test_video_audio_track.py +++ b/packages/meshbay-hub/tests/test_video_audio_track.py @@ -72,6 +72,12 @@ def _label_cases(tmp_path, app, cases, locale="en"): script = tmp_path / "label.mjs" src = "\n".join([ app[app.index("const _ISO639 = {"):app.index("};", app.index("const _ISO639 = {")) + 2], + # The language name is shared with `subtitleTrackLabel`, so it lives in + # a function of its own and has to be lifted alongside its caller. A + # lift that names one function stops exercising anything the moment + # logic moves out of it — here it throws, which is the good case; the + # bad one is a lift that still runs and no longer covers the rule. + _lift(app, "_languageName"), _lift(app, "audioTrackLabel"), ]) script.write_text( diff --git a/packages/meshbay-hub/tests/test_video_subtitles.py b/packages/meshbay-hub/tests/test_video_subtitles.py new file mode 100644 index 0000000..9f0c08d --- /dev/null +++ b/packages/meshbay-hub/tests/test_video_subtitles.py @@ -0,0 +1,232 @@ +""" +Showing a subtitle track in the player. + +The node extracts one track whole, converts it to WebVTT and caches it under +its own hash; this side asks for it by ordinal, pulls the blob through the +ordinary chunk path and hangs a `<track>` on the video element. Four things +have to hold here: + +**The selector exists only where the node said there was something to show.** +It is drawn from `subtitle_tracks` in `stream_init` and from nothing else — +there is no version check in the player — so a node too old to enumerate them +draws no selector and is never sent a `subtitle_req` it would answer "unknown +message type" to. The list also carries only the tracks the node can convert, +so what the menu offers is what will actually appear. + +**The ordinal travels untouched.** It counts every subtitle stream in the +container, including the bitmap ones that are never listed, so it is not the +track's position in the list this client received. Renumbering it here would +map a different stream on the node and show the wrong language, or nothing. + +**A seek does not take the subtitles down.** The extraction is whole-file, so +the cues are absolute: the `<track>` outlives every restart of the +MediaSource, including the one an audio-language change produces. + +**The reply is matched to the request by file *and* track.** Two extractions +for one film can be in flight when the viewer changes their mind, and the +reply that arrives first is not necessarily the one asked for first. +""" + +import json +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +STATIC = Path(__file__).resolve().parents[1] / "src" / "meshbay_hub" / "static" +APP = STATIC / "video-player.js" +TRANSPORT = STATIC / "transport.js" + +pytestmark = pytest.mark.skipif( + shutil.which("node") is None or not APP.exists(), + reason="node or the SPA sources are not available") + + +@pytest.fixture(scope="module") +def app(): + return APP.read_text() + + +@pytest.fixture(scope="module") +def transport(): + return TRANSPORT.read_text() + + +def _lift(src: str, name: str) -> str: + """One top-level function, as text, for node to execute.""" + start = src.index(f"function {name}(") + depth, i, seen = 0, start, False + while i < len(src): + if src[i] == "{": + depth += 1 + seen = True + elif src[i] == "}": + depth -= 1 + if seen and depth == 0: + return src[start:i + 1] + i += 1 + raise AssertionError(f"{name} never closes") + + +def _player(app: str) -> str: + """The whole component, by position in the file. + + Not the brace matcher below: `function VideoPlayer({ entry, ... })` + destructures its props, so counting from the first `{` closes on the + parameter list and returns the signature alone — a lift that finds + nothing it was asked about and asserts happily against an empty string. + """ + i = app.index("function VideoPlayer(") + nxt = app.find("\nfunction ", i + 1) + return app[i:nxt if nxt > 0 else len(app)] + + +def _block(src: str, opener: str) -> str: + """The body of one `x = (args) => {` assignment, braces matched.""" + start = src.index(opener) + depth, i, seen = 0, start, False + while i < len(src): + if src[i] == "{": + depth += 1 + seen = True + elif src[i] == "}": + depth -= 1 + if seen and depth == 0: + return src[start:i + 1] + i += 1 + raise AssertionError(f"{opener!r} never closes") + + +# ── The label, run rather than read ─────────────────────────────────────────── + +def _label_cases(tmp_path, app, cases, locale="en"): + script = tmp_path / "sublabel.mjs" + src = "\n".join([ + app[app.index("const _ISO639 = {"):app.index("};", app.index("const _ISO639 = {")) + 2], + _lift(app, "_languageName"), + _lift(app, "subtitleTrackLabel"), + ]) + script.write_text( + f"const getLocale = () => '{locale}';\n" + "const t = (k, p) => `${k}:${p.n}`;\n" + + src + + "\nconst out = JSON.parse(process.argv[2]).map(subtitleTrackLabel);\n" + "console.log(JSON.stringify(out));\n") + proc = subprocess.run( + ["node", str(script), json.dumps(cases)], + capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +def test_the_language_is_named_not_shown_as_a_tag(tmp_path, app): + """Same fold as the audio tracks: ffprobe reports either ISO 639-2 variant + and a library uses both.""" + out = _label_cases(tmp_path, app, [ + {"i": 0, "lang": "fre"}, + {"i": 1, "lang": "fra"}, + {"i": 2, "lang": "eng"}, + ]) + assert out[0] == out[1], f"fre and fra must agree: {out}" + assert "French" in out[0] and "English" in out[2], out + + +def test_the_container_title_tells_a_forced_track_from_a_full_one(tmp_path, app): + """The difference the title carries is not cosmetic here. + + "Forced" and "SDH" are tagged with the same language as the ordinary track + they sit beside. Dropping the title makes them one menu entry repeated, + and picking the wrong one is the difference between a full translation and + three lines of signage in a whole film. + """ + out = _label_cases(tmp_path, app, [ + {"i": 0, "lang": "eng", "title": "Forced"}, + {"i": 1, "lang": "eng", "title": "SDH"}, + {"i": 2, "lang": "eng"}, + ]) + assert len({*out}) == 3, f"three tracks must give three entries: {out}" + assert "Forced" in out[0] and "SDH" in out[1] + + +def test_an_untagged_track_is_numbered_not_called_unknown(tmp_path, app): + """A file with no language tags still needs distinguishable entries.""" + out = _label_cases(tmp_path, app, [{"i": 3, "lang": None}, {"i": 4, "lang": "und"}]) + assert out == ["video.subtitle_track_n:4", "video.subtitle_track_n:5"], out + + +# ── The shape of the feature, read from the source ──────────────────────────── + +def test_the_selector_comes_from_the_node_list_and_not_from_a_version(app): + """Discovery from the answer. There is no version check in the player, and + an empty list is the whole of "this node has no subtitles for you".""" + assert "msg.subtitle_tracks" in app + assert "subtitleTracks.length > 0 &&" in app, ( + "the selector must be drawn from the node's list") + for forbidden in ("MNP_VERSION", "v >= '3.3'", "'3.3'"): + assert forbidden not in app, ( + f"the player decided on {forbidden!r} instead of on the answer") + + +def test_the_ordinal_is_passed_through_rather_than_renumbered(app): + """`track.i` is the node's ordinal; the list index is not it.""" + assert "requestSubtitle(entry.id, track.i)" in app, ( + "the request must carry the ordinal the node published") + assert not re.search(r"requestSubtitle\([^)]*\bindex\b", app), ( + "a list position was sent where a stream ordinal belongs") + + +def test_a_seek_does_not_take_the_subtitles_down(app): + """The payoff of extracting whole-file rather than per-seek. + + `stream_init` arrives again on every seek and on every audio-language + change. It may restate the available tracks; it must not clear the chosen + one or revoke the blob, or every seek would re-extract and the track would + blink out mid-film. + """ + body = _block(app, "transport.onStreamInit = (msg) => {") + assert "setSubtitleTracks(" in body + for forbidden in ("setSubtitleUrl(", "setSubtitleTrack(", "subtitleUrlRef.current ="): + assert forbidden not in body, ( + f"stream_init touches {forbidden!r}, so a seek disturbs the track") + + +def test_the_blob_is_pulled_through_the_ordinary_chunk_path(app): + """Not a new transfer mechanism — the same indirection as a poster.""" + assert "pipelinedDownload(" in app + assert "info.hash" in app, "the cache hash from the reply is what is fetched" + + +def test_a_stale_reply_cannot_install_its_track(app): + """Two extractions in flight, and the last answered is not the last asked. + + Without the generation the older reply overwrites the newer choice, and + the menu then shows a tick against a language that is not on screen. + """ + body = _player(app) + assert "subtitleGenRef.current !== gen" in body, ( + "nothing stops an out-of-date extraction from installing its blob") + assert "URL.revokeObjectURL(url)" in body, ( + "the superseded blob must be released, not merely ignored") + + +def test_the_reply_is_matched_on_the_file_and_the_track(transport): + """One film's two tracks are exactly the pair that can be in flight + together, so the file id alone cannot route the reply.""" + assert "`subtitle:${obj.file_id}:${obj.track}`" in transport + assert "`subtitle:${msg.file_id}:${msg.track}`" in transport + + +def test_a_failed_extraction_does_not_take_the_film_down(app): + """Subtitles are an addition to a film that is already playing. + + `setError` is the player's fatal path — it replaces the picture. A text + track that could not be read must not reach it. + """ + body = _player(app) + start = body.index("const selectSubtitle = useCallback(") + end = body.index("}, [entry, transportRef, gekRef]);", start) + assert "setError(" not in body[start:end], ( + "a subtitle failure takes the whole player down") + assert "setSubtitleError(true)" in body[start:end] diff --git a/packages/meshbay-node/src/meshbay_node/media_probe.py b/packages/meshbay-node/src/meshbay_node/media_probe.py index 7858ebe..06b1e28 100644 --- a/packages/meshbay-node/src/meshbay_node/media_probe.py +++ b/packages/meshbay-node/src/meshbay_node/media_probe.py @@ -40,6 +40,37 @@ class AudioTrack: channels: int | None +# Subtitle codecs ffmpeg can convert to WebVTT, which is the only thing MSE +# can be given. An allow-list rather than a bitmap deny-list: the cost of +# wrongly excluding an exotic text codec is a track nobody can pick, and the +# cost of wrongly including a bitmap one is a track that is picked and then +# displays nothing, with no error to lead anyone back here. +TEXT_SUBTITLE_CODECS = frozenset({ + "subrip", "srt", "ass", "ssa", "mov_text", "webvtt", "text", + "subviewer", "subviewer1", "sami", "realtext", "stl", "jacosub", + "microdvd", "mpl2", "vplayer", "pjs", +}) + + +@dataclass(frozen=True) +class SubtitleTrack: + """ + One selectable subtitle track, guaranteed convertible to WebVTT. + + **`ordinal` counts every subtitle stream, including the bitmap ones this + list does not carry**, because that is what `-map 0:s:<n>` counts. The + same trap as `AudioTrack.ordinal` one level deeper: filtering the list and + numbering the survivors would give a file whose streams are PGS, SRT, SRT + the ordinals 0 and 1 for its two text tracks, and `-map 0:s:0` would then + extract the PGS stream — which produces an empty WebVTT rather than an + error, so the viewer gets a subtitle track with no subtitles in it. + """ + ordinal: int + language: str | None + title: str | None + codec_name: str | None + + @dataclass class VideoProbe: """ @@ -57,6 +88,7 @@ class VideoProbe: height: int | None raw_codec_name: str | None audio_tracks: list[AudioTrack] = field(default_factory=list) + subtitle_tracks: list[SubtitleTrack] = field(default_factory=list) async def probe_video(path: str) -> VideoProbe: @@ -93,6 +125,14 @@ async def probe_video(path: str) -> VideoProbe: group does not want. `has_audio` stays as the single question the muxing decisions ask, and is now `bool(audio_tracks)`. + **Only text subtitle tracks are reported.** A library's embedded subtitles + are roughly four-fifths text (subrip, ass) and one-fifth bitmap (PGS, + VOBSUB); a bitmap track has no path to WebVTT without OCR, so listing one + would offer a choice that silently displays nothing. A file whose only + subtitles are bitmap therefore reports none at all and gets no selector, + exactly like a file with no subtitles — which is a true statement about + what this node can serve, not a concealed failure. + width/height come from the same ffprobe call (one extra `-show_entries` field, no second process spawn) — resolution is deliberately never guessed from the filename (docs/mediacenter.md §3.5). @@ -116,6 +156,8 @@ async def probe_video(path: str) -> VideoProbe: width: int | None = None height: int | None = None audio_tracks: list[AudioTrack] = [] + subtitle_tracks: list[SubtitleTrack] = [] + subtitle_streams_seen = 0 for s in info.get("streams", []): if s.get("codec_type") == "video" and not v_codec: cn = s.get("codec_name", "") @@ -142,6 +184,20 @@ async def probe_video(path: str) -> VideoProbe: codec_name=s.get("codec_name") or None, channels=s.get("channels"), )) + elif s.get("codec_type") == "subtitle": + # Counted before the filter, never after — see SubtitleTrack. + ordinal = subtitle_streams_seen + subtitle_streams_seen += 1 + codec_name = (s.get("codec_name") or "").strip() or None + if codec_name not in TEXT_SUBTITLE_CODECS: + continue + tags = s.get("tags") or {} + subtitle_tracks.append(SubtitleTrack( + ordinal=ordinal, + language=(tags.get("language") or "").strip() or None, + title=(tags.get("title") or "").strip() or None, + codec_name=codec_name, + )) has_audio = bool(audio_tracks) codec = None @@ -155,4 +211,5 @@ async def probe_video(path: str) -> VideoProbe: height=height, raw_codec_name=raw_codec_name, audio_tracks=audio_tracks, + subtitle_tracks=subtitle_tracks, ) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py index 67ca380..f858a53 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py @@ -287,6 +287,15 @@ BROWSER_INCOMPATIBLE_AUDIO_EXTS = frozenset({".wma", ".mpc"}) # bounded generously so one slow/huge outlier can't pin a transcode slot # (shared with video, MAX_CONCURRENT_TRANSCODES above) indefinitely. AUDIO_TRANSCODE_TIMEOUT_SECS = 120 +# Extracting one subtitle track is a demux and a text conversion, not an +# encode: measured at ~1.2 s for a full film. The bound is generous against a +# pathological container rather than against the work itself, and it is short +# next to the audio one because nothing here decodes a media stream. +SUBTITLE_EXTRACT_TIMEOUT_SECS = 60 +# A subtitle file is text; a film's is ~96 KB. Anything past this is not a +# subtitle track, it is an ffmpeg that found something else to write, and it +# would sit in the media cache for ever. +SUBTITLE_MAX_BYTES = 8 * 1024 * 1024 # Bundle fetches are served in the pre-proof window (C4). Bounded and audited # until the native client removes remote keypair bundles entirely. MAX_PRE_PROOF_FETCHES = 4 @@ -675,6 +684,8 @@ class WebRTCPeerSession: self._spawn(self._do_music_meta_request(msg)) elif mtype == MNP.AUDIO_TRANSCODE_REQ: self._spawn(self._do_audio_transcode_request(msg)) + elif mtype == MNP.SUBTITLE_REQ: + self._spawn(self._do_subtitle_request(msg)) elif mtype == MNP.MEMBER_UNPIN: self._do_member_unpin(msg) elif mtype == MNP.GEK_ROTATE: @@ -4016,6 +4027,87 @@ class WebRTCPeerSession: "file_id": file_id, "hash": transcode_hash, "size": len(blob), "mime": "audio/mp4"}) + async def _do_subtitle_request(self, msg: dict) -> None: + """ + One embedded subtitle track, extracted whole-file to WebVTT and served + back through the ordinary file_req/chunk path — the same indirection + as `_do_audio_transcode_request` above, and cached the same way, so a + film's subtitles are extracted once in the life of the file rather + than once per viewing. + + The ordinal is validated against `probe_video`'s *filtered* list and + then used as the ffmpeg `-map 0:s:<n>` argument, which is only correct + because `SubtitleTrack.ordinal` counts every subtitle stream including + the bitmap ones the list omits (see media_probe.py). Checking + membership rather than range is what makes that hold: a bitmap + ordinal is in range and is not in the list, and extracting it would + produce an empty WebVTT — a subtitle track with no subtitles in it, + which reports no error anywhere. + """ + ctx = self._group_ctx() + file_id = msg.get("file_id", "") + entry = ctx["index"].get_entry(file_id) + if not entry: + self._send({"type": "error", "detail": "File not found"}) + return + file_path = entry_abs_path(ctx["roots"], entry) + if file_path is None: + self._send({"type": "error", "detail": ROOT_NOT_SERVED}) + return + if not file_path.exists(): + self._send({"type": "error", "detail": "File not on disk"}) + return + + media_cache = self._ctx.get("media_cache") + if media_cache is None: + self._send({"type": "error", "detail": "Subtitles unavailable"}) + return + + try: + ordinal = int(msg.get("track", 0) or 0) + except (TypeError, ValueError): + ordinal = -1 + + synthetic_id = f"subtitle:{entry.id}:{ordinal}" + cached_hash = await media_cache.get_thumb_hash_by_file_id(synthetic_id) + if cached_hash is not None: + blob = await media_cache.get_thumb(cached_hash) + if blob is not None: + self._send({"type": MNP.SUBTITLE_RESP, "v": MNP_VERSION, + "file_id": file_id, "track": ordinal, + "hash": cached_hash, "size": len(blob), + "mime": "text/vtt"}) + return + # Cached hash but the blob was pruned: fall through and extract + # again, same as a cold cache. + + probe = await _probe_video(str(file_path)) + if not any(tr.ordinal == ordinal for tr in probe.subtitle_tracks): + # Not a range check — see this method's docstring. + self._send({"type": "error", "detail": "No such subtitle track"}) + return + + sem = self._transcode_semaphore() + if sem.locked() and sem._value <= 0: + self._send({"type": "error", "detail": "Server busy, retry shortly"}) + return + async with sem: + try: + blob = await _extract_subtitle_to_webvtt(file_path, ordinal) + except Exception as e: + log.warning("Subtitle extract failed for %s track %d: %s", + entry.id[:12], ordinal, e) + self._send({"type": "error", "detail": f"Subtitle extraction failed: {e}"}) + return + + subtitle_hash = blake3.blake3(blob).hexdigest() + await media_cache.put_thumb(subtitle_hash, synthetic_id, blob) + self._audit("subtitle_extract", f"{entry.name} [{ordinal}]") + self._send({"type": MNP.SUBTITLE_RESP, "v": MNP_VERSION, + "file_id": file_id, "track": ordinal, + "hash": subtitle_hash, "size": len(blob), + "mime": "text/vtt"}) + async def _do_music_meta_request(self, msg: dict) -> None: """ docs/musicbay.md §4.3: MusicBrainz metadata for one track, resolved @@ -6143,6 +6235,21 @@ class WebRTCPeerSession: for tr in probe.audio_tracks ], "audio_track": audio_track if has_audio else None, + # Same discovery-from-the-answer shape as `audio_tracks`: a node + # too old to enumerate sends no list, the client shows no selector + # and never sends `subtitle_req` to a peer that would answer + # "unknown message type". Text tracks only — a bitmap one has no + # WebVTT to offer (media_probe.py), so it is absent here rather + # than present and unplayable. + "subtitle_tracks": [ + { + "i": tr.ordinal, + "lang": tr.language, + "title": tr.title, + "codec": tr.codec_name, + } + for tr in probe.subtitle_tracks + ], }) # A client that says nothing gets the old behaviour, which is why this @@ -6360,6 +6467,64 @@ async def _transcode_audio_to_aac(file_path: Path) -> bytes: tmp_path.unlink(missing_ok=True) +async def _extract_subtitle_to_webvtt(file_path: Path, ordinal: int) -> bytes: + """ + One subtitle track out of a container, whole, as WebVTT. + + Whole-file rather than following the stream, which is what makes the + result reusable: the cues carry the source's own absolute timestamps, so + the same extraction serves every seek, every audio-language change and + every later viewing, and the `<track>` the client attaches never has to be + rebuilt. It is also the only shape the cache makes sense in — a segment + keyed on a seek position would be a different blob every time. + + `-map 0:s:<ordinal>` counts subtitle streams (see media_probe.py), and + `-c:s webvtt` converts subrip/ass to text; a bitmap codec reaching here + would produce an empty file rather than an error, which is why the caller + checks membership of the probed text list first and never a range. + + Written to a temp file rather than read off a pipe: the caller wants one + complete blob to hash and cache, and there is nothing to gain from + streaming a hundred kilobytes. + """ + fd, tmp_name = tempfile.mkstemp(suffix=".vtt") + os.close(fd) + tmp_path = Path(tmp_name) + try: + proc = await asyncio.create_subprocess_exec( + platform.ffmpeg_cmd(), "-hide_banner", "-loglevel", "error", "-y", + "-i", str(file_path), + "-map", f"0:s:{ordinal}", "-c:s", "webvtt", + "-f", "webvtt", str(tmp_path), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + try: + _, stderr = await asyncio.wait_for( + proc.communicate(), timeout=SUBTITLE_EXTRACT_TIMEOUT_SECS) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise RuntimeError(f"ffmpeg timed out after {SUBTITLE_EXTRACT_TIMEOUT_SECS}s") + if proc.returncode != 0: + raise RuntimeError( + f"ffmpeg exited {proc.returncode}: {stderr.decode(errors='replace')[:300]}") + size = tmp_path.stat().st_size + if size > SUBTITLE_MAX_BYTES: + raise RuntimeError(f"subtitle track is {size} bytes, over the {SUBTITLE_MAX_BYTES} cap") + blob = tmp_path.read_bytes() + # A WebVTT file that is only its header has no cues in it. That is what + # a bitmap track extracted by mistake produces, and what a text track + # whose stream is empty produces; either way there is nothing to show, + # and an empty track attached to the player is worse than none — it + # appears in the menu and does nothing when picked. + if len(blob.strip()) <= len(b"WEBVTT"): + raise RuntimeError("extracted subtitle contains no cues") + return blob + finally: + tmp_path.unlink(missing_ok=True) + + class WebRTCTransport: """ Manages WebRTC peer connections for browser clients. diff --git a/packages/meshbay-node/tests/test_stream_subtitle_tracks.py b/packages/meshbay-node/tests/test_stream_subtitle_tracks.py new file mode 100644 index 0000000..cafb2e3 --- /dev/null +++ b/packages/meshbay-node/tests/test_stream_subtitle_tracks.py @@ -0,0 +1,327 @@ +""" +The viewer picks a subtitle track, and only the ones that can be shown. + +MSE decodes no in-band text track, so a subtitle cannot ride inside the +fragmented MP4 the player is fed: it is extracted whole, converted to WebVTT, +cached under its own hash and pulled through the ordinary chunk path. Whole, +because that makes the cue timestamps absolute — a seek re-extracts nothing +and the `<track>` survives every restart of the MediaSource underneath it. + +Two things here are about *not* offering something. Roughly a fifth of the +subtitle streams in a real library are bitmap (PGS, VOBSUB) and have no path +to WebVTT without OCR; a bitmap track extracted anyway yields a WebVTT with a +header and no cues, which is a subtitle track that appears in the menu and +does nothing. So they are not listed — and, because they still occupy a +position in `-map 0:s:<n>`, the ordinal of the tracks that *are* listed is not +their position in the list. That is the whole trap, and it is the same one +`AudioTrack.ordinal` exists for, one level deeper. + +**The fixture's unusable stream is TTML, not bitmap, and that is deliberate.** +ffmpeg refuses to encode text to bitmap, so a PGS stream cannot be synthesised +here at all; TTML is a stream this ffmpeg has no decoder for, which is the +same branch — `codec_name not in TEXT_SUBTITLE_CODECS` — reached by exactly +the same route. The real bitmap codec names are asserted against the allow-list +directly, where no fixture is needed. + +Tracks are told apart by **the words in the extracted cues**, never by their +language tags: a tag only proves the node copied a string it was handed. +""" + +import shutil +import subprocess +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from meshbay_common.crypto import generate_gek +from meshbay_common.webcrypto import chunk_key_aes, decrypt_chunk_aes +from meshbay_node.indexer.group_index import GroupIndex +from meshbay_node.media_probe import TEXT_SUBTITLE_CODECS +from meshbay_node.transport.webrtc_server import WebRTCPeerSession, _probe_video + +from conftest import needs_subprocess, one_root + +_HAVE_FFMPEG = shutil.which("ffmpeg") and shutil.which("ffprobe") +# `asyncio` is per-test rather than on the module: one test here needs no +# event loop, and a module-wide mark on a synchronous function is a warning +# that reads as a broken test every time the suite runs. +pytestmark = [ + pytest.mark.skipif(not _HAVE_FFMPEG, reason="ffmpeg/ffprobe not installed"), + needs_subprocess, +] + +# Ordinal 0 is the unusable one and is never listed; 1 and 2 are the text +# tracks. The words differ per track because that is what the assertions read. +_CUE_WORD = {1: "francaise", 2: "English"} + +_SRT_FR = """1 +00:00:01,000 --> 00:00:03,000 +Ceci est la piste francaise. +""" + +_SRT_EN = """1 +00:00:01,000 --> 00:00:03,000 +This is the English track. +""" + + +def _make_subtitled_clip(path: Path) -> None: + """~6 s of video, then three subtitle streams: TTML, then two text ones. + + The video and audio are muxed first, so the subtitle streams sit at + container indices 2, 3 and 4 while their subtitle *ordinals* are 0, 1 and + 2 — and the first ordinal belongs to a stream that is never listed, so the + listed tracks are 1 and 2 and never 0 and 1. + """ + tmp = path.parent + fr, en = tmp / "fr.srt", tmp / "en.srt" + fr.write_text(_SRT_FR, encoding="utf-8") + en.write_text(_SRT_EN, encoding="utf-8") + base = tmp / "base.mp4" + subprocess.run( + ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "testsrc=size=320x240:rate=10:duration=6", + "-f", "lavfi", "-i", "sine=duration=6", + "-c:v", "libx264", "-preset", "ultrafast", "-c:a", "aac", + "-shortest", str(base)], + check=True, capture_output=True) + subprocess.run( + ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-i", str(base), "-i", str(fr), "-i", str(en), + "-map", "0:v", "-map", "0:a", "-map", "1", "-map", "1", "-map", "2", + "-c:v", "copy", "-c:a", "copy", + "-c:s:0", "ttml", "-c:s:1", "mov_text", "-c:s:2", "mov_text", + "-metadata:s:s:0", "language=fre", + "-metadata:s:s:1", "language=fre", + "-metadata:s:s:2", "language=eng", + str(path)], + check=True, capture_output=True) + + +class _FakeMediaCache: + """The three methods `_do_subtitle_request` uses, and a count of the puts. + + A double rather than the real cache because what is under test is the + handler's use of it — that it looks before extracting, and extracts once. + """ + + def __init__(self): + self.blobs: dict[str, bytes] = {} + self.by_file_id: dict[str, str] = {} + self.puts = 0 + + async def get_thumb_hash_by_file_id(self, file_id: str) -> str | None: + return self.by_file_id.get(file_id) + + async def get_thumb(self, thumb_hash: str) -> bytes | None: + return self.blobs.get(thumb_hash) + + async def put_thumb(self, thumb_hash: str, file_id: str, blob: bytes) -> None: + self.puts += 1 + self.blobs[thumb_hash] = blob + self.by_file_id[file_id] = thumb_hash + + +def _session(video_path: Path, gek: bytes): + import blake3 + file_bytes = video_path.read_bytes() + file_id = blake3.blake3(file_bytes).hexdigest() + + sk_node = Ed25519PrivateKey.generate() + index = GroupIndex(group_id="g" * 32, sk_node=sk_node, gek=gek) + from meshbay_common.protocol import IndexEntry + index.add_entry(IndexEntry( + id=file_id, name=video_path.name, path=video_path.parent.name, + size=len(file_bytes), type="video", added_at=0)) + + session = WebRTCPeerSession.__new__(WebRTCPeerSession) + session._ctx = { + "roots": one_root(video_path.parent), + "index": index, + "gek": gek, + "sk_node": sk_node, + "max_concurrent_streams": 4, + "media_cache": _FakeMediaCache(), + } + session._group_id = None + session._user_id = "tester" + session._stream_stopped = False + session._stream_keepalives = 0 + session.sent = [] + session._send = session.sent.append + session._audit = lambda *a, **k: None + return session, file_id + + +async def _ask_for(session, file_id: str, track: int) -> dict: + before = len(session.sent) + await session._do_subtitle_request({"file_id": file_id, "track": track}) + replies = session.sent[before:] + assert len(replies) == 1, f"expected one reply, got {replies}" + return replies[0] + + +@pytest.mark.asyncio +async def test_probe_lists_only_text_tracks_and_numbers_them_by_stream(tmp_path): + """The trap this feature is one wrong line away from. + + Numbering the survivors of the filter would give the two text tracks the + ordinals 0 and 1, and `-map 0:s:0` would then extract the stream that + cannot be decoded — which produces an empty WebVTT, not an error. + """ + clip = tmp_path / "clip.mp4" + _make_subtitled_clip(clip) + + probe = await _probe_video(str(clip)) + + assert [tr.ordinal for tr in probe.subtitle_tracks] == [1, 2], ( + "the listed tracks must keep their position among all subtitle " + "streams, not be renumbered from zero") + assert [tr.language for tr in probe.subtitle_tracks] == ["fre", "eng"] + assert all(tr.codec_name == "mov_text" for tr in probe.subtitle_tracks) + + # The fixture really does carry a subtitle stream that is not listed, and + # really does put the subtitles at container indices of their own — or the + # assertion above distinguishes nothing. + raw = subprocess.run( + ["ffprobe", "-v", "error", "-select_streams", "s", + "-show_entries", "stream=index,codec_name", "-of", "csv=p=0", str(clip)], + check=True, capture_output=True, text=True) + rows = [line.split(",") for line in raw.stdout.split()] + assert [int(r[0]) for r in rows] == [2, 3, 4] + assert [r[1] for r in rows] == ["ttml", "mov_text", "mov_text"] + + +def test_bitmap_codecs_are_not_offered(): + """The 20 % no amount of ffmpeg turns into text. + + Asserted against the allow-list rather than a fixture because ffmpeg + cannot encode text to bitmap, so a PGS or VOBSUB stream cannot be built + here — while the names ffprobe reports for them are fixed and are what the + filter is actually matched against. + """ + for codec in ("hdmv_pgs_subtitle", "dvd_subtitle", "dvb_subtitle", "xsub"): + assert codec not in TEXT_SUBTITLE_CODECS + # And the two that make up four-fifths of a real library are. + assert "subrip" in TEXT_SUBTITLE_CODECS + assert "ass" in TEXT_SUBTITLE_CODECS + + +@pytest.mark.asyncio +async def test_stream_init_announces_the_tracks(tmp_path): + """How a client discovers this node can do subtitles at all. + + From the answer, never from a version number: a node too old to enumerate + sends no list, the client draws no selector and never asks. + """ + clip = tmp_path / "clip.mp4" + _make_subtitled_clip(clip) + gek = generate_gek() + session, file_id = _session(clip, gek) + + await session._stream_video_inner( + {"file_id": file_id, "start": 0, "credits": 0}) + + init = next(m for m in session.sent if m.get("type") == "stream_init") + assert [tr["i"] for tr in init["subtitle_tracks"]] == [1, 2] + assert [tr["lang"] for tr in init["subtitle_tracks"]] == ["fre", "eng"] + + +@pytest.mark.parametrize("track", [1, 2]) +@pytest.mark.asyncio +async def test_the_requested_track_is_the_one_extracted(tmp_path, track): + """Read out of the cues, not out of the reply's language tag.""" + clip = tmp_path / "clip.mp4" + _make_subtitled_clip(clip) + gek = generate_gek() + session, file_id = _session(clip, gek) + + reply = await _ask_for(session, file_id, track) + + assert reply["type"] == "subtitle_resp" + assert reply["track"] == track + assert reply["mime"] == "text/vtt" + vtt = session._ctx["media_cache"].blobs[reply["hash"]].decode("utf-8") + assert vtt.startswith("WEBVTT") + assert _CUE_WORD[track] in vtt + other = _CUE_WORD[1 if track == 2 else 2] + assert other not in vtt, ( + f"track {track} carries the other track's words, so the ordinal was " + "mapped to the wrong stream") + + +@pytest.mark.asyncio +async def test_a_track_that_cannot_be_decoded_is_refused_not_served_empty(tmp_path): + """Ordinal 0 exists in the container and is not in the list. + + A viewer cannot ask for it through the interface, which draws its menu + from the list — but the ordinal travels on the wire, and a reply carrying + a WebVTT with no cues in it would be a track that appears and shows + nothing, with no error anywhere to lead back here. + """ + clip = tmp_path / "clip.mp4" + _make_subtitled_clip(clip) + gek = generate_gek() + session, file_id = _session(clip, gek) + + reply = await _ask_for(session, file_id, 0) + + assert reply["type"] == "error" + assert session._ctx["media_cache"].puts == 0, ( + "nothing may be cached for a track that could not be extracted") + + +@pytest.mark.asyncio +async def test_an_ordinal_past_the_end_is_refused(tmp_path): + clip = tmp_path / "clip.mp4" + _make_subtitled_clip(clip) + gek = generate_gek() + session, file_id = _session(clip, gek) + + reply = await _ask_for(session, file_id, 9) + + assert reply["type"] == "error" + + +@pytest.mark.asyncio +async def test_a_second_request_is_served_from_the_cache(tmp_path): + """The reason the extraction is whole-file rather than per-seek. + + A film's subtitles are extracted once in the life of the file: the second + viewing, the second seek and the second sitting all answer from the cache, + and ffmpeg runs exactly once. + """ + clip = tmp_path / "clip.mp4" + _make_subtitled_clip(clip) + gek = generate_gek() + session, file_id = _session(clip, gek) + + first = await _ask_for(session, file_id, 1) + second = await _ask_for(session, file_id, 1) + + assert first["hash"] == second["hash"] + assert session._ctx["media_cache"].puts == 1, ( + "the second request re-extracted instead of reading the cache") + + +@pytest.mark.asyncio +async def test_the_result_is_fetched_through_the_ordinary_chunk_path(tmp_path): + """The reply names a cache hash, not a new transfer mechanism. + + Same indirection as an audio transcode or a TMDB poster — and it has to + actually resolve, or the client is handed a hash it cannot pull. + """ + clip = tmp_path / "clip.mp4" + _make_subtitled_clip(clip) + gek = generate_gek() + session, file_id = _session(clip, gek) + + reply = await _ask_for(session, file_id, 2) + chunk = await session._try_serve_thumbnail(reply["hash"], 0, gek) + + assert chunk is not None, "the hash in the reply resolves to nothing" + key = chunk_key_aes(gek, bytes.fromhex(reply["hash"]), 0) + plain = decrypt_chunk_aes(key, chunk["nonce"], chunk["ct"]) + assert plain.decode("utf-8").startswith("WEBVTT") + assert _CUE_WORD[2] in plain.decode("utf-8") |