From a98445ce246509b0d2600df14907f72b448a6d10 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Thu, 17 Sep 2026 10:12:31 +0200 Subject: feat: let the viewer pick the audio track The streaming path mapped 0:a:0 unconditionally, so a dubbed film played in whichever language was muxed first and the others were unreachable. The node now enumerates the tracks in stream_init and honours audio_track in stream_req; switching is the seek path, since one ffmpeg carries one track. MNP 3.2, additive: the player draws its selector from the node's own list and never from a version number, so an older node is never asked for a track it would ignore. MNP_MIN_SUPPORTED does not move. Co-Authored-By: Claude Opus 5 --- .../meshbay-common/src/meshbay_common/__init__.py | 22 +- .../src/meshbay_hub/static/locales/de.js | 2 + .../src/meshbay_hub/static/locales/en.js | 2 + .../src/meshbay_hub/static/locales/es.js | 2 + .../src/meshbay_hub/static/locales/fr.js | 2 + .../src/meshbay_hub/static/locales/it.js | 2 + .../src/meshbay_hub/static/locales/ja.js | 2 + .../src/meshbay_hub/static/locales/nl.js | 2 + .../src/meshbay_hub/static/locales/pl.js | 2 + .../src/meshbay_hub/static/locales/pt-BR.js | 2 + .../src/meshbay_hub/static/locales/zh-CN.js | 2 + .../src/meshbay_hub/static/transport.js | 18 +- .../src/meshbay_hub/static/video-player.js | 117 ++++++++- .../meshbay-hub/tests/test_video_audio_track.py | 213 +++++++++++++++ .../src/meshbay_node/indexer/enrich.py | 7 +- .../meshbay-node/src/meshbay_node/media_probe.py | 85 +++++- .../src/meshbay_node/transport/webrtc_server.py | 46 +++- .../tests/test_stream_audio_track_selection.py | 285 +++++++++++++++++++++ .../tests/test_stream_audio_transcode.py | 27 +- .../tests/test_stream_video_transcode.py | 16 +- 20 files changed, 808 insertions(+), 48 deletions(-) create mode 100644 packages/meshbay-hub/tests/test_video_audio_track.py create mode 100644 packages/meshbay-node/tests/test_stream_audio_track_selection.py (limited to 'packages') diff --git a/packages/meshbay-common/src/meshbay_common/__init__.py b/packages/meshbay-common/src/meshbay_common/__init__.py index 80e6d92..c4c4bf4 100644 --- a/packages/meshbay-common/src/meshbay_common/__init__.py +++ b/packages/meshbay-common/src/meshbay_common/__init__.py @@ -181,5 +181,25 @@ __version__ = "0.14.0" # next one it reaches and keeps its own copy meanwhile (§6.4). `MNP_MIN_SUPPORTED` # does not move. That is the difference from 3.0, where the requirement — not the # messages — is what made it MAJOR. -MNP_VERSION = "3.1" +# **3.2 (2026-09-17): the viewer picks the audio track.** +# +# `stream_init` carries `audio_tracks` (ordinal, language, title, codec, +# channels) and the `audio_track` actually used; `stream_req` gains an optional +# `audio_track`. Until now the streaming path mapped `0:a:0` unconditionally, +# so a dubbed film played in whichever language was muxed first — across a real +# library that is one language, and the others were unreachable. +# +# **Additive, and MINOR because nothing is required, but the reason differs +# from 3.1's.** A playlist a node cannot store is a feature a client keeps to +# itself; an `audio_track` a node ignores is the *wrong language*, served +# silently, which is a wrong answer and not a missing one. What makes this +# MINOR anyway is that the client cannot get into that position: the selector +# is drawn from `audio_tracks` in the node's own `stream_init`, so a 3.1 node +# sends no list, the client shows no selector, and no `audio_track` is ever +# sent to a peer that would ignore it. **The capability is discovered from the +# answer, never from the version number** — and that is not the opt-in +# compatibility switch 3.0 refused, because there is no second branch on the +# node: a 3.2 node always enumerates, always honours what it is asked for, and +# always says which track it used. `MNP_MIN_SUPPORTED` does not move. +MNP_VERSION = "3.2" MHP_VERSION = "0.1" diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js index cb92b18..806fa38 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js @@ -249,6 +249,8 @@ export default { 'video.buffering': 'Wird gepuffert …', 'video.resumed_at': "Fortgesetzt bei {time}", 'video.from_start': "Von vorn beginnen", + 'video.audio_track': 'Tonspur', + 'video.audio_track_n': 'Spur {n}', 'video.close': 'Schließen (Esc)', 'preview.pdf_fallback': 'Dieser Browser zeigt das PDF nicht direkt an. Laden Sie es ' + 'stattdessen herunter — entschlüsselt wurde es ohnehin hier.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js index 3b84c3c..92114c9 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js @@ -248,6 +248,8 @@ export default { 'video.buffering': 'Buffering...', 'video.resumed_at': "Resumed at {time}", 'video.from_start': "Start from the beginning", + 'video.audio_track': 'Audio track', + 'video.audio_track_n': 'Track {n}', 'video.close': 'Close (Esc)', 'preview.pdf_fallback': 'This browser will not display the PDF inline. Download it instead — it was decrypted here either way.', 'preview.too_large': 'This file is {size}, more than this page can hold in memory ({limit}). Download it instead — a download is written straight to disk.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js index 145f464..d9b8f63 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js @@ -247,6 +247,8 @@ export default { 'video.buffering': 'Almacenando en búfer...', 'video.resumed_at': "Reanudado en {time}", 'video.from_start': "Empezar desde el principio", + 'video.audio_track': 'Pista de audio', + 'video.audio_track_n': 'Pista {n}', 'video.close': 'Cerrar (Esc)', 'preview.pdf_fallback': 'Este navegador no mostrará el PDF integrado. Descárguelo ' + 'en su lugar — en cualquier caso se descifró aquí.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js index a4f32de..d507d28 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js @@ -248,6 +248,8 @@ export default { 'video.buffering': 'Mise en mémoire tampon...', 'video.resumed_at': "Reprise à {time}", 'video.from_start': "Reprendre depuis le début", + 'video.audio_track': 'Piste audio', + 'video.audio_track_n': 'Piste {n}', 'video.close': 'Fermer (Échap)', 'preview.pdf_fallback': 'Ce navigateur n’affichera pas le PDF directement. ' + 'Téléchargez-le plutôt — il a été déchiffré ici dans les deux cas.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js index 1da0793..b6c68dd 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js @@ -248,6 +248,8 @@ export default { 'video.buffering': 'Buffering in corso...', 'video.resumed_at': "Ripreso da {time}", 'video.from_start': "Riparti dall'inizio", + 'video.audio_track': 'Traccia audio', + 'video.audio_track_n': 'Traccia {n}', 'video.close': 'Chiudi (Esc)', 'preview.pdf_fallback': 'Questo browser non mostrerà il PDF nella pagina. Lo scarichi ' + 'invece — in ogni caso è stato decifrato qui.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js index 971ecb2..99e2e1f 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js @@ -245,6 +245,8 @@ export default { 'video.buffering': 'バッファリング中…', 'video.resumed_at': "{time} から再開しました", 'video.from_start': "最初から再生する", + 'video.audio_track': '音声トラック', + 'video.audio_track_n': 'トラック {n}', 'video.close': '閉じる(Esc)', 'preview.pdf_fallback': 'このブラウザーはページ内に PDF を表示しません。' + 'ダウンロードしてご覧ください。いずれにせよ復号はここで行われています。', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js index dae63d2..269fa70 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js @@ -249,6 +249,8 @@ export default { 'video.buffering': 'Bezig met bufferen...', 'video.resumed_at': "Hervat op {time}", 'video.from_start': "Vanaf het begin afspelen", + 'video.audio_track': 'Audiospoor', + 'video.audio_track_n': 'Spoor {n}', 'video.close': 'Sluiten (Esc)', 'preview.pdf_fallback': 'Deze browser toont de PDF niet in de pagina zelf. Download ' + 'hem in plaats daarvan — ontsleuteld werd hij hoe dan ook hier.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js index 639d518..363296c 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js @@ -254,6 +254,8 @@ export default { 'video.buffering': 'Buforowanie...', 'video.resumed_at': "Wznowiono od {time}", 'video.from_start': "Odtwórz od początku", + 'video.audio_track': 'Ścieżka dźwiękowa', + 'video.audio_track_n': 'Ścieżka {n}', 'video.close': 'Zamknij (Esc)', 'preview.pdf_fallback': 'Ta przeglądarka nie wyświetli pliku PDF na stronie. Proszę ' + 'go pobrać — i tak został odszyfrowany tutaj.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js index 371b84f..2a90956 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js @@ -249,6 +249,8 @@ export default { 'video.buffering': 'Armazenando em buffer...', 'video.resumed_at': "Retomado em {time}", 'video.from_start': "Começar do início", + 'video.audio_track': 'Faixa de áudio', + 'video.audio_track_n': 'Faixa {n}', 'video.close': 'Fechar (Esc)', 'preview.pdf_fallback': 'Este navegador não exibirá o PDF na própria página. Baixe ' + 'o arquivo — de todo modo ele foi descriptografado aqui.', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js index 96ba4fe..e3aa440 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js @@ -243,6 +243,8 @@ export default { 'video.buffering': '正在缓冲…', 'video.resumed_at': "已从 {time} 继续播放", 'video.from_start': "从头开始播放", + 'video.audio_track': '音轨', + 'video.audio_track_n': '音轨 {n}', 'video.close': '关闭(Esc)', 'preview.pdf_fallback': '此浏览器不会在页面内显示该 PDF。请改为下载——无论如何它都已在本地解密。', 'preview.too_large': '该文件为 {size},超出本页面可在内存中保存的上限({limit})。请改为下载——下载会直接写入磁盘。', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/transport.js b/packages/meshbay-hub/src/meshbay_hub/static/transport.js index a6b3f52..262b00e 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/transport.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/transport.js @@ -2264,12 +2264,24 @@ class MeshBayTransport { * MediaSource consumes a segment at a time — which is fine for a clip and * fatal for anything worth streaming. */ - requestStream(fileId, credits = STREAM_CREDITS, start = 0) { + requestStream(fileId, credits = STREAM_CREDITS, start = 0, audioTrack = null) { // `start` is a seek: the node retires whatever this session was streaming // and spawns ffmpeg again from there. Omitted or zero is the film's // beginning, which is what an 0.1 node understands. - console.log('[stream] sending stream_req start:', start, 'credits:', credits); - this._send({ type: 'stream_req', v: '0.1', file_id: fileId, credits, start }); + // + // `audioTrack` is omitted entirely unless the caller has one, and the + // caller only has one because a `stream_init` listed the tracks. A node + // too old to enumerate them is therefore never sent a field it would + // ignore — which matters more here than it looks: ignoring it would not + // degrade the stream, it would serve a different language without saying + // so. + const req = { type: 'stream_req', v: '0.1', file_id: fileId, credits, start }; + if (Number.isInteger(audioTrack) && audioTrack >= 0) { + req.audio_track = audioTrack; + } + console.log('[stream] sending stream_req start:', start, 'credits:', credits, + 'audio_track:', req.audio_track ?? '-'); + this._send(req); } /** Room for `n` more segments. */ diff --git a/packages/meshbay-hub/src/meshbay_hub/static/video-player.js b/packages/meshbay-hub/src/meshbay_hub/static/video-player.js index 4f8a4e4..4970262 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/video-player.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/video-player.js @@ -1,7 +1,7 @@ import { html, useState, useEffect, useCallback, useRef, } from './vendor/htm-preact.js'; -import { t } from './i18n.js'; +import { t, getLocale } from './i18n.js'; import { Icon } from './icon.js'; import { formatSize } from './file-utils.js'; import { loadAuth } from './hub-client.js'; @@ -39,6 +39,63 @@ const RESUME_MIN_S = 30; const RESUME_MAX_FRACTION = 0.97; const QUEUE_HIGH_WATER = 12; +// ffprobe reports a container's language tag as ISO 639-2, and in either of +// its two variants for the dozen languages that have both — a bibliographic +// one (fre, ger, dut) and a terminological one (fra, deu, nld), with real +// files in this library using each. `Intl.DisplayNames` wants 639-1, so both +// variants are folded onto the same two-letter code here. Only what a media +// container actually carries is listed; anything unmapped falls through to the +// raw tag, which is more useful than "Unknown". +const _ISO639 = { + ara: 'ar', ben: 'bn', bul: 'bg', cat: 'ca', ces: 'cs', cze: 'cs', + chi: 'zh', dan: 'da', deu: 'de', dut: 'nl', ell: 'el', eng: 'en', + est: 'et', fas: 'fa', fin: 'fi', fra: 'fr', fre: 'fr', ger: 'de', + gle: 'ga', gre: 'el', heb: 'he', hin: 'hi', hrv: 'hr', hun: 'hu', + ice: 'is', ind: 'id', isl: 'is', ita: 'it', jpn: 'ja', kor: 'ko', + lav: 'lv', lit: 'lt', may: 'ms', msa: 'ms', nld: 'nl', nor: 'no', + per: 'fa', pol: 'pl', por: 'pt', ron: 'ro', rum: 'ro', rus: 'ru', + slk: 'sk', slo: 'sk', slv: 'sl', spa: 'es', srp: 'sr', swe: 'sv', + tam: 'ta', tha: 'th', tur: 'tr', ukr: 'uk', urd: 'ur', vie: 'vi', + zho: 'zh', +}; + +/** + * What to call one audio track, in the reader's language. + * + * The container's own `title` tag is preferred when there is one: a muxer that + * bothered to write "VFQ" or "Director's commentary" has said something the + * language code cannot, and two tracks tagged with the same language are + * otherwise indistinguishable in the menu — which is common, since a stereo + * downmix usually sits beside the surround track it came from. + */ +function audioTrackLabel(track) { + const code = (track.lang || '').toLowerCase(); + let name = null; + const iso = _ISO639[code] || (code.length === 2 ? code : null); + if (iso) { + try { + name = new Intl.DisplayNames([getLocale()], { type: 'language' }).of(iso); + // `Intl.DisplayNames` follows each locale's prose convention, which is + // lower case in French, Spanish and Italian among others. A menu entry + // is not prose, and "français" beside "AC3 5.1" reads like a bug. Only + // this branch needs it: a raw tag is a code and is shown as written, + // and the numbered fallback comes from the catalogues already cased. + if (name) name = name.charAt(0).toUpperCase() + name.slice(1); + } catch { /* no Intl.DisplayNames, or a code it does not know */ } + } + if (!name && code && code !== 'und') name = code; + if (!name) name = t('video.audio_track_n', { n: track.i + 1 }); + // Two tracks in the same language are one menu entry repeated without + // this, and a library where a stereo downmix sits beside the surround + // track it came from is the ordinary case. The container's title wins; the + // channel layout is the fallback, written in the "5.1" notation that needs + // no catalogue entry in any of the ten languages. + let detail = track.title; + if (!detail && track.ch > 2) detail = `${track.ch - 1}.1`; + else if (!detail && track.ch) detail = `${track.ch}.0`; + return detail ? `${name} — ${detail}` : name; +} + function _mseSupported(codec) { if (!window.MediaSource) return false; const mime = `video/mp4; codecs="${codec}"`; @@ -151,6 +208,16 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { // live; the render needs to reach it for "start from the beginning". const requestSeekRef = useRef(null); const [resumedFrom, setResumedFrom] = useState(0); + // The audio tracks this node reported for this file, and which one is + // playing. An empty list means either a file with one track or a node too + // old to enumerate them — both draw no selector, which is why nothing here + // needs to know which of the two it is. + const [audioTracks, setAudioTracks] = useState([]); + const [audioTrack, setAudioTrack] = useState(0); + const [audioMenuOpen, setAudioMenuOpen] = useState(false); + // Read inside the effect's closures, which are built once and would + // otherwise capture the first track forever. + const audioTrackRef = useRef(null); const [castActive, setCastActive] = useState(false); const [castUrl, setCastUrl] = useState(null); const [castPickerOpen, setCastPickerOpen] = useState(false); @@ -412,7 +479,7 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { outstandingRef.current = STREAM_WINDOW; setPhase('loading'); console.log('[seek] request', +target.toFixed(1), 'outstanding:', STREAM_WINDOW); - t.requestStream(entry.id, STREAM_WINDOW, target); + t.requestStream(entry.id, STREAM_WINDOW, target, audioTrackRef.current); }, SEEK_DEBOUNCE_MS); }; requestSeekRef.current = requestSeek; @@ -557,6 +624,16 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { const mime = `video/mp4; codecs="${msg.codec}"`; castCodecRef.current = msg.codec; + // What the node offered, and what it actually used — which is not + // always what was asked for: a file replaced on disk since the list + // was drawn falls back to the first track, and the selector must show + // the truth rather than the request. + setAudioTracks(Array.isArray(msg.audio_tracks) ? msg.audio_tracks : []); + if (Number.isInteger(msg.audio_track)) { + audioTrackRef.current = msg.audio_track; + setAudioTrack(msg.audio_track); + } + if (!window.MediaSource || !MediaSource.isTypeSupported(mime)) { setError(t('video.err_mse', { codec: msg.codec })); setPhase('error'); @@ -769,7 +846,7 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { outstandingRef.current = STREAM_WINDOW; const resumeAt = readResumePosition(entry.id); if (resumeAt) setResumedFrom(resumeAt); - transport.requestStream(entry.id, STREAM_WINDOW, resumeAt); + transport.requestStream(entry.id, STREAM_WINDOW, resumeAt, audioTrackRef.current); }; // Closing the tab, or backgrounding it on a phone, never runs a React @@ -954,6 +1031,40 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { }}>
${entry.name} (${formatSize(entry.size)}) + ${audioTracks.length > 1 && html` +
+ + ${audioMenuOpen && html` +
+ ${audioTracks.map((track) => html` + + `)} +
+ `} +
+ `} ${platform.capabilities.lanCast && html`