diff options
| -rw-r--r-- | docs/MESHBAY_DESIGN.md | 19 | ||||
| -rw-r--r-- | packages/meshbay-client/src/cast-chromecast.js | 89 | ||||
| -rw-r--r-- | packages/meshbay-client/src/cast-relay.js | 114 | ||||
| -rw-r--r-- | packages/meshbay-client/src/main.js | 19 | ||||
| -rw-r--r-- | packages/meshbay-client/src/preload.js | 1 | ||||
| -rw-r--r-- | packages/meshbay-hub/src/meshbay_hub/static/platform.js | 4 | ||||
| -rw-r--r-- | packages/meshbay-hub/src/meshbay_hub/static/video-player.js | 124 | ||||
| -rw-r--r-- | packages/meshbay-hub/tests/test_cast_subtitles.py | 393 | ||||
| -rw-r--r-- | packages/meshbay-hub/tests/test_video_subtitles.py | 13 |
9 files changed, 741 insertions, 35 deletions
diff --git a/docs/MESHBAY_DESIGN.md b/docs/MESHBAY_DESIGN.md index 8f91a87..781438a 100644 --- a/docs/MESHBAY_DESIGN.md +++ b/docs/MESHBAY_DESIGN.md @@ -2596,6 +2596,25 @@ any LAN renderer can play; the relay is device-agnostic. Chromecast discovery an control ship. DLNA/UPnP is designed and not built: it is a second device backend beside the first, not a second relay. +**Subtitles are rebased onto the relay's clock before they are sent.** The node +extracts a track whole, so its cues carry the film's timeline, and the player +can use them unchanged because its SourceBuffer is given `timestampOffset = +start`. The relay has no such offset: it forwards the node's fragments, which +begin at zero at the seek point. A receiver is therefore sent the cues shifted +by `-start`, recomputed at every restart of the relay, and cues that end before +the stream begins are dropped rather than clamped to zero. + +**The subtitle is served from the relay's own port, behind the same token as the +stream, and with CORS.** A receiver fetches a side-loaded track with XHR from +its own origin rather than handing it to a media element, so without +`Access-Control-Allow-Origin` it fails as a network error and the film plays on +with no subtitles and no message. The stream carries the same headers, because a +receiver given a side-loaded track reads the media through the same checked +path: on one and not the other, the load fails whole. They widen nothing the +token does not already govern. The subtitle URL carries a version because a +receiver caches a track by address: changing the cues behind a fixed URL leaves +the previous language on screen. + --- ## 12. Testing posture diff --git a/packages/meshbay-client/src/cast-chromecast.js b/packages/meshbay-client/src/cast-chromecast.js index 50f79fd..2dd210f 100644 --- a/packages/meshbay-client/src/cast-chromecast.js +++ b/packages/meshbay-client/src/cast-chromecast.js @@ -6,6 +6,56 @@ const DefaultMediaReceiver = require('castv2-client').DefaultMediaReceiver; const SCAN_DURATION_MS = 6000; +// The one track the receiver is ever told about. A side-loaded track cannot +// have its cues replaced in place, so changing subtitles means loading again +// with a new URL under this same id rather than adding a second track. +const TEXT_TRACK_ID = 1; + +// White on nothing is unreadable over a bright scene, and the receiver's +// default style is exactly that. An outline costs no bandwidth. +const TEXT_TRACK_STYLE = Object.freeze({ + backgroundColor: '#00000000', + foregroundColor: '#FFFFFFFF', + edgeType: 'OUTLINE', + edgeColor: '#000000FF', + fontScale: 1.0, + fontFamily: 'SANS_SERIF', +}); + +/** + * What to hand `player.load()` for a stream, with or without subtitles. + * + * `streamType: 'LIVE'` because the relay has no beginning to seek back to — + * the film's own timeline lives on this side, and a seek restarts the relay. + */ +function mediaFor(mediaUrl, subtitle) { + const media = { + contentId: mediaUrl, + contentType: 'video/mp4', + streamType: 'LIVE', + }; + if (subtitle && subtitle.url) { + media.tracks = [{ + trackId: TEXT_TRACK_ID, + type: 'TEXT', + trackContentId: subtitle.url, + trackContentType: 'text/vtt', + subtype: 'SUBTITLES', + name: subtitle.label || 'Subtitles', + language: subtitle.language || 'und', + }]; + media.textTrackStyle = TEXT_TRACK_STYLE; + } + return media; +} + +function loadOptionsFor(subtitle) { + return { + autoplay: true, + activeTrackIds: subtitle && subtitle.url ? [TEXT_TRACK_ID] : [], + }; +} + class CastChromecast { constructor() { this._bonjour = null; @@ -56,7 +106,7 @@ class CastChromecast { }); } - async connect(deviceId, mediaUrl) { + async connect(deviceId, mediaUrl, subtitle) { const device = this._devices.get(deviceId); if (!device) throw new Error(`Unknown device: ${deviceId}`); @@ -91,36 +141,29 @@ class CastChromecast { console.log(`[cast-chromecast] player status: ${status.playerState}`); }); - const media = { - contentId: mediaUrl, - contentType: 'video/mp4', - streamType: 'LIVE', - }; - + console.log(`[cast-chromecast] loading with ${subtitle?.url ? 'subtitles' : 'no subtitles'}`); const status = await new Promise((resolve, reject) => { - player.load(media, { autoplay: true }, (err, s) => { - if (err) return reject(err); - resolve(s); - }); + player.load(mediaFor(mediaUrl, subtitle), loadOptionsFor(subtitle), + (err, s) => { + if (err) return reject(err); + resolve(s); + }); }); console.log(`[cast-chromecast] loaded on "${device.name}", state: ${status.playerState}`); return { deviceName: device.name, playerState: status.playerState }; } - async reload(mediaUrl) { + async reload(mediaUrl, subtitle) { if (!this._player) throw new Error('Not connected'); - console.log(`[cast-chromecast] reloading stream on "${this._connectedDevice?.name}"`); - const media = { - contentId: mediaUrl, - contentType: 'video/mp4', - streamType: 'LIVE', - }; + console.log(`[cast-chromecast] reloading stream on "${this._connectedDevice?.name}"` + + ` with ${subtitle?.url ? 'subtitles' : 'no subtitles'}`); const status = await new Promise((resolve, reject) => { - this._player.load(media, { autoplay: true }, (err, s) => { - if (err) return reject(err); - resolve(s); - }); + this._player.load(mediaFor(mediaUrl, subtitle), loadOptionsFor(subtitle), + (err, s) => { + if (err) return reject(err); + resolve(s); + }); }); console.log(`[cast-chromecast] reloaded, state: ${status.playerState}`); return { playerState: status.playerState }; @@ -155,3 +198,5 @@ class CastChromecast { } module.exports = CastChromecast; +module.exports.mediaFor = mediaFor; +module.exports.loadOptionsFor = loadOptionsFor; diff --git a/packages/meshbay-client/src/cast-relay.js b/packages/meshbay-client/src/cast-relay.js index 12939c4..71c07a4 100644 --- a/packages/meshbay-client/src/cast-relay.js +++ b/packages/meshbay-client/src/cast-relay.js @@ -10,10 +10,16 @@ * properly framed fMP4 fragments. The BoxAccumulator reassembles the byte * stream and emits complete moof+mdat pairs. * + * Subtitles ride alongside the video as a side-loaded WebVTT file at + * `/subs.vtt`. A receiver fetches that one with XHR rather than handing it to + * a media element, so unlike the stream it needs CORS headers to be readable + * at all. + * * Mitigations: * · Bind to the LAN interface, never 0.0.0.0 * · Fixed port range (19550-19553), opened only during active cast * · Unguessable token in the URL (32 hex chars) + * · CORS is granted on the subtitle path only, and still behind the token * · Cache-Control: no-store on every response * · Server destroyed when playback stops — zero residual surface */ @@ -30,6 +36,16 @@ const MOOF = 0x6d6f6f66; const PORT_BASE = 19550; const PORT_COUNT = 4; +// A receiver reads a side-loaded subtitle with XHR, from its own origin, so +// the three headers it sends have to be allowed by name — `Range` included, +// which it sends even for a document it will read whole. +const CORS_HEADERS = Object.freeze({ + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Accept-Encoding, Range', + 'Access-Control-Expose-Headers': 'Content-Length, Content-Range', +}); + function lanAddress() { const ifaces = os.networkInterfaces(); for (const name of Object.keys(ifaces)) { @@ -122,6 +138,8 @@ class CastRelay { this._fragCount = 0; this._chunkCount = 0; this._bytesSent = 0; + this._subtitle = null; + this._subtitleVersion = 0; } get active() { return this._server !== null; } @@ -131,7 +149,54 @@ class CastRelay { return `http://${this._lanIP}:${this._port}/stream.mp4?t=${this._token}`; } - async start({ codec, initSegment }) { + /** + * Where the current subtitle can be fetched, or null when there is none. + * + * The version is part of the URL because a receiver caches a side-loaded + * track by its address: changing the cues behind a fixed URL would leave the + * old ones on screen. + */ + get subtitleUrl() { + if (!this._server || !this._subtitle) return null; + return `http://${this._lanIP}:${this._port}/subs.vtt` + + `?t=${this._token}&v=${this._subtitleVersion}`; + } + + get subtitle() { + if (!this._subtitle) return null; + return { + url: this.subtitleUrl, + language: this._subtitle.language, + label: this._subtitle.label, + }; + } + + /** + * Carry a WebVTT document for the receiver to side-load. + * + * The cues must already be expressed on the *stream's* timeline, which + * starts at zero at the seek point — not on the film's. The renderer shifts + * them before they get here; the relay only serves bytes. + */ + setSubtitle(sub) { + if (!sub || !sub.vtt) { + this._subtitle = null; + this._subtitleVersion++; + console.log('[cast-relay] subtitle cleared'); + return null; + } + this._subtitle = { + vtt: Buffer.from(sub.vtt, 'utf8'), + language: sub.language || '', + label: sub.label || '', + }; + this._subtitleVersion++; + console.log(`[cast-relay] subtitle set: ${this._subtitle.vtt.length} bytes` + + `, lang "${this._subtitle.language}", v${this._subtitleVersion}`); + return this.subtitle; + } + + async start({ codec, initSegment, subtitle }) { if (this._server) await this.stop(); this._token = crypto.randomBytes(16).toString('hex'); @@ -143,6 +208,8 @@ class CastRelay { this._fragCount = 0; this._chunkCount = 0; this._bytesSent = 0; + this._subtitle = null; + this.setSubtitle(subtitle); const server = http.createServer((req, res) => this._handle(req, res)); @@ -173,7 +240,12 @@ class CastRelay { this._server = server; console.log(`[cast-relay] started on ${this.url}`); console.log(`[cast-relay] init segment: ${this._initSegment ? this._initSegment.length + ' bytes' : 'none'}`); - return { url: this.url, port: this._port, token: this._token }; + return { + url: this.url, + port: this._port, + token: this._token, + subtitle: this.subtitle, + }; } pushSegment(data) { @@ -227,6 +299,7 @@ class CastRelay { this._port = null; this._token = null; this._initSegment = null; + this._subtitle = null; this._ring = []; this._accum.reset(); this._fragCount = 0; @@ -235,7 +308,7 @@ class CastRelay { } _handle(req, res) { - if (req.method !== 'GET') { + if (req.method !== 'GET' && req.method !== 'OPTIONS') { res.writeHead(405); res.end(); return; @@ -250,12 +323,26 @@ class CastRelay { return; } + // The preflight is answered before the token is examined: a browser sends + // it without credentials and rejecting it here would read, on the receiver, + // as a network failure rather than as a refusal. + if (req.method === 'OPTIONS') { + res.writeHead(204, CORS_HEADERS); + res.end(); + return; + } + if (url.searchParams.get('t') !== this._token) { res.writeHead(403); res.end(); return; } + if (url.pathname === '/subs.vtt') { + this._handleSubtitle(res); + return; + } + if (url.pathname !== '/stream.mp4') { res.writeHead(404); res.end(); @@ -264,6 +351,11 @@ class CastRelay { let sent = 0; res.writeHead(200, { + // A receiver that has been given a side-loaded track reads the media + // through the same CORS-checked path as the track, so the headers go on + // both or neither. They widen nothing: the URL is already unguessable, + // and a page that has it could embed it in a media element regardless. + ...CORS_HEADERS, 'Content-Type': 'video/mp4', 'Cache-Control': 'no-store', 'Accept-Ranges': 'none', @@ -288,6 +380,22 @@ class CastRelay { console.log(`[cast-relay] client disconnected, ${this._clients.size} remaining`); }); } + + _handleSubtitle(res) { + if (!this._subtitle) { + res.writeHead(404, CORS_HEADERS); + res.end(); + return; + } + res.writeHead(200, { + ...CORS_HEADERS, + 'Content-Type': 'text/vtt; charset=utf-8', + 'Content-Length': this._subtitle.vtt.length, + 'Cache-Control': 'no-store', + }); + res.end(this._subtitle.vtt); + console.log(`[cast-relay] subtitle served: ${this._subtitle.vtt.length} bytes`); + } } module.exports = CastRelay; diff --git a/packages/meshbay-client/src/main.js b/packages/meshbay-client/src/main.js index f201a21..122a302 100644 --- a/packages/meshbay-client/src/main.js +++ b/packages/meshbay-client/src/main.js @@ -1770,9 +1770,17 @@ function registerBridge() { return castRelay.start({ codec: opts.codec, initSegment: opts.initSegment ? Buffer.from(opts.initSegment) : null, + subtitle: opts.subtitle || null, }); }); + // Carried separately from `cast:start` for the viewer who turns subtitles on + // without seeking: the relay keeps serving the same video while the receiver + // is told to load again with the new track. + ipcMain.handle('cast:subtitle', async (_e, sub) => { + return castRelay.setSubtitle(sub || null); + }); + ipcMain.handle('cast:push', async (_e, data) => { castRelay.pushSegment(Buffer.from(data)); return true; @@ -1791,6 +1799,7 @@ function registerBridge() { ipcMain.handle('cast:status', async () => ({ active: castRelay.active, url: castRelay.url, + subtitle: castRelay.subtitle, chromecast: castChromecast.getStatus(), })); @@ -1800,12 +1809,14 @@ function registerBridge() { return castChromecast.discover(); }); - ipcMain.handle('cast:chromecast:connect', async (_e, { deviceId, mediaUrl }) => { - return castChromecast.connect(deviceId, mediaUrl); + ipcMain.handle('cast:chromecast:connect', async (_e, { deviceId, mediaUrl, subtitle }) => { + return castChromecast.connect(deviceId, mediaUrl, + subtitle === undefined ? castRelay.subtitle : subtitle); }); - ipcMain.handle('cast:chromecast:reload', async (_e, { mediaUrl }) => { - return castChromecast.reload(mediaUrl); + ipcMain.handle('cast:chromecast:reload', async (_e, { mediaUrl, subtitle }) => { + return castChromecast.reload(mediaUrl, + subtitle === undefined ? castRelay.subtitle : subtitle); }); ipcMain.handle('cast:chromecast:disconnect', async () => { diff --git a/packages/meshbay-client/src/preload.js b/packages/meshbay-client/src/preload.js index c9c9fe0..c2a2bd4 100644 --- a/packages/meshbay-client/src/preload.js +++ b/packages/meshbay-client/src/preload.js @@ -139,6 +139,7 @@ contextBridge.exposeInMainWorld('meshbay', { start: (opts) => ipcRenderer.invoke('cast:start', opts), push: (data) => ipcRenderer.invoke('cast:push', data), stop: () => ipcRenderer.invoke('cast:stop'), + subtitle: (sub) => ipcRenderer.invoke('cast:subtitle', sub), finish: () => ipcRenderer.invoke('cast:finish'), status: () => ipcRenderer.invoke('cast:status'), discover: () => ipcRenderer.invoke('cast:discover'), diff --git a/packages/meshbay-hub/src/meshbay_hub/static/platform.js b/packages/meshbay-hub/src/meshbay_hub/static/platform.js index 43b8ba8..07e5b6f 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/platform.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/platform.js @@ -441,6 +441,10 @@ export const cast = { if (!bridge || !bridge.cast) return false; return bridge.cast.stop(); }, + async subtitle(sub) { + if (!bridge || !bridge.cast || !bridge.cast.subtitle) return null; + return bridge.cast.subtitle(sub); + }, async finish() { if (!bridge || !bridge.cast) return false; return bridge.cast.finish(); diff --git a/packages/meshbay-hub/src/meshbay_hub/static/video-player.js b/packages/meshbay-hub/src/meshbay_hub/static/video-player.js index 1522539..a07558e 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/video-player.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/video-player.js @@ -124,6 +124,75 @@ function subtitleTrackLabel(track) { return detail ? `${name} — ${detail}` : name; } +/** + * The same cues, moved onto a stream that begins somewhere else. + * + * The node extracts a subtitle whole, so its cues carry the film's own + * timeline. That is what the player wants — the SourceBuffer is given + * `timestampOffset = start`, so the element's `currentTime` is film time and + * the cues need no adjustment. + * + * A cast has no such offset. The relay hands the receiver the node's fragments + * untouched, and those are rebased to zero at the seek point, so film time and + * stream time differ by exactly `start`. Sending the unshifted document to a + * receiver would put the subtitles out by however far the viewer had seeked — + * an hour into a film, an hour wrong. + * + * Cues that end before the stream does are dropped rather than clamped: a cue + * pinned to 0 would show a line from before the seek over the first frames + * after it. + */ +function shiftWebVtt(text, delta) { + const TIMING = /^((?:\d+:)?\d{1,2}:\d{2}[.,]\d{1,3})\s*-->\s*((?:\d+:)?\d{1,2}:\d{2}[.,]\d{1,3})(.*)$/; + const parse = (stamp) => { + const parts = stamp.replace(',', '.').split(':'); + const secs = parseFloat(parts.pop()); + const mins = parseInt(parts.pop() || '0', 10); + const hours = parseInt(parts.pop() || '0', 10); + return hours * 3600 + mins * 60 + secs; + }; + const pad = (n, width) => String(n).padStart(width, '0'); + const format = (t) => { + const ms = Math.round(t * 1000); + return `${pad(Math.floor(ms / 3600000), 2)}:${pad(Math.floor(ms / 60000) % 60, 2)}` + + `:${pad(Math.floor(ms / 1000) % 60, 2)}.${pad(ms % 1000, 3)}`; + }; + + const kept = []; + for (const block of String(text).split(/\r?\n\r?\n/)) { + const lines = block.split(/\r?\n/); + const at = lines.findIndex((line) => TIMING.test(line)); + // The header, NOTE, STYLE and REGION blocks carry no timing and travel + // unchanged — dropping them would take the cue positioning with them. + if (at === -1) { + kept.push(block); + continue; + } + const m = lines[at].match(TIMING); + const from = parse(m[1]) + delta; + const to = parse(m[2]) + delta; + if (to <= 0) continue; + lines[at] = `${format(Math.max(0, from))} --> ${format(to)}${m[3]}`; + kept.push(lines.join('\n')); + } + return kept.join('\n\n'); +} + +/** + * What the cast relay should serve for the track now showing, or null. + * + * `start` is where the stream the relay is being fed begins, in film time, so + * the shift is its negation: film time minus start is stream time. + */ +function castSubtitleFor(sub, start) { + if (!sub || !sub.text) return null; + return { + vtt: shiftWebVtt(sub.text, -(start || 0)), + language: sub.language || '', + label: sub.label || '', + }; +} + function _mseSupported(codec) { if (!window.MediaSource) return false; const mime = `video/mp4; codecs="${codec}"`; @@ -265,6 +334,12 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { // the first one asked for is not necessarily the first one answered. Only // the newest request may install its blob. const subtitleGenRef = useRef(0); + // The cues as text, on the film's own timeline. Kept because a cast needs + // them shifted onto the relay's, and that shift changes at every seek. + const subtitleTextRef = useRef(null); + // Where the stream the node is sending begins, in film time. The same number + // the SourceBuffer gets as its `timestampOffset`. + const streamStartRef = useRef(0); const [castActive, setCastActive] = useState(false); const [castUrl, setCastUrl] = useState(null); const [castPickerOpen, setCastPickerOpen] = useState(false); @@ -694,6 +769,10 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { } durationRef.current = msg.duration || 0; + // Recorded before either branch below: both restart the relay, and the + // subtitle sent with it has to be shifted by *this* start, not the one + // the previous stream had. + streamStartRef.current = msg.start || 0; // A second init on a live SourceBuffer is a seek landing, not a new // film. Reuse what is there: rebuilding the MediaSource would reset the @@ -841,6 +920,8 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { platform.cast.start({ codec: castCodecRef.current, initSegment: plaintext, + subtitle: castSubtitleFor( + subtitleTextRef.current, streamStartRef.current), }).then(async (result) => { if (castRestartGenRef.current !== gen) return; if (!result) return; @@ -1091,6 +1172,36 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { * film that is already running, and taking the film down because a text * track could not be read would be a worse answer than no subtitles. */ + /** + * Put the chosen track in front of the receiver, when one is casting. + * + * A seek already carries the subtitle with it — the relay restarts and is + * handed the shifted cues. This covers the other case: the viewer turns + * subtitles on, off, or swaps languages while the picture keeps running. The + * relay keeps serving the same video; only the receiver has to be told, and + * a side-loaded track cannot be changed in place, so it is told by loading + * the same stream URL again with a new track address. + * + * Never allowed to disturb playback. A receiver that refuses the track keeps + * showing the film without subtitles, which is what it was doing anyway. + */ + const sendSubtitleToCast = useCallback(async (sub) => { + if (!castActiveRef.current || !platform.cast.available) return; + try { + const payload = castSubtitleFor(sub, streamStartRef.current); + await platform.cast.subtitle(payload); + const status = await platform.cast.status(); + if (status && status.chromecast && status.chromecast.connected + && status.url) { + await platform.cast.chromecastReload({ mediaUrl: status.url }); + } + console.log('[cast] subtitle', payload ? 'sent' : 'cleared', + '— stream starts at', streamStartRef.current.toFixed(1)); + } catch (err) { + console.warn('[cast] subtitle not sent:', err); + } + }, []); + const selectSubtitle = useCallback(async (track) => { const gen = ++subtitleGenRef.current; if (subtitleUrlRef.current) { @@ -1102,6 +1213,8 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { if (track === null) { setSubtitleTrack(null); setSubtitleBusy(false); + subtitleTextRef.current = null; + sendSubtitleToCast(null); return; } const transport = transportRef.current; @@ -1137,6 +1250,12 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { subtitleUrlRef.current = url; setSubtitleUrl(url); console.log('[MeshBay] subtitle: track attached'); + subtitleTextRef.current = { + text: await new Blob(chunks).text(), + language: track.lang || '', + label: subtitleTrackLabel(track), + }; + sendSubtitleToCast(subtitleTextRef.current); } catch (err) { if (subtitleGenRef.current !== gen) return; console.warn('[MeshBay] subtitle track', track.i, 'failed after', @@ -1146,7 +1265,7 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { } finally { if (subtitleGenRef.current === gen) setSubtitleBusy(false); } - }, [entry, transportRef, gekRef]); + }, [entry, transportRef, gekRef, sendSubtitleToCast]); // The mode is set here rather than left to the `default` attribute. // @@ -1175,6 +1294,9 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) { URL.revokeObjectURL(subtitleUrlRef.current); subtitleUrlRef.current = null; } + // A whole film's cues, held as a string for the cast path. Nothing else + // drops it, and the next film's are a different document. + subtitleTextRef.current = null; }; }, []); diff --git a/packages/meshbay-hub/tests/test_cast_subtitles.py b/packages/meshbay-hub/tests/test_cast_subtitles.py new file mode 100644 index 0000000..722a56a --- /dev/null +++ b/packages/meshbay-hub/tests/test_cast_subtitles.py @@ -0,0 +1,393 @@ +""" +Subtitles on a cast receiver. + +Casting takes a different road than the player does, and the difference that +matters is the clock. The node extracts a subtitle whole, so its cues carry the +film's own timeline; the player feeds its SourceBuffer a `timestampOffset` +equal to the stream's start, which puts the element's `currentTime` on that +same timeline and lets the cues be used as they arrive. + +A receiver has no such offset. The relay hands it the node's fragments +untouched, and those are rebased to zero at the seek point. Film time and +stream time therefore differ by exactly the start, and a document sent across +unshifted is wrong by however far the viewer had seeked — an hour into a film, +an hour wrong, with nothing in any log to say so. `shiftWebVtt` is the whole +correction and the first half of this file tests it by running it. + +The second half runs the relay itself. Unlike the Electron shell it is plain +Node with no dependencies, so it can be started, served from and stopped here. +Three of its properties are load-bearing and invisible when broken: the +subtitle sits behind the same token as the stream, it carries CORS headers +because a receiver fetches it with XHR rather than handing it to a media +element, and its URL changes when its content does — a side-loaded track is +cached by address, so a fixed URL would leave the old language on screen. +""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +STATIC = Path(__file__).resolve().parents[1] / "src" / "meshbay_hub" / "static" +APP = STATIC / "video-player.js" +CLIENT = Path(__file__).resolve().parents[2] / "meshbay-client" +RELAY = CLIENT / "src" / "cast-relay.js" +CHROMECAST = CLIENT / "src" / "cast-chromecast.js" + +pytestmark = pytest.mark.skipif( + shutil.which("node") is None or not APP.exists(), + reason="node or the SPA sources are not available") + + +def _lift(src: str, name: str) -> str: + """One top-level function, as text, for node to execute.""" + start = src.index(f"function {name}(") + depth, i, seen = 0, start, False + while i < len(src): + if src[i] == "{": + depth += 1 + seen = True + elif src[i] == "}": + depth -= 1 + if seen and depth == 0: + return src[start:i + 1] + i += 1 + raise AssertionError(f"{name} never closes") + + +def _run(tmp_path, body: str, *args: str): + script = tmp_path / "cast.mjs" + app = APP.read_text(encoding="utf-8") + script.write_text( + _lift(app, "shiftWebVtt") + + "\n" + _lift(app, "castSubtitleFor") + + "\n" + body, encoding="utf-8") + proc = subprocess.run( + ["node", str(script), *args], + capture_output=True, text=True, timeout=30) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +VTT = """WEBVTT + +NOTE this file was converted from an embedded track + +1 +00:00:10.000 --> 00:00:12.500 +Before the seek. + +2 +01:23:45.000 --> 01:23:47.250 line:90% +Hello. + +3 +01:30:00.000 --> 01:30:02.000 +Goodbye. +""" + + +def _timings(vtt: str): + return [line.strip() for line in vtt.splitlines() if "-->" in line] + + +# ── The clock ──────────────────────────────────────────────────────────────── + +def test_cues_move_back_by_the_streams_start(tmp_path): + """ + A stream that begins at 01:20:00 makes a cue at 01:23:45 land at 00:03:45. + + This is the correction the whole feature rests on. Without it the cues keep + the film's timeline while the receiver counts from zero, and the error is + the size of the seek rather than a small drift — invisible in code review, + unmistakable on screen. + """ + out = _run(tmp_path, """ + const shifted = shiftWebVtt(process.argv[2], -4800); + console.log(JSON.stringify(shifted)); + """, VTT) + assert "00:03:45.000 --> 00:03:47.250 line:90%" in out + assert "00:10:00.000 --> 00:10:02.000" in out + + +def test_a_stream_from_the_top_leaves_every_cue_where_it_was(tmp_path): + """No seek means no shift, and the document must come back unchanged.""" + out = _run(tmp_path, """ + console.log(JSON.stringify(shiftWebVtt(process.argv[2], 0))); + """, VTT) + assert _timings(out) == _timings(VTT) + + +def test_cues_before_the_stream_are_dropped_not_clamped(tmp_path): + """ + A cue that has already finished when the stream starts must disappear. + + Clamping it to zero instead would print a line from before the seek over + the first frames after it — the one failure mode that looks like a bug in + the extraction rather than in the arithmetic. + """ + out = _run(tmp_path, """ + console.log(JSON.stringify(shiftWebVtt(process.argv[2], -4800))); + """, VTT) + assert "Before the seek." not in out + assert len(_timings(out)) == 2 + + +def test_a_cue_straddling_the_seek_survives_and_starts_at_zero(tmp_path): + """ + Someone is mid-sentence when the viewer lands. The line is still owed to + them, so the cue is kept with its start pulled up to the stream's own. + """ + out = _run(tmp_path, """ + const vtt = 'WEBVTT\\n\\n1\\n00:00:08.000 --> 00:00:14.000\\nMid-sentence.\\n'; + console.log(JSON.stringify(shiftWebVtt(vtt, -10))); + """) + assert "00:00:00.000 --> 00:00:04.000" in out + assert "Mid-sentence." in out + + +def test_the_header_and_its_notes_travel_unchanged(tmp_path): + """ + A document that loses its `WEBVTT` line is not a WebVTT document, and a + receiver rejects it whole rather than complaining about one cue. + """ + out = _run(tmp_path, """ + console.log(JSON.stringify(shiftWebVtt(process.argv[2], -4800))); + """, VTT) + assert out.startswith("WEBVTT") + assert "NOTE this file was converted" in out + + +def test_comma_decimals_are_read_and_written_back_as_points(tmp_path): + """ + SRT writes `00:00:10,000` and some converters leave the comma in place. + Reading it and emitting the WebVTT spelling costs nothing and saves a + document that would otherwise be silently dropped cue by cue. + """ + out = _run(tmp_path, """ + const vtt = 'WEBVTT\\n\\n1\\n00:01:00,500 --> 00:01:02,000\\nComma.\\n'; + console.log(JSON.stringify(shiftWebVtt(vtt, -30))); + """) + assert "00:00:30.500 --> 00:00:32.000" in out + + +def test_the_payload_negates_the_start_it_is_given(tmp_path): + """ + `castSubtitleFor` takes where the stream *begins* and must subtract it. + Getting the sign wrong doubles the error instead of cancelling it, and both + directions produce a plausible-looking document. + """ + out = _run(tmp_path, """ + const sub = { text: process.argv[2], language: 'fr', label: 'French' }; + const payload = castSubtitleFor(sub, 4800); + console.log(JSON.stringify(payload)); + """, VTT) + assert "00:03:45.000 --> 00:03:47.250" in out["vtt"] + assert out["language"] == "fr" + assert out["label"] == "French" + + +def test_no_track_showing_means_no_payload(tmp_path): + """ + Subtitles off has to reach the relay as null, not as an empty document: an + empty WebVTT file is a track the receiver will happily display nothing + from, and its menu would still offer it. + """ + out = _run(tmp_path, """ + console.log(JSON.stringify([ + castSubtitleFor(null, 0), castSubtitleFor({ text: '' }, 0), + ])); + """) + assert out == [None, None] + + +# ── The relay ──────────────────────────────────────────────────────────────── + +relay_only = pytest.mark.skipif( + not RELAY.exists(), reason="desktop client sources not present") + +RELAY_SCRIPT = """ +const CastRelay = require(process.argv[2]); + +(async () => { + const relay = new CastRelay(); + const started = await relay.start({ + codec: 'avc1.640028', + initSegment: Buffer.from([0, 0, 0, 8, 102, 116, 121, 112]), + subtitle: { vtt: 'WEBVTT\\n\\n1\\n00:00:01.000 --> 00:00:02.000\\nHi.\\n', + language: 'fr', label: 'French' }, + }); + + const out = { streamUrl: started.url, subtitleUrl: started.subtitle.url }; + + const get = await fetch(out.subtitleUrl); + out.status = get.status; + out.contentType = get.headers.get('content-type'); + out.allowOrigin = get.headers.get('access-control-allow-origin'); + out.allowHeaders = get.headers.get('access-control-allow-headers'); + out.body = await get.text(); + + const noToken = await fetch(out.subtitleUrl.replace(/t=[0-9a-f]+/, 't=0')); + out.withoutToken = noToken.status; + + const preflight = await fetch(out.subtitleUrl, { method: 'OPTIONS' }); + out.preflight = preflight.status; + out.preflightAllowOrigin = + preflight.headers.get('access-control-allow-origin'); + + const second = relay.setSubtitle({ vtt: 'WEBVTT\\n\\n', language: 'en' }); + out.secondUrl = second.url; + + relay.setSubtitle(null); + out.afterClear = relay.subtitleUrl; + out.afterClearStatus = (await fetch(out.secondUrl)).status; + const stream = await fetch(out.streamUrl); + out.streamStillServes = stream.status; + out.streamAllowOrigin = stream.headers.get('access-control-allow-origin'); + + await relay.stop(); + console.log(JSON.stringify(out)); + process.exit(0); +})().catch((err) => { console.error(err); process.exit(1); }); +""" + + +@pytest.fixture(scope="module") +def served(tmp_path_factory): + script = tmp_path_factory.mktemp("relay") / "serve.cjs" + script.write_text(RELAY_SCRIPT, encoding="utf-8") + proc = subprocess.run( + ["node", str(script), str(RELAY)], + capture_output=True, text=True, timeout=60) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout.strip().splitlines()[-1]) + + +@relay_only +def test_the_subtitle_is_served_as_webvtt(served): + assert served["status"] == 200 + assert served["contentType"].startswith("text/vtt") + assert served["body"].startswith("WEBVTT") + + +@relay_only +def test_the_subtitle_sits_behind_the_same_token_as_the_stream(served): + """ + The relay's only defence is that its URL cannot be guessed. A subtitle + path exempt from the token would hand the film's dialogue — often the + whole script — to anything on the Wi-Fi. + """ + assert served["withoutToken"] == 403 + + +@relay_only +def test_the_receiver_is_allowed_to_read_it(served): + """ + A side-loaded track is fetched with XHR from the receiver's own origin, so + without CORS it fails as a network error and the film plays on with no + subtitles and no message. `Range` is named because the receiver sends it + even for a document it reads whole. + """ + assert served["allowOrigin"] == "*" + for header in ("Content-Type", "Accept-Encoding", "Range"): + assert header in served["allowHeaders"] + + +@relay_only +def test_the_stream_carries_the_same_headers_as_its_subtitle(served): + """ + A receiver given a side-loaded track reads the media through the same + CORS-checked path, so headers on one and not the other fails the load + entirely rather than losing the subtitles alone. They widen nothing the + token does not already govern: a page holding the URL could put it in a + media element with or without them. + """ + assert served["streamAllowOrigin"] == "*" + + +@relay_only +def test_the_preflight_is_answered_before_the_token_is_checked(served): + """ + A browser sends `OPTIONS` without the credentials that would let it pass a + token check, so refusing it there would deny every well-formed request. + """ + assert served["preflight"] == 204 + assert served["preflightAllowOrigin"] == "*" + + +@relay_only +def test_a_new_subtitle_gets_a_new_address(served): + """ + A receiver caches a side-loaded track by its URL. Serving different cues + from a fixed address leaves the previous language on screen, which reads + as the switch having been ignored. + """ + assert served["secondUrl"] != served["subtitleUrl"] + + +@relay_only +def test_turning_subtitles_off_takes_the_file_away_but_not_the_film(served): + """ + Clearing the track must not disturb playback: the stream is the reason the + relay exists, and losing the picture to a subtitle change would be a far + worse failure than the one being fixed. + """ + assert served["afterClear"] is None + assert served["afterClearStatus"] == 404 + assert served["streamStillServes"] == 200 + + +# ── What the receiver is told ──────────────────────────────────────────────── + +chromecast_only = pytest.mark.skipif( + not CHROMECAST.exists(), reason="desktop client sources not present") + +CHROMECAST_SCRIPT = """ +const m = require(process.argv[2]); +const sub = { url: 'http://10.0.0.2:19550/subs.vtt?t=ab&v=3', + language: 'fr', label: 'French — Forced' }; +console.log(JSON.stringify({ + with: { media: m.mediaFor('http://10.0.0.2:19550/stream.mp4?t=ab', sub), + options: m.loadOptionsFor(sub) }, + without: { media: m.mediaFor('http://10.0.0.2:19550/stream.mp4?t=ab', null), + options: m.loadOptionsFor(null) }, +})); +""" + + +@pytest.fixture(scope="module") +def loaded(tmp_path_factory): + script = tmp_path_factory.mktemp("cc") / "media.cjs" + script.write_text(CHROMECAST_SCRIPT, encoding="utf-8") + proc = subprocess.run( + ["node", str(script), str(CHROMECAST)], + capture_output=True, text=True, timeout=60) + if proc.returncode != 0: + pytest.skip(f"cast-chromecast.js is not loadable here: {proc.stderr}") + return json.loads(proc.stdout) + + +@chromecast_only +def test_the_track_is_declared_and_switched_on(loaded): + """ + Declaring a track without naming it in `activeTrackIds` loads it and shows + nothing, which is the same symptom as not declaring it at all. + """ + track = loaded["with"]["media"]["tracks"][0] + assert track["type"] == "TEXT" + assert track["subtype"] == "SUBTITLES" + assert track["trackContentType"] == "text/vtt" + assert track["language"] == "fr" + assert loaded["with"]["options"]["activeTrackIds"] == [track["trackId"]] + + +@chromecast_only +def test_a_stream_without_subtitles_declares_none(loaded): + """ + A stale id in `activeTrackIds` is a load error on the receiver, and the + load error takes the film with it. + """ + assert "tracks" not in loaded["without"]["media"] + assert loaded["without"]["options"]["activeTrackIds"] == [] diff --git a/packages/meshbay-hub/tests/test_video_subtitles.py b/packages/meshbay-hub/tests/test_video_subtitles.py index 6867076..7fdc9cd 100644 --- a/packages/meshbay-hub/tests/test_video_subtitles.py +++ b/packages/meshbay-hub/tests/test_video_subtitles.py @@ -259,9 +259,12 @@ def test_a_failed_extraction_does_not_take_the_film_down(app): `setError` is the player's fatal path — it replaces the picture. A text track that could not be read must not reach it. """ - body = _player(app) - start = body.index("const selectSubtitle = useCallback(") - end = body.index("}, [entry, transportRef, gekRef]);", start) - assert "setError(" not in body[start:end], ( + # Matched on braces rather than on the dependency list. The deps are the + # part of a callback most likely to change for reasons unrelated to what is + # asserted here, and a slice keyed to them stops at `.index()` raising + # rather than at the property being broken — a guard that reports the wrong + # thing is barely better than one that reports nothing. + body = _block(_player(app), "const selectSubtitle = useCallback(") + assert "setError(" not in body, ( "a subtitle failure takes the whole player down") - assert "setSubtitleError(true)" in body[start:end] + assert "setSubtitleError(true)" in body |