diff options
| -rw-r--r-- | docs/MESHBAY_DESIGN.md | 15 | ||||
| -rw-r--r-- | docs/MESHBAY_NODE_PROTOCOL.md | 5 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py | 42 | ||||
| -rw-r--r-- | packages/meshbay-node/tests/test_stream_seek_audio_alignment.py | 238 |
4 files changed, 292 insertions, 8 deletions
diff --git a/docs/MESHBAY_DESIGN.md b/docs/MESHBAY_DESIGN.md index e7d4a76..ef65995 100644 --- a/docs/MESHBAY_DESIGN.md +++ b/docs/MESHBAY_DESIGN.md @@ -2031,6 +2031,21 @@ MSE string) fed to a source buffer, with the node holding one slot per viewer. - **Seeking restarts the source with an index seek before the input**, clamped away from the end and echoed back; the client supplies the timestamp offset, because copying timestamps does not preserve position. +- **A seek trims what it can, and what it can differs per stream — which is a + desync, not an inconvenience.** Accurate seeking cannot trim copied video, which + must begin on a keyframe, but it does trim re-encoded audio to the exact request. + The output then carries video from the keyframe and audio from the request, a + whole GOP apart, with a hole between them. So **accurate seeking is off wherever + video is copied and on wherever it is re-encoded**, which is the only place the + two can agree on where to begin. + + > **Every timestamp was correct while this was happening.** First PTS per stream, + > durations, spans, and the browser's own audio/video delta through MediaSource + > all agreed, because both streams genuinely sat where the container said. Only + > the *content* at a given instant was displaced. A property that every + > timestamp-shaped check confirms is not thereby true: this one needed the output + > decoded and compared against the source, frame against frame and envelope + > against envelope. - **The viewer picks the audio track, and picking one is a seek.** One ffmpeg carries one audio track, so there is nothing to switch inside a running stream: the node is asked again at the current position and the source buffer is reset diff --git a/docs/MESHBAY_NODE_PROTOCOL.md b/docs/MESHBAY_NODE_PROTOCOL.md index 19cced9..d4b9552 100644 --- a/docs/MESHBAY_NODE_PROTOCOL.md +++ b/docs/MESHBAY_NODE_PROTOCOL.md @@ -1636,7 +1636,9 @@ array while MediaSource consumes it a segment at a time. | | ffprobe: codec, duration, the | | audio tracks | | spawn ffmpeg - | | -ss before -i (index seek) + | | -ss before -i (index seek), + | | -noaccurate_seek when the + | | video is copied | | video: copy, or libx264 when | | the browser cannot decode | | audio: always AAC, 2 ch, @@ -1667,6 +1669,7 @@ array while MediaSource consumes it a segment at a time. | `n == 0` | keepalive: a viewer buffered 90 s ahead grants nothing and must still be able to say it is there | | Concurrent transcodes | 8 node-wide, semaphore on the transport context | | Seeking | a new `stream_req` with `start`; the previous stream is retired first, ffmpeg respawned with `-ss` | +| Accurate seek | **off when the video is copied, on when it is re-encoded.** Copied video has to begin on a keyframe and cannot be trimmed to the request; re-encoded audio can, and is. Leaving both at the default put a whole GOP of silence at the head of every seek and left sound and picture a GOP apart — with correct timestamps throughout, so nothing downstream could detect it | | `start` in `stream_init` | the value actually used — seeking lands on the keyframe at or before the request, and the client adds it back as `SourceBuffer.timestampOffset` | | `audio_tracks` in `stream_init` | every audio track: `i` (the **audio ordinal**, what `-map 0:a:<n>` takes, never the container stream index), `lang`, `title`, `codec`, `ch`. Empty for a file with no audio | | `audio_track` in `stream_req` | which ordinal to map. Absent, out of range or malformed is the first track | diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py index b39941d..67ca380 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py @@ -5979,13 +5979,6 @@ class WebRTCPeerSession: start = max(0.0, duration - 5) start = max(0.0, start) - # -ss BEFORE -i, which seeks by the container index rather than by - # decoding up to the point: milliseconds on a 500 MB film instead of - # tens of seconds. It lands on the keyframe at or before `start`, so - # the picture can begin a few seconds earlier than asked — which is - # what every streaming player does, and why the client is told the - # value used rather than left to assume its own. - seek_args = ["-ss", f"{start:.3f}"] if start > 0 else [] # Video is copied whenever the browser can decode it directly — # re-encoding it is the expensive thing this pipeline exists to avoid, # and H264/VP9/AV1 already decode fine in-browser. HEVC is the one @@ -6036,6 +6029,41 @@ class WebRTCPeerSession: "detail": "This video needs transcoding, which the " "operator has turned off"}) return + # Seeking, and the trap that made a seek on a copied stream unwatchable. + # + # -ss BEFORE -i seeks by the container index rather than by decoding up + # to the point: milliseconds on a 500 MB film instead of tens of + # seconds. It lands on the keyframe at or before `start`, so the + # picture can begin a few seconds earlier than asked — which is what + # every streaming player does. + # + # **`-accurate_seek` is on by default, and it trims what it can.** It + # cannot trim copied video, which has to begin on a keyframe; it does + # trim the re-encoded audio, to exactly `start`. So the output began + # with video from the keyframe and audio from `start` — correct + # timestamps, both streams honestly placed, and **a hole in the audio + # one whole GOP wide**. Measured on a real film with a 10 s keyframe + # interval: seeking to 609 s against a keyframe at 599.104 s left + # 9.979 s of silence, after which sound and picture were a GOP apart + # for the rest of the film. + # + # Nothing downstream could see it. Every timestamp check passes — the + # first PTS of each stream, their durations, their spans, the browser's + # own A/V delta through MediaSource — because the timestamps were never + # wrong. Only the *content* at a given instant was, which is why this + # was found by decoding the output and comparing it against the source: + # the first frame is byte-identical to the source frame at the + # keyframe, and with the fix the audio's energy envelope matches the + # source at that same instant (r = 0.97) instead of one GOP later. + # + # This is also why re-encoded video never showed the fault, and why a + # library's HEVC files looked like the only ones that worked: video + # that is re-encoded *can* start exactly at `start`, so accurate + # seeking is right there and stays on. + seek_args: list[str] = [] + if start > 0: + seek_args = ["-ss", f"{start:.3f}"] if transcode_video else [ + "-noaccurate_seek", "-ss", f"{start:.3f}"] map_args = ["-map", "0:v:0"] if transcode_video: log.info("stream: re-encoding %s (%s) to H264", entry.name, diff --git a/packages/meshbay-node/tests/test_stream_seek_audio_alignment.py b/packages/meshbay-node/tests/test_stream_seek_audio_alignment.py new file mode 100644 index 0000000..c1d0c66 --- /dev/null +++ b/packages/meshbay-node/tests/test_stream_seek_audio_alignment.py @@ -0,0 +1,238 @@ +""" +A seek must not leave the audio a GOP behind the picture. + +`-ss` before `-i` lands on the keyframe at or before the requested position. +`-accurate_seek` is on by default and trims each stream to exactly that +position *where it can*: it cannot trim copied video, which has to begin on a +keyframe, but it does trim the re-encoded audio. The output then begins with +video from the keyframe and audio from the request, with a hole between them +one whole GOP wide — and after the hole, sound and picture belong to different +moments for the rest of the film. + +**Every timestamp is correct while this happens**, which is why it survived: +the first PTS of each stream, their durations, their spans and the browser's +own A/V delta through MediaSource all agree, because both streams really are +where the container says. Only the content is displaced. So these tests assert +on the *shape of the audio track* — the hole — rather than on timestamps, which +cannot see it. + +Found against a real film with a 10 s keyframe interval: seeking to 609 s +against a keyframe at 599.104 s left 9.979 s of silence. Re-encoded video never +showed it, since video that is re-encoded can begin exactly where it was asked +to — which is why the one part of a library that worked was the part the node +transcodes. +""" + +import shutil +import subprocess +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from meshbay_common.crypto import generate_gek +from meshbay_common.webcrypto import chunk_key_aes, decrypt_chunk_aes +from meshbay_node.indexer.group_index import GroupIndex +from meshbay_node.transport.webrtc_server import WebRTCPeerSession + +from conftest import needs_subprocess, one_root + +_HAVE_FFMPEG = shutil.which("ffmpeg") and shutil.which("ffprobe") +pytestmark = [ + pytest.mark.asyncio, + pytest.mark.skipif(not _HAVE_FFMPEG, reason="ffmpeg/ffprobe not installed"), + needs_subprocess, +] + +FPS = 24 +GOP_S = 10 # keyframe every 10 s, as real films in the wild are +CLIP_S = 40 +SEEK_S = 25.0 # mid-GOP: the keyframe before it is at 20 s +EXPECTED_PREROLL = 5.0 # SEEK_S - 20.0 + + +def _make_long_gop_clip(path: Path) -> None: + """Video with a 10 s keyframe interval and continuous audio. + + The interval is what gives the fault room to appear: with a keyframe every + frame there is nothing for the audio to be trimmed past. + """ + subprocess.run( + ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", f"testsrc=size=320x240:rate={FPS}:duration={CLIP_S}", + "-f", "lavfi", "-i", + f"sine=frequency=440:duration={CLIP_S}:sample_rate=48000", + "-c:v", "libx264", "-preset", "ultrafast", + "-g", str(GOP_S * FPS), "-keyint_min", str(GOP_S * FPS), + "-sc_threshold", "0", + "-c:a", "aac", str(path)], + check=True, capture_output=True, + ) + + +def _session(tmp_path: Path, video_path: Path, gek: bytes): + import blake3 + file_bytes = video_path.read_bytes() + file_id = blake3.blake3(file_bytes).hexdigest() + + sk_node = Ed25519PrivateKey.generate() + index = GroupIndex(group_id="g" * 32, sk_node=sk_node, gek=gek) + from meshbay_common.protocol import IndexEntry + index.add_entry(IndexEntry( + id=file_id, name=video_path.name, path=video_path.parent.name, + size=len(file_bytes), type="video", added_at=0)) + + session = WebRTCPeerSession.__new__(WebRTCPeerSession) + session._ctx = { + "roots": one_root(video_path.parent), + "index": index, + "gek": gek, + "sk_node": sk_node, + "max_concurrent_streams": 4, + } + session._group_id = None + session._user_id = "tester" + session._stream_stopped = False + session._stream_keepalives = 0 + session.sent = [] + session._send = session.sent.append + session._audit = lambda *a, **k: None + return session, file_id + + +def _reassemble(sent, gek, file_id) -> bytes: + file_hash = bytes.fromhex(file_id) + out = b"" + for m in sorted((m for m in sent if m.get("type") == "stream_data"), + key=lambda m: m["segment_index"]): + key = chunk_key_aes(gek, file_hash, m["segment_index"]) + out += decrypt_chunk_aes(key, m["nonce"], m["ct"]) + return out + + +def _packet_times(path: Path, kind: str) -> list[float]: + out = subprocess.run( + ["ffprobe", "-v", "error", "-select_streams", kind, + "-show_entries", "packet=pts_time", "-of", "csv=p=0", str(path)], + capture_output=True, text=True).stdout + return sorted(float(x.rstrip(",")) for x in out.split() if x.strip()) + + +def _largest_audio_gap(path: Path) -> float: + t = _packet_times(path, "a") + return max((t[i + 1] - t[i] for i in range(len(t) - 1)), default=0.0) + + +async def _stream_to_file(tmp_path: Path, clip: Path, start: float, tag: str) -> Path: + gek = generate_gek() + session, file_id = _session(tmp_path, clip, gek) + await session._stream_video_inner( + {"file_id": file_id, "start": start, "credits": 0}) + errors = [m for m in session.sent if m.get("type") == "error"] + assert not errors, f"streaming must not fail: {errors}" + out = tmp_path / f"seek-{tag}.mp4" + out.write_bytes(_reassemble(session.sent, gek, file_id)) + return out + + +async def test_a_seek_mid_gop_leaves_no_hole_in_the_audio(tmp_path): + """The regression itself. + + Without the fix the audio begins one whole GOP after the video and the gap + is the distance from the keyframe to the request — here five seconds. + """ + clip = tmp_path / "longgop.mkv" + _make_long_gop_clip(clip) + + out = await _stream_to_file(tmp_path, clip, SEEK_S, "midgop") + gap = _largest_audio_gap(out) + + assert gap < 0.5, ( + f"the audio track has a {gap:.3f}s hole after a seek to {SEEK_S}s. " + "Video was copied from the keyframe before it while the re-encoded " + "audio was trimmed to the request, so sound and picture are a GOP " + "apart for the rest of the stream") + + +async def test_the_fixture_really_has_a_long_gop(tmp_path): + """Otherwise the test above proves nothing: with a keyframe every frame + there is no interval for the audio to be trimmed past, and it would pass + against the broken code.""" + clip = tmp_path / "longgop.mkv" + _make_long_gop_clip(clip) + + keys = subprocess.run( + ["ffprobe", "-v", "error", "-select_streams", "v:0", + "-skip_frame", "nokey", "-show_entries", "frame=pts_time", + "-of", "csv=p=0", str(clip)], + capture_output=True, text=True).stdout.split() + times = sorted(float(x.rstrip(",")) for x in keys if x.strip()) + gaps = [times[i + 1] - times[i] for i in range(len(times) - 1)] + + assert gaps, "no keyframe interval to measure" + assert max(gaps) >= GOP_S - 1, ( + f"fixture keyframe interval is {max(gaps):.1f}s, too short to expose " + "the fault this file exists to guard") + assert SEEK_S not in times, "the seek must land between keyframes" + + +async def test_the_stream_carries_as_much_sound_as_picture(tmp_path): + """The positive form, and it has to be measured on **decoded audio**. + + Comparing the first packet of each stream proves nothing here: the hole + leaves one audio packet at zero and resumes a GOP later, so both streams + still *start* at the same timestamp and every timestamp comparison agrees. + What is actually missing is a GOP of sound, so that is what is counted. + """ + clip = tmp_path / "longgop.mkv" + _make_long_gop_clip(clip) + + out = await _stream_to_file(tmp_path, clip, SEEK_S, "together") + decoded = subprocess.run( + ["ffmpeg", "-v", "error", "-i", str(out), "-map", "0:a:0", + "-ac", "1", "-ar", "8000", "-f", "s16le", "-"], + capture_output=True).stdout + audio_s = len(decoded) / 2 / 8000 + video_s = max(_packet_times(out, "v")) - min(_packet_times(out, "v")) + + assert audio_s >= video_s - 0.5, ( + f"{video_s:.2f}s of picture arrived with only {audio_s:.2f}s of sound: " + f"{video_s - audio_s:.2f}s of audio is missing from the head of the " + "stream, which is the GOP the seek copied video across") + + +async def test_playing_from_the_beginning_is_untouched(tmp_path): + """No `-ss` at all, so no seek behaviour to change — and the fault never + appeared there, which is why a film watched straight through looked fine.""" + clip = tmp_path / "longgop.mkv" + _make_long_gop_clip(clip) + + out = await _stream_to_file(tmp_path, clip, 0, "zero") + + assert _largest_audio_gap(out) < 0.5 + + +async def test_a_transcoded_video_keeps_accurate_seeking(tmp_path): + """Re-encoded video can begin exactly where it was asked to, so the + trimming that breaks a copy is right there. Turning it off everywhere + would have cost seek precision on the one path that has it. + """ + clip = tmp_path / "longgop.mkv" + _make_long_gop_clip(clip) + gek = generate_gek() + session, file_id = _session(tmp_path, clip, gek) + # Force the re-encode branch the way a browser-incompatible source would. + session._ctx["transcode_incompatible_video"] = True + import meshbay_node.transport.webrtc_server as ws + real = ws.BROWSER_INCOMPATIBLE_VIDEO_CODECS + ws.BROWSER_INCOMPATIBLE_VIDEO_CODECS = frozenset({"h264"}) + try: + await session._stream_video_inner( + {"file_id": file_id, "start": SEEK_S, "credits": 0}) + finally: + ws.BROWSER_INCOMPATIBLE_VIDEO_CODECS = real + + assert not [m for m in session.sent if m.get("type") == "error"] + out = tmp_path / "seek-transcoded.mp4" + out.write_bytes(_reassemble(session.sent, gek, file_id)) + assert _largest_audio_gap(out) < 0.5 |