aboutsummaryrefslogtreecommitdiffstats
path: root/packages/meshbay-node/src
diff options
context:
space:
mode:
authorChristophe Besson <cbesson@gmail.com>2026-09-19 13:24:53 +0200
committerChristophe Besson <cbesson@gmail.com>2026-09-19 13:24:53 +0200
commiteca01c7f970d2d3ab2298f934da9776fe01179c6 (patch)
tree4a106ec9fd13bb91eed236c07d821ab50747d5cc /packages/meshbay-node/src
parent5d5d55b588401cdd304922b58e2af2fb18c63648 (diff)
downloadmeshbay-eca01c7f970d2d3ab2298f934da9776fe01179c6.tar.gz
fix(node): bound what a transcode may produce, and read ffmpeg's output off the loop
Two things about the same three functions, which write to a temp file with ffmpeg and then read it back. **The read was on the event loop.** These files are ffmpeg's own, under `tempfile.mkstemp` on the system disk, so they are not a group root and there is no spun-down platter to serialise against — which is why they go through `asyncio.to_thread` and not `roots.off_disk`. But a whole transcode read inline is still tens of megabytes of blocking read while nothing else in the node is served. The AST guard now covers the module with no exemption at all, and a second check refuses a direct call to the reading helper: passed to `to_thread` it appears in the syntax tree as a name, called inline it appears as a call. **The audio transcode had no size ceiling**, where the subtitle path beside it has had one all along. The bound is the media cache's rather than memory's: `put_thumb` writes one SQLite row, and the store is 512 MB with least-recently-used eviction, sized for thumbnails, posters and short transcodes. At 192 kbit/s a three-hour source is ~260 MB — one row that evicts most of the cache to fit and is evicted again by the next few thumbnails. Not a size this store can hold usefully. 64 MiB, about forty-five minutes: past any track, any single piece, most sets. It takes away nothing that worked. `AUDIO_TRANSCODE_TIMEOUT_SECS` is 120, so a source long enough to reach this was already liable to be killed mid-transcode; what changes is that the refusal now names the limit it met and the size that met it. Serving audio of that length properly means streaming the conversion instead of buffering it, which is a different feature — recorded in §9.8 rather than left as an implied promise. The stat comes before the read, so an oversized result costs a stat rather than the read and the memory behind it. Twelve `test_sticky_header.py[firefox]` setup errors again: Firefox is still open on this machine, and its `[chrome]` half passes. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Diffstat (limited to 'packages/meshbay-node/src')
-rw-r--r--packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py58
1 files changed, 50 insertions, 8 deletions
diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py
index 773d3dc..d2ec1de 100644
--- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py
+++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py
@@ -339,6 +339,24 @@ SEEK_PROBE_MAX_BACKOFF_SECS = 60
# subtitle track, it is an ffmpeg that found something else to write, and it
# would sit in the media cache for ever.
SUBTITLE_MAX_BYTES = 8 * 1024 * 1024
+
+# What a whole-file audio transcode may produce. The output is AAC at 192 kbit/s,
+# so this is about forty-five minutes of source — past any track, any single
+# piece, most sets.
+#
+# The bound is the media cache's, not memory's. `put_thumb` writes one SQLite row
+# and the store is 512 MB with least-recently-used eviction, sized for what it
+# holds: thumbnails, posters, subtitle tracks, short transcodes. A three-hour
+# audiobook at this bitrate is ~260 MB — a single row that would evict most of
+# the cache to make room for itself, and be evicted in turn by the next few
+# thumbnails. It is not a size this store can hold usefully.
+#
+# It does not take away something that worked: `AUDIO_TRANSCODE_TIMEOUT_SECS` is
+# 120, so a source long enough to reach this cap was already liable to be killed
+# mid-transcode. What changes is that the refusal now says which limit was met.
+# Serving audio of that length properly is streaming the transcode rather than
+# buffering it, which is a different feature from this one.
+AUDIO_TRANSCODE_MAX_BYTES = 64 * 1024 * 1024
# Bundle fetches are served in the pre-proof window (C4). Bounded and audited
# until the native client removes remote keypair bundles entirely.
MAX_PRE_PROOF_FETCHES = 4
@@ -6721,6 +6739,30 @@ def _locate(roots: RootSet, entry) -> tuple[Path | None, str | None]:
return path, None
+def _read_scratch_capped(tmp_path: Path, cap: int, what: str) -> bytes:
+ """
+ Stat ffmpeg's output, refuse it if it is too big, read it. Blocking.
+
+ Run through `asyncio.to_thread` and not `off_disk`: this file is ffmpeg's
+ own, under `tempfile.mkstemp` on the system disk, so it is not a group root
+ and there is no spun-down platter to serialise against — it only has to be
+ off the event loop. A whole transcode read inline is tens of megabytes of
+ blocking read while nothing else in the node is served.
+
+ The size is checked before the bytes are asked for, so an oversized result
+ costs a stat rather than the read *and* the memory.
+ """
+ size = tmp_path.stat().st_size
+ if size > cap:
+ raise RuntimeError(f"{what} is {size} bytes, over the {cap} cap")
+ return tmp_path.read_bytes()
+
+
+async def _discard_scratch(tmp_path: Path) -> None:
+ """Remove one of ffmpeg's temp files, off the loop like the read of it."""
+ await asyncio.to_thread(tmp_path.unlink, True)
+
+
def _append_chunk(tmp_path: Path, chunk_bytes: bytes, first: bool) -> None:
"""Add one chunk to a partial upload. Blocking; called through `off_disk`."""
with open(tmp_path, "wb" if first else "ab") as f:
@@ -6777,9 +6819,11 @@ async def _transcode_audio_to_aac(file_path: Path) -> bytes:
if proc.returncode != 0:
raise RuntimeError(
f"ffmpeg exited {proc.returncode}: {stderr.decode(errors='replace')[:300]}")
- return tmp_path.read_bytes()
+ return await asyncio.to_thread(
+ _read_scratch_capped, tmp_path, AUDIO_TRANSCODE_MAX_BYTES,
+ "transcoded audio")
finally:
- tmp_path.unlink(missing_ok=True)
+ await _discard_scratch(tmp_path)
async def _seek_lands_at(file_path: Path, t: float, map_args: list[str]) -> float | None:
@@ -6833,7 +6877,7 @@ async def _seek_lands_at(file_path: Path, t: float, map_args: list[str]) -> floa
log.warning("stream: seek probe failed at %.1fs: %r", t, e)
return None
finally:
- tmp_path.unlink(missing_ok=True)
+ await _discard_scratch(tmp_path)
text = stdout.decode(errors="replace").strip().rstrip(",")
try:
landed = float(text)
@@ -6891,10 +6935,8 @@ async def _extract_subtitle_to_webvtt(file_path: Path, ordinal: int,
if proc.returncode != 0:
raise RuntimeError(
f"ffmpeg exited {proc.returncode}: {stderr.decode(errors='replace')[:300]}")
- size = tmp_path.stat().st_size
- if size > SUBTITLE_MAX_BYTES:
- raise RuntimeError(f"subtitle track is {size} bytes, over the {SUBTITLE_MAX_BYTES} cap")
- blob = tmp_path.read_bytes()
+ blob = await asyncio.to_thread(
+ _read_scratch_capped, tmp_path, SUBTITLE_MAX_BYTES, "subtitle track")
# A WebVTT file that is only its header has no cues in it. That is what
# a bitmap track extracted by mistake produces, and what a text track
# whose stream is empty produces; either way there is nothing to show,
@@ -6904,7 +6946,7 @@ async def _extract_subtitle_to_webvtt(file_path: Path, ordinal: int,
raise RuntimeError("extracted subtitle contains no cues")
return blob
finally:
- tmp_path.unlink(missing_ok=True)
+ await _discard_scratch(tmp_path)
class WebRTCTransport: