From 2ef0c215ea4af421a4b22ab543a5999320998f68 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Sun, 11 Oct 2026 20:47:42 +0200 Subject: fix(node): a receive buffer that holds an upload's window Uploads ran at 6.5 MB/s because the kernel dropped datagrams at the node's 256 KB socket. Asking for 4 MB gave 19.5 MB/s, no drop. Co-Authored-By: Claude Opus 5.5 --- .../src/meshbay_node/transport/webrtc/core.py | 1 + .../meshbay_node/transport/webrtc/udp_buffer.py | 34 ++++++++++++++++++++++ .../src/meshbay_node/transport/webrtc/worker.py | 6 +++- .../src/meshbay_node/transport/webrtc_server.py | 12 ++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py (limited to 'packages/meshbay-node/src/meshbay_node') diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py index c48e9ca..d197044 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py @@ -17,6 +17,7 @@ from meshbay_node import transfers as transfers_mod from meshbay_node.transport.webrtc import ( sctp_fix, # noqa: F401 (patches aiortc on import) sctp_stats, # noqa: F401 (patches aiortc on import) + udp_buffer, # noqa: F401 (patches aioice on import) ) from meshbay_node.transport.webrtc.channel import ( _REPLY_TO, diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py new file mode 100644 index 0000000..8a2db31 --- /dev/null +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py @@ -0,0 +1,34 @@ +"""A larger receive buffer on the node's ICE sockets, applied on import. + +aioice asks the kernel for 256 KB per socket. The node reads its sockets from +Python, more slowly than a line delivers, and a browser sending a file keeps up +to 1.5 MB in flight (32 chunks of 48 KB, `transport-upload.js`). What does not +fit in the buffer is dropped by the kernel of the node's own machine +(`UdpRcvbufErrors`), the browser's SCTP sender takes each drop for congestion, +and an upload settles around 6.5 MB/s on a line that carries 47 +(docs/MESHBAY_DESIGN.md ยง5.5). + +The buffer has to hold what one upload may have in flight, counted the way the +kernel counts it, which is about twice the bytes for packets of this size. The +kernel grants no more than `net.core.rmem_max` allows, and on a machine where +that is left at its default of 208 KB this changes nothing: `granted()` says +what a socket really gets, and the node logs it once. +""" + +import socket + +import aioice.turn + +WANTED = 4 * 1024 * 1024 + +aioice.turn.UDP_SOCKET_BUFFER_SIZE = WANTED + + +def granted() -> int: + """What the kernel gives a socket that asks for `WANTED`, in bytes.""" + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, WANTED) + except OSError: + pass + return sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py index 072c849..832428a 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py @@ -226,7 +226,11 @@ async def _main() -> None: # Patches aiortc on import, as the node does for the connections it holds # itself. - from meshbay_node.transport.webrtc import sctp_fix, sctp_stats # noqa: F401 + from meshbay_node.transport.webrtc import ( # noqa: F401 + sctp_fix, + sctp_stats, + udp_buffer, + ) worker = Worker(reader, writer) worker.configure(hello) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py index 0baaf19..1976ad4 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py @@ -42,6 +42,7 @@ from meshbay_node.indexer import GroupIndex from meshbay_node.roots import ( RootSet, ) +from meshbay_node.transport.webrtc import udp_buffer from meshbay_node.transport.webrtc.admin import AdminMixin from meshbay_node.transport.webrtc.admission import AdmissionMixin from meshbay_node.transport.webrtc.apps.music import MusicMixin @@ -152,6 +153,17 @@ class WebRTCTransport: # Above one, the peer connections live in that many processes of their # own and the sessions here speak to them (transport/webrtc/workers.py). # At one, this process holds them itself and none is started. + got = udp_buffer.granted() + if got < udp_buffer.WANTED: + # Said once, with what to change: nothing else would tell an + # operator why uploads to this node stay around 6 MB/s. + log.warning( + "UDP receive buffer is %d KB per socket, below the %d KB an " + "upload needs to run at full speed; raise net.core.rmem_max " + "to %d to lift it", got // 1024, udp_buffer.WANTED // 1024, + udp_buffer.WANTED) + else: + log.info("UDP receive buffer: %d KB per socket", got // 1024) self._workers = WorkerPool( min(max(1, int(connection_processes or 1)), max_processes()), self._stun, ice_interfaces) -- cgit v1.2.3