diff options
| -rw-r--r-- | CLAUDE.md | 10 | ||||
| -rw-r--r-- | docs/MESHBAY_DESIGN.md | 13 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py | 1 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py | 34 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py | 6 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py | 12 | ||||
| -rw-r--r-- | packages/meshbay-node/tests/test_udp_buffer.py | 38 |
7 files changed, 113 insertions, 1 deletions
@@ -749,6 +749,16 @@ do. Read them before writing anything that touches the same mechanism. core at 131 MB/s. Splitting there also means no key leaves the node's own process +- **A transfer capped with no core busy and no loss in the log: read the + kernel's drop counter on the receiving machine.** Uploads ran at 6.5 MB/s. + The chunk handler was timed (0.7 ms a chunk, 30 % of the time), the window + the node advertises was read (1 MB), and neither was it. `nstat -az | grep + RcvbufErrors` before and after one upload was: 2460 datagrams dropped at the + node's own socket. Same cause as downloads, mirrored, and the node's log + cannot show it because the sender that backs off is the browser. A 4 MB + socket buffer gave 19.5 MB/s (`udp_buffer.py`); it needs + `net.core.rmem_max` raised on a default machine + ### More of the same, from four rounds of live testing - **A protocol harness cannot test the SPA.** Any second implementation of the diff --git a/docs/MESHBAY_DESIGN.md b/docs/MESHBAY_DESIGN.md index 233bcc8..9c63c52 100644 --- a/docs/MESHBAY_DESIGN.md +++ b/docs/MESHBAY_DESIGN.md @@ -1465,6 +1465,19 @@ TCP because that is the one transport the event loop offers identically on every platform; a process proves it is the one the node started by returning 32 random bytes handed to it on its standard input. +**Uploads met the same wall from the other side.** A browser 22 ms away uploaded +at 6.5 MB/s, with no core busy on either machine and no loss in the node's log. +The loss was on the node's machine: its kernel dropped 2460 of 264000 datagrams at +the socket (`UdpRcvbufErrors`), because aioice asks for a 256 KB receive buffer, +the node reads its sockets from Python, and the browser keeps 1.5 MB of an upload +in flight. Its SCTP sender took each drop for congestion. The node now asks for +4 MB per socket (`transport/webrtc/udp_buffer.py`): the same upload ran at 19.3 and +19.7 MB/s with no datagram dropped. The kernel grants no more than +`net.core.rmem_max`, which many distributions leave at 208 KB, and on such a +machine nothing changes until the operator raises it; the node says so at startup. +The limit that remains is the node's own receive path, about 5 % of a core per +MB/s. + The lease module is free of asyncio and of the transport: it decides, and the caller does the I/O. A queue that reveals itself only through a DataChannel is a queue nobody can prove things about. diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py index c48e9ca..d197044 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py @@ -17,6 +17,7 @@ from meshbay_node import transfers as transfers_mod from meshbay_node.transport.webrtc import ( sctp_fix, # noqa: F401 (patches aiortc on import) sctp_stats, # noqa: F401 (patches aiortc on import) + udp_buffer, # noqa: F401 (patches aioice on import) ) from meshbay_node.transport.webrtc.channel import ( _REPLY_TO, diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py new file mode 100644 index 0000000..8a2db31 --- /dev/null +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py @@ -0,0 +1,34 @@ +"""A larger receive buffer on the node's ICE sockets, applied on import. + +aioice asks the kernel for 256 KB per socket. The node reads its sockets from +Python, more slowly than a line delivers, and a browser sending a file keeps up +to 1.5 MB in flight (32 chunks of 48 KB, `transport-upload.js`). What does not +fit in the buffer is dropped by the kernel of the node's own machine +(`UdpRcvbufErrors`), the browser's SCTP sender takes each drop for congestion, +and an upload settles around 6.5 MB/s on a line that carries 47 +(docs/MESHBAY_DESIGN.md §5.5). + +The buffer has to hold what one upload may have in flight, counted the way the +kernel counts it, which is about twice the bytes for packets of this size. The +kernel grants no more than `net.core.rmem_max` allows, and on a machine where +that is left at its default of 208 KB this changes nothing: `granted()` says +what a socket really gets, and the node logs it once. +""" + +import socket + +import aioice.turn + +WANTED = 4 * 1024 * 1024 + +aioice.turn.UDP_SOCKET_BUFFER_SIZE = WANTED + + +def granted() -> int: + """What the kernel gives a socket that asks for `WANTED`, in bytes.""" + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, WANTED) + except OSError: + pass + return sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py index 072c849..832428a 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py @@ -226,7 +226,11 @@ async def _main() -> None: # Patches aiortc on import, as the node does for the connections it holds # itself. - from meshbay_node.transport.webrtc import sctp_fix, sctp_stats # noqa: F401 + from meshbay_node.transport.webrtc import ( # noqa: F401 + sctp_fix, + sctp_stats, + udp_buffer, + ) worker = Worker(reader, writer) worker.configure(hello) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py index 0baaf19..1976ad4 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py @@ -42,6 +42,7 @@ from meshbay_node.indexer import GroupIndex from meshbay_node.roots import ( RootSet, ) +from meshbay_node.transport.webrtc import udp_buffer from meshbay_node.transport.webrtc.admin import AdminMixin from meshbay_node.transport.webrtc.admission import AdmissionMixin from meshbay_node.transport.webrtc.apps.music import MusicMixin @@ -152,6 +153,17 @@ class WebRTCTransport: # Above one, the peer connections live in that many processes of their # own and the sessions here speak to them (transport/webrtc/workers.py). # At one, this process holds them itself and none is started. + got = udp_buffer.granted() + if got < udp_buffer.WANTED: + # Said once, with what to change: nothing else would tell an + # operator why uploads to this node stay around 6 MB/s. + log.warning( + "UDP receive buffer is %d KB per socket, below the %d KB an " + "upload needs to run at full speed; raise net.core.rmem_max " + "to %d to lift it", got // 1024, udp_buffer.WANTED // 1024, + udp_buffer.WANTED) + else: + log.info("UDP receive buffer: %d KB per socket", got // 1024) self._workers = WorkerPool( min(max(1, int(connection_processes or 1)), max_processes()), self._stun, ice_interfaces) diff --git a/packages/meshbay-node/tests/test_udp_buffer.py b/packages/meshbay-node/tests/test_udp_buffer.py new file mode 100644 index 0000000..2159663 --- /dev/null +++ b/packages/meshbay-node/tests/test_udp_buffer.py @@ -0,0 +1,38 @@ +""" +The node's ICE sockets ask for a receive buffer that holds an upload's window. + +Measured on a node in service: during a 200 MB upload from a browser 22 ms +away the node's machine dropped 2460 of 264000 datagrams at the socket +(`Udp6RcvbufErrors`), with a buffer of 512 KB, and the upload ran at 6.4 MB/s. +The browser keeps 32 chunks of 48 KB in flight, 1.5 MB, and the kernel counts +about twice the bytes it holds. +""" + +import socket + +import aioice +import pytest +from meshbay_node.transport.webrtc import udp_buffer + + +def test_the_request_covers_what_an_upload_keeps_in_flight(): + in_flight = 32 * 48 * 1024 + assert udp_buffer.WANTED >= 2 * in_flight + assert aioice.turn.UDP_SOCKET_BUFFER_SIZE == udp_buffer.WANTED + + +@pytest.mark.asyncio +async def test_a_gathered_socket_gets_what_the_kernel_allows(): + """aioice reads the constant when it gathers, not when it is imported: the + socket of a real candidate has the buffer, up to the machine's ceiling.""" + conn = aioice.Connection(ice_controlling=True) + try: + await conn.gather_candidates() + sizes = [p.transport.get_extra_info("socket").getsockopt( + socket.SOL_SOCKET, socket.SO_RCVBUF) + for p in conn._protocols] + assert sizes, "no candidate was gathered" + assert min(sizes) == udp_buffer.granted() + assert min(sizes) > 262144, "the machine grants no more than aioice asked before" + finally: + await conn.close() |