fix(node): a receive buffer that holds an upload's window

Uploads ran at 6.5 MB/s because the kernel dropped datagrams at the
node's 256 KB socket. Asking for 4 MB gave 19.5 MB/s, no drop.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

7 files changed, 113 insertions, 1 deletions

diff --git a/CLAUDE.md b/CLAUDE.md
index 460a709..f889685 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -749,6 +749,16 @@ do. Read them before writing anything that touches the same mechanism.
   core at 131 MB/s. Splitting there also means no key leaves the node's own
   process
 
+- **A transfer capped with no core busy and no loss in the log: read the
+  kernel's drop counter on the receiving machine.** Uploads ran at 6.5 MB/s.
+  The chunk handler was timed (0.7 ms a chunk, 30 % of the time), the window
+  the node advertises was read (1 MB), and neither was it. `nstat -az | grep
+  RcvbufErrors` before and after one upload was: 2460 datagrams dropped at the
+  node's own socket. Same cause as downloads, mirrored, and the node's log
+  cannot show it because the sender that backs off is the browser. A 4 MB
+  socket buffer gave 19.5 MB/s (`udp_buffer.py`); it needs
+  `net.core.rmem_max` raised on a default machine
+
 ### More of the same, from four rounds of live testing
 
 - **A protocol harness cannot test the SPA.** Any second implementation of the
diff --git a/docs/MESHBAY_DESIGN.md b/docs/MESHBAY_DESIGN.md
index 233bcc8..9c63c52 100644
--- a/docs/MESHBAY_DESIGN.md
+++ b/docs/MESHBAY_DESIGN.md
@@ -1465,6 +1465,19 @@ TCP because that is the one transport the event loop offers identically on every
 platform; a process proves it is the one the node started by returning 32 random
 bytes handed to it on its standard input.
 
+**Uploads met the same wall from the other side.** A browser 22 ms away uploaded
+at 6.5 MB/s, with no core busy on either machine and no loss in the node's log.
+The loss was on the node's machine: its kernel dropped 2460 of 264000 datagrams at
+the socket (`UdpRcvbufErrors`), because aioice asks for a 256 KB receive buffer,
+the node reads its sockets from Python, and the browser keeps 1.5 MB of an upload
+in flight. Its SCTP sender took each drop for congestion. The node now asks for
+4 MB per socket (`transport/webrtc/udp_buffer.py`): the same upload ran at 19.3 and
+19.7 MB/s with no datagram dropped. The kernel grants no more than
+`net.core.rmem_max`, which many distributions leave at 208 KB, and on such a
+machine nothing changes until the operator raises it; the node says so at startup.
+The limit that remains is the node's own receive path, about 5 % of a core per
+MB/s.
+
 The lease module is free of asyncio and of the transport: it decides, and the
 caller does the I/O. A queue that reveals itself only through a DataChannel is a
 queue nobody can prove things about.
diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py
index c48e9ca..d197044 100644
--- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py
+++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/core.py
@@ -17,6 +17,7 @@ from meshbay_node import transfers as transfers_mod
 from meshbay_node.transport.webrtc import (
     sctp_fix,  # noqa: F401  (patches aiortc on import)
     sctp_stats,  # noqa: F401  (patches aiortc on import)
+    udp_buffer,  # noqa: F401  (patches aioice on import)
 )
 from meshbay_node.transport.webrtc.channel import (
     _REPLY_TO,
diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py
new file mode 100644
index 0000000..8a2db31
--- /dev/null
+++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/udp_buffer.py
@@ -0,0 +1,34 @@
+"""A larger receive buffer on the node's ICE sockets, applied on import.
+
+aioice asks the kernel for 256 KB per socket. The node reads its sockets from
+Python, more slowly than a line delivers, and a browser sending a file keeps up
+to 1.5 MB in flight (32 chunks of 48 KB, `transport-upload.js`). What does not
+fit in the buffer is dropped by the kernel of the node's own machine
+(`UdpRcvbufErrors`), the browser's SCTP sender takes each drop for congestion,
+and an upload settles around 6.5 MB/s on a line that carries 47
+(docs/MESHBAY_DESIGN.md §5.5).
+
+The buffer has to hold what one upload may have in flight, counted the way the
+kernel counts it, which is about twice the bytes for packets of this size. The
+kernel grants no more than `net.core.rmem_max` allows, and on a machine where
+that is left at its default of 208 KB this changes nothing: `granted()` says
+what a socket really gets, and the node logs it once.
+"""
+
+import socket
+
+import aioice.turn
+
+WANTED = 4 * 1024 * 1024
+
+aioice.turn.UDP_SOCKET_BUFFER_SIZE = WANTED
+
+
+def granted() -> int:
+    """What the kernel gives a socket that asks for `WANTED`, in bytes."""
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+        try:
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, WANTED)
+        except OSError:
+            pass
+        return sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)
diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py
index 072c849..832428a 100644
--- a/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py
+++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc/worker.py
@@ -226,7 +226,11 @@ async def _main() -> None:
 
     # Patches aiortc on import, as the node does for the connections it holds
     # itself.
-    from meshbay_node.transport.webrtc import sctp_fix, sctp_stats  # noqa: F401
+    from meshbay_node.transport.webrtc import (  # noqa: F401
+        sctp_fix,
+        sctp_stats,
+        udp_buffer,
+    )
 
     worker = Worker(reader, writer)
     worker.configure(hello)
diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py
index 0baaf19..1976ad4 100644
--- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py
+++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py
@@ -42,6 +42,7 @@ from meshbay_node.indexer import GroupIndex
 from meshbay_node.roots import (
     RootSet,
 )
+from meshbay_node.transport.webrtc import udp_buffer
 from meshbay_node.transport.webrtc.admin import AdminMixin
 from meshbay_node.transport.webrtc.admission import AdmissionMixin
 from meshbay_node.transport.webrtc.apps.music import MusicMixin
@@ -152,6 +153,17 @@ class WebRTCTransport:
         # Above one, the peer connections live in that many processes of their
         # own and the sessions here speak to them (transport/webrtc/workers.py).
         # At one, this process holds them itself and none is started.
+        got = udp_buffer.granted()
+        if got < udp_buffer.WANTED:
+            # Said once, with what to change: nothing else would tell an
+            # operator why uploads to this node stay around 6 MB/s.
+            log.warning(
+                "UDP receive buffer is %d KB per socket, below the %d KB an "
+                "upload needs to run at full speed; raise net.core.rmem_max "
+                "to %d to lift it", got // 1024, udp_buffer.WANTED // 1024,
+                udp_buffer.WANTED)
+        else:
+            log.info("UDP receive buffer: %d KB per socket", got // 1024)
         self._workers = WorkerPool(
             min(max(1, int(connection_processes or 1)), max_processes()),
             self._stun, ice_interfaces)
diff --git a/packages/meshbay-node/tests/test_udp_buffer.py b/packages/meshbay-node/tests/test_udp_buffer.py
new file mode 100644
index 0000000..2159663
--- /dev/null
+++ b/packages/meshbay-node/tests/test_udp_buffer.py
@@ -0,0 +1,38 @@
+"""
+The node's ICE sockets ask for a receive buffer that holds an upload's window.
+
+Measured on a node in service: during a 200 MB upload from a browser 22 ms
+away the node's machine dropped 2460 of 264000 datagrams at the socket
+(`Udp6RcvbufErrors`), with a buffer of 512 KB, and the upload ran at 6.4 MB/s.
+The browser keeps 32 chunks of 48 KB in flight, 1.5 MB, and the kernel counts
+about twice the bytes it holds.
+"""
+
+import socket
+
+import aioice
+import pytest
+from meshbay_node.transport.webrtc import udp_buffer
+
+
+def test_the_request_covers_what_an_upload_keeps_in_flight():
+    in_flight = 32 * 48 * 1024
+    assert udp_buffer.WANTED >= 2 * in_flight
+    assert aioice.turn.UDP_SOCKET_BUFFER_SIZE == udp_buffer.WANTED
+
+
+@pytest.mark.asyncio
+async def test_a_gathered_socket_gets_what_the_kernel_allows():
+    """aioice reads the constant when it gathers, not when it is imported: the
+    socket of a real candidate has the buffer, up to the machine's ceiling."""
+    conn = aioice.Connection(ice_controlling=True)
+    try:
+        await conn.gather_candidates()
+        sizes = [p.transport.get_extra_info("socket").getsockopt(
+                     socket.SOL_SOCKET, socket.SO_RCVBUF)
+                 for p in conn._protocols]
+        assert sizes, "no candidate was gathered"
+        assert min(sizes) == udp_buffer.granted()
+        assert min(sizes) > 262144, "the machine grants no more than aioice asked before"
+    finally:
+        await conn.close()