diff options
Diffstat (limited to 'packages')
| -rw-r--r-- | packages/meshbay-common/src/meshbay_common/protocol.py | 14 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/ops.py | 36 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/transfers.py | 354 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py | 249 | ||||
| -rw-r--r-- | packages/meshbay-node/src/meshbay_node/ui/app.py | 6 | ||||
| -rw-r--r-- | packages/meshbay-node/tests/test_transfer_slots.py | 308 | ||||
| -rw-r--r-- | packages/meshbay-node/tests/test_transfer_slots_wire.py | 298 |
7 files changed, 1264 insertions, 1 deletions
diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index e092353..4890d3b 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -96,6 +96,20 @@ class MNP: # never serve the GEK in plaintext. Members obtain it by unwrapping their own # ECIES bundle. The constants lingered after the handlers were deleted, leaving # the wire contract looking as though the endpoint still existed. + # Transfer slots. A download is otherwise invisible to the node -- a series + # of independent file_req messages, with nothing saying one started or + # ended -- so there is nothing to count and nothing to cap. The lease is + # that missing object: `tr` is drawn by the client like `upload_id`, covers + # a job rather than a file, and dies with the connection. + # + # One reply type with a state field, not four: a client that must switch on + # the message type to discover it is still waiting is a client that will get + # one branch wrong. Carries no filename and no path -- `tr` is opaque, + # `bytes` and `chunks` are numbers -- so it stays in clear like + # INDEX_PROGRESS, for the same stated reason. + TRANSFER_OPEN = "transfer_open" # client -> node: I want a slot + TRANSFER_CLOSE = "transfer_close" # client -> node: I am done with it + TRANSFER_STATE = "transfer_state" # node -> client: granted/queued/closed FILE_UPLOAD = "file_upload" # client pushes file chunk to node FILE_UPLOAD_ACK = "file_upload_ack" # node acknowledges chunk receipt DIR_CREATE = "dir_create" # client → node: make a directory diff --git a/packages/meshbay-node/src/meshbay_node/ops.py b/packages/meshbay-node/src/meshbay_node/ops.py index 7557302..9fcbc21 100644 --- a/packages/meshbay-node/src/meshbay_node/ops.py +++ b/packages/meshbay-node/src/meshbay_node/ops.py @@ -1368,6 +1368,15 @@ async def set_node_settings(state: dict, settings: dict) -> dict: if webrtc is not None: webrtc.set_capacity( max_concurrent_streams=updated["max_concurrent_streams"]) + if ("max_concurrent_downloads" in updated + or "max_concurrent_uploads" in updated): + webrtc = state.get("webrtc") + if webrtc is not None: + webrtc.set_capacity( + max_concurrent_downloads=updated.get( + "max_concurrent_downloads"), + max_concurrent_uploads=updated.get( + "max_concurrent_uploads")) if "stun_servers" in updated: webrtc = state.get("webrtc") if webrtc and hasattr(webrtc, '_stun'): @@ -1382,6 +1391,33 @@ async def set_node_settings(state: dict, settings: dict) -> dict: return {"updated": updated} +# ── Transfers ──────────────────────────────────────────────────────────────── + +async def list_transfers(state: dict) -> dict: + """Live transfer leases and queue depth. + + The operator's window into "is anything actually holding a slot". When + somebody reports a transfer stuck at waiting, this is the only thing that + says whether the node ever had them in a queue — the alternative is reading + a log for a line that, by definition, is not being printed. + + Carries no filename and no path: a lease holds neither, and this is exactly + where it would be tempting to add one. + """ + webrtc = state.get("webrtc") + slots = getattr(webrtc, "_ctx", {}).get("_transfer_slots") if webrtc else None + if slots is None: + from meshbay_node.transfers import ( + DEFAULT_MAX_CONCURRENT, DEFAULT_MAX_PER_MEMBER, KINDS) + # No pool built means nothing has transferred since the daemon started, + # which is a real answer and not an error. + return {"pools": {k: {"in_use": 0, "cap": DEFAULT_MAX_CONCURRENT, + "per_member": DEFAULT_MAX_PER_MEMBER, + "queued": 0} for k in KINDS}, + "leases": []} + return slots.snapshot() + + # ── Applications ───────────────────────────────────────────────────────────── async def set_enabled_apps(state: dict, group_id: str, apps: list[str]) -> dict: diff --git a/packages/meshbay-node/src/meshbay_node/transfers.py b/packages/meshbay-node/src/meshbay_node/transfers.py new file mode 100644 index 0000000..0689ec8 --- /dev/null +++ b/packages/meshbay-node/src/meshbay_node/transfers.py @@ -0,0 +1,354 @@ +""" +Transfer slots: how many downloads and uploads a node runs at once. + +A download is invisible to the node today. `pipelinedDownload` sends eight +independent `file_req` messages and reassembles the answers; nothing tells the +node a transfer started, and nothing tells it one ended. There is nothing to +count and so nothing to cap — which is why this exists before any cap does. + +The unit is the **lease**: the node's record that a peer is transferring +something, held for the length of the transfer and released by name. Six +properties are load-bearing, and each one is a decision: + + - **`tr` is drawn by the client**, like `upload_id`. Re-opening after a + reconnect with the same `tr` is idempotent, so a reconnect cannot charge a + member twice for one transfer. + - **A lease is scoped to the connection**, never to the account. It dies with + the session, which is what makes the primary reclaim deterministic. + - **A lease covers a job, not a file.** A directory zip is dozens of files and + one lease. + - **Nothing is persisted.** A restart drops every session anyway; a lease that + outlived the process would be a slot nothing can release. + - **Leases are counted, not bytes.** What a slot protects is concurrency — + open file handles, disk seeks, the channel buffer each transfer keeps full. + - **Per-member first, then node-wide.** A member at their own cap queues + behind their own transfers and never holds a node-wide slot a second member + has none of. Reversed, whoever arrives first takes everything. + +This module is deliberately free of asyncio and of the transport: it decides, +and the caller does the I/O. `sweep()` is called on a clock the caller owns, and +every method returns what changed so the caller can push it. That is what makes +the failure modes in §5 of ~/next/improve-downloads.md testable at all — a +queue that only reveals itself through a DataChannel is a queue nobody can +prove things about. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass, field + +log = logging.getLogger(__name__) + +DOWNLOAD = "download" +UPLOAD = "upload" +KINDS = (DOWNLOAD, UPLOAD) + +# Node-wide defaults. The operator's own values arrive from node.toml/roster.db +# via `set_capacity` — these apply when they have said nothing. +DEFAULT_MAX_CONCURRENT = 8 +# Per account, per group. Absent means this, not "unlimited": a group that +# predates the setting coming back unlimited would leave the node-wide cap as +# the only control, which is the situation this exists to end. +DEFAULT_MAX_PER_MEMBER = 2 + +# A grant nobody takes up is a slot nobody can use. Long enough for a client to +# send its first chunk request, short enough that a browser that died between +# the grant and that request does not hold a slot until the idle timeout. +GRANT_DEADLINE_SECS = 30.0 +# Silence on a granted lease. The session dying is the primary reclaim and is +# immediate; this only catches a peer that vanished without the connection +# noticing, so it can afford to be generous. +IDLE_TIMEOUT_SECS = 120.0 +# Per account, per kind. Unbounded queues are how a node runs out of memory +# politely; past this the client keeps the rest in its own list. +MAX_QUEUED_PER_MEMBER = 32 + +# Why a lease ended, as it reaches the peer. +REASON_DONE = "done" +REASON_CANCELLED = "cancelled" +REASON_PAUSED = "paused" +REASON_FAILED = "failed" +REASON_SESSION_GONE = "session_gone" +REASON_IDLE = "idle" +REASON_NOT_TAKEN_UP = "not_taken_up" + + +@dataclass +class Lease: + tr: str + kind: str + session_key: str + user_id: str + group_id: str + bytes: int = 0 + chunks: int = 0 + state: str = "queued" # "queued" | "granted" + created_at: float = 0.0 + granted_at: float | None = None + # Set the first time anything happens under this lease. Distinguishes "the + # client never came back for its slot" from "the client went quiet": the + # first is a grant to revoke and pass on, the second a transfer to reclaim. + used: bool = False + last_seen: float = 0.0 + + @property + def member(self) -> tuple[str, str]: + return (self.group_id, self.user_id) + + +@dataclass +class TransferSlots: + """Every lease on this node, and the queues behind them.""" + + caps: dict[str, int] = field( + default_factory=lambda: {k: DEFAULT_MAX_CONCURRENT for k in KINDS}) + per_member: dict[str, int] = field( + default_factory=lambda: {k: DEFAULT_MAX_PER_MEMBER for k in KINDS}) + leases: dict[str, Lease] = field(default_factory=dict) + # FIFO of `tr`, per kind. Order is arrival; a member at their own cap is + # skipped rather than blocking the head, or one member's limit would stall + # the whole node. + queues: dict[str, list[str]] = field( + default_factory=lambda: {k: [] for k in KINDS}) + + # ── counting ──────────────────────────────────────────────────────────── + + def in_use(self, kind: str) -> int: + return sum(1 for x in self.leases.values() + if x.kind == kind and x.state == "granted") + + def member_in_use(self, kind: str, member: tuple[str, str]) -> int: + return sum(1 for x in self.leases.values() + if x.kind == kind and x.state == "granted" + and x.member == member) + + def queued_for(self, kind: str, member: tuple[str, str]) -> int: + return sum(1 for tr in self.queues[kind] + if (x := self.leases.get(tr)) and x.member == member) + + def ahead_of(self, lease: Lease) -> int: + """How many are in front of this one in its queue.""" + try: + return self.queues[lease.kind].index(lease.tr) + except ValueError: + return 0 + + def _has_room(self, kind: str, member: tuple[str, str]) -> bool: + # Per-member first: see the module docstring. + if self.member_in_use(kind, member) >= self.per_member.get( + kind, DEFAULT_MAX_PER_MEMBER): + return False + return self.in_use(kind) < self.caps.get(kind, DEFAULT_MAX_CONCURRENT) + + # ── the operations a peer asks for ────────────────────────────────────── + + def open(self, *, tr: str, kind: str, session_key: str, user_id: str, + group_id: str, bytes: int = 0, chunks: int = 0, + now: float | None = None) -> tuple[Lease | None, str]: + """Ask for a slot. Returns (lease, error_code); one of them is falsy. + + Idempotent on `tr`: re-opening a lease this session already holds + returns it unchanged rather than charging for a second one. That is what + makes a client's reconnect safe, and it is checked before anything else + because every other branch below would otherwise double-count. + """ + now = time.monotonic() if now is None else now + if kind not in KINDS: + return None, "bad_kind" + existing = self.leases.get(tr) + if existing is not None: + if existing.session_key != session_key: + # Someone else's lease id. Refused rather than adopted: a `tr` + # is drawn at random by its owner, so a collision is either a + # bug or a peer guessing, and neither should move a slot between + # connections. + return None, "not_your_transfer" + return existing, "" + + member = (group_id, user_id) + if self.queued_for(kind, member) >= MAX_QUEUED_PER_MEMBER: + return None, "too_many_queued" + + lease = Lease(tr=tr, kind=kind, session_key=session_key, + user_id=user_id, group_id=group_id, + bytes=int(bytes or 0), chunks=int(chunks or 0), + created_at=now, last_seen=now) + self.leases[tr] = lease + if self._has_room(kind, member): + self._grant(lease, now) + else: + self.queues[kind].append(tr) + return lease, "" + + def _grant(self, lease: Lease, now: float) -> None: + lease.state = "granted" + lease.granted_at = now + lease.last_seen = now + lease.used = False + + def touch(self, tr: str, now: float | None = None) -> bool: + """Something happened under this lease. False if it is not granted.""" + lease = self.leases.get(tr) + if lease is None or lease.state != "granted": + return False + lease.used = True + lease.last_seen = time.monotonic() if now is None else now + return True + + def close(self, tr: str, reason: str = REASON_DONE, + now: float | None = None) -> tuple[Lease | None, list[Lease]]: + """Give a slot back. Returns (the closed lease, newly granted ones). + + The only place a lease is destroyed, and the only caller of the pump — + two functions that both released would be this repo's flow-control + lesson one feature later. + """ + lease = self.leases.pop(tr, None) + if lease is None: + return None, [] + if lease.tr in self.queues[lease.kind]: + self.queues[lease.kind].remove(lease.tr) + log.debug("transfer: closed %s (%s, %s)", tr[:8], lease.kind, reason) + return lease, self._pump(lease.kind, now) + + def release_session(self, session_key: str, + now: float | None = None) -> tuple[list[Lease], list[Lease]]: + """The connection is gone; everything it held goes with it. + + The deterministic reclaim, and the reason a lease is scoped to a + connection rather than to an account: a tab closed, a browser quit and a + network that dropped all arrive here, and none of them needs a timer. + """ + gone = [x for x in self.leases.values() if x.session_key == session_key] + for lease in gone: + self.leases.pop(lease.tr, None) + if lease.tr in self.queues[lease.kind]: + self.queues[lease.kind].remove(lease.tr) + granted: list[Lease] = [] + for kind in KINDS: + if any(x.kind == kind for x in gone): + granted.extend(self._pump(kind, now)) + return gone, granted + + def sweep(self, now: float | None = None) -> tuple[list[tuple[Lease, str]], + list[Lease]]: + """Reclaim what the session teardown cannot see. + + Two different failures, deliberately told apart: + a grant nobody took up (the client died between asking and starting) + goes back to the tail of the queue; a granted transfer that has gone + quiet is closed, and the peer is told, so its widget can offer a resume + rather than sit on a lie. + """ + now = time.monotonic() if now is None else now + ended: list[tuple[Lease, str]] = [] + requeued = False + for lease in list(self.leases.values()): + if lease.state != "granted": + continue + if not lease.used and lease.granted_at is not None \ + and now - lease.granted_at > GRANT_DEADLINE_SECS: + lease.state = "queued" + lease.granted_at = None + self.queues[lease.kind].append(lease.tr) + ended.append((lease, REASON_NOT_TAKEN_UP)) + requeued = True + elif lease.used and now - lease.last_seen > IDLE_TIMEOUT_SECS: + self.leases.pop(lease.tr, None) + ended.append((lease, REASON_IDLE)) + granted: list[Lease] = [] + if ended or requeued: + for kind in KINDS: + granted.extend(self._pump(kind, now)) + return ended, granted + + # ── the queue ─────────────────────────────────────────────────────────── + + def _pump(self, kind: str, now: float | None = None) -> list[Lease]: + """Grant to whoever can start, in arrival order, skipping who cannot. + + Called from exactly one place per release. Walking past a member who is + at their own cap is the whole reason this is a walk and not a `pop(0)`: + granting strictly in order lets one member's limit stall every other + member behind them. + """ + now = time.monotonic() if now is None else now + granted: list[Lease] = [] + for tr in list(self.queues[kind]): + lease = self.leases.get(tr) + if lease is None: # closed while queued + self.queues[kind].remove(tr) + continue + if self.in_use(kind) >= self.caps.get(kind, DEFAULT_MAX_CONCURRENT): + break # the node is full; stop + if not self._has_room(kind, lease.member): + continue # this member is; skip them + self.queues[kind].remove(tr) + self._grant(lease, now) + granted.append(lease) + return granted + + # ── what the operator sees ────────────────────────────────────────────── + + def set_caps(self, *, node: dict[str, int] | None = None, + per_member: dict[str, int] | None = None, + now: float | None = None) -> list[Lease]: + """Change a cap live. Raising one may start queued transfers at once. + + Lowering never interrupts a transfer that is running, for the same + reason lowering the stream cap does not stop a film: the new value + governs what starts next. + """ + for kind, value in (node or {}).items(): + if kind in KINDS: + self.caps[kind] = max(1, int(value)) + for kind, value in (per_member or {}).items(): + if kind in KINDS: + self.per_member[kind] = max(1, int(value)) + granted: list[Lease] = [] + for kind in KINDS: + granted.extend(self._pump(kind, now)) + return granted + + def snapshot(self) -> dict: + """The whole picture, for `GET /api/transfers` and the summary log. + + When somebody reports a transfer stuck at "waiting", this is the only + thing that will say whether the node ever had them in a queue. + """ + return { + "pools": { + kind: { + "in_use": self.in_use(kind), + "cap": self.caps.get(kind, DEFAULT_MAX_CONCURRENT), + "per_member": self.per_member.get(kind, + DEFAULT_MAX_PER_MEMBER), + "queued": len(self.queues[kind]), + } for kind in KINDS + }, + "leases": [ + { + "tr": x.tr[:12], + "kind": x.kind, + "state": x.state, + "user_id": x.user_id, + "group_id": x.group_id, + "bytes": x.bytes, + "used": x.used, + "ahead": self.ahead_of(x) if x.state == "queued" else 0, + } + # Never a filename or a path: a lease carries none, and this is + # the one place it would be tempting to add one for a prettier + # log line. + for x in sorted(self.leases.values(), + key=lambda l: (l.kind, l.state, l.created_at)) + ], + } + + def summary(self) -> str: + p = self.snapshot()["pools"] + return " ".join( + f"{kind[0]}={p[kind]['in_use']}/{p[kind]['cap']}" + f"(q{p[kind]['queued']})" for kind in KINDS) diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py index 33f5474..164cc62 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py @@ -127,6 +127,8 @@ from meshbay_node.transport.wire import index_sync_message from meshbay_node.indexer import GroupIndex from meshbay_node.indexer.indexer import DirectoryIndexer from meshbay_node import linkpreview, ops, platform +from meshbay_node import transfers as transfers_mod +from meshbay_node.transfers import TransferSlots # Re-imported under its original name: every call site and existing test in # this module still refers to it as `_probe_video`. The implementation lives # in media_probe.py so the indexer package (imported just above) can call it @@ -258,6 +260,11 @@ STREAM_CREDIT_TIMEOUT = 120 # How often that budget is re-examined. A viewer who left stops being # charged for a slot within this, rather than within the timeout. STREAM_CREDIT_POLL = 3 +# How often transfer leases are swept. Nothing depends on it being +# prompt -- the session teardown is the reclaim that matters and is +# immediate; this catches peers that vanished without the connection +# noticing, so it trades latency for a timer that hardly ever runs. +TRANSFER_SWEEP_SECS = 15 def _pack(obj: dict) -> bytes: @@ -524,6 +531,10 @@ class WebRTCPeerSession: self._spawn(self._do_link_preview_request(msg)) elif mtype == MNP.PING: self._do_ping(msg) + elif mtype == MNP.TRANSFER_OPEN: + self._do_transfer_open(msg) + elif mtype == MNP.TRANSFER_CLOSE: + self._do_transfer_close(msg) elif mtype == MNP.FILE_UPLOAD: self._do_file_upload(msg) elif mtype == MNP.DIR_CREATE: @@ -3309,6 +3320,187 @@ class WebRTCPeerSession: "total_bytes": progress.total_bytes, } + # ── Transfer slots ─────────────────────────────────────────────────────── + + def _slots(self) -> "TransferSlots": + """The node's transfer pools, shared across every peer and every group. + + On the transport context, not the session: it counts the node's + transfers, not one browser's. Built once, for the same reason the + transcode semaphore is — rebuilding it per call would hand every caller + its own budget and cap nothing at all. + """ + slots = self._ctx.get("_transfer_slots") + if slots is None: + slots = TransferSlots() + n = self._ctx.get("max_concurrent_downloads") + u = self._ctx.get("max_concurrent_uploads") + if n: + slots.caps[transfers_mod.DOWNLOAD] = int(n) + if u: + slots.caps[transfers_mod.UPLOAD] = int(u) + self._ctx["_transfer_slots"] = slots + log.info("transfer: %s", slots.summary()) + return slots + + def _transfer_state_msg(self, lease, state: str, reason: str = "") -> dict: + slots = self._slots() + out = { + "type": MNP.TRANSFER_STATE, + "v": MNP_VERSION, + "tr": lease.tr, + "state": state, + "kind": lease.kind, + "used": slots.member_in_use(lease.kind, lease.member), + "cap": slots.per_member.get(lease.kind, + transfers_mod.DEFAULT_MAX_PER_MEMBER), + "node_used": slots.in_use(lease.kind), + "node_cap": slots.caps.get(lease.kind, + transfers_mod.DEFAULT_MAX_CONCURRENT), + } + if state == "queued": + out["ahead"] = slots.ahead_of(lease) + if reason: + out["reason"] = reason + return out + + def _notify_transfer(self, lease, state: str, reason: str = "") -> None: + """Push a lease's state to the connection that owns it. + + By session key, never by account: a lease belongs to one connection, and + telling a member's other device that *its* transfer was granted is how a + queue starts lying. + """ + session = self._peer_registry().get(lease.session_key) + for candidate in ([session] if session else + self._sessions_everywhere(lease.session_key)): + try: + candidate._send(self._transfer_state_msg(lease, state, reason)) + except Exception: + pass + + def _sessions_everywhere(self, session_key: str) -> list["WebRTCPeerSession"]: + """The session with this key, whichever group it is in. + + `_peer_registry` is per group (finding H1) and the pools are node-wide, + so a slot freed in one group can grant one in another: the peer to tell + is not necessarily in this session's own registry. + """ + groups = self._ctx.get("groups") + registries = ([g.get("_peers", {}) for g in groups.values()] + if groups else [self._ctx.get("_peers", {})]) + return [reg[session_key] for reg in registries if session_key in reg] + + def _announce(self, granted: list, ended: list | None = None) -> None: + for lease, reason in (ended or []): + self._notify_transfer( + lease, "queued" if lease.state == "queued" else "closed", reason) + for lease in granted: + self._notify_transfer(lease, "granted") + + def _do_transfer_open(self, msg: dict) -> None: + tr = str(msg.get("tr") or "")[:64] + kind = str(msg.get("kind") or transfers_mod.DOWNLOAD) + if not tr: + self._send({"type": "error", "detail": "Missing transfer id", + "code": "bad_transfer_id"}) + return + slots = self._slots() + try: + nbytes = int(msg.get("bytes") or 0) + chunks = int(msg.get("chunks") or 0) + except (TypeError, ValueError): + self._send({"type": "error", "detail": "Invalid transfer size", + "code": "bad_transfer_size", "tr": tr}) + return + lease, err = slots.open( + tr=tr, kind=kind, session_key=self._registry_key, + user_id=self._user_id or "", group_id=self._group_id or "", + bytes=nbytes, chunks=chunks) + if err: + self._send({"type": "error", "detail": err, "code": err, "tr": tr}) + return + self._send(self._transfer_state_msg(lease, lease.state)) + log.debug("transfer: open %s %s -> %s (%s)", + kind, tr[:8], lease.state, slots.summary()) + self._ensure_transfer_sweeper() + + def _do_transfer_close(self, msg: dict) -> None: + tr = str(msg.get("tr") or "")[:64] + reason = str(msg.get("reason") or transfers_mod.REASON_DONE)[:32] + slots = self._slots() + held = slots.leases.get(tr) + if held is not None and held.session_key != self._registry_key: + # Closing somebody else's transfer would be a denial of service one + # random id away. + self._send({"type": "error", "detail": "not_your_transfer", + "code": "not_your_transfer", "tr": tr}) + return + lease, granted = slots.close(tr, reason) + if lease is not None: + self._send(self._transfer_state_msg(lease, "closed", reason)) + self._announce(granted) + + def _release_transfers(self) -> None: + """Give back everything this connection held. Called from teardown.""" + slots = self._ctx.get("_transfer_slots") + if slots is None: + return + gone, granted = slots.release_session(self._registry_key) + if gone: + log.info("transfer: session gone, released %d (%s)", + len(gone), slots.summary()) + self._announce(granted) + + def _ensure_transfer_sweeper(self) -> None: + """Start the maintenance task, once, and only while it has work. + + It reclaims what a session teardown cannot see — a grant nobody took up, + a transfer that went quiet — and logs the one line that answers "was + this peer ever in a queue" when somebody reports a stuck transfer. It + stops when the last lease goes, so an idle node runs no timer. + """ + running = self._ctx.get("_transfer_sweeper") + if running is not None and not running.done(): + return + + ctx = self._ctx + + async def _sweep_loop() -> None: + while True: + await asyncio.sleep(TRANSFER_SWEEP_SECS) + slots = ctx.get("_transfer_slots") + if slots is None or not slots.leases: + return + ended, granted = slots.sweep() + for lease, reason in ended: + log.info("transfer: reclaimed %s (%s)", lease.tr[:8], reason) + self._announce(granted, ended) + log.debug("transfer: %s", slots.summary()) + + # Deliberately NOT `self._spawn`, which is otherwise the only way to + # start a task here. `_spawn` ties a task to *this session's* set, and + # `shutdown_tasks` cancels those when the peer leaves — so the sweeper + # would die with whichever connection happened to open the first + # transfer, and every other peer's abandoned lease would then never be + # reclaimed. It belongs to the node, so the strong reference that keeps + # it off the garbage collector lives on the transport context; the rule + # `_spawn` exists for (asyncio holds only a weak reference) is satisfied + # by that reference, not by which set it is in. + task = asyncio.ensure_future(_sweep_loop()) + ctx["_transfer_sweeper"] = task + + def _finished(done: asyncio.Task) -> None: + if ctx.get("_transfer_sweeper") is done: + ctx["_transfer_sweeper"] = None + if not done.cancelled() and done.exception() is not None: + # Nothing awaits this task, so an exception here would otherwise + # be swallowed and idle leases would silently stop being + # reclaimed — the failure mode is a node that fills up over days. + log.error("transfer: sweeper died: %r", done.exception()) + + task.add_done_callback(_finished) + def _register_peer(self) -> None: """Add this connection to its group's peer set. @@ -5507,6 +5699,12 @@ class WebRTCPeerSession: here: cancelling the task runs the exit of its `async with sem`. """ self._stop_stream() + # Before the tasks are cancelled: a lease is not held by a task, so + # nothing else would give it back, and this hook is the one place every + # way of walking away arrives at (see the connectionstatechange handler, + # which calls it for a closed tab, a quit browser and a dead network + # alike). + self._release_transfers() for task in list(self._tasks): task.cancel() if self._tasks: @@ -5514,6 +5712,7 @@ class WebRTCPeerSession: async def close(self) -> None: self._audit("disconnect") + self._release_transfers() if self._user_id: self._unregister_peer() await self.shutdown_tasks() @@ -5620,7 +5819,9 @@ class WebRTCTransport: self._stun = stun_servers or list(DEFAULT_STUN_SERVERS) self._sessions: dict[str, WebRTCPeerSession] = {} - def set_capacity(self, *, max_concurrent_streams: int | None = None) -> dict: + def set_capacity(self, *, max_concurrent_streams: int | None = None, + max_concurrent_downloads: int | None = None, + max_concurrent_uploads: int | None = None) -> dict: """Resize a live pool without restarting the daemon. `ops.set_node_settings` used to do this by assigning @@ -5660,8 +5861,54 @@ class WebRTCTransport: log.info("stream: capacity %s -> %d (no pool built yet)", before, n) changed["max_concurrent_streams"] = n + + pools = {} + if max_concurrent_downloads is not None: + pools[transfers_mod.DOWNLOAD] = int(max_concurrent_downloads) + if max_concurrent_uploads is not None: + pools[transfers_mod.UPLOAD] = int(max_concurrent_uploads) + for key, value in pools.items(): + if value < 1: + raise ValueError(f"max_concurrent_{key}s must be positive") + if pools: + # Kept on the context whether or not a pool exists yet: the pools + # are built on the first transfer, and would otherwise come up with + # the defaults after an operator had already changed them. + for key, value in pools.items(): + self._ctx[f"max_concurrent_{key}s"] = value + changed[f"max_concurrent_{key}s"] = value + slots = self._ctx.get("_transfer_slots") + if slots is not None: + granted = slots.set_caps(node=pools) + log.info("transfer: capacity now %s (%d started at once)", + slots.summary(), len(granted)) + # Raising a cap can start queued transfers immediately, and the + # peers waiting on them have to be told: a grant nobody hears + # about is the "stuck at waiting" report this design exists to + # prevent. + for lease in granted: + self._notify_granted(lease) return changed + def _notify_granted(self, lease) -> None: + """Tell the connection that owns `lease` it may start. + + On the transport rather than the session because a cap change has no + session behind it — it arrives from the loopback API. + """ + groups = self._ctx.get("groups") + registries = ([g.get("_peers", {}) for g in groups.values()] + if groups else [self._ctx.get("_peers", {})]) + for reg in registries: + session = reg.get(lease.session_key) + if session is not None: + try: + session._send( + session._transfer_state_msg(lease, "granted")) + except Exception: + pass + return + async def handle_offer( self, offer_sdp: str, peer_id: str, ) -> tuple[str, list[dict]]: diff --git a/packages/meshbay-node/src/meshbay_node/ui/app.py b/packages/meshbay-node/src/meshbay_node/ui/app.py index 2b99f20..9b9307d 100644 --- a/packages/meshbay-node/src/meshbay_node/ui/app.py +++ b/packages/meshbay-node/src/meshbay_node/ui/app.py @@ -509,4 +509,10 @@ def create_ui_app(state: dict) -> FastAPI: async def update_node_settings(payload: dict): return await _op(lambda: ops.set_node_settings(state, payload)) + # ── Transfers (operator only, localhost) ─────────────────────────────── + + @app.get("/api/transfers") + async def get_transfers(): + return await _op(lambda: ops.list_transfers(state)) + return app diff --git a/packages/meshbay-node/tests/test_transfer_slots.py b/packages/meshbay-node/tests/test_transfer_slots.py new file mode 100644 index 0000000..a0ef381 --- /dev/null +++ b/packages/meshbay-node/tests/test_transfer_slots.py @@ -0,0 +1,308 @@ +""" +Transfer slots: the caps, the queue, and every way a slot can be lost. + +The requirement this is written against is not "a cap exists". It is that +**nobody stays stuck** — neither a slot the node never gets back, which fills +the node and queues everyone for ever, nor a transfer a client shows as waiting +that the node has already forgotten. + +`TransferSlots` has no asyncio and no transport in it precisely so that those +failures can be driven here instead of through a DataChannel, where they are +rare, timing-dependent and unprovable. The clock is passed in, so the two +timeouts are exercised without a test that sleeps for two minutes. + +`test_the_counter_never_drifts` is the one that matters most: a stuck slot is a +race by nature, "it works now" is not evidence against a race, and the earlier +flow-control bugs in this repo (window_leak.mjs, the discarded segment that +leaked a slot per discard) were all found by forcing the worst case rather than +by reasoning about it. +""" + +import random + +import pytest + +from meshbay_node.transfers import ( + DOWNLOAD, GRANT_DEADLINE_SECS, IDLE_TIMEOUT_SECS, KINDS, + MAX_QUEUED_PER_MEMBER, REASON_IDLE, REASON_NOT_TAKEN_UP, TransferSlots, + UPLOAD, +) + + +def _slots(node=8, per_member=2) -> TransferSlots: + s = TransferSlots() + s.caps = {k: node for k in KINDS} + s.per_member = {k: per_member for k in KINDS} + return s + + +def _open(s, tr, *, session="s1", user="u1", group="g1", kind=DOWNLOAD, now=0.0): + lease, err = s.open(tr=tr, kind=kind, session_key=session, user_id=user, + group_id=group, now=now) + assert not err, err + return lease + + +# ── the caps ──────────────────────────────────────────────────────────────── + +def test_a_member_is_held_to_their_own_cap_first(_=None): + s = _slots(node=8, per_member=2) + assert _open(s, "a").state == "granted" + assert _open(s, "b").state == "granted" + assert _open(s, "c").state == "queued", ( + "a third transfer for one member must queue even though the node has " + "six free slots — otherwise one member takes the node") + + +def test_the_member_cap_spans_their_devices(_=None): + """Per account, not per connection: two browsers and a desktop client + signed in as the same person share the two slots, or the cap becomes a + function of how many tabs somebody opens.""" + s = _slots(per_member=2) + _open(s, "a", session="laptop") + _open(s, "b", session="phone") + assert _open(s, "c", session="desktop").state == "queued" + + +def test_the_node_cap_holds_across_members(_=None): + s = _slots(node=3, per_member=2) + _open(s, "a", user="u1") + _open(s, "b", user="u1") + _open(s, "c", user="u2") + assert _open(s, "d", user="u2").state == "queued" + assert s.in_use(DOWNLOAD) == 3 + + +def test_downloads_and_uploads_have_separate_pools(_=None): + s = _slots(node=2, per_member=2) + _open(s, "a", kind=DOWNLOAD) + _open(s, "b", kind=DOWNLOAD) + assert _open(s, "c", kind=UPLOAD).state == "granted", ( + "a full download pool must not stop an upload") + + +# ── the queue ─────────────────────────────────────────────────────────────── + +def test_a_freed_slot_goes_to_whoever_was_waiting(_=None): + s = _slots(node=1, per_member=2) + _open(s, "a", user="u1") + queued = _open(s, "b", user="u2") + assert queued.state == "queued" + _, granted = s.close("a") + assert [x.tr for x in granted] == ["b"] + assert s.leases["b"].state == "granted" + + +def test_a_member_at_their_cap_is_skipped_not_waited_for(_=None): + """Granting strictly in arrival order lets one member's own limit stall + every other member behind them.""" + s = _slots(node=3, per_member=2) + _open(s, "a", user="u1") + _open(s, "b", user="u1") + hog = _open(s, "c", user="u1") # u1 is at their cap + other = _open(s, "d", user="u2") # arrives later + assert hog.state == "queued" + assert other.state == "granted", "u2 was made to wait behind u1's own limit" + + +def test_position_is_reported_from_the_queue_itself(_=None): + s = _slots(node=1, per_member=8) + _open(s, "a") + b, c = _open(s, "b"), _open(s, "c") + assert (s.ahead_of(b), s.ahead_of(c)) == (0, 1) + + +def test_a_member_cannot_queue_without_end(_=None): + s = _slots(node=1, per_member=1) + _open(s, "granted") + for i in range(MAX_QUEUED_PER_MEMBER): + _open(s, f"q{i}") + lease, err = s.open(tr="one-too-many", kind=DOWNLOAD, session_key="s1", + user_id="u1", group_id="g1") + assert lease is None and err == "too_many_queued" + + +# ── every way a slot comes back (§5.1) ────────────────────────────────────── + +def test_closing_returns_the_slot(_=None): + s = _slots(node=1) + _open(s, "a") + s.close("a") + assert s.in_use(DOWNLOAD) == 0 + + +def test_losing_the_session_returns_everything_it_held(_=None): + """The primary reclaim, and the reason a lease is scoped to a connection: + a closed tab, a quit browser and a dropped network all arrive here, and + none of them needs a timer.""" + s = _slots(node=8, per_member=8) + _open(s, "a", session="doomed") + _open(s, "b", session="doomed") + _open(s, "c", session="other") + gone, _ = s.release_session("doomed") + assert sorted(x.tr for x in gone) == ["a", "b"] + assert s.in_use(DOWNLOAD) == 1 + + +def test_a_queued_lease_dies_with_its_session_too(_=None): + s = _slots(node=1, per_member=8) + _open(s, "a", session="s1") + _open(s, "waiting", session="doomed") + s.release_session("doomed") + assert "waiting" not in s.leases + assert s.queues[DOWNLOAD] == [] + + +def test_a_grant_nobody_takes_up_is_passed_on(_=None): + s = _slots(node=1, per_member=8) + _open(s, "a", now=0.0) + _open(s, "b", now=0.0) + ended, granted = s.sweep(now=GRANT_DEADLINE_SECS + 1) + assert [(x.tr, r) for x, r in ended] == [("a", REASON_NOT_TAKEN_UP)] + assert [x.tr for x in granted] == ["b"], "the slot was not passed on" + assert s.leases["a"].state == "queued", "the abandoned one goes to the tail" + + +def test_a_transfer_that_started_is_not_mistaken_for_an_abandoned_grant(_=None): + s = _slots(node=1) + _open(s, "a", now=0.0) + s.touch("a", now=1.0) + ended, _ = s.sweep(now=GRANT_DEADLINE_SECS + 2) + assert ended == [], "a transfer that is running was revoked" + + +def test_a_transfer_that_goes_quiet_is_reclaimed(_=None): + s = _slots(node=1) + _open(s, "a", now=0.0) + s.touch("a", now=1.0) + ended, _ = s.sweep(now=1.0 + IDLE_TIMEOUT_SECS + 1) + assert [(x.tr, r) for x, r in ended] == [("a", REASON_IDLE)] + assert "a" not in s.leases + + +def test_activity_keeps_a_slow_transfer_alive(_=None): + """A slow reader is not an absent one. The idle clock follows the lease's + own activity, not the wall since it started.""" + s = _slots(node=1) + _open(s, "a", now=0.0) + t = 0.0 + for _ in range(10): + t += IDLE_TIMEOUT_SECS - 1 + s.touch("a", now=t) + assert s.sweep(now=t)[0] == [] + assert "a" in s.leases + + +# ── idempotence, which is what makes a reconnect safe ─────────────────────── + +def test_reopening_the_same_transfer_does_not_charge_twice(_=None): + s = _slots(node=8, per_member=2) + first = _open(s, "a") + again = _open(s, "a") + assert again is first + assert s.in_use(DOWNLOAD) == 1 + + +def test_another_session_cannot_adopt_a_lease(_=None): + s = _slots() + _open(s, "a", session="mine") + lease, err = s.open(tr="a", kind=DOWNLOAD, session_key="theirs", + user_id="u1", group_id="g1") + assert lease is None and err == "not_your_transfer" + + +# ── caps changed live ─────────────────────────────────────────────────────── + +def test_raising_a_cap_starts_what_was_waiting(_=None): + s = _slots(node=1, per_member=8) + _open(s, "a") + _open(s, "b") + granted = s.set_caps(node={DOWNLOAD: 4}) + assert [x.tr for x in granted] == ["b"] + + +def test_lowering_a_cap_does_not_interrupt_anything(_=None): + s = _slots(node=4, per_member=4) + for tr in "abcd": + _open(s, tr) + s.set_caps(node={DOWNLOAD: 1}) + assert s.in_use(DOWNLOAD) == 4, "a running transfer was taken away" + assert _open(s, "e").state == "queued" + + +# ── the property that matters (§5.3) ──────────────────────────────────────── + +@pytest.mark.parametrize("seed", range(25)) +def test_the_counter_never_drifts(seed): + """ + Random open/close/drop/sweep/resize, checked after every single step. + + A leaked slot is a race, and a test that reasons about the happy path + agrees with a broken implementation by construction. Two invariants, both + of which a real leak breaks: what the pool says is in use is exactly the + set of granted leases, and no queue entry names a lease that no longer + exists — the second being how "waiting for ever behind a ghost" starts. + """ + rng = random.Random(seed) + s = _slots(node=rng.randint(1, 4), per_member=rng.randint(1, 3)) + sessions = [f"s{i}" for i in range(4)] + users = ["u1", "u2", "u3"] + live: list[str] = [] + now = 0.0 + counter = 0 + + for _ in range(400): + before = {k: s.in_use(k) for k in KINDS} + member_before = {(k, m): s.member_in_use(k, m) + for k in KINDS + for m in {x.member for x in s.leases.values()}} + now += rng.uniform(0.0, 40.0) + action = rng.choice( + ["open", "open", "open", "close", "touch", "drop", "sweep", "caps"]) + if action == "open": + counter += 1 + tr = f"t{counter}" + lease, err = s.open( + tr=tr, kind=rng.choice(KINDS), session_key=rng.choice(sessions), + user_id=rng.choice(users), group_id="g1", now=now) + if lease is not None: + live.append(tr) + elif action == "close" and live: + s.close(live.pop(rng.randrange(len(live))), now=now) + elif action == "touch" and live: + s.touch(rng.choice(live), now=now) + elif action == "drop": + s.release_session(rng.choice(sessions), now=now) + elif action == "sweep": + s.sweep(now=now) + elif action == "caps": + s.set_caps(node={rng.choice(KINDS): rng.randint(1, 5)}, now=now) + live = [tr for tr in live if tr in s.leases] + + for kind in KINDS: + granted = [x for x in s.leases.values() + if x.kind == kind and x.state == "granted"] + assert s.in_use(kind) == len(granted) + # Not `in_use <= cap`: lowering a cap never interrupts a transfer + # that is running, so the count legitimately sits above the new + # value until those finish. What must never happen is a *new* grant + # while the pool is at or over its cap -- so the count may fall or + # hold, and may only rise while there was room. + assert s.in_use(kind) <= max(s.caps[kind], before[kind]), ( + f"{kind}: {before[kind]} -> {s.in_use(kind)} granted with a cap " + f"of {s.caps[kind]} — a slot was handed out past the cap") + for tr in s.queues[kind]: + assert tr in s.leases, "a queue entry outlived its lease" + assert s.leases[tr].state == "queued" + for member in {x.member for x in granted}: + assert s.member_in_use(kind, member) <= max( + s.per_member[kind], member_before.get((kind, member), 0)) + + # And at the end: drop every session and nothing may be left holding + # anything. A slot that survives the last connection is a slot nothing can + # ever release. + for session in sessions: + s.release_session(session, now=now) + assert s.leases == {} + assert all(q == [] for q in s.queues.values()) + assert all(s.in_use(k) == 0 for k in KINDS) diff --git a/packages/meshbay-node/tests/test_transfer_slots_wire.py b/packages/meshbay-node/tests/test_transfer_slots_wire.py new file mode 100644 index 0000000..afa4582 --- /dev/null +++ b/packages/meshbay-node/tests/test_transfer_slots_wire.py @@ -0,0 +1,298 @@ +""" +Transfer leases over the session, rather than over `TransferSlots` alone. + +test_transfer_slots.py proves the decisions; this proves the seam. Both exist +because the seam is where this repo's defects have actually lived — a reply +routed by arrival order, a session popped from a dict without its work being +stopped, a slot released by a `finally` nobody reached. + +Three things can only be checked here: + + - the handlers answer under the right shape, and refuse another connection's + transfer id; + - **losing the connection gives everything back.** That is the primary + reclaim, and it is a hook (`shutdown_tasks`) rather than a timer, so a test + of the pool alone would never touch it; + - a slot freed by one peer is *announced* to the peer waiting on it. A grant + nobody hears about is precisely the "stuck at waiting" report the design + exists to prevent, and it would look correct in the pool. +""" + +import pytest + +# Every test drives a message handler, and in the node a message handler always +# runs inside the event loop: `_do_transfer_open` starts the sweeper task there. +# Calling these synchronously tested a situation that cannot happen and failed +# on "no current event loop" the moment the sweeper stopped being faked. +pytestmark = pytest.mark.asyncio + +from meshbay_common.protocol import MNP +from meshbay_node.transfers import DOWNLOAD, UPLOAD +from meshbay_node.transport.webrtc_server import WebRTCPeerSession + + +class _Session(WebRTCPeerSession): + """A session with the DataChannel replaced by a list, and nothing else.""" + + def __init__(self, ctx, *, key, user, group="g1"): + self._ctx = ctx + self._registry_key = key + self._user_id = user + self._group_id = group + self.sent: list[dict] = [] + + def _send(self, msg): + self.sent.append(msg) + + def _spawn(self, coro): # pragma: no cover - not used by these tests + coro.close() + return None + + def last(self, mtype=MNP.TRANSFER_STATE): + return next(m for m in reversed(self.sent) if m.get("type") == mtype) + + +@pytest.fixture +def ctx(): + """A transport context, with the sweeper stopped on the way out. + + A task left running past the end of its test is a warning in the next one + and a hang in the worst case; the sweeper is started on demand by design, so + tearing it down is the test's job. + """ + c: dict = {"_peers": {}} + yield c + task = c.get("_transfer_sweeper") + if task is not None: + task.cancel() + + +def _join(ctx, key, user, group="g1") -> _Session: + s = _Session(ctx, key=key, user=user, group=group) + ctx["_peers"][key] = s + return s + + +async def test_a_granted_transfer_is_answered_as_granted(ctx): + peer = _join(ctx, "s1", "alice") + peer._do_transfer_open({"tr": "t1", "kind": DOWNLOAD, "bytes": 10}) + reply = peer.last() + assert reply["state"] == "granted" + assert reply["tr"] == "t1" + assert reply["kind"] == DOWNLOAD + assert reply["used"] == 1 and reply["cap"] >= 1 + + +async def test_a_queued_transfer_is_told_how_many_are_ahead(ctx): + peer = _join(ctx, "s1", "alice") + peer._slots().per_member[DOWNLOAD] = 1 + peer._do_transfer_open({"tr": "t1"}) + peer._do_transfer_open({"tr": "t2"}) + peer._do_transfer_open({"tr": "t3"}) + assert [m["state"] for m in peer.sent] == ["granted", "queued", "queued"] + assert peer.sent[-1]["ahead"] == 1 + + +async def test_the_reply_carries_no_name_and_no_path(ctx): + """A lease holds neither, and `transfer_state` stays in clear — so this is + the message where a filename would quietly become metadata on the wire.""" + peer = _join(ctx, "s1", "alice") + peer._do_transfer_open({"tr": "t1", "name": "Some Saga.mkv", + "path": "/srv/films"}) + assert set(peer.last()) <= { + "type", "v", "tr", "state", "kind", "used", "cap", "node_used", + "node_cap", "ahead", "reason"} + + +async def test_closing_frees_the_slot(ctx): + peer = _join(ctx, "s1", "alice") + peer._do_transfer_open({"tr": "t1"}) + peer._do_transfer_close({"tr": "t1", "reason": "done"}) + assert peer.last()["state"] == "closed" + assert peer._slots().in_use(DOWNLOAD) == 0 + + +async def test_one_peer_cannot_close_anothers_transfer(ctx): + """A denial of service one random id away, otherwise.""" + alice = _join(ctx, "s1", "alice") + bob = _join(ctx, "s2", "bob") + alice._do_transfer_open({"tr": "t1"}) + bob._do_transfer_close({"tr": "t1"}) + assert bob.last("error")["code"] == "not_your_transfer" + assert "t1" in alice._slots().leases + + +async def test_one_peer_cannot_open_on_anothers_id(ctx): + alice = _join(ctx, "s1", "alice") + bob = _join(ctx, "s2", "bob") + alice._do_transfer_open({"tr": "t1"}) + bob._do_transfer_open({"tr": "t1"}) + assert bob.last("error")["code"] == "not_your_transfer" + + +async def test_a_transfer_with_no_id_is_refused(ctx): + peer = _join(ctx, "s1", "alice") + peer._do_transfer_open({"kind": DOWNLOAD}) + assert peer.last("error")["code"] == "bad_transfer_id" + + +# ── the reclaim that matters ──────────────────────────────────────────────── + +async def test_losing_the_connection_gives_everything_back(ctx): + peer = _join(ctx, "s1", "alice") + peer._do_transfer_open({"tr": "t1"}) + peer._do_transfer_open({"tr": "t2", "kind": UPLOAD}) + peer._release_transfers() + slots = peer._slots() + assert slots.leases == {} + assert slots.in_use(DOWNLOAD) == 0 and slots.in_use(UPLOAD) == 0 + + +async def test_the_freed_slot_reaches_the_peer_that_was_waiting(ctx): + """ + The seam this file exists for. In the pool, granting is correct; if the + grant is not pushed, the waiting client sits on "waiting" for ever with a + node that believes it is streaming — and every unit test still passes. + """ + alice = _join(ctx, "s1", "alice") + bob = _join(ctx, "s2", "bob") + alice._slots().caps[DOWNLOAD] = 1 + + alice._do_transfer_open({"tr": "a1"}) + bob._do_transfer_open({"tr": "b1"}) + assert bob.last()["state"] == "queued" + + alice._release_transfers() + assert bob.last()["state"] == "granted", ( + "bob was granted the slot and never told") + assert bob.last()["tr"] == "b1" + + +async def test_a_grant_crossing_groups_still_reaches_its_peer(ctx): + """The pools are node-wide and `_peer_registry` is per group (finding H1), + so the peer to notify is not necessarily in the notifier's own registry.""" + groups = {"g1": {"_peers": {}}, "g2": {"_peers": {}}} + ctx = {"groups": groups} + alice = _Session(ctx, key="s1", user="alice", group="g1") + groups["g1"]["_peers"]["s1"] = alice + bob = _Session(ctx, key="s2", user="bob", group="g2") + groups["g2"]["_peers"]["s2"] = bob + + alice._slots().caps[DOWNLOAD] = 1 + alice._do_transfer_open({"tr": "a1"}) + bob._do_transfer_open({"tr": "b1"}) + assert bob.last()["state"] == "queued" + + alice._release_transfers() + assert bob.last()["state"] == "granted", ( + "a slot freed in one group never reached the peer waiting in another") + + +async def test_raising_the_cap_notifies_who_it_starts(ctx): + """`set_capacity` arrives from the loopback API, with no session behind it — + the grants it produces still have to be pushed.""" + from meshbay_node.transport.webrtc_server import WebRTCTransport + + transport = WebRTCTransport.__new__(WebRTCTransport) + transport._ctx = ctx + peer = _join(ctx, "s1", "alice") + peer._slots().caps[DOWNLOAD] = 1 + peer._do_transfer_open({"tr": "t1"}) + peer._do_transfer_open({"tr": "t2"}) + assert peer.last()["state"] == "queued" + + transport.set_capacity(max_concurrent_downloads=4) + assert peer.last()["state"] == "granted" and peer.last()["tr"] == "t2" + + +async def test_the_operator_can_see_the_queue(ctx): + """`GET /api/transfers` is the answer to "was this peer ever queued", which + a log line cannot give when the symptom is that nothing is happening.""" + from meshbay_node import ops + + peer = _join(ctx, "s1", "alice") + peer._slots().caps[DOWNLOAD] = 1 + peer._do_transfer_open({"tr": "t1", "bytes": 5}) + peer._do_transfer_open({"tr": "t2", "bytes": 7}) + + class _T: + _ctx = ctx + + snapshot = await ops.list_transfers({"webrtc": _T()}) + assert snapshot["pools"][DOWNLOAD]["in_use"] == 1 + assert snapshot["pools"][DOWNLOAD]["queued"] == 1 + assert {x["state"] for x in snapshot["leases"]} == {"granted", "queued"} + assert all("name" not in x and "path" not in x for x in snapshot["leases"]) + + +async def test_asking_before_anything_has_transferred_is_not_an_error(): + from meshbay_node import ops + + class _T: + _ctx: dict = {} + + snapshot = await ops.list_transfers({"webrtc": _T()}) + assert snapshot["leases"] == [] + assert snapshot["pools"][DOWNLOAD]["in_use"] == 0 + + +# ── the sweeper's lifetime ────────────────────────────────────────────────── + +async def test_the_sweeper_outlives_the_session_that_started_it(ctx): + """ + It was started with `self._spawn`, which ties a task to one session's set — + so it was cancelled the moment that peer left, and every other peer's + abandoned lease stopped being reclaimed. Nothing else would have noticed: + the node simply fills up over days. + """ + import asyncio + + from meshbay_node.transport import webrtc_server as ws + + alice = _join(ctx, "s1", "alice") + bob = _join(ctx, "s2", "bob") + # The real _spawn, so the session genuinely owns what it starts. + alice._tasks = set() + alice._spawn = ws.WebRTCPeerSession._spawn.__get__(alice) + bob._tasks = set() + bob._spawn = ws.WebRTCPeerSession._spawn.__get__(bob) + + alice._do_transfer_open({"tr": "a1"}) + bob._do_transfer_open({"tr": "b1"}) + sweeper = ctx["_transfer_sweeper"] + assert sweeper is not None and not sweeper.done() + + # Alice leaves, exactly as shutdown_tasks does it. + alice._release_transfers() + for task in list(alice._tasks): + task.cancel() + await asyncio.gather(*alice._tasks, return_exceptions=True) + await asyncio.sleep(0) + + assert not sweeper.done(), ( + "the sweeper died with the session that happened to start it; bob's " + "lease would never be reclaimed") + sweeper.cancel() + + +async def test_the_sweeper_stops_when_the_last_lease_goes(ctx): + """An idle node must run no timer — the reason this is started on demand + rather than at boot.""" + import asyncio + + from meshbay_node.transport import webrtc_server as ws + + peer = _join(ctx, "s1", "alice") + peer._tasks = set() + peer._spawn = ws.WebRTCPeerSession._spawn.__get__(peer) + + original = ws.TRANSFER_SWEEP_SECS + ws.TRANSFER_SWEEP_SECS = 0.01 + try: + peer._do_transfer_open({"tr": "t1"}) + peer._do_transfer_close({"tr": "t1"}) + sweeper = ctx["_transfer_sweeper"] + await asyncio.wait_for(sweeper, timeout=2) + assert ctx.get("_transfer_sweeper") is None + finally: + ws.TRANSFER_SWEEP_SECS = original |