From 1a495f5ed3f8a55222d406152c833882264dc377 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Tue, 8 Sep 2026 22:54:16 +0200 Subject: feat(hub): client-side transfer leases and the transfers panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 5 and 6 of ~/next/improve-downloads.md. The node has handed out slots since step 2 and nothing asked for one; now the client does, and the panel shows what is happening. `transport.openTransfer()` returns a Lease: `acquire()` resolves when the node grants, `release()` gives it back exactly once, and nothing else in the client speaks to the node about slots. Whether a node hands out slots is read from the handshake ack rather than guessed from a timeout — "no answer yet" and "this node will never answer" are indistinguishable in time, and guessing wrong either stalls every download or defeats the cap. Two things exist only because a queue can lie: a watchdog re-asks when a pushed grant does not arrive (the node is idempotent on `tr`, so asking again is free), and a grant for a transfer the page has forgotten is handed straight back rather than held until the node's deadline. The slot is asked for **after** there is somewhere to write, and that ordering is load-bearing: opening a target takes thirty seconds of streamed-download timeouts, or as long as somebody leaves a Save As dialog open, and a grant not taken up in time is revoked. Moving it earlier looked better and broke three downloads into one. Pinned by a test. The panel groups by state — running, waiting, finished — rather than re-sorting a flat list, so a row moves only when its own state does. The ETA is withheld until the speed window holds real measurement: a figure from the first two chunks swings between four seconds and an hour, and people plan around the first number they see. One live region announces state changes and not progress. Three silent paths closed on the way: a download refused for want of a user gesture (a browser grants one file picker per gesture, and downloading three files is one gesture) now falls back to the streamed path, which needs none; a click with no connection says so instead of doing nothing at all; and a queued transfer counts as busy, so a transport is never closed under one that is waiting for a grant that could then never arrive. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HCGdheDLxGReuKHga3BtST --- packages/meshbay-hub/tests/test_transfers.py | 213 +++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) (limited to 'packages/meshbay-hub/tests/test_transfers.py') diff --git a/packages/meshbay-hub/tests/test_transfers.py b/packages/meshbay-hub/tests/test_transfers.py index 3316615..e805afc 100644 --- a/packages/meshbay-hub/tests/test_transfers.py +++ b/packages/meshbay-hub/tests/test_transfers.py @@ -222,3 +222,216 @@ def test_a_folder_name_carries_no_trailing_slash(): assert "dir-row" in row, "the anchor no longer lands on the directory row" assert "${d}/" not in row, "the folder name is rendered with a trailing slash" assert "${d}" in row + + +# ── Transfer slots, client side ───────────────────────────────────────────── +# +# A queue can lie in two directions, and both are worse than no queue: a +# transfer that shows "waiting" on a node that already granted it, and a slot +# the page holds after it has stopped using it. Everything below is one of +# those two. + +def _lease_stub(): + """A Lease as the store sees it, driveable from the test.""" + return """ +class L { + constructor() { + this.state = 'queued'; this.ahead = 2; this.closed = false; + this.released = []; this.tr = 'tr1'; + this._wait = new Promise(r => { this._go = r; }); + } + acquire() { return this._wait; } + release(reason) { if (!this.closed) { this.closed = true; this.released.push(reason); } } + grant() { this.state = 'granted'; if (this._onState) this._onState(this); this._go(); } + push(state, ahead) { this.state = state; this.ahead = ahead; if (this._onState) this._onState(this); } +} +""" + + +def test_a_transfer_waiting_for_a_slot_is_queued_not_running(tmp_path): + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'f', total: 10, lease, + run: async () => { say('ran'); } }); + say(t.list()[0].status, t.list()[0].ahead); + await new Promise(r => setTimeout(r, 0)); + say('still:' + t.list()[0].status); + """, tmp_path) + assert out[:2] == ["queued", 2] + assert "ran" not in out, "the work started before the slot was granted" + assert out[-1] == "still:queued" + + +def test_the_grant_starts_the_work(tmp_path): + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'f', total: 10, lease, + run: async () => { say('ran:' + t.list()[0].status); } }); + lease.grant(); + await new Promise(r => setTimeout(r, 10)); + say('after:' + t.list()[0].status); + """, tmp_path) + assert out[0] == "ran:running" + assert out[1] == "after:done" + + +def test_the_slot_comes_back_however_the_transfer_ends(tmp_path): + """A slot not returned is a member who cannot transfer again until the node + times it out — so this must hold for a throw as much as for a success.""" + out = _run(_lease_stub() + """ + for (const mode of ['ok', 'throw']) { + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'f', total: 10, lease, + run: async () => { if (mode === 'throw') throw new Error('x'); } }); + lease.grant(); + await new Promise(r => setTimeout(r, 10)); + say(mode + ':' + lease.released.join(',') + ':' + t.list()[0].status); + } + """, tmp_path) + assert out == ["ok:done:done", "throw:done:failed"] + + +def test_cancelling_while_queued_gives_the_slot_back(tmp_path): + """The transfer somebody is most likely to give up on is the one that has + not started. Its queue entry has to go, or the node grants a slot to a + transfer that will never use it.""" + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + const id = t.start({ kind: 'download', name: 'f', total: 10, lease, + run: async () => { say('ran'); } }); + t.cancel(id); + say(t.list()[0].status, lease.released.join(',')); + lease.grant(); + await new Promise(r => setTimeout(r, 10)); + say('ran?', out.includes('ran')); + """, tmp_path) + assert out[0] == "cancelled" + assert out[1] == "cancelled" + assert out[-1] is False, "a cancelled transfer ran anyway once granted" + + +def test_a_queue_position_update_reaches_the_view(tmp_path): + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + const seen = []; + t.subscribe(items => seen.push(items[0].ahead)); + t.start({ kind: 'download', name: 'f', total: 10, lease, run: async () => {} }); + lease.push('queued', 1); + lease.push('queued', 0); + say(seen.join('>')); + """, tmp_path) + assert out[0].endswith("1>0"), "the widget never learns it is moving up" + + +def test_a_transport_with_a_queued_transfer_is_not_closed(tmp_path): + """Closing it would leave the transfer waiting for a grant that can never + arrive — waiting for ever, with nothing left to answer.""" + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + let closed = false; + const transport = { close() { closed = true; } }; + t.start({ kind: 'download', name: 'f', total: 10, transport, lease, + run: async () => {} }); + t.releaseWhenIdle(transport); + say('closed while queued:', closed); + lease.grant(); + await new Promise(r => setTimeout(r, 10)); + say('closed after:', closed); + """, tmp_path) + assert out[1] is False + assert out[3] is True + + +def test_clearing_finished_keeps_what_is_waiting(tmp_path): + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'waiting', total: 1, lease, run: async () => {} }); + t.start({ kind: 'download', name: 'done', total: 1, run: async () => {} }); + await new Promise(r => setTimeout(r, 10)); + t.clearFinished(); + say(t.list().map(i => i.name + ':' + i.status).join(',')); + """, tmp_path) + assert out[0] == "waiting:queued" + + +def test_asking_for_a_slot_on_a_dead_channel_does_not_throw(tmp_path): + """ + The transport reconnects on its own and re-asks for every live lease when it + does, so a closed channel at the moment a transfer starts is a wait, not a + failure. `_fetchChunkResilient` has always treated it that way — and before + leases existed a chunk request was the first thing to touch the channel, so + a download begun on a briefly dead connection simply retried. + + Asking for a slot first made `_send` the first contact. It threw + "DataChannel not open (state: closed)" straight out of `downloadEntry`, + where nothing catches it: a download that used to recover became an error + with no row in the widget to show it. Found live, by downloading a file + just after a connection dropped. + """ + module = tmp_path / "transport_lease.mjs" + # The real Lease, lifted out as text — the class is not exported, and a + # second copy of it here would agree with whatever it was copied from. + src = (STATIC / "transport.js").read_text() + # From the constant the class depends on, not from the class: lifting only + # the class left LEASE_WATCHDOG_MS undefined, which the class reads the + # first time it arms its watchdog. + start = src.index("const LEASE_WATCHDOG_MS") + end = src.index("\nclass MeshBayTransport") + module.write_text(src[start:end] + "\nexport { Lease };\n") + + script = tmp_path / "case.mjs" + script.write_text(f""" +import {{ Lease }} from '{module.as_posix()}'; +const out = []; +const transport = {{ + supportsTransferSlots: true, + _leases: new Map(), + _send() {{ throw new Error('DataChannel not open (state: closed)'); }}, +}}; +let threw = null; +const lease = new Lease(transport, 'tr1', 'download', 10, 1, null); +try {{ lease._request(); }} catch (e) {{ threw = e.message; }} +out.push(threw); +// And releasing one must be just as safe: a lease not released is a member who +// cannot start another transfer until the node times it out. +try {{ lease.release('cancelled'); out.push('release ok'); }} +catch (e) {{ out.push('release threw: ' + e.message); }} +clearTimeout(lease._watchdog); +console.log(JSON.stringify(out)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + out = json.loads(proc.stdout) + assert out[0] is None, f"asking for a slot threw: {out[0]}" + assert out[1] == "release ok" + + +def test_the_slot_is_asked_for_after_there_is_somewhere_to_write(): + """ + A granted slot has to be taken up within the node's acceptance deadline, so + it must not be asked for until the download can actually start. + + Asking first reads better — the widget could draw a row while the target is + being chosen — and is wrong: opening a target takes thirty seconds of + streamed-download timeouts, or as long as somebody leaves a Save As dialog + open. The node revokes the grant, passes it to the next in the queue + (`transfer: reclaimed … (not_taken_up)` in its log), and the download then + fetches under a `tr` that is no longer granted. Three downloads started, one + arrived. + + Source-reading, because the ordering is the whole property and it has no + behaviour of its own to drive: what matters is which call comes first. + """ + src = (STATIC / "file-utils.js").read_text() + fn = src[src.index("async function downloadEntry"):] + fn = fn[:fn.index("\n}\n")] + assert fn.index("_openDownloadTarget") < fn.index("openTransfer"), ( + "downloadEntry asks for a transfer slot before it has anywhere to " + "write — the grant expires before the download can use it") -- cgit v1.2.3 From 77615ddb5fead3e74751a94847d3bcc99fc0a96d Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Tue, 8 Sep 2026 23:43:46 +0200 Subject: fix(hub): the transfers row exists from the click MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clicking Download produced nothing — no row, no icon, no panel — for as long as it took to open somewhere to write, and then several rows at once. The streamed path waits for the worker twice; a Save As dialog waits for a person. The row was created after that, so the slowest part of a download happened with nothing on screen to say it had begun. The store gains a `prepare` step, distinct from `run`, and the order is now: row, then target, then slot. That last part is why the obvious fix was wrong. Taking the slot first would let the row appear immediately, and it was tried this morning: a granted slot has to be taken up within the node's deadline, opening a target can outlast it, and three downloads became one. (The diagnosis at the time blamed that ordering for revocations which were in fact a missing `touch()` call — the revert was right for the wrong reason.) `makeLease` is called after `prepare` succeeds, never before. Three behaviours fall out, each with a test: - a dismissed dialog leaves nothing behind. `prepare` returning false drops the row: nothing started, so nothing should remain on screen to explain it; - the row takes the name the file was actually saved under, once known; - a refusal above the memory ceiling fails the row that is already there, rather than creating one to kill it. `preparing` counts as live everywhere — badge, cancel, clearFinished, and `_busy`, since closing a transport under a preparing transfer strands it exactly as under a queued one. Six places asked "is this finished?" and were drifting apart; there is one definition now. Two mistakes in the tests, worth the note: one counted positions in an output array by hand and was one out, which reads exactly like a failing assertion about the code — the values are tagged now, not indexed. And test_zip_size_limit.py's stub did not run `prepare`, so it no longer reached the size check the file is about; it now behaves like the real store. 819 hub, 1169 node, 0 failed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HCGdheDLxGReuKHga3BtST --- packages/meshbay-hub/src/meshbay_hub/static/app.js | 23 +++- .../src/meshbay_hub/static/file-utils.js | 111 +++++++++----------- .../src/meshbay_hub/static/locales/de.js | 1 + .../src/meshbay_hub/static/locales/en.js | 1 + .../src/meshbay_hub/static/locales/es.js | 1 + .../src/meshbay_hub/static/locales/fr.js | 1 + .../src/meshbay_hub/static/locales/it.js | 1 + .../src/meshbay_hub/static/locales/ja.js | 1 + .../src/meshbay_hub/static/locales/nl.js | 1 + .../src/meshbay_hub/static/locales/pl.js | 1 + .../src/meshbay_hub/static/locales/pt-BR.js | 1 + .../src/meshbay_hub/static/locales/zh-CN.js | 1 + .../src/meshbay_hub/static/transfers.js | 90 ++++++++++++---- packages/meshbay-hub/tests/test_transfers.py | 116 +++++++++++++++++++++ packages/meshbay-hub/tests/test_zip_size_limit.py | 17 ++- 15 files changed, 280 insertions(+), 87 deletions(-) (limited to 'packages/meshbay-hub/tests/test_transfers.py') diff --git a/packages/meshbay-hub/src/meshbay_hub/static/app.js b/packages/meshbay-hub/src/meshbay_hub/static/app.js index ff066cf..6f4bb92 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/app.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/app.js @@ -157,9 +157,11 @@ function TransferWidget() { }, [open]); const running = items.filter(i => i.status === 'running'); - const waiting = items.filter(i => i.status === 'queued'); + const waiting = items.filter( + i => i.status === 'queued' || i.status === 'preparing'); const finished = items.filter( - i => i.status !== 'running' && i.status !== 'queued'); + i => i.status !== 'running' && i.status !== 'queued' + && i.status !== 'preparing'); const active = running.length + waiting.length; // Grouped, and in this order: what is moving, what is waiting, what is over. @@ -240,7 +242,8 @@ function TransferRow({ it }) { ? html` { e.preventDefault(); transfers.open(it.id); }}>${it.name}` : html`${it.name}`} - ${(it.status === 'running' || it.status === 'queued') && html` + ${(it.status === 'running' || it.status === 'queued' + || it.status === 'preparing') && html` + `} ${(it.status === 'running' || it.status === 'queued' - || it.status === 'preparing') && html` + || it.status === 'preparing' || it.status === 'paused') && html` `} + ${!it.pausable && downloads.SUPPORTED && it.kind === 'download' + && (it.status === 'running' || it.status === 'queued') && html` + ${/* Say why, rather than leaving a gap where a button is on the row + above. Without a granted folder this browser writes through the + service worker — a download it already owns, which cannot be + paused — so the button is absent for a reason nobody can see, + and an upload beside it has one. Shown only where choosing a + folder is actually possible: on Firefox and Safari there is no + folder to choose and this hint would be a lie. */''} + + <${Icon} name="pause" /> + + `} ${(it.status === 'running' || it.status === 'queued' || it.status === 'preparing' || it.status === 'paused') && html`