diff options
| author | Christophe Besson <cbesson@gmail.com> | 2026-09-09 14:28:40 +0200 |
|---|---|---|
| committer | Christophe Besson <cbesson@gmail.com> | 2026-09-09 14:28:40 +0200 |
| commit | 7e2d078fe1870d256ae47781bee6ac4f454edf24 (patch) | |
| tree | 05689049fd48f01ebbdb9995d5189cba15cee052 /packages/meshbay-hub/tests | |
| parent | 813d18424ec57963bb56e6f40824a2db0ccce50d (diff) | |
| parent | e6f895c473a0b19e7b186889c1836d3945bc880b (diff) | |
| download | meshbay-7e2d078fe1870d256ae47781bee6ac4f454edf24.tar.gz | |
Merge branch 'fix/large-download-paths'
Concurrent-transfer limits, with the queue, the pause and the flag day.
A node now caps how many transfers it runs at once (8 downloads, 8 uploads,
node-wide) and how many one member may run in one group (2 by default,
operator-signed). Beyond that the node answers "queued" and the client waits its
turn, visibly, in the transfers panel — and a slot that frees starts whatever is
next, skipping past a member who is at their own cap rather than letting them
stall everyone behind them.
Browsing is never subject to a slot: not the poster grid, not the covers, not
opening a photo to look at it. That is structural — a transfer is what the
transfers widget shows — and the exemption is bounded rather than open, at two
files in flight per session, because an exemption with no bound is a leaseless
branch under another name.
Transfers can be cancelled, and now paused and resumed. A paused one holds
nothing: its slot goes back at once and resuming rejoins the queue at the tail.
Uploads survive the connection that started them and resume where the node
stopped, asked for inside the seal rather than on a clear message. What they
leave behind when they are abandoned is reaped, which closes a disk leak that
predates this work.
MNP 3.0 makes the lease compulsory and refuses 2.x at the handshake, with the
desktop client checking `client.minimum` before connecting so an un-updated one
says "update" instead of failing every connection in a protocol vocabulary.
Fourteen defects were found on the way, eight of them by a person clicking
Download and pasting a console — none of which 2075 tests could reach. Section
12 of ~/next/improve-downloads.md is that report, including the three this work
introduced itself and the one that turned out to be caused by an instruction to
hard-reload after each deployment.
Node suite 1209 passed, hub suite 866 passed.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HCGdheDLxGReuKHga3BtST
Diffstat (limited to 'packages/meshbay-hub/tests')
17 files changed, 2603 insertions, 38 deletions
diff --git a/packages/meshbay-hub/tests/harness/chat_scroll_probe.py b/packages/meshbay-hub/tests/harness/chat_scroll_probe.py index 7373976..17ec554 100644 --- a/packages/meshbay-hub/tests/harness/chat_scroll_probe.py +++ b/packages/meshbay-hub/tests/harness/chat_scroll_probe.py @@ -193,7 +193,15 @@ class H(http.server.BaseHTTPRequestHandler): def main() -> int: with socketserver.TCPServer(("127.0.0.1", PORT), H) as srv: threading.Thread(target=srv.serve_forever, daemon=True).start() - with tempfile.TemporaryDirectory() as profile: + # ignore_cleanup_errors: Chrome's children (zygote, renderer, gpu) + # outlive terminate() on the parent by a moment and go on writing into + # the profile. rmtree then walks a directory that gains a file between + # its readdir and its rmdir and raises "Directory not empty" -- which + # failed the probe, which failed every test in the file, intermittently + # and for a reason nowhere near the chat code they were testing. A few + # bytes left in a throwaway profile are harmless; failing the run is not. + with tempfile.TemporaryDirectory( + ignore_cleanup_errors=True) as profile: # Real time, not `--virtual-time-budget`: the defect is a feedback # loop between layout and an event, and a virtual clock does not # run it. @@ -211,6 +219,7 @@ def main() -> int: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() + proc.wait() if not RECORDS: print(json.dumps({"error": "no measurement"}), file=sys.stderr) return 1 diff --git a/packages/meshbay-hub/tests/harness/chat_send_probe.py b/packages/meshbay-hub/tests/harness/chat_send_probe.py index 5f99beb..28635b0 100644 --- a/packages/meshbay-hub/tests/harness/chat_send_probe.py +++ b/packages/meshbay-hub/tests/harness/chat_send_probe.py @@ -297,7 +297,15 @@ class H(http.server.BaseHTTPRequestHandler): def main() -> int: with socketserver.TCPServer(("127.0.0.1", PORT), H) as srv: threading.Thread(target=srv.serve_forever, daemon=True).start() - with tempfile.TemporaryDirectory() as profile: + # ignore_cleanup_errors: Chrome's children (zygote, renderer, gpu) + # outlive terminate() on the parent by a moment and go on writing into + # the profile. rmtree then walks a directory that gains a file between + # its readdir and its rmdir and raises "Directory not empty" -- which + # failed the probe, which failed every test in the file, intermittently + # and for a reason nowhere near the chat code they were testing. A few + # bytes left in a throwaway profile are harmless; failing the run is not. + with tempfile.TemporaryDirectory( + ignore_cleanup_errors=True) as profile: proc = subprocess.Popen( ["google-chrome", "--headless=new", "--disable-gpu", "--no-sandbox", f"--user-data-dir={profile}", "--window-size=1100,800", @@ -312,6 +320,7 @@ def main() -> int: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() + proc.wait() if not RECORDS: print(json.dumps({"error": "no measurement"}), file=sys.stderr) return 1 diff --git a/packages/meshbay-hub/tests/harness/group_tab_probe.py b/packages/meshbay-hub/tests/harness/group_tab_probe.py index 5e4e452..b6d2bcc 100644 --- a/packages/meshbay-hub/tests/harness/group_tab_probe.py +++ b/packages/meshbay-hub/tests/harness/group_tab_probe.py @@ -156,7 +156,15 @@ class H(http.server.BaseHTTPRequestHandler): def main() -> int: with socketserver.TCPServer(("127.0.0.1", PORT), H) as srv: threading.Thread(target=srv.serve_forever, daemon=True).start() - with tempfile.TemporaryDirectory() as profile: + # ignore_cleanup_errors: Chrome's children (zygote, renderer, gpu) + # outlive terminate() on the parent by a moment and go on writing into + # the profile. rmtree then walks a directory that gains a file between + # its readdir and its rmdir and raises "Directory not empty" -- which + # failed the probe, which failed every test in the file, intermittently + # and for a reason nowhere near the chat code they were testing. A few + # bytes left in a throwaway profile are harmless; failing the run is not. + with tempfile.TemporaryDirectory( + ignore_cleanup_errors=True) as profile: proc = subprocess.Popen( ["google-chrome", "--headless=new", "--disable-gpu", "--no-sandbox", f"--user-data-dir={profile}", "--window-size=1100,900", @@ -171,6 +179,7 @@ def main() -> int: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() + proc.wait() if not RECORDS: print(json.dumps({"error": "no measurement"}), file=sys.stderr) return 1 diff --git a/packages/meshbay-hub/tests/harness/layout_probe.py b/packages/meshbay-hub/tests/harness/layout_probe.py index 530b3f0..65b3083 100644 --- a/packages/meshbay-hub/tests/harness/layout_probe.py +++ b/packages/meshbay-hub/tests/harness/layout_probe.py @@ -19,6 +19,7 @@ single pass. Launching Chrome per width put three minutes on the test suite. """ import http.server import json +import shutil import socketserver import subprocess import sys @@ -118,16 +119,25 @@ def main() -> int: srv = S(("127.0.0.1", PORT), H) threading.Thread(target=srv.serve_forever, daemon=True).start() + # mkdtemp left a Chrome profile in /tmp on every run, for ever, and nothing + # waited for Chrome to exit. Same cleanup rule as the other probes. + profile = tempfile.mkdtemp(prefix="chrome-layout-") chrome = subprocess.Popen([ "google-chrome", "--headless=new", "--no-sandbox", "--window-size=1000,900", - "--user-data-dir=" + tempfile.mkdtemp(prefix="chrome-layout-"), + "--user-data-dir=" + profile, f"http://127.0.0.1:{PORT}/", ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) deadline = time.time() + 45 while time.time() < deadline and not RECORDS: time.sleep(0.2) chrome.terminate() + try: + chrome.wait(timeout=10) + except subprocess.TimeoutExpired: + chrome.kill() + chrome.wait() + shutil.rmtree(profile, ignore_errors=True) srv.shutdown() if not RECORDS: print(json.dumps({"error": "no measurement"})) diff --git a/packages/meshbay-hub/tests/harness/scroll_probe.py b/packages/meshbay-hub/tests/harness/scroll_probe.py index 46d8357..ae407b8 100644 --- a/packages/meshbay-hub/tests/harness/scroll_probe.py +++ b/packages/meshbay-hub/tests/harness/scroll_probe.py @@ -151,7 +151,15 @@ class H(http.server.BaseHTTPRequestHandler): def main() -> int: with socketserver.TCPServer(("127.0.0.1", PORT), H) as srv: threading.Thread(target=srv.serve_forever, daemon=True).start() - with tempfile.TemporaryDirectory() as profile: + # ignore_cleanup_errors: Chrome's children (zygote, renderer, gpu) + # outlive terminate() on the parent by a moment and go on writing into + # the profile. rmtree then walks a directory that gains a file between + # its readdir and its rmdir and raises "Directory not empty" -- which + # failed the probe, which failed every test in the file, intermittently + # and for a reason nowhere near the chat code they were testing. A few + # bytes left in a throwaway profile are harmless; failing the run is not. + with tempfile.TemporaryDirectory( + ignore_cleanup_errors=True) as profile: subprocess.run( ["google-chrome", "--headless", "--disable-gpu", "--no-sandbox", f"--user-data-dir={profile}", "--window-size=1100,1300", diff --git a/packages/meshbay-hub/tests/harness/upload_seal_probe.mjs b/packages/meshbay-hub/tests/harness/upload_seal_probe.mjs index 0b77e42..a6008c2 100644 --- a/packages/meshbay-hub/tests/harness/upload_seal_probe.mjs +++ b/packages/meshbay-hub/tests/harness/upload_seal_probe.mjs @@ -72,13 +72,34 @@ tp._nodeVersion = input.node_version; const frames = []; let uploadId = null; +let answered = 0; tp._send = (msg) => { frames.push(toHex(msgpack_encode(msg))); if (msg.upload_id) uploadId = msg.upload_id; - if (input.mode !== 'receive') return; + if (input.mode !== 'receive') { + // Nothing answers in this mode -- except the probe, which the client waits + // five seconds for. A node that predates it refuses the index, and that + // refusal is a plain error rather than a sealed ack, so the harness can + // produce it honestly. It is also the degradation path worth exercising. + if (msg.chunk_index === -1) { + // With `probe_ack`, answer it the way a node holding part of this file + // does; without, the way one that predates the probe does. + const reply = input.probe_ack + ? Object.assign(msgpack_decode(hex(input.probe_ack)), + { upload_id: uploadId }) + : { type: 'error', upload_id: uploadId, + code: 'bad_chunk_index', detail: 'Unexpected chunk index' }; + setImmediate(() => tp._dispatch(reply)); + } + return; + } // Answer as the node did, on the next turn of the loop so the send path // finishes first — which is also how a real ack arrives. - const ack = msgpack_decode(hex(input.acks[msg.chunk_index])); + // + // By position, not by `chunk_index`: the node answers every frame including + // the probe, whose index is -1, and the two lists are built from the same + // sequence of frames. + const ack = msgpack_decode(hex(input.acks[answered++])); ack.upload_id = uploadId; // Through the real `_dispatch`, so the routing under test — matching an // ack to its uploader by `upload_id` — is the shipped one. diff --git a/packages/meshbay-hub/tests/test_client_version_gate.py b/packages/meshbay-hub/tests/test_client_version_gate.py new file mode 100644 index 0000000..69ca062 --- /dev/null +++ b/packages/meshbay-hub/tests/test_client_version_gate.py @@ -0,0 +1,141 @@ +""" +The desktop client refuses to start when the hub will no longer talk to it. + +The SPA is served by the hub, so a browser picks up a new client on reload. The +desktop application **ships its own interface**, so on a flag day an un-updated +one can still sign in, still list groups, and then fail every connection with +`version_too_old` — a refusal in a protocol vocabulary, surfacing as a node that +will not talk, with nothing anyone can act on. §12.3 of +~/next/improve-downloads.md named this as the thing that had to exist before +MNP 3.0 could ship. + +`compareVersions` and `refuseIfTooOld` are lifted out of `main.js` **as text** +and executed against a modelled environment, on the rule this repo follows +elsewhere: model the environment, never the code under test. The rest of +`test_desktop_shell.py` can only read the source, because there is no npm here +to launch Electron with; these two are ordinary functions and can be run. +""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +CLIENT = Path(__file__).resolve().parents[2] / "meshbay-client" +MAIN = CLIENT / "src" / "main.js" + +pytestmark = pytest.mark.skipif( + shutil.which("node") is None or not MAIN.exists(), + reason="node or the desktop client sources are not available") + + +def _lift(name: str) -> str: + src = MAIN.read_text() + cut = src[src.index(name):] + return cut[:cut.index("\n}\n") + 2] + + +def _run(tmp_path, *, mine="1.1.0", hub_base="https://hub.example", + answer=None, status=200, throws=False): + """Drive the gate against one hub. + + `answer` is what `/v1/hub/version` returns; None means the field is absent + entirely, which is what an older hub sends. + """ + script = tmp_path / "gate.mjs" + script.write_text(f""" +const out = {{ dialogs: 0, opened: null }}; +const config = {{ hubBase: {json.dumps(hub_base)} }}; +const app = {{ getVersion: () => {json.dumps(mine)} }}; +const dialog = {{ + showMessageBox: async () => {{ out.dialogs += 1; return {{ response: 0 }}; }}, +}}; +const shell = {{ openExternal: async (u) => {{ out.opened = u; }} }}; +globalThis.fetch = async () => {{ + if ({json.dumps(throws)}) throw new Error('unreachable'); + return {{ ok: {json.dumps(status)} === 200, + json: async () => ({json.dumps(answer)}) }}; +}}; +""" + _lift("function compareVersions") + _lift("async function refuseIfTooOld") + """ +out.refused = await refuseIfTooOld(); +out.compare = [ + compareVersions('1.0.0', '1.1.0'), + compareVersions('1.1.0', '1.1.0'), + compareVersions('1.2.0', '1.1.0'), + compareVersions('1.10.0', '1.9.0'), + compareVersions('1.1', '1.1.0'), + compareVersions('nonsense', '1.1.0'), +]; +console.log(JSON.stringify(out)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +OK = {"client": {"minimum": "1.1.0", "recommended": "1.1.0"}} + + +# ── the comparison ────────────────────────────────────────────────────────── + +def test_versions_compare_by_number_and_not_by_string(tmp_path): + """`1.10.0` is newer than `1.9.0`, which string comparison gets backwards — + and that mistake locks out exactly the people who did update.""" + assert _run(tmp_path, answer=OK)["compare"] == [-1, 0, 1, 1, 0, 0] + + +# ── the gate ──────────────────────────────────────────────────────────────── + +def test_a_client_older_than_the_minimum_is_stopped(tmp_path): + out = _run(tmp_path, mine="1.0.0", answer=OK) + assert out["refused"] is True + assert out["dialogs"] == 1, "it stopped without saying why" + assert out["opened"] == "https://hub.example", ( + "the offer to download the update led nowhere") + + +def test_a_current_client_starts_normally(tmp_path): + out = _run(tmp_path, mine="1.1.0", answer=OK) + assert out["refused"] is False + assert out["dialogs"] == 0 + + +def test_a_newer_client_is_not_stopped(tmp_path): + """A development build ahead of the hub is not a reason to refuse to open + the application.""" + assert _run(tmp_path, mine="2.0.0", answer=OK)["refused"] is False + + +def test_an_unreachable_hub_is_not_too_old(tmp_path): + """A hub that is down, a laptop with no network, a captive portal. Treating + any of those as "you are out of date" would make an offline start + impossible for ever, and would do it at the worst moment.""" + assert _run(tmp_path, mine="1.0.0", throws=True)["refused"] is False + assert _run(tmp_path, mine="1.0.0", status=503, answer=OK)["refused"] is False + + +def test_a_hub_that_states_no_minimum_stops_nothing(tmp_path): + """An older hub answers without the field. Absent must read as "no opinion", + never as a refusal.""" + assert _run(tmp_path, mine="0.0.1", answer={"hub": "1.2.3"})["refused"] is False + + +def test_a_first_run_with_no_hub_yet_is_not_stopped(tmp_path): + """There is nothing to ask, and the first-run screen is where the address + gets typed.""" + assert _run(tmp_path, mine="0.0.1", hub_base="", answer=OK)["refused"] is False + + +# ── where it is called ────────────────────────────────────────────────────── + +def test_the_gate_runs_before_the_window_is_built(): + """A window that opens and then cannot connect is the failure this + replaces, so the order is the whole point.""" + src = MAIN.read_text() + ready = src[src.index("app.whenReady().then("):] + ready = ready[:ready.index("createWindow();")] + assert "await refuseIfTooOld()" in ready, ( + "the version check does not run before the window is created") + assert "app.quit()" in ready diff --git a/packages/meshbay-hub/tests/test_downloads.py b/packages/meshbay-hub/tests/test_downloads.py index 32d3e11..afb85d6 100644 --- a/packages/meshbay-hub/tests/test_downloads.py +++ b/packages/meshbay-hub/tests/test_downloads.py @@ -149,8 +149,15 @@ def test_a_length_is_only_promised_when_it_is_known(tmp_path): # and was lifted into file-utils.js's downloadDirectory (docs/photos.md # §3) so photos-app.js's own "zip this album" button calls the same # implementation rather than a second one. + # Anchored on the call, not on how its result is bound: the assignment + # became a bare `target = ...` inside a try when _openDownloadTarget gained + # the ability to refuse an oversized download (test_memory_ceiling.py). + # What this test is about -- the `0` -- did not move. app = (STATIC / "file-utils.js").read_text() - zip_call = app[app.index("const target = await _openDownloadTarget(suggested"):] + # Anchored on the argument list, not on the function name: the call became + # `_openTargetInTurn(suggested, …)` when target openings were serialised. + # The `0` this test is about did not move. + zip_call = app[app.index("(suggested, totalBytes"):] zip_call = zip_call[:zip_call.index(");") + 2] assert zip_call.rstrip().endswith(", 0);"), ( "the zip download announces a Content-Length it will not match") @@ -168,9 +175,23 @@ def test_backpressure_is_real(tmp_path): # The transfer list may carry more than the stream — a reply port rides # along now — so this asserts that `readable` is transferred, not the exact # shape of the list. - transfer = fn[fn.index("worker.postMessage("):] - transfer = transfer[transfer.index("["):transfer.index("]") + 1] - assert "readable" in transfer, "the readable half must be transferred, not copied" + # + # And every `postMessage` in here, not the first: a ping is sent to wake the + # worker before it is handed anything, and it carries only a port. Reading + # the first one would have moved this check onto the ping the day it was + # added, leaving the stream unguarded while still passing. + posts = [] + rest = fn + while "worker.postMessage(" in rest: + rest = rest[rest.index("worker.postMessage("):] + # Bounded by the call's own end: the keep-alive ping transfers nothing + # at all, and reaching past it for a `[` would read the next call's. + posts.append(rest[:rest.index(");") + 2]) + rest = rest[len("worker.postMessage("):] + lists = [c[c.index("["):c.index("]") + 1] for c in posts if "[" in c] + assert len(posts) >= 2, "the wake-up and the stream are both posted from here" + assert any("readable" in t for t in lists), ( + "the readable half must be transferred, not copied") assert "writer.write(bytes)" in fn assert "return null" in fn, "a browser that cannot transfer streams must say so" @@ -201,9 +222,352 @@ def test_the_streamed_path_gives_up_rather_than_blocking_for_ever(): def test_an_uncontrolled_page_is_not_treated_as_ready(): """`registration.active` says a worker exists, not that it will see our fetch.""" + # Anchored on the streaming section rather than on one function: waiting + # for control moved into `_awaitControl`/`_claimController` when the budget + # became a parameter, and `serviceWorker()` no longer contains the words. + # The behaviour itself is executed in test_streamed_download_reliability.py; + # this stays as the cheap guard on the module's shape. src = DOWNLOADS.read_text() - fn = src[src.index("async function serviceWorker()"):] - fn = fn[:fn.index("\n}")] - assert "navigator.serviceWorker.controller" in fn - assert "controllerchange" in fn, ( + section = src[src.index("// ── Streaming to disk"):] + assert "navigator.serviceWorker.controller" in section + assert "controllerchange" in section, ( "control can arrive a tick after registration; waiting beats refusing") + assert "mbdl-claim" in section, ( + "an active-but-uncontrolled page must ask for a claim, not give up") + + +def test_an_apostrophe_in_a_name_does_not_lose_the_name(tmp_path): + """ + `encodeURIComponent` leaves `'` alone, and `'` is the delimiter in RFC + 5987's `filename*=<charset>'<lang>'<value>`. One apostrophe made the header + unparseable, and a browser that cannot parse it names the file after the + last segment of the URL — which for this worker is a made-up id. The file + arrived complete and 449 MB of it was called "mtsshk9w-ohqty535". + + Found by downloading three files where exactly one had an apostrophe. + Nothing in the suite could have: the header was built correctly for every + name anybody had tested with. + + The real function is lifted out of sw.js and run — a second copy here would + have the same blind spot as the first. + """ + src = SW.read_text() + fn = src[src.index("function contentDisposition"):] + fn = fn[:fn.index("\n}") + 2] + + script = tmp_path / "case.mjs" + script.write_text(fn + """ +const out = {}; +for (const name of ["S03E02. Queen's Landing.mp4", 'Caf\\u00e9 (2019).mkv', + 'plain.mp4', 'quote".mp4', 'star*.mp4']) { + out[name] = contentDisposition(name); +} +console.log(JSON.stringify(out)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + out = json.loads(proc.stdout) + + for name, header in out.items(): + starred = header.split("filename*=UTF-8''", 1)[1] + assert "'" not in starred, ( + f"{name!r}: an apostrophe survived into the starred value, which " + f"is where RFC 5987 puts its delimiter — the name is lost") + for forbidden in "()*": + assert forbidden not in starred, ( + f"{name!r}: {forbidden!r} is not an attr-char and must be " + f"percent-encoded") + # The starred value has to decode back to the real name, or the escaping + # fixed the parse and broke the result. + from urllib.parse import unquote + assert unquote(starred) == name + + # The ASCII fallback must not end its own quoted string. + for name, header in out.items(): + ascii_part = header.split('filename="', 1)[1].split('";', 1)[0] + assert '"' not in ascii_part and "\\" not in ascii_part + + +def test_a_sink_that_stops_consuming_fails_instead_of_hanging(tmp_path): + """ + `writable.write()` was the one await on the download path with no bound. + + Every other one reports itself: `_sendAndWait` logs a Response timeout, + `_fetchChunkResilient` retries and throws. A sink that stops consuming — a + service-worker stream the browser has stopped reading — leaves `write()` + pending for ever. It never rejects, so there is no error, no log and no + failed transfer: the progress bar stops, the console stays empty, and the + node is healthy throughout. + + That combination is what made it unfindable: three separate measurements + cleared the node, the transport and the worker, because none of them was + wrong. Bounding it does not fix whatever stopped the sink — it turns an + unexplainable freeze into a failed transfer that names itself. + """ + src = (STATIC / "file-utils.js").read_text() + fn = src[src.index("async function _writeOrStall"):] + fn = fn[:fn.index("\n}\n") + 2] + + script = tmp_path / "case.mjs" + script.write_text(""" +const t = (key, vars) => key + ' ' + JSON.stringify(vars); +const WRITE_STALL_MS = 300; // the real value is 60s; the shape is the test +""" + fn + """ +const out = {}; +// A sink that never resolves — the frozen download, exactly. +const dead = { write: () => new Promise(() => {}) }; +const t0 = Date.now(); +try { + await _writeOrStall(dead, new Uint8Array(4), 41); + out.threw = null; +} catch (e) { out.threw = e.message; } +out.ms = Date.now() - t0; + +// And a working sink is not slowed down or wrapped in anything. +const live = { write: async () => {} }; +await _writeOrStall(live, new Uint8Array(4), 0); +out.liveOk = true; +console.log(JSON.stringify(out)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + out = json.loads(proc.stdout) + assert out["threw"], "a dead sink hung for ever instead of failing" + assert "group.download_write_stalled" in out["threw"], ( + "the failure must name itself in the transfers panel") + assert "41" in out["threw"], "and say which chunk it stopped at" + assert out["ms"] < 3000 + assert out["liveOk"] is True + + +def test_the_worker_is_kept_alive_while_it_streams(): + """ + A service worker with no event for ~30 s is terminated — Firefox does it, + and `respondWith(new Response(stream))` does not extend its life while the + response is still being written. The reader vanishes mid-file, the page's + next `write()` never resolves and never rejects: the progress bar stops, + the console stays empty, and the node looks healthy throughout. + + Measured in real Firefox 154 on 2026-09-08, writing 1 MB every 2 s: + without the ping it stalled at 17 MB after 59 s; with it, 40 MB in 80 s, + complete. The first version of that probe wrote 450 MB in two seconds and + passed — fast enough to hide the bug entirely, which is why the pacing + matters and is written down here. + + Source-reading, because the behaviour needs a browser and a minute of wall + clock. What it protects is that the ping exists at all, is cleared on both + exits, and is answered by the worker. + """ + dl = DOWNLOADS.read_text() + sw = SW.read_text() + + assert "SW_KEEPALIVE_MS" in dl and "mbdl-ping" in dl, ( + "nothing keeps the worker alive; downloads longer than ~30 s will " + "stall on Firefox with no error anywhere") + fn = dl[dl.index("async function _attemptStreamedDownload"):] + interval = fn[fn.index("setInterval"):] + assert "mbdl-ping" in interval[:200] + + # Cleared on both ways out, or a finished download leaves a timer pinging a + # worker for the life of the page. + for exit_path in ("close:", "abort:"): + block = fn[fn.index(exit_path):] + assert "clearInterval(keepAlive)" in block[:220], ( + f"the keep-alive is not cleared in {exit_path} — it outlives the " + f"download") + + # And the worker has to answer it: a message it ignores still counts as an + # event, but the reply is what tells the page it is talking to the worker + # that holds its stream. + assert "mbdl-ping" in sw and "mbdl-pong" in sw + + +def _turn_harness(tmp_path, name, body, *, picker=True, budget_ms=90000): + """Run the real `_openTargetInTurn` against a stubbed opener. + + Both it and `_waitBriefly` are lifted out of `file-utils.js` as text; only + the budget is supplied here, so a case about the budget need not wait a + minute and a half for it. + """ + src = (STATIC / "file-utils.js").read_text() + + def lift(decl): + cut = src[src.index(decl):] + return cut[:cut.index("\n}\n") + 2] + + picker_js = ("window.showSaveFilePicker = async () => ({});" + if picker else "") + script = tmp_path / f"{name}.mjs" + script.write_text(f""" +const out = []; +let live = 0, peak = 0; +const asked = []; +// Stands in for _openDownloadTarget: records how many are open at once, and +// whether each was told it is not the first of its batch. +const _openDownloadTarget = async (name, size, opts, swSize, flags) => {{ + live += 1; peak = Math.max(peak, live); + asked.push(!!(flags && flags.batched)); + if (name === 'stuck') return await new Promise(() => {{}}); + await new Promise(r => setTimeout(r, 20)); + live -= 1; + if (name === 'boom') throw new Error('refused'); + return {{ name }}; +}}; +// Only a browser with a Save As dialog has anything to serialise. +globalThis.window = {{}}; +{picker_js} +let _targetQueue = Promise.resolve(); +let _targetsInFlight = 0; +const TARGET_QUEUE_BUDGET_MS = {budget_ms}; +""" + lift("function _openTargetInTurn") + lift("function _waitBriefly") + f""" +{body} +console.log(JSON.stringify(out)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +def test_targets_are_opened_one_at_a_time(tmp_path): + """ + A browser shows one file picker at a time and grants one per user gesture, + so four downloads asking at once get one dialog and three failures. + + That used to be prevented by accident: `downloadEntry` awaited the target + inline and files-app.js's `for (…) await downloadFile(e)` serialised them. + Opening the target inside `prepare` — so the row appears at the click rather + than tens of seconds later — removed the accident, and four pickers raced. + Reported from Chrome: one file downloaded, a prompt for the second, the + other two timed out. + + The queue is on the *targets*, never on the rows: every download still + appears the moment it is asked for. + + Queueing alone was not enough: a second dialog with no gesture behind it + still waits for a human, and the two behind it wait for the dialog. So + everything that has to wait its turn is also marked `batched`, which the + opener reads as "do not ask" — see the streamed-path branch in + test_memory_ceiling.py. + """ + peak, statuses, batched = _turn_harness(tmp_path, "one_at_a_time", """ +const results = await Promise.allSettled( + ['a', 'boom', 'c', 'd'].map(n => _openTargetInTurn(n))); +out.push(peak); +out.push(results.map(r => r.status).join(',')); +out.push(asked); +""") + assert peak == 1, f"{peak} targets were being opened at once" + # And one refusal must not stop the rest: a chain that breaks on a rejection + # leaves every later download unable to open anything at all. + assert statuses == "fulfilled,rejected,fulfilled,fulfilled" + assert batched == [False, True, True, True], ( + "only the first of a batch holds the user's gesture; the rest must be " + "opened without asking") + + +def test_a_browser_with_no_dialog_does_not_queue_at_all(tmp_path): + """Firefox and Safari have no `showSaveFilePicker`, so no two openings there + can race a dialog and there is nothing for a queue to protect. + + Queueing them anyway was a regression: four downloads that had always opened + their targets at the same time began waiting on the slowest, and all four + sat at "preparing". A queue that buys nothing must not be paid for. + """ + peak, = _turn_harness(tmp_path, "no_picker", """ +await Promise.all(['a', 'b', 'c', 'd'].map(n => _openTargetInTurn(n))); +out.push(peak); +""", picker=False) + assert peak == 4, ( + f"only {peak} target opening(s) ran at once; without a dialog to " + "serialise, all four must proceed together as they did before") + + +def test_one_stuck_opening_does_not_hold_the_others_for_ever(tmp_path): + """`_targetQueue` is never reset, so an opening that never settles would + otherwise leave the page unable to start any download again — a panel that + only a reload can fix. + + The budget is 60 ms here; in the page it is ninety seconds, long enough that + a real dialog is never cut in front of. + """ + statuses, batched = _turn_harness(tmp_path, "stuck", """ +const first = _openTargetInTurn('stuck'); +first.catch(() => {}); +const rest = await Promise.allSettled( + ['b', 'c'].map(n => _openTargetInTurn(n))); +out.push(rest.map(r => r.status).join(',')); +out.push(asked); +""", budget_ms=60) + assert statuses == "fulfilled,fulfilled", ( + "an opening that never settles must not strand the ones behind it") + assert batched == [False, True, True], ( + "the stuck one is still the only holder of the gesture, so the released " + "openings must not try for a dialog of their own") + + +def test_a_pause_falls_between_chunks_and_resumes_at_one(tmp_path): + """What makes resuming exact rather than approximate. + + Everything written is a whole number of chunks, because the loop checks for + a pause between two of them and never inside one. So `fromChunk` is a + position, not an estimate, and a resumed download is never appended to at an + offset nobody verified — the failure mode being avoided is a file that looks + complete and is quietly corrupt. + + The real `pipelinedDownload` is lifted out and run against stubs, on the + rule this repo follows for the video player: model the environment, never + the code under test. + """ + src = (STATIC / "file-utils.js").read_text() + fn = src[src.index("async function pipelinedDownload"):] + fn = fn[:fn.index("\n}\n") + 2] + + script = tmp_path / "pipeline.mjs" + script.write_text(""" +const CHUNK_SIZE = 8; +const PIPELINE_WINDOW = 4; +const written = []; +// `ct` has to be truthy: chunk 0 with a falsy body is refused as undecryptable, +// which is the guard working, not the harness. +const _fetchChunkResilient = async (transport, fileId, i) => + ({ ct: new Uint8Array([i & 0xff]), nonce: new Uint8Array(12) }); +const _writeOrStall = async (w, bytes, index) => { written.push(index); }; +globalThis.window = { MeshBayCrypto: { + // The plaintext carries its own index, so what lands where can be checked. + decryptChunkBin: async (k, id, index) => ({ byteLength: CHUNK_SIZE, index }), +} }; +""" + fn + """ +const out = {}; +const signal = { aborted: false, paused: false }; +// Stop it part way, the way the store does. +let seen = 0; +const onChunk = () => { if (++seen === 3) signal.paused = true; }; +try { + await pipelinedDownload({}, 'k', 'file', 10, onChunk, {}, signal, '', 0); + out.threw = 'no'; +} catch (err) { + out.threw = err.name; +} +out.resumeFrom = signal.resumeFrom; +out.writtenBeforePause = written.slice(); + +// And again, from where it said. +signal.paused = false; +written.length = 0; +await pipelinedDownload({}, 'k', 'file', 10, () => {}, {}, signal, '', + out.resumeFrom); +out.writtenAfterResume = written.slice(); +console.log(JSON.stringify(out)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + out = json.loads(proc.stdout) + + assert out["threw"] == "PausedError", out + # Whole chunks only, in order, with nothing skipped. + assert out["writtenBeforePause"] == list(range(len(out["writtenBeforePause"]))) + assert out["resumeFrom"] == len(out["writtenBeforePause"]), ( + f"stopped after {len(out['writtenBeforePause'])} chunks but asked to " + f"resume at {out['resumeFrom']} — that gap is a hole in the file") + # The resumed run covers exactly the rest, and repeats nothing. + assert out["writtenAfterResume"] == list(range(out["resumeFrom"], 10)), out diff --git a/packages/meshbay-hub/tests/test_layout_measured.py b/packages/meshbay-hub/tests/test_layout_measured.py index 91f1ed0..a71b6b9 100644 --- a/packages/meshbay-hub/tests/test_layout_measured.py +++ b/packages/meshbay-hub/tests/test_layout_measured.py @@ -54,7 +54,7 @@ NAV = textwrap.dedent(""" <div class="transfer-item"> <div class="transfer-line"> <span class="transfer-kind">↓</span> - <span class="transfer-name">S03E01. Salt and Sea, Fire and Blood.mp4</span> + <span class="transfer-name">Some Saga S03E01 - A Long Enough Title.mp4</span> <button class="transfer-cancel">✕</button> </div> <div class="dl-progress"><div class="dl-fill" style="width:42%"></div></div> @@ -144,3 +144,120 @@ def test_the_page_does_not_scroll_sideways(measured, width): r = measured[str(width)] assert r["docScrollW"] <= r["viewport"]["w"], ( f"the document scrolls to {r['docScrollW']} px on a {width} px screen") + + +# ── The grouped panel (§8.2) ──────────────────────────────────────────────── +# +# The panel gained groups, a header summary and a waiting row. Every one of +# those can push something off a 320 px screen, and none of it can be seen by +# reading the stylesheet: what decides where the panel lands is the button it +# hangs from, which is not at the right edge. That is the defect this file was +# written for, and it comes back with any change to the header's width. + +GROUPED = textwrap.dedent(""" + <nav class="nav"> + <div class="nav-left"> + <button class="nav-hamburger">☰</button> + <a class="nav-brand" href="#/">MeshBay</a> + </div> + <div class="nav-right"> + <div class="transfer-wrap"> + <button class="nav-notif transfer-btn">↓</button> + <div class="transfer-panel"> + <div class="transfer-head"> + <span class="transfer-head-title">Transfers</span> + <span class="transfer-head-summary">2 running · 3 waiting</span> + <button class="btn-secondary">Clear finished</button> + </div> + <div class="transfer-group"> + <div class="transfer-group-head">Running</div> + <div class="transfer-item transfer-running"> + <div class="transfer-line"> + <span class="transfer-kind">↓</span> + <span class="transfer-name">Some Saga S03E01 - A Long Enough Title.mp4</span> + <button class="transfer-cancel">✕</button> + </div> + <div class="dl-progress"><div class="dl-fill" style="width:42%"></div></div> + <div class="transfer-meta"><span>210 MB / 493 MB</span><span>3.1 MB/s · 4 min left</span></div> + </div> + </div> + <div class="transfer-group"> + <div class="transfer-group-head">Waiting</div> + <div class="transfer-item transfer-queued"> + <div class="transfer-line"> + <span class="transfer-kind">↓</span> + <span class="transfer-name">Another File With A Long Name.mkv</span> + <button class="transfer-cancel">✕</button> + </div> + <div class="dl-progress dl-waiting"></div> + <div class="transfer-meta"><span>Waiting — your slots are busy</span><span>1.2 GB</span></div> + </div> + </div> + </div> + </div> + <a class="nav-notif" href="#/">🔔</a> + <div class="user-menu"><button class="nav-btn">someone</button></div> + </div> + </nav> +""") + +GROUPED_SELECTORS = [".transfer-panel", ".transfer-head", ".transfer-head-summary", + ".transfer-group-head", ".transfer-name", + ".transfer-item.transfer-queued .dl-progress"] + + +@pytest.fixture(scope="module") +def grouped(tmp_path_factory): + fragment = tmp_path_factory.mktemp("grouped") / "fragment.html" + fragment.write_text(GROUPED) + proc = subprocess.run( + ["python3", str(HARNESS), ",".join(str(w) for w in WIDTHS), + str(fragment), *GROUPED_SELECTORS], + capture_output=True, text=True, timeout=180) + assert proc.returncode == 0, f"probe failed: {proc.stdout}{proc.stderr}" + out = json.loads(proc.stdout) + assert "error" not in out, f"no measurement: {out}" + return out + + +def test_the_grouped_panel_stays_on_a_phone_screen(grouped): + for width in WIDTHS: + box = grouped[str(width)]["boxes"][".transfer-panel"] + assert box["offLeft"] == 0, ( + f"at {width} px the panel hangs {box['offLeft']} px off the left — " + "which is where the file names are") + assert box["offRight"] == 0, ( + f"at {width} px the panel hangs {box['offRight']} px off the right") + + +def test_the_file_name_is_on_screen_in_every_group(grouped): + for width in WIDTHS: + box = grouped[str(width)]["boxes"][".transfer-name"] + assert box["offLeft"] == 0 and box["offRight"] == 0, ( + f"at {width} px a file name is cut off: {box}") + assert box["width"] > 40, "the name column collapsed to nothing" + + +def test_the_header_summary_does_not_push_the_header_taller(grouped): + """It is the one part of the header that grows with what is happening. If + it wraps, the header changes height as transfers come and go and every row + below it moves — on the narrowest screen, repeatedly.""" + for width in WIDTHS: + head = grouped[str(width)]["boxes"][".transfer-head"] + summary = grouped[str(width)]["boxes"][".transfer-head-summary"] + assert head["height"] <= 48, ( + f"at {width} px the header is {head['height']} px tall — it wrapped") + assert summary["height"] <= 24, ( + f"at {width} px the summary wrapped to {summary['height']} px") + + +def test_the_waiting_bar_is_as_wide_as_a_progress_bar(grouped): + """A waiting row has no inner fill element — the stripes are on the track + itself. Getting that wrong renders a zero-width bar, which reads as a + transfer stuck at 0% rather than one that has not started.""" + for width in WIDTHS: + bar = grouped[str(width)]["boxes"][ + ".transfer-item.transfer-queued .dl-progress"] + assert bar["width"] > 100, ( + f"at {width} px the waiting bar is {bar['width']} px wide") + assert bar["height"] >= 3, "the waiting bar has no height" diff --git a/packages/meshbay-hub/tests/test_memory_ceiling.py b/packages/meshbay-hub/tests/test_memory_ceiling.py new file mode 100644 index 0000000..1654825 --- /dev/null +++ b/packages/meshbay-hub/tests/test_memory_ceiling.py @@ -0,0 +1,329 @@ +""" +No download above the ceiling is ever collected in the page. + +`pipelinedDownload` with no `writable` allocates `new Array(totalChunks)` and +keeps every decrypted chunk, so whatever `_openDownloadTarget` returns `null` +for is a file held whole in RAM. That floor had no upper bound: the +`!window.showSaveFilePicker` branch returned `null` at any size, so on a browser +without the File System Access API a 20 GB film went to memory whenever the +service-worker path did not answer — which happens for ordinary reasons. The +symptom was the tab dying, with nothing in the source to lead back here. + +The real `_openDownloadTarget` is lifted out of `file-utils.js` **as text** and +executed against stubbed browsers, on the rule this repo already follows for the +video player: model the environment, never the code under test. A test that +transcribed the decision tree would agree with a broken version of it by +construction. + +`test_no_unguarded_memory_floor` is the one that outlives today's branches: it +reads the function and fails if a `return null` appears in it that does not go +through the guard — which is what a fourth fallback added in a hurry would look +like. +""" + +import json +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +STATIC = Path(__file__).resolve().parents[1] / "src" / "meshbay_hub" / "static" +FILE_UTILS = STATIC / "file-utils.js" + +pytestmark = pytest.mark.skipif( + shutil.which("node") is None or not FILE_UTILS.exists(), + reason="node or the SPA sources are not available") + +CEILING = 100 * 1024 * 1024 +GB = 1024 * 1024 * 1024 + + +def _lift(name, source): + """The text of one top-level declaration, from its opening line to the + column-0 brace that closes it. Nothing is re-typed into this test.""" + start = source.index(name) + end = source.index("\n}\n", start) + len("\n}\n") + return source[start:end] + + +@pytest.fixture(scope="module") +def target_fn(): + """The ceiling, its error and the real function — read, never re-typed.""" + src = FILE_UTILS.read_text() + ceiling = re.search(r"^const MEMORY_CEILING = .*?;$", src, re.M) + assert ceiling, "MEMORY_CEILING is gone from file-utils.js" + # The test's own CEILING constant must agree with the source's, or every + # boundary case below is asserting against a number nothing uses. + assert str(CEILING) in ceiling.group(0).replace(" ", "") or \ + eval(ceiling.group(0).split("=")[1].strip(" ;")) == CEILING + return "\n".join([ + ceiling.group(0), + _lift("class TooLargeForMemoryError", src), + _lift("async function _openDownloadTarget", src), + ]) + + +def _run(target_fn, tmp_path, *, size, native=False, granted=False, + streamed=False, picker=False, mode="auto", batched=False): + """Drive the real function against one browser shape.""" + script = tmp_path / "case.mjs" + script.write_text(f""" +// Stubs for everything the lifted function reaches. `formatSize` and `t` only +// build the message; the assertions are about which branch was taken. +// +// stdout carries the outcome and nothing else, so the function's own logging +// goes to stderr -- where it is still shown when a case fails. +console.info = (...a) => console.error(...a); +const formatSize = (n) => `${{n}} B`; +const t = (key, vars) => key + ' ' + JSON.stringify(vars); +const platform = {{ + capabilities: {{ nativeSave: {json.dumps(native)} }}, + nativeSave: async () => ({{ name: 'n', writable: {{}} }}), + bridgeMessage: (e) => String(e), +}}; +const downloads = {{ + BLOB_LIMIT: 512 * 1024 * 1024, + // Called by the refusal to name why the streamed path declined -- absent + // from this stub, the error constructor threw TypeError and the test saw the + // wrong failure entirely. + lastStreamFailure: () => 'stubbed: no streamed target in this harness', + getMode: () => {json.dumps(mode)}, + // `pausable` mirrors the real modules: a granted folder is a held-open file + // handle, a service-worker stream is a download the browser already owns. + openTarget: async () => + ({json.dumps(granted)} ? {{ name: 'g', writable: {{}}, pausable: true }} : null), + openStreamedDownload: async () => + ({json.dumps(streamed)} ? {{ name: 's', writable: {{}}, pausable: false }} : null), +}}; +globalThis.window = {{}}; +if ({json.dumps(picker)}) {{ + window.showSaveFilePicker = async () => {{ + if ({json.dumps(picker)} === 'no-gesture') {{ + const e = new Error("Failed to execute 'showSaveFilePicker' on 'Window': " + + "Must be handling a user gesture to show a file picker."); + e.name = 'SecurityError'; + throw e; + }} + return {{ name: 'p', createWritable: async () => ({{}}) }}; + }}; +}} + +{target_fn} + +let outcome; +try {{ + const r = await _openDownloadTarget('film.mkv', {size}, {{}}, {size}, + {{ batched: {json.dumps(batched)} }}); + outcome = r === null ? {{ kind: 'memory' }} + : r === false ? {{ kind: 'cancelled' }} + : {{ kind: 'stream', name: r.name, pausable: !!r.pausable }}; +}} catch (err) {{ + outcome = {{ kind: 'refused', name: err.name, message: err.message }}; +}} +console.log(JSON.stringify(outcome)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +# ── The hole this was written for ─────────────────────────────────────────── + +def test_a_film_is_refused_rather_than_collected_in_memory(target_fn, tmp_path): + """Firefox/Safari shape: no picker, no granted folder, the worker did not + answer. This returned null — 20 GB into a tab.""" + out = _run(target_fn, tmp_path, size=20 * GB) + assert out["kind"] == "refused", out + assert out["name"] == "TooLargeForMemoryError" + + +def test_the_refusal_says_how_big_and_what_the_limit_is(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=20 * GB) + assert "download.too_large_for_memory" in out["message"] + assert str(20 * GB) in out["message"] + assert str(CEILING) in out["message"] + + +def test_the_same_browser_in_ask_mode_is_refused_too(target_fn, tmp_path): + """'ask' skips the service-worker block entirely, so it reached the + unguarded branch without even trying to stream.""" + out = _run(target_fn, tmp_path, size=20 * GB, mode="ask") + assert out["kind"] == "refused", out + + +# ── What must keep working ────────────────────────────────────────────────── + +def test_something_small_still_uses_the_memory_floor(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=4 * 1024 * 1024) + assert out["kind"] == "memory", out + + +def test_the_boundary_is_the_ceiling_itself(target_fn, tmp_path): + assert _run(target_fn, tmp_path, size=CEILING)["kind"] == "memory" + assert _run(target_fn, tmp_path, size=CEILING + 1)["kind"] == "refused" + + +def test_a_granted_folder_streams_whatever_the_size(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=20 * GB, granted=True) + assert (out["kind"], out["name"]) == ("stream", "g") + + +def test_the_service_worker_streams_whatever_the_size(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=20 * GB, streamed=True) + assert (out["kind"], out["name"]) == ("stream", "s") + + +def test_the_desktop_app_streams_whatever_the_size(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=20 * GB, native=True) + assert (out["kind"], out["name"]) == ("stream", "n") + + +def test_a_browser_with_a_picker_is_offered_one_instead_of_being_refused( + target_fn, tmp_path): + """Chrome/Edge: the file is large, nothing streamed yet, but Save As does. + A refusal here would be this fix breaking a path that was never broken.""" + out = _run(target_fn, tmp_path, size=20 * GB, picker=True) + assert (out["kind"], out["name"]) == ("stream", "p") + + +# ── One dialog per gesture, not one per file ──────────────────────────────── + +def test_the_first_of_a_batch_still_asks_where_to_save(target_fn, tmp_path): + """The preference is not being taken away. Someone who asked to choose the + folder chooses it, for the download they actually clicked.""" + out = _run(target_fn, tmp_path, size=20 * GB, mode="ask", picker=True, + streamed=True) + assert (out["kind"], out["name"]) == ("stream", "p") + + +def test_the_rest_of_a_batch_stream_instead_of_asking(target_fn, tmp_path): + """A browser grants one picker per user gesture and selecting four files is + one gesture. Chrome showed the dialog for the second file anyway and then + waited for a human, so the third and fourth sat behind it until they timed + out — reported as three downloads frozen. + + There is no gesture left to spend, so nothing is lost by streaming: the file + still lands on disk, in the browser's own download folder. Only the choice + of folder goes, and it was not on offer. + """ + out = _run(target_fn, tmp_path, size=20 * GB, mode="ask", picker=True, + streamed=True, batched=True) + assert (out["kind"], out["name"]) == ("stream", "s") + + +def test_a_batched_download_falls_back_to_the_dialog_rather_than_failing( + target_fn, tmp_path): + """When the worker does not answer, asking is better than refusing: a + dialog that has to be answered is still a download, and the alternative + here is losing the file. A preference must not cost a capability, and + neither must the fix for one.""" + out = _run(target_fn, tmp_path, size=20 * GB, mode="ask", picker=True, + streamed=False, batched=True) + assert (out["kind"], out["name"]) == ("stream", "p") + + +def test_batching_never_pushes_a_large_file_into_memory(target_fn, tmp_path): + """Firefox shape — no picker at all. Nothing about the batch flag may reach + the memory floor above the ceiling.""" + out = _run(target_fn, tmp_path, size=20 * GB, mode="ask", picker=False, + streamed=False, batched=True) + assert out["kind"] == "refused", out + + +# ── The one that outlives today's branches ────────────────────────────────── + +def test_no_unguarded_memory_floor(target_fn): + """Every `return null` in the function goes through the guard. + + A fourth fallback appended to the chain — which is exactly how the third one + got here — is caught by this even though no case above covers it. + """ + body = target_fn[target_fn.index("async function _openDownloadTarget"):] + lines = body.splitlines() + # The guard's own `return null` is the one legitimate instance, so cut its + # definition out before looking. Comments go too — the branch that used to + # be the bug is now described in one, and a test that reads prose is the + # mistake already recorded in CLAUDE.md for the packaged systemd unit. + start = next(n for n, l in enumerate(lines) if "const _memoryFloor" in l) + end = next(n for n in range(start, len(lines)) if lines[n].strip() == "};") + rest = lines[:start] + lines[end + 1:] + code = [re.sub(r"//.*$", "", l) for l in rest] + bare = [l.strip() for l in code if re.search(r"\breturn null\b", l)] + assert bare == [], ( + "an unguarded in-memory fallback was added to _openDownloadTarget; " + "return _memoryFloor() instead: " + "; ".join(bare)) + + +def test_the_guard_is_what_the_preview_uses_too(target_fn): + """`FilePreview` decrypts a whole entry with no writable at all, so it needs + the same ceiling — and must import it rather than keep a second number.""" + files_app = (STATIC / "files-app.js").read_text() + assert "MEMORY_CEILING" in files_app + assert re.search(r"entry\.size\s*>\s*MEMORY_CEILING", files_app), ( + "the preview modal must refuse an oversized entry before fetching it") + assert not re.search(r"100\s*\*\s*1024\s*\*\s*1024", files_app), ( + "the ceiling is defined once, in file-utils.js") + + +def test_a_lost_gesture_streams_instead_of_failing(target_fn, tmp_path): + """ + A browser grants one file picker per user gesture, and downloading three + files is one gesture — so the second and third throw "Must be handling a + user gesture". The person sees a failed transfer, with a message from Chrome + about gestures, for having done something entirely reasonable. + + The streamed path needs no gesture, so it is the right answer rather than a + consolation: the file lands on disk either way, and the only thing lost is + the choice of folder, which there was no picker to make anyway. + """ + out = _run(target_fn, tmp_path, size=20 * GB, + picker="no-gesture", streamed=True, mode="ask") + assert (out["kind"], out["name"]) == ("stream", "s"), out + + +def test_a_lost_gesture_with_nothing_to_stream_to_still_refuses(target_fn, tmp_path): + """And the ceiling still holds underneath: no gesture and no stream is not + a reason to put twenty gigabytes in the page.""" + out = _run(target_fn, tmp_path, size=20 * GB, + picker="no-gesture", streamed=False, mode="ask") + assert out["kind"] == "refused", out + + +# ── Which targets can be paused ───────────────────────────────────────────── +# +# `pausable` travels with the target rather than with the platform, because the +# same browser yields both answers on the same page: a granted folder is a +# held-open file, and a service-worker stream is a download the browser already +# owns. The widget draws its button from this and nothing else. + + +def test_a_granted_folder_can_be_paused(tmp_path, target_fn): + out = _run(target_fn, tmp_path, size=20 * GB, granted=True) + assert out["pausable"] is True + + +def test_a_save_dialog_can_be_paused(tmp_path, target_fn): + out = _run(target_fn, tmp_path, size=20 * GB, picker=True) + assert out["pausable"] is True + + +def test_the_desktop_sink_can_be_paused(tmp_path, target_fn): + out = _run(target_fn, tmp_path, size=20 * GB, native=True) + assert out["pausable"] is True + + +def test_a_service_worker_stream_cannot_be_paused(tmp_path, target_fn): + """Not a shortcoming of this code. The browser is already writing an HTTP + response into its own download folder: not feeding the stream stalls that + download where we can neither see nor resume it, and an idle worker is + terminated within seconds. Firefox and Safari have no other target, so they + get cancel and no pause — the browser's own download manager is where a + pause lives there, for as long as it works. + + This is also why Chrome shows no pause button until a download folder has + been granted: without one, "save automatically" means the service worker. + """ + out = _run(target_fn, tmp_path, size=20 * GB, streamed=True) + assert out["pausable"] is False diff --git a/packages/meshbay-hub/tests/test_security_headers.py b/packages/meshbay-hub/tests/test_security_headers.py index b4d7e6d..44dd7e8 100644 --- a/packages/meshbay-hub/tests/test_security_headers.py +++ b/packages/meshbay-hub/tests/test_security_headers.py @@ -24,7 +24,7 @@ async def test_the_spa_shell_carries_the_policy(client): r = await client.get("/") assert r.headers["content-security-policy"] == CSP assert r.headers["x-content-type-options"] == "nosniff" - assert r.headers["x-frame-options"] == "DENY" + assert r.headers["x-frame-options"] == "SAMEORIGIN" assert "referrer-policy" in r.headers @@ -42,12 +42,17 @@ async def test_even_a_404_carries_the_headers(client): # cannot be framed or content-sniffed either. r = await client.get("/no/such/path") assert r.status_code == 404 - assert r.headers["x-frame-options"] == "DENY" + assert r.headers["x-frame-options"] == "SAMEORIGIN" def test_the_policy_is_locked_down_where_it_matters(): assert "default-src 'none'" in CSP # covers object-src, etc. - assert _directive(CSP, "frame-ancestors") == "frame-ancestors 'none'" + # `'self'`, not `'none'`: every foreign origin is still refused, which is + # the whole of the clickjacking protection. What `'self'` adds is this + # origin framing itself, which the streamed download needs — see + # test_the_streamed_download_frame_is_allowed. Under `'none'` Firefox + # blocked it and large downloads there had no path to disk at all. + assert _directive(CSP, "frame-ancestors") == "frame-ancestors 'self'" assert _directive(CSP, "base-uri") == "base-uri 'none'" script = _directive(CSP, "script-src") @@ -63,3 +68,66 @@ def test_recaptcha_is_the_only_external_origin(): for tok in part.strip().split()[1:]: if tok.startswith(("http://", "https://")): assert tok in hosts, f"unexpected external origin in CSP: {tok}" + + +def test_the_streamed_download_frame_is_allowed(): + """ + `frame-src` must carry `'self'`, and this is not a preference. + + The streamed-download path works by navigating a hidden iframe to + `/_mbdl/<id>` so the service worker is asked for the response it is already + holding. `frame-src` was tightened to reCAPTCHA's two origins when the + captcha needed a frame, and nobody connected the two: Chrome refused the + frame, the worker was never asked, and the page waited out its timeout for + a download that could not happen. On Firefox and Safari that is the *only* + way to write a large file to disk — there is no File System Access API and + OPFS is capped at 10% of the volume — so the whole path was dead, silently, + on the deployed hub. + + Found by clicking Download three times and watching nothing happen, with + the reason in the browser console and nowhere else. + """ + frame_src = _directive(CSP, "frame-src") + assert "'self'" in frame_src, ( + "the same-origin download frame is blocked; large downloads fall back " + "to memory, or are refused outright above the ceiling") + # And still no wildcard: `'self'` is what the download needs, nothing more. + assert "*" not in frame_src + + +def test_no_foreign_origin_may_frame_this_page(): + """The clickjacking property, stated separately from how it is spelled. + + `frame-ancestors` moved from `'none'` to `'self'` so the streamed download + could frame its own URL. That must not become a list of origins, and it must + never become `*`: the threat is a foreign page framing this one and stealing + clicks, and `'self'` is the most permissive value that still refuses every + one of them. + """ + value = _directive(CSP, "frame-ancestors").split(" ", 1)[1].strip() + assert value in ("'none'", "'self'"), ( + f"frame-ancestors is {value!r}: anything naming an origin lets that " + f"origin frame this page") + + +def test_the_two_framing_headers_agree(): + """X-Frame-Options and CSP must say the same thing. + + They did not: the CSP let this origin frame itself (which the streamed + download needs) while `X-Frame-Options: DENY` forbade all framing. The spec + says a browser must ignore the header when frame-ancestors is present, and + counting on that while shipping a contradiction is how an afternoon goes: + the CSP was fixed, the download stayed broken, and the header was why. + + Checked as a pair rather than one value apiece, because the defect was the + disagreement and either one alone reads as correct. + """ + import asyncio + + from meshbay_hub.app import create_app # noqa: F401 (import check) + + ancestors = _directive(CSP, "frame-ancestors").split(" ", 1)[1].strip() + expected = {"'none'": "DENY", "'self'": "SAMEORIGIN"}[ancestors] + assert expected == "SAMEORIGIN", ( + "if frame-ancestors goes back to 'none', X-Frame-Options must go back " + "to DENY in app.py — and the streamed download will stop working again") diff --git a/packages/meshbay-hub/tests/test_streamed_download_reliability.py b/packages/meshbay-hub/tests/test_streamed_download_reliability.py new file mode 100644 index 0000000..e1b3800 --- /dev/null +++ b/packages/meshbay-hub/tests/test_streamed_download_reliability.py @@ -0,0 +1,589 @@ +""" +The service-worker download path, which on Firefox and Safari is the only +unbounded way to write a file to disk. + +Neither of those browsers has the File System Access API, and OPFS is not a +substitute: measured on Firefox 154, its quota is exactly 10% of the volume's +size (389,233,459 bytes on a 3,892,334,592-byte volume, refused to the byte), +which a film exceeds. So when this path declines, a large download has nowhere +left to go — there is no floor under it that can hold a film. That is what makes +its reliability a correctness property rather than a nicety. + +The real module is imported under Node with the browser pieces it reaches +stubbed — `navigator.serviceWorker`, a document that "navigates" an iframe, and +Node's own TransformStream and MessageChannel, which are the real ones. What is +modelled is the environment; `serviceWorker()` and `openStreamedDownload()` are +executed, never reimplemented. + +Three failures are pinned, all of which shipped: + + - registration happened inside the first click, so that click paid install, + activate and claim while somebody watched a button do nothing; + - a null result was cached for the life of the page, so one slow first click + left the tab unable to stream anything again, curable only by a reload + nobody knew to do; + - one missed navigation fell straight through instead of retrying. +""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +STATIC = Path(__file__).resolve().parents[1] / "src" / "meshbay_hub" / "static" +DOWNLOADS = STATIC / "downloads.js" + +pytestmark = pytest.mark.skipif( + shutil.which("node") is None or not DOWNLOADS.exists(), + reason="node or the SPA sources are not available") + +# The stub browser. `plan` decides how the fake worker behaves, so one harness +# covers every case below. +PRELUDE = """ +const store = new Map(); +globalThis.localStorage = { + getItem: k => (store.has(k) ? store.get(k) : null), + setItem: (k, v) => store.set(k, String(v)), + removeItem: k => store.delete(k), +}; +const PLAN = %(plan)s; +const log = { registers: 0, claims: 0, navigations: 0, served: 0, + unregisters: 0, wakes: 0 }; + +// The worker as the page sees it: something with postMessage. It answers a +// navigation by posting mbdl-serving back on the port it was handed, which is +// exactly the confirmation the real sw.js sends from its fetch handler. +let controller = null; +const pendingByFrame = new Map(); +// Set before the controller exists, because the declaration below is what +// the temporal dead zone protects. +let asleep = PLAN.workerAsleep; +const makeController = () => ({ + postMessage: (msg, transfer) => { + // A worker with nothing to do is terminated, and `pending` goes with it. + // A ping wakes it; anything else posted while it sleeps is simply lost, + // which is what makes this failure silent. + if (asleep) { + if (msg.type === 'mbdl-ping') { + asleep = false; + log.wakes += 1; + if (msg.ports || (transfer && transfer[0])) { + const port = (transfer && transfer[0]) || null; + if (port) setTimeout(() => port.postMessage({type: 'mbdl-pong'}), 0); + } + } + return; + } + if (msg.type === 'mbdl-ping') { + const port = (transfer && transfer[0]) || null; + if (port) setTimeout(() => port.postMessage({type: 'mbdl-pong'}), 0); + return; + } + if (msg.type === 'mbdl-claim') { + log.claims += 1; + // A worker that actually claims when asked, which is what sw.js does. + if (PLAN.controlOnClaim) { + controller = makeController(); + for (const fn of listeners) fn(); + } + return; + } + if (msg.type !== 'mbdl') return; + pendingByFrame.set('/_mbdl/' + msg.id, msg.port); + // The worker says it has it, which is what the page waits for. + if (msg.port) setTimeout(() => msg.port.postMessage({type: 'mbdl-ready', + id: msg.id}), 0); + }, +}); + +const listeners = new Set(); +// `globalThis.navigator` is read-only from Node 22 -- assigning to it is the +// mistake CLAUDE.md already records against test_locales.py. Define it. +Object.defineProperty(globalThis, 'navigator', { + configurable: true, + value: { + serviceWorker: { + get controller() { return controller; }, + // What the document started with, which is the whole of the repair's + // evidence now. `getRegistration` is asked before anything registers. + getRegistration: async () => (PLAN.registeredAtLoad + ? {active: makeController()} : undefined), + register: async () => { + log.registers += 1; + if (PLAN.registerThrows) throw new Error('registration blocked'); + // A registration that never answers at all. Distinct from one that + // rejects: nothing is reported, nothing fails, the caller just waits. + if (PLAN.registerHangs) await new Promise(() => {}); + // A worker that only becomes installable once the stuck registration + // has been thrown away -- the browser this was reported from. + const healed = PLAN.activeAfterUnregister && log.unregisters > 0; + if (PLAN.controlAfterMs !== null || healed) { + setTimeout(() => { + controller = makeController(); + for (const fn of listeners) fn(); + }, healed ? 0 : PLAN.controlAfterMs); + } + return { + active: (PLAN.active || healed) ? makeController() : null, + unregister: async () => { log.unregisters += 1; return true; }, + }; + }, + // `register()` resolves as soon as the registration object exists, with + // nothing but an installing worker; `ready` is what waits for an active + // one. Measured on Firefox 154: an install handler that rejects leaves + // `ready` unsettled past ten seconds while `register()` returns in 7 ms. + get ready() { + const healed = PLAN.activeAfterUnregister && log.unregisters > 0; + return (PLAN.readySettles || healed) + ? Promise.resolve({}) : new Promise(() => {}); + }, + addEventListener: (type, fn) => { if (type === 'controllerchange') listeners.add(fn); }, + removeEventListener: (type, fn) => { listeners.delete(fn); }, + }, + }, +}); + +globalThis.window = globalThis; +globalThis.isSecureContext = true; +// Set before the module is imported, because it reads it at evaluation. +controller = %(controlled)s ? makeController() : null; +// The self-test's repair reloads once and remembers it for the tab; both have +// to exist here or priming the worker throws instead of repairing. +const session = new Map(); +globalThis.sessionStorage = { + getItem: k => (session.has(k) ? session.get(k) : null), + setItem: (k, v) => session.set(k, String(v)), + removeItem: k => session.delete(k), +}; +log.reloads = 0; +globalThis.location = { reload: () => { log.reloads += 1; } }; +globalThis.document = { + createElement: () => ({ hidden: false, src: '', remove() {} }), + body: { + appendChild: (frame) => { + log.navigations += 1; + const port = pendingByFrame.get(frame.src); + const answer = PLAN.serveOnNavigation === 'always' + || (PLAN.serveOnNavigation === 'second' && log.navigations >= 2); + if (port && answer) { + log.served += 1; + setTimeout(() => { + port.postMessage({type: 'mbdl-serving', id: frame.src}); + // The worker's own copy of the port, dropped once answered. sw.js + // drops it with the pending entry; here it has to be explicit or the + // harness process never exits. + port.close(); + }, 0); + } + }, + }, +}; + +const M = await import('%(module)s'); +// Production waits 15 s for each; these cases are about which branch runs. +const FAST = {controlMs: %(control)d, servedMs: 400}; +const out = {}; +""" + + +def _run(tmp_path, body, *, control_after_ms=0, active=True, + serve="always", register_throws=False, control_budget_ms=800, + ready_settles=True, register_hangs=False, + active_after_unregister=False, control_on_claim=False, + controlled_at_load=False, registered_at_load=False, + worker_asleep=False): + module = tmp_path / "downloads.mjs" + module.write_text(DOWNLOADS.read_text()) + (tmp_path / "package.json").write_text('{"type":"module"}') + plan = { + "controlAfterMs": control_after_ms, + "active": active, + "serveOnNavigation": serve, + "registerThrows": register_throws, + "readySettles": ready_settles, + "registerHangs": register_hangs, + "activeAfterUnregister": active_after_unregister, + "controlOnClaim": control_on_claim, + "registeredAtLoad": registered_at_load, + "workerAsleep": worker_asleep, + } + script = tmp_path / "case.mjs" + script.write_text( + (PRELUDE % {"plan": json.dumps(plan), "module": module.as_posix(), + "control": control_budget_ms, + "controlled": json.dumps(controlled_at_load)}) + + body + + "\nout.log = log;\nconsole.log(JSON.stringify(out));\n") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True, + timeout=120) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +# ── A failure must never be cached ────────────────────────────────────────── + +def test_a_missed_claim_does_not_poison_the_page(tmp_path): + """ + The bug: `_swReady` held the null, so every later download in that tab got + it back without trying. One slow first click and the tab could not stream + again — on Firefox, that is every large download for the rest of the visit. + + Here the worker never takes control, so the first call fails; the second + must register again rather than return a remembered null. + """ + r = _run(tmp_path, """ + out.first = await M.openStreamedDownload('a.bin', 10, FAST) !== null; + const after = log.registers; + out.second = await M.openStreamedDownload('b.bin', 10, FAST) !== null; + out.registeredAgain = log.registers > after; + """, control_after_ms=None) + assert r["first"] is False and r["second"] is False + assert r["registeredAgain"] is True, "a failed attempt was cached" + + +def test_a_success_is_reused_rather_than_re_registered(tmp_path): + """The other half: once controlled, it must not re-register per download.""" + r = _run(tmp_path, """ + // Closed, like a real caller: an open target holds a keep-alive interval + // for the worker, and a test that leaks one never lets Node exit. + for (const name of ['a.bin', 'b.bin']) { + const t = await M.openStreamedDownload(name, 10, FAST); + out[name[0]] = t !== null; + if (t) await t.writable.close(); + } + """) + assert r["a"] and r["b"] + assert r["log"]["registers"] <= 1, "re-registered on a page already controlled" + + +# ── Waiting for control, rather than giving up ────────────────────────────── + +def test_control_arriving_late_is_still_used(tmp_path): + """ + Control used to be waited for with a 3 s cap, inside the click. A cold + worker on a busy machine can take longer, and the old code called that a + browser that cannot stream. Scaled down here — the budget is a parameter, so + what is pinned is that a claim arriving after the first check is still used, + not the particular number of seconds. + """ + r = _run(tmp_path, """ + const t0 = Date.now(); + const target = await M.openStreamedDownload('film.mkv', 20e9, FAST); + out.ok = target !== null; + out.waitedMs = Date.now() - t0; + if (target) await target.writable.close(); + """, control_after_ms=1200, control_budget_ms=6000) + assert r["ok"] is True, "gave up on a claim that arrived late" + assert r["waitedMs"] >= 1100, "did not actually wait for the claim" + + +def test_an_uncontrolled_page_asks_the_worker_to_claim_again(tmp_path): + """ + Active but not controlling — a page loaded before any worker existed, whose + claim was missed. Rather than declare the path unavailable, ask again. + """ + r = _run(tmp_path, """ + out.ok = await M.openStreamedDownload('a.bin', 10, FAST) !== null; + """, control_after_ms=None, active=True) + assert r["log"]["claims"] >= 1, "never asked the active worker to claim" + + +# ── Retrying a missed navigation ──────────────────────────────────────────── + +def test_a_missed_navigation_is_retried(tmp_path): + """ + The worker takes the stream and is then never asked for the URL. The page + used to give up at once; on Firefox that sends a film to the in-memory + floor. It gets a second go, with a fresh id and a fresh iframe. + """ + r = _run(tmp_path, """ + const t = await M.openStreamedDownload('film.mkv', 20e9, FAST); + out.ok = t !== null; + if (t) await t.writable.close(); + """, serve="second") + assert r["ok"] is True, "one missed navigation ended the download" + assert r["log"]["navigations"] == 2 + + +def test_giving_up_says_why(tmp_path): + """ + A silent null is what made the original defect invisible. Whatever happens, + the reason has to be readable afterwards — it is what the refusal quotes. + """ + r = _run(tmp_path, """ + out.target = await M.openStreamedDownload('a.bin', 10, FAST); + out.why = M.lastStreamFailure(); + """, control_after_ms=None) + assert r["target"] is None + assert r["why"], "declined with no stated reason" + + +def test_a_registration_that_throws_is_reported_not_swallowed(tmp_path): + r = _run(tmp_path, """ + out.target = await M.openStreamedDownload('a.bin', 10, FAST); + out.why = M.lastStreamFailure(); + """, register_throws=True) + assert r["target"] is None + assert "registration" in r["why"] + + +# ── Wiring that the behavioural cases cannot see ──────────────────────────── + +def test_the_worker_is_primed_at_boot_not_at_the_first_click(tmp_path): + """ + Registration inside the first download is the whole reason the claim was + ever raced. `primeServiceWorker` has to be called where the app starts, and + from a module that actually imports it — `node --check` would not notice a + missing import, which is a mistake this repo has already shipped once. + """ + app = (STATIC / "app.js").read_text() + assert "downloads.primeServiceWorker()" in app, "nothing primes the worker" + assert "import * as downloads from './downloads.js'" in app, ( + "app.js calls downloads.primeServiceWorker() without importing downloads") + # In mount(), which runs at start-up — not inside a component or a handler. + mount = app[app.index("const mount = () => {"):] + assert "downloads.primeServiceWorker()" in mount[:mount.index("\n};")] + + +def test_the_worker_answers_a_re_claim(tmp_path): + """The page's last resort before declaring the path unavailable only works + if sw.js implements the other half.""" + sw = (STATIC / "sw.js").read_text() + assert "mbdl-claim" in sw and "clients.claim()" in sw + + +# ── Nothing on this path may wait for ever ────────────────────────────────── + +def test_a_worker_that_never_installs_does_not_hang_every_download(tmp_path): + """The one that reached a person: four downloads stuck at "preparing", for + ever, with nothing in the node's journal because no transfer had been asked + for yet. + + `register()` resolves as soon as the registration object exists — with + nothing but an *installing* worker — and `ready` waits for an active one. + Measured on Firefox 154: an install handler that rejects leaves `ready` + unsettled past ten seconds while `register()` returns in seven + milliseconds. Neither had a deadline, and `_swPromise` is shared, so every + download on the page waited on the same promise that would never settle. + """ + out = _run(tmp_path, """ +const t0 = Date.now(); +out.worker = await M.openStreamedDownload('film.mkv', 1, FAST); +out.ms = Date.now() - t0; +out.why = M.lastStreamFailure(); +""", ready_settles=False, active=False, control_after_ms=None, + control_budget_ms=300) + assert out["worker"] is None + assert out["ms"] < 8000, ( + f"gave up after {out['ms']}ms — a budget that is not enforced is not a " + "budget, and the row above it says 'preparing' the whole time") + assert "active" in out["why"], out["why"] + + +def test_a_stuck_ready_does_not_throw_away_a_working_worker(tmp_path): + """`ready` can be waiting on a *newer* worker that cannot install while an + older one is perfectly able to serve. Giving up then would cost Firefox the + only unbounded way it has to write a download to disk — a deadline must + bound the waiting, never remove the capability.""" + out = _run(tmp_path, """ +const target = await M.openStreamedDownload('film.mkv', 1, FAST); +out.target = target !== null; +// Closing stops the keep-alive; left open, its interval keeps this process +// alive well past the test's own timeout. +if (target) await target.writable.close(); +""", ready_settles=False, active=True, control_budget_ms=300) + assert out["target"] is True + + +def test_a_registration_that_never_answers_gives_up_too(tmp_path): + """The other unbounded await. It rejects loudly in the case above; this is + the case where it says nothing at all.""" + out = _run(tmp_path, """ +const t0 = Date.now(); +out.worker = await M.openStreamedDownload('film.mkv', 1, FAST); +out.ms = Date.now() - t0; +out.why = M.lastStreamFailure(); +""", register_hangs=True, active=False, control_after_ms=None, + control_budget_ms=300) + assert out["worker"] is None + assert out["ms"] < 8000, f"gave up after {out['ms']}ms" + assert "register" in out["why"], out["why"] + + +def test_a_registration_stuck_installing_is_discarded_and_asked_for_again(tmp_path): + """A deadline turns an invisible hang into a named failure, which is better + but is not a fix: a registration stuck with nothing but an installing worker + does not heal on its own. Every later visit finds the same registration and + waits on the same `ready`, so the browser stays unable to stream a download + until somebody opens developer tools — and on Firefox there is nothing else + that can write a film to disk. + + So the stuck registration is thrown away and asked for once more. + """ + out = _run(tmp_path, """ +const target = await M.openStreamedDownload('film.mkv', 1, FAST); +out.target = target !== null; +if (target) await target.writable.close(); +""", ready_settles=False, active=False, control_after_ms=None, + active_after_unregister=True, control_budget_ms=300) + assert out["log"]["unregisters"] == 1, ( + "the stuck registration was left in place") + assert out["target"] is True, ( + "discarding it did not get the page a worker it could stream to") + + +# ── A page loaded with the worker bypassed ────────────────────────────────── + + +def test_a_hard_reloaded_page_reloads_itself_once(tmp_path): + """Uncontrolled at load while an active registration already exists is a + document fetched by a hard reload — Ctrl+F5, Ctrl+Shift+R — and nothing + else. Measured on Chrome at document start: a first visit has neither, an + ordinary reload has both, a hard reload has the registration and no + controller. + + Such a page can still be claimed, so every control check passes; but the + navigations it starts keep missing the worker, and the hidden iframe a + streamed download needs is one. On Firefox and Safari that is the only way + to write a file too large to hold in memory. An ordinary reload undoes it. + """ + out = _run(tmp_path, """ + M.primeServiceWorker(); + await new Promise((r) => setTimeout(r, 200)); + out.reloads = log.reloads; + """, controlled_at_load=False, registered_at_load=True) + assert out["reloads"] == 1 + + +def test_a_first_visit_is_not_a_bypass(tmp_path): + """Also uncontrolled at load, and perfectly healthy: the worker is being + installed right now and will claim the page in a moment. Reloading here + would be a flicker on everybody's first visit — and it was, taking the + group's WebRTC session down with it when it landed mid-connection.""" + out = _run(tmp_path, """ + M.primeServiceWorker(); + await new Promise((r) => setTimeout(r, 200)); + out.reloads = log.reloads; + """, controlled_at_load=False, registered_at_load=False) + assert out["reloads"] == 0 + + +def test_a_controlled_page_does_not_reload(tmp_path): + """The ordinary case, which must cost nothing at all: no reload, and no + download spent asking. Chrome rations the downloads a page may start + without a user gesture to about three, and the first version of this check + asked its question by performing one — competing with the person's own + downloads for that budget.""" + out = _run(tmp_path, """ + M.primeServiceWorker(); + await new Promise((r) => setTimeout(r, 200)); + out.reloads = log.reloads; + out.navigations = log.navigations; + """, controlled_at_load=True, registered_at_load=True) + assert out["reloads"] == 0 + assert out["navigations"] == 0, ( + "priming performed a download; that budget belongs to the person") + + +def test_the_repair_happens_at_most_once(tmp_path): + """The flag is in sessionStorage rather than a variable because the point is + to survive the reload it triggers, and because a page that is still bypassed + afterwards must stop rather than reload again, and again.""" + out = _run(tmp_path, """ + sessionStorage.setItem('meshbay.sw-repaired', '1'); + M.primeServiceWorker(); + await new Promise((r) => setTimeout(r, 200)); + out.reloads = log.reloads; + """, controlled_at_load=False, registered_at_load=True) + assert out["reloads"] == 0 + + +# ── The claim is asked for, not waited for ────────────────────────────────── + +def test_an_uncontrolled_page_asks_at_once_rather_than_after_the_budget(tmp_path): + """A page that is uncontrolled while an active worker exists will not be + claimed on its own — a document fetched by a hard reload is exactly that + shape. Waiting the whole control budget first spends it on something that + is not coming: about thirty seconds, measured, during which the person + clicks download and watches four rows hang before the page repairs itself. + """ + out = _run(tmp_path, """ + const t0 = Date.now(); + const target = await M.openStreamedDownload('film.mkv', 20e9, FAST); + out.ms = Date.now() - t0; + out.target = target !== null; + out.claims = log.claims; + // Closing stops the keep-alive; left open, its interval outlives the test. + if (target) await target.writable.close(); + """, control_after_ms=None, control_on_claim=True, control_budget_ms=6000) + assert out["target"] is True + assert out["claims"] >= 1 + assert out["ms"] < 3000, ( + f"took {out['ms']}ms of a 6000ms budget — the claim was asked for only " + "after the wait, not before it") + + +def test_a_download_waits_for_priming(tmp_path): + """A click that lands while priming is still running must not race it: on a + page about to reload, the attempt would fail for nothing.""" + out = _run(tmp_path, """ + M.primeServiceWorker(); + const t0 = Date.now(); + const target = await M.openStreamedDownload('film.mkv', 20e9, FAST); + out.ms = Date.now() - t0; + out.target = target !== null; + if (target) await target.writable.close(); + """) + assert out["target"] is True + assert out["ms"] >= 1, "the download did not wait for priming at all" + + +def test_the_streamed_target_says_it_cannot_be_paused(tmp_path): + """The value the widget's pause button is drawn from, read off the real + module rather than a stub of it. + + It is false for a reason that is not about this code: the browser is already + writing an HTTP response into its own download folder, so not feeding the + stream stalls a download we can neither see nor resume, and an idle worker + is terminated within seconds. Firefox and Safari therefore get cancel and no + pause; Chrome gets one as soon as a download folder has been granted, which + yields a held-open file instead of this. + """ + out = _run(tmp_path, """ + const target = await M.openStreamedDownload('film.mkv', 20e9, FAST); + out.pausable = target && target.pausable; + if (target) await target.writable.close(); + """) + assert out["pausable"] is False + + +# ── a worker that was asleep when we posted ───────────────────────────────── + +def test_a_sleeping_worker_is_woken_before_it_is_handed_a_stream(tmp_path): + """Reported from Chrome: a download started while an upload was running took + thirty seconds to begin, every time. + + `pending` lives in the worker's memory and a worker with nothing to do is + terminated — which is what a long upload leaves it, for minutes, since a + WebRTC transfer gives it no events at all. The stream posted to it was lost; + the iframe then woke it with nothing to find and the request fell through to + the network, measured in the console as a 404 from the hub and fifteen + seconds of silence, twice. + + `mbdl-ping` already existed — sent every ten seconds *while* writing, for + the same reason. Nothing sent one before *starting*. + """ + out = _run(tmp_path, """ + const target = await M.openStreamedDownload('film.mkv', 20e9, FAST); + out.target = target !== null; + out.wakes = log.wakes; + out.navigations = log.navigations; + if (target) await target.writable.close(); + """, worker_asleep=True) + assert out["target"] is True, "the download never started" + assert out["wakes"] == 1, "the worker was handed a stream while asleep" + assert out["navigations"] == 1, ( + f"took {out['navigations']} attempts — the first one was wasted on a " + "worker that had not been woken") diff --git a/packages/meshbay-hub/tests/test_transfers.py b/packages/meshbay-hub/tests/test_transfers.py index 3316615..d93cf80 100644 --- a/packages/meshbay-hub/tests/test_transfers.py +++ b/packages/meshbay-hub/tests/test_transfers.py @@ -222,3 +222,695 @@ def test_a_folder_name_carries_no_trailing_slash(): assert "dir-row" in row, "the anchor no longer lands on the directory row" assert "${d}/" not in row, "the folder name is rendered with a trailing slash" assert "${d}" in row + + +# ── Transfer slots, client side ───────────────────────────────────────────── +# +# A queue can lie in two directions, and both are worse than no queue: a +# transfer that shows "waiting" on a node that already granted it, and a slot +# the page holds after it has stopped using it. Everything below is one of +# those two. + +def _lease_stub(): + """A Lease as the store sees it, driveable from the test.""" + return """ +class L { + constructor() { + this.state = 'queued'; this.ahead = 2; this.closed = false; + this.released = []; this.tr = 'tr1'; + this._wait = new Promise(r => { this._go = r; }); + } + acquire() { return this._wait; } + release(reason) { if (!this.closed) { this.closed = true; this.released.push(reason); } } + grant() { this.state = 'granted'; if (this._onState) this._onState(this); this._go(); } + push(state, ahead) { this.state = state; this.ahead = ahead; if (this._onState) this._onState(this); } +} +""" + + +def test_a_transfer_waiting_for_a_slot_is_queued_not_running(tmp_path): + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'f', total: 10, lease, + run: async () => { say('ran'); } }); + say(t.list()[0].status, t.list()[0].ahead); + await new Promise(r => setTimeout(r, 0)); + say('still:' + t.list()[0].status); + """, tmp_path) + assert out[:2] == ["queued", 2] + assert "ran" not in out, "the work started before the slot was granted" + assert out[-1] == "still:queued" + + +def test_the_grant_starts_the_work(tmp_path): + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'f', total: 10, lease, + run: async () => { say('ran:' + t.list()[0].status); } }); + lease.grant(); + await new Promise(r => setTimeout(r, 10)); + say('after:' + t.list()[0].status); + """, tmp_path) + assert out[0] == "ran:running" + assert out[1] == "after:done" + + +def test_the_slot_comes_back_however_the_transfer_ends(tmp_path): + """A slot not returned is a member who cannot transfer again until the node + times it out — so this must hold for a throw as much as for a success.""" + out = _run(_lease_stub() + """ + for (const mode of ['ok', 'throw']) { + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'f', total: 10, lease, + run: async () => { if (mode === 'throw') throw new Error('x'); } }); + lease.grant(); + await new Promise(r => setTimeout(r, 10)); + say(mode + ':' + lease.released.join(',') + ':' + t.list()[0].status); + } + """, tmp_path) + assert out == ["ok:done:done", "throw:done:failed"] + + +def test_cancelling_while_queued_gives_the_slot_back(tmp_path): + """The transfer somebody is most likely to give up on is the one that has + not started. Its queue entry has to go, or the node grants a slot to a + transfer that will never use it.""" + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + const id = t.start({ kind: 'download', name: 'f', total: 10, lease, + run: async () => { say('ran'); } }); + t.cancel(id); + say(t.list()[0].status, lease.released.join(',')); + lease.grant(); + await new Promise(r => setTimeout(r, 10)); + say('ran?', out.includes('ran')); + """, tmp_path) + assert out[0] == "cancelled" + assert out[1] == "cancelled" + assert out[-1] is False, "a cancelled transfer ran anyway once granted" + + +def test_a_queue_position_update_reaches_the_view(tmp_path): + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + const seen = []; + t.subscribe(items => seen.push(items[0].ahead)); + t.start({ kind: 'download', name: 'f', total: 10, lease, run: async () => {} }); + lease.push('queued', 1); + lease.push('queued', 0); + say(seen.join('>')); + """, tmp_path) + assert out[0].endswith("1>0"), "the widget never learns it is moving up" + + +def test_a_transport_with_a_queued_transfer_is_not_closed(tmp_path): + """Closing it would leave the transfer waiting for a grant that can never + arrive — waiting for ever, with nothing left to answer.""" + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + let closed = false; + const transport = { close() { closed = true; } }; + t.start({ kind: 'download', name: 'f', total: 10, transport, lease, + run: async () => {} }); + t.releaseWhenIdle(transport); + say('closed while queued:', closed); + lease.grant(); + await new Promise(r => setTimeout(r, 10)); + say('closed after:', closed); + """, tmp_path) + assert out[1] is False + assert out[3] is True + + +def test_clearing_finished_keeps_what_is_waiting(tmp_path): + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'waiting', total: 1, lease, run: async () => {} }); + t.start({ kind: 'download', name: 'done', total: 1, run: async () => {} }); + await new Promise(r => setTimeout(r, 10)); + t.clearFinished(); + say(t.list().map(i => i.name + ':' + i.status).join(',')); + """, tmp_path) + assert out[0] == "waiting:queued" + + +def test_asking_for_a_slot_on_a_dead_channel_does_not_throw(tmp_path): + """ + The transport reconnects on its own and re-asks for every live lease when it + does, so a closed channel at the moment a transfer starts is a wait, not a + failure. `_fetchChunkResilient` has always treated it that way — and before + leases existed a chunk request was the first thing to touch the channel, so + a download begun on a briefly dead connection simply retried. + + Asking for a slot first made `_send` the first contact. It threw + "DataChannel not open (state: closed)" straight out of `downloadEntry`, + where nothing catches it: a download that used to recover became an error + with no row in the widget to show it. Found live, by downloading a file + just after a connection dropped. + """ + module = tmp_path / "transport_lease.mjs" + # The real Lease, lifted out as text — the class is not exported, and a + # second copy of it here would agree with whatever it was copied from. + src = (STATIC / "transport.js").read_text() + # From the constant the class depends on, not from the class: lifting only + # the class left LEASE_WATCHDOG_MS undefined, which the class reads the + # first time it arms its watchdog. + start = src.index("const LEASE_WATCHDOG_MS") + end = src.index("\nclass MeshBayTransport") + module.write_text(src[start:end] + "\nexport { Lease };\n") + + script = tmp_path / "case.mjs" + script.write_text(f""" +import {{ Lease }} from '{module.as_posix()}'; +const out = []; +const transport = {{ + supportsTransferSlots: true, + _leases: new Map(), + _send() {{ throw new Error('DataChannel not open (state: closed)'); }}, +}}; +let threw = null; +const lease = new Lease(transport, 'tr1', 'download', 10, 1, null); +try {{ lease._request(); }} catch (e) {{ threw = e.message; }} +out.push(threw); +// And releasing one must be just as safe: a lease not released is a member who +// cannot start another transfer until the node times it out. +try {{ lease.release('cancelled'); out.push('release ok'); }} +catch (e) {{ out.push('release threw: ' + e.message); }} +clearTimeout(lease._watchdog); +console.log(JSON.stringify(out)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + out = json.loads(proc.stdout) + assert out[0] is None, f"asking for a slot threw: {out[0]}" + assert out[1] == "release ok" + + +def test_the_slot_is_asked_for_after_there_is_somewhere_to_write(): + """ + A granted slot has to be taken up within the node's acceptance deadline, so + it must not be asked for until the download can actually start. + + Asking first reads better — the widget could draw a row while the target is + being chosen — and is wrong: opening a target takes thirty seconds of + streamed-download timeouts, or as long as somebody leaves a Save As dialog + open. The node revokes the grant, passes it to the next in the queue + (`transfer: reclaimed … (not_taken_up)` in its log), and the download then + fetches under a `tr` that is no longer granted. Three downloads started, one + arrived. + + Source-reading, because the ordering is the whole property and it has no + behaviour of its own to drive: what matters is which call comes first. + """ + src = (STATIC / "file-utils.js").read_text() + fn = src[src.index("async function downloadEntry"):] + fn = fn[:fn.index("\n}\n")] + # `_openTargetInTurn` since target openings were serialised — same call, + # queued. What is pinned is that it comes before the slot is asked for. + assert fn.index("_openTargetInTurn") < fn.index("openTransfer"), ( + "downloadEntry asks for a transfer slot before it has anywhere to " + "write — the grant expires before the download can use it") + + +# ── the row exists from the click ─────────────────────────────────────────── + +def test_the_row_appears_before_the_target_is_open(tmp_path): + """ + Opening a target is the slow part — the streamed path waits for the worker + twice, a Save As dialog waits for a person — and the row used to be created + only after it returned. Three clicks produced no panel at all, not even the + icon, and then several rows at once. + """ + out = _run(""" + const t = new TransferStore(); + let release; + const opened = new Promise(r => { release = r; }); + t.start({ kind: 'download', name: 'film.mkv', total: 10, + prepare: async () => { await opened; return { name: 'saved.mkv' }; }, + run: async () => { say('ran'); } }); + const shot = (when) => say(when + '=' + t.list().length + ':' + + t.list().map(i => i.status + '/' + i.name).join(',')); + shot('click'); + release(); + await new Promise(r => setTimeout(r, 10)); + shot('after'); + """, tmp_path) + # Tagged, not indexed. An earlier version counted pushes by hand and was one + # out, which reads exactly like a failing assertion about the code. + seen = dict(line.split("=", 1) for line in out + if isinstance(line, str) and "=" in line) + assert {"click", "after"} <= set(seen), f"probe produced: {out}" + assert seen["click"] == "1:preparing/film.mkv", ( + f"no row, or the wrong one, at the moment of the click: {seen['click']}") + assert seen["after"] == "1:done/saved.mkv", ( + f"the row must keep the name it was saved under: {seen['after']}") + + +def test_a_dismissed_dialog_leaves_nothing_behind(tmp_path): + """Dismissing a Save As dialog is not a failure and not a cancellation: + nothing was started, so nothing should be left on screen explaining it.""" + out = _run(""" + const t = new TransferStore(); + t.start({ kind: 'download', name: 'film.mkv', total: 10, + prepare: async () => false, + run: async () => { say('ran'); } }); + say('at click:', t.list().length); + await new Promise(r => setTimeout(r, 10)); + say('after:', t.list().length, out.includes('ran')); + """, tmp_path) + assert out[1] == 1 + assert out[3] == 0, "a dismissed dialog left a row behind" + assert out[4] is False + + +def test_the_slot_is_only_asked_for_once_there_is_somewhere_to_write(tmp_path): + """ + A granted slot must be taken up within the node's deadline, and opening a + target can outlast it. Asking first cost two of three downloads. + """ + out = _run(""" + const t = new TransferStore(); + let release; + const opened = new Promise(r => { release = r; }); + let asked = false; + t.start({ kind: 'download', name: 'f', total: 10, + prepare: async () => { await opened; return true; }, + makeLease: () => { asked = true; return { + state: 'granted', ahead: 0, tr: 'x', + acquire: () => Promise.resolve(), release: () => {} }; }, + run: async () => {} }); + say('while preparing, asked?', asked); + release(); + await new Promise(r => setTimeout(r, 10)); + say('after preparing, asked?', asked, t.list()[0].status); + """, tmp_path) + assert out[1] is False, "the slot was taken before there was a target" + assert out[3] is True + assert out[4] == "done" + + +def test_a_target_that_cannot_be_opened_fails_the_row_it_already_has(tmp_path): + """The refusal above the memory ceiling lands in the panel, on the row that + is already there, rather than in a console nobody opens.""" + out = _run(""" + const t = new TransferStore(); + t.start({ kind: 'download', name: 'film.mkv', total: 10, + prepare: async () => { throw new Error('too large for memory'); }, + run: async () => { say('ran'); } }); + await new Promise(r => setTimeout(r, 10)); + const it = t.list()[0]; + say(it.status, it.error, out.includes('ran')); + """, tmp_path) + assert out[0] == "failed" + assert "too large" in out[1] + assert out[2] is False + + +def test_a_transport_is_not_closed_under_a_preparing_transfer(tmp_path): + """It has no lease yet and has moved no bytes, but closing its transport + would strand it exactly like a queued one.""" + out = _run(""" + const t = new TransferStore(); + let release; + const opened = new Promise(r => { release = r; }); + let closed = false; + const transport = { close() { closed = true; } }; + t.start({ kind: 'download', name: 'f', total: 10, transport, + prepare: async () => { await opened; return true; }, + run: async () => {} }); + t.releaseWhenIdle(transport); + say('closed while preparing:', closed); + release(); + await new Promise(r => setTimeout(r, 10)); + say('closed after:', closed); + """, tmp_path) + assert out[1] is False + assert out[3] is True + + +# ── Pause and resume ──────────────────────────────────────────────────────── +# +# The rule the whole design turns on: **a paused transfer holds nothing.** Its +# slot goes back to the node the moment it stops, and resuming rejoins the queue +# at the tail. Anything else lets one member close a node by pausing four +# downloads and going to lunch (§6.2 of ~/next/improve-downloads.md). + + +def _pausable_run(): + """A `run` that stops where it is told and reports where it resumed.""" + return """ +const mkStore = () => { + const t = new TransferStore(); + const leases = []; + const state = { starts: [], paused: null, aborted: false }; + t.start({ + kind: 'download', name: 'f', total: 1000, + prepare: async () => ({ name: 'f', pausable: true }), + makeLease: () => { const l = new L(); leases.push(l); return l; }, + run: async ({ signal, from }) => { + state.starts.push(from); + state.running = true; + try { + // Runs until told to stop, one "chunk" at a time. + for (let i = from; i < 10; i++) { + await new Promise(r => setTimeout(r, 5)); + if (signal.aborted) { const e = new Error('c'); e.name = 'AbortError'; throw e; } + if (signal.paused) { + signal.resumeFrom = i; + const e = new Error('p'); e.name = 'PausedError'; throw e; + } + } + } finally { state.running = false; } + }, + }); + return { t, leases, state }; +}; +""" + + +def test_pausing_gives_the_slot_back(tmp_path): + """The node has to get it back at once, not when the person resumes: the + whole point of a queue is that a slot nobody is using is a slot somebody + else can have.""" + out = _run(_lease_stub() + _pausable_run() + """ + const { t, leases } = mkStore(); + await new Promise(r => setTimeout(r, 5)); + leases[0].grant(); + await new Promise(r => setTimeout(r, 20)); + say('before:' + t.list()[0].status); + t.pause(t.list()[0].id); + await new Promise(r => setTimeout(r, 30)); + say('after:' + t.list()[0].status); + say('released:' + leases[0].released.join(',')); + say('leases:' + leases.length); + """, tmp_path) + assert out[0] == "before:running" + assert out[1] == "after:paused" + assert out[2] == "released:paused", "a paused transfer kept its slot" + assert out[3] == "leases:1" + + +def test_resuming_asks_for_a_new_slot_and_continues_where_it_stopped(tmp_path): + """Rejoining at the tail is the design, not an accident: a paused transfer + that could reclaim its old place would be a way to hold one.""" + out = _run(_lease_stub() + _pausable_run() + """ + const { t, leases, state } = mkStore(); + await new Promise(r => setTimeout(r, 5)); + leases[0].grant(); + await new Promise(r => setTimeout(r, 20)); + const id = t.list()[0].id; + t.pause(id); + await new Promise(r => setTimeout(r, 30)); + t.resume(id); + await new Promise(r => setTimeout(r, 10)); + say('queued:' + t.list()[0].status, 'leases:' + leases.length); + leases[1].grant(); + await new Promise(r => setTimeout(r, 120)); + say('end:' + t.list()[0].status); + say('starts:' + state.starts.join(',')); + """, tmp_path) + assert out[0] == "queued:queued", "a resumed transfer skipped the queue" + assert out[1] == "leases:2", "resuming did not ask for a slot again" + assert out[2] == "end:done" + starts = out[3].split(":")[1].split(",") + assert starts[0] == "0" and int(starts[1]) > 0, ( + f"resumed from {starts} — it started again from the beginning") + + +def test_a_transfer_whose_target_cannot_pause_is_not_paused(tmp_path): + """A service-worker stream is a download the browser already owns: not + writing to it stalls it outside our control and an idle worker is killed + within seconds. A button that silently restarts from zero is worse than no + button, so `pause` refuses rather than pretending.""" + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'download', name: 'f', total: 10, lease, + prepare: async () => ({ name: 'f' }), + run: async ({ signal }) => { + while (!signal.aborted) await new Promise(r => setTimeout(r, 5)); + } }); + lease.grant(); + await new Promise(r => setTimeout(r, 20)); + const id = t.list()[0].id; + say('pausable:' + t.list()[0].pausable); + t.pause(id); + await new Promise(r => setTimeout(r, 20)); + say('status:' + t.list()[0].status); + t.cancel(id); + """, tmp_path) + assert out == ["pausable:false", "status:running"] + + +def test_cancelling_a_paused_transfer_actually_ends_it(tmp_path): + """A paused run is parked on a promise. Without waking it, cancel marks the + row and leaves the work parked for the life of the page, holding its target + open — a button that lies, in the same way the first test in this file + describes.""" + out = _run(_lease_stub() + _pausable_run() + """ + const { t, leases, state } = mkStore(); + await new Promise(r => setTimeout(r, 5)); + leases[0].grant(); + await new Promise(r => setTimeout(r, 20)); + const id = t.list()[0].id; + t.pause(id); + await new Promise(r => setTimeout(r, 30)); + t.cancel(id); + // What matters is whether the store's own loop ends, not whether the row + // says so: the row is marked at once either way. + const settled = await Promise.race([ + t._items[0].promise.then(() => 'settled', () => 'settled'), + new Promise(r => setTimeout(() => r('parked'), 60)), + ]); + say('status:' + t.list()[0].status); + say('loop:' + settled); + say('resumed:' + state.starts.length); + """, tmp_path) + assert out[0] == "status:cancelled" + assert out[1] == "loop:settled", ( + "the run was still parked on the resume promise after a cancel — the " + "row said cancelled over work that had not stopped") + assert out[2] == "resumed:1", "cancelling started the work again" + + +def test_a_paused_transfer_still_counts_as_live(tmp_path): + """It is not finished, and its transport must not be closed under it — the + person is coming back to it.""" + out = _run(_lease_stub() + _pausable_run() + """ + const { t, leases } = mkStore(); + await new Promise(r => setTimeout(r, 5)); + leases[0].grant(); + await new Promise(r => setTimeout(r, 20)); + t.pause(t.list()[0].id); + await new Promise(r => setTimeout(r, 30)); + say('pending:' + t.pending); + """, tmp_path) + assert out == ["pending:1"] + + +def test_an_upload_can_be_paused_without_a_prepare_step(tmp_path): + """A download learns whether it can pause from its target, because only the + target knows. An upload has no target to ask: a `File` is seekable and the + node keeps the position, so it says so outright. + + This was missed when pause shipped — the button appeared on downloads and + nowhere else, including in the desktop app where everything else works. + """ + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const leases = []; + t.start({ + kind: 'upload', name: 'f', total: 100, pausable: true, + makeLease: () => { const l = new L(); leases.push(l); return l; }, + run: async ({ signal, from }) => { + for (let i = from || 0; i < 10; i++) { + await new Promise(r => setTimeout(r, 5)); + if (signal.paused) { + signal.resumeFrom = i; + const e = new Error('p'); e.name = 'PausedError'; throw e; + } + } + }, + }); + await new Promise(r => setTimeout(r, 5)); + leases[0].grant(); + await new Promise(r => setTimeout(r, 20)); + say('pausable:' + t.list()[0].pausable); + t.pause(t.list()[0].id); + await new Promise(r => setTimeout(r, 30)); + say('status:' + t.list()[0].status); + say('released:' + leases[0].released.join(',')); + """, tmp_path) + assert out == ["pausable:true", "status:paused", "released:paused"] + + +def test_an_upload_handed_a_lease_it_cannot_recreate_is_not_offered_pause(tmp_path): + """Pausing gives the slot back. A transfer that cannot ask for another one + would pause once and wait for ever, so the button is refused instead.""" + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const lease = new L(); + t.start({ kind: 'upload', name: 'f', total: 100, pausable: true, lease, + run: async ({ signal }) => { + while (!signal.aborted) await new Promise(r => setTimeout(r, 5)); + } }); + lease.grant(); + await new Promise(r => setTimeout(r, 20)); + const id = t.list()[0].id; + t.pause(id); + await new Promise(r => setTimeout(r, 20)); + say('status:' + t.list()[0].status); + t.cancel(id); + """, tmp_path) + assert out == ["status:running"] + + +def test_pausing_one_transfer_leaves_the_others_alone(tmp_path): + """Reported: three downloads running, one upload paused, and the three + downloads lost their pause buttons. + + The button is drawn from `pausable` and the status, so this asks the store + what it says about the other three at the moment one of them pauses. + """ + out = _run(_lease_stub() + """ + const t = new TransferStore(); + const leases = []; + const mk = (kind, name) => t.start({ + kind, name, total: 100, pausable: true, + makeLease: () => { const l = new L(); leases.push(l); return l; }, + run: async ({ signal, from }) => { + for (let i = from || 0; i < 40; i++) { + await new Promise(r => setTimeout(r, 5)); + if (signal.aborted) { const e = new Error('c'); e.name = 'AbortError'; throw e; } + if (signal.paused) { + signal.resumeFrom = i; + const e = new Error('p'); e.name = 'PausedError'; throw e; + } + } + }, + }); + mk('download', 'd1'); mk('download', 'd2'); mk('download', 'd3'); + mk('upload', 'u1'); + await new Promise(r => setTimeout(r, 5)); + for (const l of leases) l.grant(); + await new Promise(r => setTimeout(r, 20)); + const up = t.list().find(i => i.kind === 'upload'); + say('before:' + t.list().filter( + i => i.kind === 'download' && i.pausable && i.status === 'running').length); + t.pause(up.id); + await new Promise(r => setTimeout(r, 40)); + const rows = t.list(); + say('after:' + rows.filter( + i => i.kind === 'download' && i.pausable && i.status === 'running').length); + say('statuses:' + rows.map(i => i.kind[0] + ':' + i.status).join(',')); + for (const r of rows) t.cancel(r.id); + """, tmp_path) + assert out[0] == "before:3" + assert out[1] == "after:3", ( + f"pausing the upload changed the downloads — {out[2]}") + + +def test_a_paused_transfer_is_not_filed_under_finished(tmp_path): + """"Finished" was defined by exclusion — everything that is not running, + queued or preparing — so it quietly swallowed `paused` the day pausing + shipped. A transfer somebody stopped on purpose then sat beside the ones + that are actually over, offering a resume button in the section of things + that cannot be resumed. + + The three filters are lifted out of `app.js` and run, rather than described + here: a copy of them in this file would agree with a broken version by + construction. + """ + src = (STATIC / "app.js").read_text() + start = src.index(" const running = items.filter(") + block = src[start:src.index("const active =", start)] + + script = tmp_path / "groups.mjs" + script.write_text(""" +const items = [ + { id: 1, status: 'running' }, + { id: 2, status: 'queued' }, + { id: 3, status: 'preparing' }, + { id: 4, status: 'paused' }, + { id: 5, status: 'done' }, + { id: 6, status: 'failed' }, + { id: 7, status: 'cancelled' }, +]; +""" + block + """ +const seen = { running, waiting, paused, finished }; +console.log(JSON.stringify(Object.fromEntries( + Object.entries(seen).map(([k, v]) => [k, v.map(i => i.id)])))); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + groups = json.loads(proc.stdout) + + assert groups["paused"] == [4] + assert groups["finished"] == [5, 6, 7], ( + f"paused landed in {groups['finished']}") + assert groups["running"] == [1] and groups["waiting"] == [2, 3] + # Every row appears exactly once: a state added later that lands in no group + # is a transfer the panel simply does not show. + placed = sum((groups[k] for k in groups), []) + assert sorted(placed) == [1, 2, 3, 4, 5, 6, 7] + + +def test_a_paused_transfer_still_counts_as_active(tmp_path): + """The badge says how much is going on. A paused transfer is not over — the + person means to come back to it — so counting it as nothing would be a + panel that says "0" over work that is still there.""" + src = (STATIC / "app.js").read_text() + start = src.index(" const running = items.filter(") + block = src[start:src.index("\n\n", src.index("const active =", start))] + + script = tmp_path / "active.mjs" + script.write_text(""" +const items = [{ id: 1, status: 'paused' }, { id: 2, status: 'done' }]; +""" + block + """ +console.log(JSON.stringify({ active })); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + assert json.loads(proc.stdout)["active"] == 1 + + +def test_a_row_that_cannot_pause_says_so_where_the_button_would_be(): + """Reported from Chrome: four downloads with no pause button and an upload + with one, and no way to tell why. + + The reason is real — without a granted folder the browser writes through the + service worker, a download it already owns and cannot pause — but it was + stated only in a Settings line nobody reads on the way to a download. A gap + where the row above has a button is not an explanation. + + Shown only where a folder can actually be chosen: Firefox and Safari have + none to choose, and "choose a folder" would be advice that cannot be taken. + """ + src = (STATIC / "app.js").read_text() + row = src[src.index("function TransferRow"):] + row = row[:row.index("\n}\n")] + + hint = row[row.index("!it.pausable"):] + hint = hint[:hint.index("`}")] + assert "downloads.SUPPORTED" in hint, ( + "the hint would tell a Firefox user to choose a folder it cannot offer") + assert "it.kind === 'download'" in hint, ( + "an upload is always pausable; this is about download targets") + assert "transfers.not_pausable" in hint, "the reason is not stated" + # Not a button. There is nothing to click, and a disabled one invites the + # click anyway. + assert "<button" not in hint + + +def test_the_reason_is_translated_everywhere(): + """`t()` falls back to the key, so a missing catalogue entry shows + `transfers.not_pausable` in a tooltip rather than a sentence.""" + for path in sorted((STATIC / "locales").glob("*.js")): + assert "'transfers.not_pausable'" in path.read_text(), path.name diff --git a/packages/meshbay-hub/tests/test_transport_contracts.py b/packages/meshbay-hub/tests/test_transport_contracts.py index 879062b..fe550f9 100644 --- a/packages/meshbay-hub/tests/test_transport_contracts.py +++ b/packages/meshbay-hub/tests/test_transport_contracts.py @@ -349,16 +349,29 @@ def test_the_upload_itself_is_sealed(transport): "the upload must be sealed under the group key") assert "openGroup(" in body and "'file_upload_ack'" in body, ( "the ack carries the stored name and must be opened, not read") - # The message the node actually receives: everything between `this._send({` - # and its close. Read on its own, because the same field names appear a few - # lines above inside `msgpack_encode({...})`, which is the sealed half. - sent = body[body.index("this._send({"):] - sent = sent[:sent.index("});")] - assert "filename" not in sent, "the filename is on the message in clear" - assert "data" not in sent, "the bytes are on the message in clear" - assert "dir" not in sent and "root" not in sent, ( - "the destination is on the message in clear") - assert "...sealed," in sent, "the message must carry the sealed pair" + # The messages the node actually receives: everything between each + # `this._send({` and its close. Read on their own, because the same field + # names appear a few lines above inside `msgpack_encode({...})`, which is + # the sealed half. + # + # Every one of them, not the first: `uploadFile` sends a probe chunk before + # the file ("where am I?", UPLOAD_PROBE_INDEX) and it names the file too, so + # a check that stopped at the first message would have moved off the one it + # was written for the day the second appeared. + sends = [] + rest = body + while "this._send({" in rest: + rest = rest[rest.index("this._send({"):] + sends.append(rest[:rest.index("});")]) + rest = rest[len("this._send({"):] + assert len(sends) >= 2, "the probe and the chunks are both sent from here" + for sent in sends: + assert "filename" not in sent, "the filename is on the message in clear" + assert "data" not in sent, "the bytes are on the message in clear" + assert "dir" not in sent and "root" not in sent, ( + "the destination is on the message in clear") + assert "...sealed," in sent or "...probeSealed," in sent, ( + "the message must carry the sealed pair") assert "supportsSealedUpload" in body, ( "an older node must be refused before a chunk is sent, not after") diff --git a/packages/meshbay-hub/tests/test_upload_seal_client.py b/packages/meshbay-hub/tests/test_upload_seal_client.py index d6f9156..2e4bfb5 100644 --- a/packages/meshbay-hub/tests/test_upload_seal_client.py +++ b/packages/meshbay-hub/tests/test_upload_seal_client.py @@ -167,3 +167,41 @@ def test_the_client_refuses_an_older_node_before_sending_a_chunk(_gek): assert result["state"] == "rejected" assert "older MeshBay" in result["message"] assert result["frames"] == [], "a chunk was sent to a node that cannot open it" + + +def test_an_interrupted_upload_resumes_where_the_node_stopped(tmp_path, _gek): + """ + The browser asks, the node answers, and the second attempt sends only what + is missing. + + Both halves are the shipped ones: the frames come from the real + `uploadFile`, the answer comes from the real node handler. What is asserted + is the thing that used to be impossible — an upload interrupted at chunk two + of five that sends three chunks instead of five. + """ + body = bytes(range(256)) * ((CHUNK * 5) // 256 + 1) + body = body[:CHUNK * 5] + first = _run_probe(_probe_input(_gek, "send", + file={"name": "film.mkv", "data": body.hex()})) + frames = [msgpack.unpackb(bytes.fromhex(f), raw=False) + for f in first["frames"]] + assert [f["chunk_index"] for f in frames] == [-1, 0, 1, 2, 3, 4] + + # The link drops after two chunks. + session = _node_session(tmp_path, _gek) + for frame in frames[1:3]: + session._do_file_upload(frame) + assert not [m for m in session.sent if m.get("type") == "error"] + + # It comes back and asks. + session.sent.clear() + session._do_file_upload(frames[0]) + probe_ack = msgpack.packb(session.sent[-1], use_bin_type=True).hex() + + second = _run_probe(_probe_input( + _gek, "send", file={"name": "film.mkv", "data": body.hex()}, + probe_ack=probe_ack)) + resumed = [msgpack.unpackb(bytes.fromhex(f), raw=False)["chunk_index"] + for f in second["frames"]] + assert resumed == [-1, 2, 3, 4], ( + f"sent {resumed} — the answer to the probe was not used") diff --git a/packages/meshbay-hub/tests/test_versions_agree.py b/packages/meshbay-hub/tests/test_versions_agree.py new file mode 100644 index 0000000..4466c93 --- /dev/null +++ b/packages/meshbay-hub/tests/test_versions_agree.py @@ -0,0 +1,74 @@ +""" +Every package in this repository carries the same version. + +They are built, deployed and updated together — hub, node, common and the +desktop client — so a version that differs is not a statement about that +package, it is a mistake nobody has noticed yet. + +**Found on 2026-09-09, on the MNP 3.0 flag day.** `meshbay-client`'s +`package.json` had drifted to `1.0.0` while every Python package was on +`0.12.0`. That was invisible until the hub started publishing a minimum client +version and the client started comparing itself against it — at which point an +installed client announcing `1.0.0` sorted *above* a minimum of `0.13.0` and +walked straight through the gate meant to stop it. A version nobody reads is +free to be wrong; the moment something compares it, it is load-bearing. +""" + +import json +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +PACKAGES = ROOT / "packages" + + +def _python_versions() -> dict[str, str]: + found = {} + for pyproject in sorted(PACKAGES.glob("*/pyproject.toml")): + m = re.search(r'^version = "([^"]+)"', pyproject.read_text(), re.M) + if m: + found[f"{pyproject.parent.name}/pyproject.toml"] = m.group(1) + for init in sorted(PACKAGES.glob("*/src/*/__init__.py")): + m = re.search(r'^__version__ = "([^"]+)"', init.read_text(), re.M) + if m: + found[f"{init.parent.name}/__init__.py"] = m.group(1) + return found + + +def _client_version() -> str | None: + pkg = PACKAGES / "meshbay-client" / "package.json" + if not pkg.exists(): + return None + return json.loads(pkg.read_text()).get("version") + + +@pytest.mark.skipif(not PACKAGES.is_dir(), reason="package layout not present") +def test_every_package_carries_the_same_version(): + versions = _python_versions() + assert versions, "no package versions found at all — has the layout moved?" + client = _client_version() + if client is not None: + versions["meshbay-client/package.json"] = client + distinct = sorted(set(versions.values())) + assert len(distinct) == 1, ( + "packages disagree about the version: " + + ", ".join(f"{k}={v}" for k, v in sorted(versions.items()))) + + +@pytest.mark.skipif(not PACKAGES.is_dir(), reason="package layout not present") +def test_the_hub_will_not_refuse_the_client_it_ships_with(): + """`MIN_CLIENT_VERSION` is compared against a client's own version, so a + minimum above the version being built would lock out the very build being + released — the one failure this field can cause that nobody would think to + test for by hand.""" + from meshbay_hub.api.hub import MIN_CLIENT_VERSION + + client = _client_version() + if client is None: + pytest.skip("desktop client sources not present") + as_numbers = lambda v: [int(n) for n in v.split(".")] # noqa: E731 + assert as_numbers(MIN_CLIENT_VERSION) <= as_numbers(client), ( + f"the hub requires client {MIN_CLIENT_VERSION} but this tree builds " + f"{client}") diff --git a/packages/meshbay-hub/tests/test_zip_size_limit.py b/packages/meshbay-hub/tests/test_zip_size_limit.py index 203c10c..9471b8a 100644 --- a/packages/meshbay-hub/tests/test_zip_size_limit.py +++ b/packages/meshbay-hub/tests/test_zip_size_limit.py @@ -6,11 +6,16 @@ folder as a zip" button — Files' single folder, Files' multi-folder selection, and the Photos album button (docs/photos.md §3) — so the limit is checked once, there, and holds for all of them. -Two things are worth pinning. That an oversized folder is refused *before* +Three things are worth pinning. That an oversized folder is refused *before* `_openDownloadTarget`, because a save dialog for an archive that will never be -written is worse than no dialog at all. And that a folder at exactly the limit +written is worse than no dialog at all. That a folder at exactly the limit still goes through, since an off-by-one here silently costs a whole megabyte -of allowance and nobody would ever notice. +of allowance and nobody would ever notice. And that the two limits in play do +not contradict each other: ZIP_MAX_BYTES (512 MB) bounds the archive, while +MEMORY_CEILING (100 MB, test_memory_ceiling.py) bounds what may be built in the +page — so a 400 MB zip is allowed when there is somewhere to stream it and +refused when the only route left is memory. The `confirm()` that offers the +build-in-memory path therefore only ever appears below the ceiling. """ import json @@ -30,7 +35,7 @@ pytestmark = pytest.mark.skipif( MIB = 1024 * 1024 -def _run(total_bytes, tmp_path): +def _run(total_bytes, tmp_path, picker=False): """ Call downloadDirectory over one folder holding `total_bytes`, and report what it did: the errors it set, how many times it put a question to the @@ -44,6 +49,7 @@ def _run(total_bytes, tmp_path): (tmp_path / "package.json").write_text('{"type":"module"}') script = tmp_path / "case.mjs" + picker_js = "true" if picker else "false" script.write_text(f""" const store = new Map(); globalThis.localStorage = {{ @@ -54,17 +60,53 @@ globalThis.localStorage = {{ // Node 22 defines `navigator` itself, so it is left alone; `window` is what // platform.js reaches for to decide it is not running in the desktop app. globalThis.window = globalThis; -const out = {{ errors: [], started: 0, asked: 0 }}; +// stdout carries the outcome and nothing else, so file-utils' own logging goes +// to stderr -- where it is still shown when a case fails. It logs before every +// save dialog, which is exactly what this harness provokes. +console.info = (...a) => console.error(...a); +const out = {{ errors: [], started: 0, asked: 0, dropped: 0 }}; // Reached only once the size check has passed: with no File System Access API // under Node, downloadDirectory falls through to its build-in-memory path and // asks first. Answering yes is what lets the at-the-limit case get as far as // starting a transfer, and `asked` is how the refusal proves it never did. globalThis.confirm = () => {{ out.asked += 1; return true; }}; +// With `picker`, the browser can stream to a file the person chooses, which is +// the only legal route for an archive over MEMORY_CEILING. Never exercised — +// the stubbed `transfers.start` below does not run the job — it just has to be +// a target rather than null. +if ({picker_js}) {{ + window.showSaveFilePicker = async () => ({{ + name: 'album.zip', + createWritable: async () => ({{ write: async () => {{}}, close: async () => {{}}, + abort: async () => {{}} }}), + }}); +}} const M = await import('{(sandbox / "file-utils.js").as_posix()}'); -const transfers = {{ start: () => {{ out.started += 1; }} }}; -const transport = {{ connected: true }}; +// Faithful enough to the real store: it runs `prepare` and honours what it +// returns. The target is opened there now — the row exists from the click and +// the slow part happens behind it — so a stub that only counts calls would +// never reach the size check this file is about. +const transfers = {{ start: (opts) => {{ + out.started += 1; + if (!opts.prepare) return; + Promise.resolve() + .then(() => opts.prepare()) + .then((ready) => {{ if (ready === false) {{ out.started -= 1; out.dropped += 1; }} }}) + .catch((e) => {{ out.started -= 1; out.errors.push(e.message); }}); +}} }}; +// A transport hands out transfer slots now (transfers.py's leases). The stub +// grants at once, which is what a node with no caps does: what this file is +// about is the archive limit, not the queue. +const transport = {{ + connected: true, + openTransfer: () => ({{ + tr: 'stub', state: 'granted', ahead: 0, + acquire: () => Promise.resolve(), + release: () => {{}}, + }}), +}}; // One file, in the folder itself — entriesUnder keys on `path`. const entries = [{{ id: 'f1', name: 'big.bin', path: 'album', size: {total_bytes}, added_at: 0 }}]; @@ -73,6 +115,8 @@ await M.downloadDirectory(transfers, transport, null, entries, 'album', {{ setError: (m) => out.errors.push(m), }}); +// `prepare` runs on a microtask, so let it. +await new Promise(r => setTimeout(r, 10)); out.limit = M.ZIP_MAX_BYTES; console.log(JSON.stringify(out)); """, encoding="utf-8") @@ -101,7 +145,37 @@ def test_an_oversized_folder_is_refused_before_anything_opens(tmp_path): def test_a_folder_exactly_at_the_limit_still_downloads(tmp_path): - """The bound is inclusive: `> ZIP_MAX_BYTES`, not `>=`.""" - result = _run(512 * MIB, tmp_path) + """The bound is inclusive: `> ZIP_MAX_BYTES`, not `>=`. + + Given somewhere to stream to, because 512 MB is five times MEMORY_CEILING + and building it in the page is no longer a route this code will take. That + is what the next test is about; this one is still only about the off-by-one. + """ + result = _run(512 * MIB, tmp_path, picker=True) assert result["errors"] == [] assert result["started"] == 1 + assert result["asked"] == 0, "nothing is built in memory when it can stream" + + +def test_a_zip_over_the_memory_ceiling_is_refused_when_nothing_streams(tmp_path): + """ + Between the two limits — larger than the page may hold, smaller than the + archive limit — and no way to stream it. Before the ceiling existed this + asked "build it in memory?" and, on yes, held 400 MB in the tab. + + The refusal names the memory ceiling, not the zip limit: quoting 512 MB at + someone whose folder is under 512 MB would be a message about the wrong + rule. + """ + result = _run(400 * MIB, tmp_path) + assert result["started"] == 0 + assert result["asked"] == 0, ( + "the person must not be offered a build-in-memory path above the ceiling") + assert result["errors"] and "group.zip_too_large" not in result["errors"][0] + + +def test_a_small_folder_may_still_be_built_in_memory(tmp_path): + """The floor is intact below the ceiling — that is what it is for.""" + result = _run(4 * MIB, tmp_path) + assert result["errors"] == [] + assert result["asked"] == 1 and result["started"] == 1 |