diff options
| author | Christophe Besson <cbesson@gmail.com> | 2026-10-02 17:44:00 +0200 |
|---|---|---|
| committer | Christophe Besson <cbesson@gmail.com> | 2026-10-03 14:24:54 +0200 |
| commit | ede70b0fcc006d3ced8ef0d1d7115593f6627aef (patch) | |
| tree | 043734f7cb5641f7086be8f2efcf9f2fbf707c13 /packages/meshbay-hub | |
| parent | 78f3208db5e7285e62151cecb06c2d3c9eb4d2ae (diff) | |
| download | meshbay-ede70b0fcc006d3ced8ef0d1d7115593f6627aef.tar.gz | |
fix(client): a download written to disk no longer holds the whole file
pipelinedDownload kept every chunk's resolved promise until the end; a 2 GB
download held 2 GB in the page. Each is released once read.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Diffstat (limited to 'packages/meshbay-hub')
| -rw-r--r-- | packages/meshbay-hub/src/meshbay_hub/static/file-utils.js | 4 | ||||
| -rw-r--r-- | packages/meshbay-hub/tests/test_downloads.py | 53 |
2 files changed, 57 insertions, 0 deletions
diff --git a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js index aa45a25..2ff2bd4 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js @@ -392,6 +392,10 @@ async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk throw err; } const chunkMsg = await inflight[nextRecv]; + // Released as soon as it is read: a resolved promise left here holds its + // ciphertext, and the array holds every chunk of the file — so a download + // written straight to disk was also held whole in the page until it ended. + inflight[nextRecv] = undefined; // One shape, and a refusal for anything else. There used to be two fallbacks // below this: a base64 `ct_b64` chunk, which was the real wire format until // the binary switch in Phase 9.15 and which no node has sent since, and a diff --git a/packages/meshbay-hub/tests/test_downloads.py b/packages/meshbay-hub/tests/test_downloads.py index 3716f4c..7062d39 100644 --- a/packages/meshbay-hub/tests/test_downloads.py +++ b/packages/meshbay-hub/tests/test_downloads.py @@ -602,3 +602,56 @@ console.log(JSON.stringify(out)); f"resume at {out['resumeFrom']} — that gap is a hole in the file") # The resumed run covers exactly the rest, and repeats nothing. assert out["writtenAfterResume"] == list(range(out["resumeFrom"], 10)), out + + +def test_a_written_chunk_is_not_held_until_the_download_ends(tmp_path): + """A file streamed to disk holds one pipeline window in the page, not the + whole file. + + `pipelinedDownload` kept every chunk's resolved promise in `inflight` for + the length of the call, and each promise held its ciphertext — so a file + written straight to disk was also held whole in memory until the last + chunk landed. Measured in the Android application on a 2 GB download: the + page's JS heap tracked the bytes already written, 905 MB at 928 MB, and the + renderer reached 2.1 GB before dropping to 82 MB at the end. Every target + that writes as it goes (a granted folder, a service worker, the desktop's + native save) had the same cost. The real function runs here, with each + chunk message weakly referenced and the collector forced between writes. + """ + src = (STATIC / "file-utils.js").read_text(encoding="utf-8") + fn = src[src.index("async function pipelinedDownload"):] + fn = fn[:fn.index("\n}\n") + 2] + + script = tmp_path / "retention.mjs" + script.write_text(""" +const PIPELINE_WINDOW = 4; +const TOTAL = 40; +const refs = []; +let worst = 0; +const _fetchChunkResilient = async (transport, fileId, i) => { + const msg = { ct: new Uint8Array(64 * 1024), nonce: new Uint8Array(12) }; + refs[i] = new WeakRef(msg); + return msg; +}; +const _writeOrStall = async (w, bytes, index) => { + // A macrotask ends the job that keeps WeakRef targets alive; then collect. + await new Promise((r) => setTimeout(r, 0)); + globalThis.gc(); + let alive = 0; + for (let j = 0; j < index - PIPELINE_WINDOW; j++) if (refs[j] && refs[j].deref()) alive++; + worst = Math.max(worst, alive); +}; +globalThis.window = { MeshBayCrypto: { + decryptChunkBin: async () => ({ byteLength: 64 * 1024 }), +} }; +""" + fn + """ +await pipelinedDownload({}, 'k', 'file', TOTAL, () => {}, {}, { aborted: false }, '', 0); +console.log(JSON.stringify({ worst })); +""", encoding="utf-8") + proc = subprocess.run(["node", "--expose-gc", str(script)], capture_output=True, + text=True, encoding="utf-8", timeout=60) + assert proc.returncode == 0, proc.stderr + worst = json.loads(proc.stdout)["worst"] + assert worst == 0, ( + f"{worst} chunks already written were still held when a later one was — " + "the download keeps the whole file in memory until it ends") |