From 3f2bb22586d3e1aef765149b555ccc8e174ce7eb Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Tue, 8 Sep 2026 13:20:29 +0200 Subject: fix(hub): make the streamed download path reliable, and clean up after an abort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Firefox and Safari the service worker is the only unbounded way to write a download to disk: neither has the File System Access API, and OPFS is not a substitute — measured on Firefox 154, its quota is exactly 10% of the volume's size (389,233,459 bytes of a 3,892,334,592-byte volume, refused to the byte), which a film exceeds. So when this path declines, a large download has nowhere left to go, which makes its reliability a correctness property. Four ways it declined, all of them avoidable: - it was registered inside the first click on Download, so that click paid install, activate and claim while somebody watched a button do nothing; - `_swReady` cached a null for the life of the page. One slow first click left the tab unable to stream anything again, curable only by a reload nobody knew to do. Only a successful controller is remembered now; - control was waited for with a 3 s cap. It is 15 s, and a page that is active but not controlled asks the worker to claim again (`mbdl-claim`) instead of declaring the path unavailable; - a missed navigation gave up at once. It gets a second attempt with a fresh id and iframe, the failed one torn down completely first. Also closes a MessagePort leaked per download, and gives the reason a name (`lastStreamFailure`) so a refusal can say what happened. The timeouts became parameters: the defaults are the production values, no caller passes any, and the tests do not spend a minute waiting. `openTarget` gets an unrelated but adjacent fix, in the same file: it creates the destination with `getFileHandle({create: true})`, so an empty file exists before the first byte, and `abort()` leaves the target untouched — every cancelled download left a 0-byte file behind, and since `freeName` avoids collisions, three cancels left film.mkv, film (2).mkv and film (3).mkv, all empty. Its `abort()` now removes the entry. Safe here and only here, because `freeName` guarantees the name was not taken: the `showSaveFilePicker` path must not do the same, where the person may have picked an existing file whose contents `abort()` correctly preserves. Verified by hand in Chrome. test_streamed_download_reliability.py runs the real module under Node against a stubbed browser — it fails if the null is cached again. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HCGdheDLxGReuKHga3BtST --- packages/meshbay-hub/src/meshbay_hub/static/app.js | 9 + .../src/meshbay_hub/static/downloads.js | 203 +++++++++++++++++---- packages/meshbay-hub/src/meshbay_hub/static/sw.js | 8 + 3 files changed, 183 insertions(+), 37 deletions(-) (limited to 'packages/meshbay-hub/src/meshbay_hub') diff --git a/packages/meshbay-hub/src/meshbay_hub/static/app.js b/packages/meshbay-hub/src/meshbay_hub/static/app.js index c2858f4..9d64292 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/app.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/app.js @@ -6,6 +6,7 @@ import { t, getLocale, setLocale, initLocale, LOCALES } from './i18n.js'; import { ZipStream, entriesUnder } from './zipstream.js'; import { transfers, formatSpeed } from './transfers.js'; import * as platform from './platform.js'; +import * as downloads from './downloads.js'; import { Icon } from './icon.js'; import { formatSize } from './file-utils.js'; import { @@ -941,6 +942,14 @@ const trayLabels = () => ({ // falls back to English rather than rejecting, so this cannot strand the page. const mount = () => { render(html`<${App} />`, document.getElementById('app')); + // Get the download worker registered and this page under its control now, + // rather than inside the first click on Download. On Firefox and Safari it is + // the only unbounded way to write a file to disk, and it used to be + // registered lazily — so the first download of a session paid install, + // activate and claim while somebody watched, and a claim that missed its + // budget sent the file to a path that cannot hold a film. Fire-and-forget: + // nothing renders differently for it, and a failure is retried on demand. + downloads.primeServiceWorker(); // After the catalogue, so the labels are in the right language. A no-op in a // browser and on macOS. A language change reloads the page, which comes back // through here, so nothing else has to watch for it. diff --git a/packages/meshbay-hub/src/meshbay_hub/static/downloads.js b/packages/meshbay-hub/src/meshbay_hub/static/downloads.js index 90f88b0..f620e15 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/downloads.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/downloads.js @@ -155,8 +155,28 @@ export async function openTarget(filename) { }; const name = await freeName(filename, exists); const handle = await dir.getFileHandle(name, { create: true }); + const writable = await handle.createWritable(); return { - writable: await handle.createWritable(), + // `abort()` on a FileSystemWritableFileStream discards the swap file and + // leaves the target as it was — which here is the empty file + // `getFileHandle({create: true})` just made, before a single byte arrived. + // So every cancelled or failed download left a 0-byte file behind, and + // because `freeName` avoids collisions, three cancels left `film.mkv`, + // `film (2).mkv` and `film (3).mkv`, all empty, in the person's folder. + // + // Removing it is safe *here and only here*: `freeName` guarantees this name + // was not taken, so the file being deleted is one we created moments ago + // and nothing else. The `showSaveFilePicker` path in file-utils.js must not + // do the same — there the person may have picked an existing file, whose + // contents `abort()` correctly preserves. + writable: { + write: (bytes) => writable.write(bytes), + close: () => writable.close(), + abort: async (reason) => { + try { await writable.abort(reason); } catch { /* already gone */ } + try { await dir.removeEntry(name); } catch { /* already gone */ } + }, + }, name, // Reading it back is the only way a page can "open" a file it wrote: hand // the bytes to a tab and let the browser decide what to do with them. No @@ -180,40 +200,114 @@ export const BLOB_LIMIT = 512 * 1024 * 1024; // ── Streaming to disk without the File System Access API ──────────────────── const SW_PATH = '/sw.js'; -let _swReady = null; + +// On Firefox and Safari this worker is not a nicety, it is the only unbounded +// way to write a download to disk: the File System Access API does not exist +// there, and OPFS is capped at 10% of the volume's size (measured on Firefox +// 154: 389,233,459 bytes on a 3,892,334,592-byte volume, refused to the byte), +// which a film can exceed. Everything below exists to make sure this path is +// available when it is needed, because there is nothing underneath it. +// +// How long to wait for this page to become *controlled*. Generous on purpose: +// the cost of waiting is a spinner, and the cost of giving up is a download +// this browser then cannot do at all. +const SW_CONTROL_BUDGET_MS = 15000; +// How long to wait for the worker to confirm it answered the iframe. +const SW_SERVED_BUDGET_MS = 15000; +// A transient miss gets a second go with a fresh id and a fresh iframe. +const SW_ATTEMPTS = 2; + +// Holds a *successful* controller, or an in-flight attempt. Never a failure — +// see serviceWorker(). The previous version cached the rejected/null result +// for the life of the page, so one slow first click (a cold worker, a busy +// phone) left the tab unable to stream anything ever again, with no way back +// but a reload nobody knew to do. +let _swPromise = null; +let _lastFailure = ''; + +/** Why the streamed path last declined, for a message worth reading. */ +export function lastStreamFailure() { return _lastFailure; } export const STREAMS_VIA_SW = typeof window !== 'undefined' && 'serviceWorker' in navigator && typeof TransformStream === 'function' && window.isSecureContext; -async function serviceWorker() { - if (!STREAMS_VIA_SW) return null; - if (!_swReady) { - _swReady = navigator.serviceWorker.register(SW_PATH, { scope: '/' }) - .then(() => navigator.serviceWorker.ready) - .then(async (reg) => { - // `reg.active` is not enough. A worker can be active while this page is - // still uncontrolled, and an uncontrolled page's requests are never - // handed to its fetch handler — so the worker would take our stream and - // then never be asked for it. The iframe would 404, nothing would read - // the stream, and the first write() would block for good: a download - // stuck at one chunk. - if (navigator.serviceWorker.controller) return navigator.serviceWorker.controller; - // sw.js claims clients on activate, so control usually arrives within a - // tick of registration. Wait briefly rather than give up at once. - return await new Promise((resolve) => { - const done = () => resolve(navigator.serviceWorker.controller || null); - navigator.serviceWorker.addEventListener('controllerchange', done, { once: true }); - setTimeout(done, 3000); - }); - }) - .catch(err => { - console.warn('[MeshBay] service worker unavailable:', err.message); - return null; - }); +/** Resolves with the controller, or null once `budgetMs` is spent. */ +function _awaitControl(budgetMs) { + if (navigator.serviceWorker.controller) { + return Promise.resolve(navigator.serviceWorker.controller); + } + return new Promise((resolve) => { + let timer = 0; + const done = () => { + clearTimeout(timer); + navigator.serviceWorker.removeEventListener('controllerchange', done); + resolve(navigator.serviceWorker.controller || null); + }; + navigator.serviceWorker.addEventListener('controllerchange', done); + timer = setTimeout(done, budgetMs); + }); +} + +async function _claimController(budgetMs) { + const reg = await navigator.serviceWorker.register(SW_PATH, { scope: '/' }); + // `ready` resolves on an *active* registration; being active is not being in + // control. An uncontrolled page's requests never reach the fetch handler, so + // the worker would take our stream and never be asked for it — the download + // then freezes after exactly one chunk, which is how this was found. + await navigator.serviceWorker.ready; + const controller = await _awaitControl(budgetMs); + if (controller) return controller; + // Active but not controlling after the whole budget. `sw.js` calls + // `clients.claim()` on activate, so this is rare; when it happens the page + // was loaded before any worker existed and the claim was missed. Ask the + // active worker to claim again rather than declare the path unavailable. + if (reg.active) { + try { reg.active.postMessage({ type: 'mbdl-claim' }); } catch { /* gone */ } + return await _awaitControl(2000); + } + return null; +} + +/** + * Register the worker and get this page controlled, now. + * + * Called at application start, not at the first download. Registration used to + * happen inside the first click, so that click paid install, activate and claim + * while somebody watched a button do nothing — and if the claim did not land + * inside the budget, the download fell through to a path that cannot hold a + * film. By the time anyone clicks anything, this has long since finished. + * + * Fire-and-forget by design: nothing waits on it, and a failure here is not + * fatal because `serviceWorker()` will simply try again. + */ +export function primeServiceWorker() { + if (!STREAMS_VIA_SW) return; + serviceWorker().catch(() => {}); +} + +async function serviceWorker(controlMs = SW_CONTROL_BUDGET_MS) { + if (!STREAMS_VIA_SW) { + _lastFailure = 'no service worker support in this browser'; + return null; } - return _swReady; + if (navigator.serviceWorker.controller) return navigator.serviceWorker.controller; + if (!_swPromise) { + _swPromise = _claimController(controlMs).catch((err) => { + _lastFailure = 'service worker registration failed: ' + err.message; + console.warn('[MeshBay]', _lastFailure); + return null; + }); + } + const controller = await _swPromise; + if (!controller) { + // Not remembered. The next attempt starts from scratch, which is the whole + // point: these failures are transient far more often than they are final. + _swPromise = null; + if (!_lastFailure) _lastFailure = 'the page did not come under the worker’s control'; + } + return controller; } /** @@ -229,8 +323,29 @@ async function serviceWorker() { * Returns {writable, name} shaped like the File System Access one, or null if * this browser cannot do it either. */ -export async function openStreamedDownload(filename, size = 0) { - const worker = await serviceWorker(); +export async function openStreamedDownload(filename, size = 0, { + controlMs = SW_CONTROL_BUDGET_MS, + servedMs = SW_SERVED_BUDGET_MS, + attempts = SW_ATTEMPTS, +} = {}) { + for (let attempt = 1; attempt <= attempts; attempt++) { + const target = await _attemptStreamedDownload( + filename, size, attempt, controlMs, servedMs); + if (target) return target; + // A miss is usually the worker having been asleep or the navigation losing + // a race, not this browser being unable. Falling through on the first miss + // is what sent large downloads to the in-memory floor. + if (attempt < attempts) { + console.warn(`[MeshBay] streamed download attempt ${attempt} missed ` + + `(${_lastFailure}); retrying`); + } + } + return null; +} + +async function _attemptStreamedDownload(filename, size, attempt, + controlMs, servedMs) { + const worker = await serviceWorker(controlMs); if (!worker) return null; const id = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; @@ -252,8 +367,11 @@ export async function openStreamedDownload(filename, size = 0) { [readable, chan.port2]); } catch (err) { // Transferable streams are what makes the backpressure work; without them - // this would be a memory buffer wearing a stream's clothes. - console.warn('[MeshBay] streams cannot be transferred here:', err.message); + // this would be a memory buffer wearing a stream's clothes. This one is + // final rather than transient — a browser does not grow the capability + // between two attempts — so it is reported as such. + _lastFailure = 'this browser cannot transfer a stream to the worker: ' + err.message; + console.warn('[MeshBay]', _lastFailure); return null; } @@ -264,19 +382,30 @@ export async function openStreamedDownload(filename, size = 0) { const answered = await Promise.race([ serving, - new Promise((r) => setTimeout(() => r(false), 8000)), + new Promise((r) => setTimeout(() => r(false), servedMs)), ]); if (!answered) { // Some browsers refuse a download started from a hidden iframe, and an - // uncontrolled page never reaches the worker at all. Say so and let the - // caller fall back rather than hand back a sink nothing drains. - console.warn('[MeshBay] the service worker never served the download; ' - + 'falling back'); + // uncontrolled page never reaches the worker at all. Tear this attempt + // down completely — the stream, the port and the frame — so a retry starts + // clean rather than leaving a half-open sink behind. + _lastFailure = `the worker did not answer the download within ` + + `${servedMs / 1000}s (attempt ${attempt})`; + console.warn('[MeshBay]', _lastFailure); frame.remove(); + try { chan.port1.close(); } catch { /* already gone */ } try { await writable.abort('not served'); } catch { /* already gone */ } return null; } + _lastFailure = ''; + // The port has delivered the one message it exists for. Closing it matters: + // an open MessagePort is a live handle, and one was leaked per download for + // the life of the page. (It is also what hung the Node harness in + // test_streamed_download_reliability.py — there the leak is a process that + // never exits, which is the same defect wearing a louder symptom.) + try { chan.port1.close(); } catch { /* already gone */ } + const writer = writable.getWriter(); return { name: filename, diff --git a/packages/meshbay-hub/src/meshbay_hub/static/sw.js b/packages/meshbay-hub/src/meshbay_hub/static/sw.js index 119687d..0dd87d8 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/sw.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/sw.js @@ -24,6 +24,14 @@ self.addEventListener('activate', (event) => event.waitUntil(self.clients.claim( self.addEventListener('message', (event) => { const data = event.data || {}; + // A page that loaded before any worker existed can miss the claim on + // activate. Rather than declare the streamed path unavailable — which on + // Firefox and Safari means the download cannot happen at all — the page asks + // for another claim and waits a moment longer. + if (data.type === 'mbdl-claim') { + event.waitUntil(self.clients.claim()); + return; + } if (data.type !== 'mbdl' || !data.id || !data.readable) return; pending.set(data.id, { readable: data.readable, -- cgit v1.2.3 From bb8aad22c4d41dd1b89c85c8d46878a31a9530e5 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Tue, 8 Sep 2026 13:21:02 +0200 Subject: fix(hub): never collect a large download in the page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pipelinedDownload` with no writable allocates `new Array(totalChunks)` and keeps every decrypted chunk, so whatever `_openDownloadTarget` returns null for is held whole in RAM. That floor had no upper bound: the `!window.showSaveFilePicker` branch returned null at any size, so on a browser without the File System Access API a 20 GB film went to memory whenever the streamed path did not answer. Nothing logged, nothing refused; the symptom was the tab dying, with no error attributable to this code. MEMORY_CEILING is 100 MB and every `return null` in that chain now goes through a guard that throws above it. The refusal names the size, the limit and why the streamed path declined, and lands in the transfers panel as a failed transfer rather than in a console nobody opens. This is a guard, not a limit on what can be downloaded: with the streamed path primed and retried (previous commit), a file of any size still goes to disk progressively on every browser. Two things had to change for that to be true: - the streamed path is now tried in "ask" mode too, for a file over the ceiling on a browser with no Save As of its own. The mode decides whether to show a dialog; it was silently deciding whether a film could be downloaded at all; - FilePreview had no size check whatsoever — a multi-gigabyte PDF or .csv was fetched whole, and the text branch decoded all of it to keep 500 000 characters. It refuses above the same ceiling and offers the download. ZIP_MAX_BYTES (512 MB) and the ceiling do not contradict: the archive limit bounds the archive, the ceiling bounds what may be built in the page, so a 400 MB zip is allowed when there is somewhere to stream it and refused when the only route left is memory. The build-in-memory confirmation only appears below the ceiling now. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HCGdheDLxGReuKHga3BtST --- .../src/meshbay_hub/static/file-utils.js | 103 +++++++++- .../src/meshbay_hub/static/files-app.js | 20 +- .../src/meshbay_hub/static/locales/de.js | 2 + .../src/meshbay_hub/static/locales/en.js | 2 + .../src/meshbay_hub/static/locales/es.js | 2 + .../src/meshbay_hub/static/locales/fr.js | 2 + .../src/meshbay_hub/static/locales/it.js | 2 + .../src/meshbay_hub/static/locales/ja.js | 2 + .../src/meshbay_hub/static/locales/nl.js | 2 + .../src/meshbay_hub/static/locales/pl.js | 2 + .../src/meshbay_hub/static/locales/pt-BR.js | 2 + .../src/meshbay_hub/static/locales/zh-CN.js | 2 + packages/meshbay-hub/tests/test_memory_ceiling.py | 209 +++++++++++++++++++++ packages/meshbay-hub/tests/test_zip_size_limit.py | 59 +++++- 14 files changed, 395 insertions(+), 16 deletions(-) create mode 100644 packages/meshbay-hub/tests/test_memory_ceiling.py (limited to 'packages/meshbay-hub/src/meshbay_hub') diff --git a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js index 241d761..a942fd5 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js @@ -47,15 +47,64 @@ const CHUNK_SIZE = 1024 * 1024; // that hesitates, and looks like a hang while it is quiet. const PIPELINE_WINDOW = 8; +// The most this code will ever collect in the page. +// +// Below every streaming target there is a floor: `pipelinedDownload` with no +// `writable` allocates `new Array(totalChunks)` and keeps every decrypted +// chunk, and `_saveBlob` hands the lot to the browser. That floor is fine for +// something small and is a dead tab for a film. It had **no upper bound**: the +// `!window.showSaveFilePicker` branch below returned null at any size, so on a +// browser without the File System Access API (Firefox, Safari) a 20 GB film +// went to RAM whenever the service-worker path did not answer — which happens +// for ordinary reasons (an uncontrolled page, a stream that cannot be +// transferred, the 8 s timeout). Nothing logged, nothing refused; the symptom +// was the tab dying, with no error attributable to this code. +// +// So: above this, there is no floor. A refusal naming what happened is +// recoverable and a dead tab is not. `CLAUDE.md`'s standing lesson is that a +// fallback chain reaches its floor silently — this is that floor being given a +// bottom. +const MEMORY_CEILING = 100 * 1024 * 1024; + +/** + * Thrown instead of falling through to the in-memory floor. + * + * This should now be unreachable in ordinary use: the streamed path is primed + * at application start and retried on demand, so a browser with a service + * worker has somewhere to write whatever the size. If it is ever raised, the + * reason the streamed path declined is appended — untranslated, because it is a + * diagnostic and a vague failure is what made the original bug invisible. + */ +class TooLargeForMemoryError extends Error { + constructor(filename, size) { + const why = downloads.lastStreamFailure(); + super(t('download.too_large_for_memory', { + name: filename, size: formatSize(size), limit: formatSize(MEMORY_CEILING), + }) + (why ? ` (${why})` : '')); + this.name = 'TooLargeForMemoryError'; + } +} + /** * Open somewhere to write, honouring the user's download setting. * * Returns a target ({writable, name}), null for "no stream available — collect * it and hand the browser a blob", or false for "the person dismissed the * dialog", which is not an error and must not start a transfer. + * + * **Never returns null above MEMORY_CEILING.** Every `return null` below is + * guarded by `_memoryFloor`, which throws instead. A fourth fallback added + * later must go through it too — `test_memory_ceiling.py` fails the build if a + * bare `return null` appears in this function. */ async function _openDownloadTarget(filename, size = 0, pickerOpts = {}, swSize = size) { + // "Collect it in the page", or a refusal when that would be too much. + const _memoryFloor = () => { + if (size > MEMORY_CEILING) throw new TooLargeForMemoryError(filename, size); + return null; + }; + // On a desktop build this is the whole answer, and it comes first. // // The two browser paths below are both unavailable there — `showDirectoryPicker` @@ -87,14 +136,27 @@ async function _openDownloadTarget(filename, size = 0, pickerOpts = {}, // write, which is how this works at all in Firefox: the alternative there is // to collect gigabytes in a tab. It goes to the browser's own download // folder, without a dialog, which is what "save automatically" meant. - if (downloads.getMode() === 'auto') { + // + // Tried in "ask" mode too when the file is large and this browser has no Save + // As of its own. The mode is about whether to show a dialog; it was never + // meant to decide whether a 20 GB film can be downloaded at all, and on + // Firefox and Safari — where `showSaveFilePicker` does not exist — skipping + // this block left nothing but the in-memory floor. A preference must not cost + // a capability. + const canPick = typeof window.showSaveFilePicker === 'function'; + if (downloads.getMode() === 'auto' || (size > MEMORY_CEILING && !canPick)) { const streamed = await downloads.openStreamedDownload(filename, swSize); if (streamed) return streamed; - // Nothing to stream to: small enough for memory, and no dialog. - if (size < downloads.BLOB_LIMIT) return null; + // Nothing to stream to: small enough for memory, and no dialog. The old + // comparison here was against downloads.BLOB_LIMIT (512 MB), five times + // this ceiling — and it was the *only* size test in the whole chain, with + // the branch below it unguarded. + if (size <= MEMORY_CEILING) return _memoryFloor(); } - if (!window.showSaveFilePicker) return null; + // No File System Access API — Firefox, Safari. This is the branch that used + // to return null at any size. + if (!canPick) return _memoryFloor(); try { const handle = await window.showSaveFilePicker({ suggestedName: filename, ...pickerOpts, @@ -210,7 +272,20 @@ async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk * download button — both just want "get this entry to disk". */ async function downloadEntry(transfers, transport, gek, entry) { - const target = await _openDownloadTarget(entry.name, entry.size); + let target; + try { + target = await _openDownloadTarget(entry.name, entry.size); + } catch (err) { + // The refusal belongs in the transfers panel, not in a console nobody + // opens: that is where someone who just clicked Download is looking, and a + // failed row naming the reason is the whole point of refusing rather than + // filling the tab. Started only to be failed, deliberately. + transfers.start({ + kind: 'download', name: entry.name, total: entry.size, transport, + run: async () => { throw err; }, + }); + return; + } if (target === false) return; // the picker was dismissed const openRef = { url: null }; @@ -292,10 +367,19 @@ async function downloadDirectory(transfers, transport, gek, entries, dir, { setE // totalBytes decides how this is delivered, but it is not the archive's // size — headers and the central directory come on top — so it is not // announced as a Content-Length that the download would then miss. - const target = await _openDownloadTarget(suggested, totalBytes, { - types: [{ description: 'ZIP archive', - accept: { 'application/zip': ['.zip'] } }], - }, 0); + let target; + try { + target = await _openDownloadTarget(suggested, totalBytes, { + types: [{ description: 'ZIP archive', + accept: { 'application/zip': ['.zip'] } }], + }, 0); + } catch (err) { + // Reported beside the folder that was clicked, like zip_too_large just + // above — this function is called in a loop over a selection, and the + // sibling folders must still download. + setError(err.message); + return; + } if (target === false) return; if (!target && !confirm(t('group.zip_no_stream', { size: formatSize(totalBytes), name: suggested, @@ -351,6 +435,7 @@ async function downloadDirectory(transfers, transport, gek, entries, dir, { setE export { FILE_ICONS, formatSize, formatDate, PREVIEWABLE_TEXT, canPreview, CHUNK_SIZE, ZIP_MAX_BYTES, + MEMORY_CEILING, TooLargeForMemoryError, _openDownloadTarget, _saveBlob, pipelinedDownload, downloadEntry, downloadDirectory, }; diff --git a/packages/meshbay-hub/src/meshbay_hub/static/files-app.js b/packages/meshbay-hub/src/meshbay_hub/static/files-app.js index 4947f8c..36c5af3 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/files-app.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/files-app.js @@ -6,7 +6,7 @@ import { Icon } from './icon.js'; import { entriesUnder } from './zipstream.js'; import { transfers } from './transfers.js'; import { - FILE_ICONS, formatSize, formatDate, canPreview, CHUNK_SIZE, + FILE_ICONS, formatSize, formatDate, canPreview, CHUNK_SIZE, MEMORY_CEILING, pipelinedDownload, downloadEntry, downloadDirectory as sharedDownloadDirectory, } from './file-utils.js'; @@ -598,6 +598,24 @@ function FilePreview({ entry, transportRef, gekRef, onClose, onDownload }) { useEffect(() => { let cancelled = false; const load = async () => { + // Nothing here streams: a preview is decrypted whole, held as an array of + // chunks, and turned into a blob. That is right for a page of text and a + // photograph, and it is a dead tab for the things that also reach here — + // a scanned PDF, a multi-gigabyte .csv or .log. There was no size test at + // all, and the text branch is the sharpest illustration: it decoded the + // entire file and then kept 500 000 characters of it. + // + // Films and music never arrive (group-page.js's onPreview routes video to + // the MSE player and audio to the music queue), so this guard is only ever + // met by a document somebody clicked without knowing how big it was. It + // offers the download instead, which does stream. + if (entry.size > MEMORY_CEILING) { + setError(t('preview.too_large', { + size: formatSize(entry.size), limit: formatSize(MEMORY_CEILING), + })); + setPhase('error'); + return; + } const transport = transportRef.current; if (!transport || !transport.connected) { setError(t('video.err_transport')); diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js index 97a835d..1ebab8b 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js @@ -216,6 +216,8 @@ export default { 'video.close': 'Schließen (Esc)', 'preview.pdf_fallback': 'Dieser Browser zeigt das PDF nicht direkt an. Laden Sie es ' + 'stattdessen herunter — entschlüsselt wurde es ohnehin hier.', + 'preview.too_large': 'Diese Datei ist {size} groß, mehr als diese Seite im Arbeitsspeicher halten kann ({limit}). Laden Sie sie stattdessen herunter — ein Download wird direkt auf die Festplatte geschrieben.', + 'download.too_large_for_memory': '„{name}“ ist {size} groß. Dieser Browser kann eine Datei dieser Größe nur speichern, indem er sie direkt auf die Festplatte schreibt, und das ist hier nicht möglich — er müsste die ganze Datei im Arbeitsspeicher halten. Verwenden Sie die Desktop-App oder Chrome bzw. Edge.', 'group.upload_indexing': 'wird indiziert …', 'video.err_transport': 'Transport nicht verbunden', 'video.err_mse': 'Codec wird für das Streaming nicht unterstützt: {codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js index c55a702..d85f51b 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js @@ -214,6 +214,8 @@ export default { 'video.from_start': "Start from the beginning", 'video.close': 'Close (Esc)', 'preview.pdf_fallback': 'This browser will not display the PDF inline. Download it instead — it was decrypted here either way.', + 'preview.too_large': 'This file is {size}, more than this page can hold in memory ({limit}). Download it instead — a download is written straight to disk.', + 'download.too_large_for_memory': '"{name}" is {size}. This browser can only save a file that large by streaming it to disk, and it has no way to do that here — it would have to hold the whole file in memory. Use the desktop app, or Chrome or Edge.', 'group.upload_indexing': 'indexing…', 'video.err_transport': 'Transport not connected', 'video.err_mse': 'Codec not supported for streaming: {codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js index 9ce3900..bfe4112 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js @@ -214,6 +214,8 @@ export default { 'video.close': 'Cerrar (Esc)', 'preview.pdf_fallback': 'Este navegador no mostrará el PDF integrado. Descárguelo ' + 'en su lugar — en cualquier caso se descifró aquí.', + 'preview.too_large': 'Este archivo ocupa {size}, más de lo que esta página puede mantener en memoria ({limit}). Descárguelo en su lugar — una descarga se escribe directamente en disco.', + 'download.too_large_for_memory': '«{name}» ocupa {size}. Este navegador solo puede guardar un archivo así transmitiéndolo al disco, y aquí no puede hacerlo — tendría que mantener el archivo entero en memoria. Use la aplicación de escritorio, o Chrome o Edge.', 'group.upload_indexing': 'indexando…', 'video.err_transport': 'Transporte no conectado', 'video.err_mse': 'Códec no compatible con la reproducción en continuo: {codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js index d612f71..9addec5 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js @@ -215,6 +215,8 @@ export default { 'video.close': 'Fermer (Échap)', 'preview.pdf_fallback': 'Ce navigateur n’affichera pas le PDF directement. ' + 'Téléchargez-le plutôt — il a été déchiffré ici dans les deux cas.', + 'preview.too_large': 'Ce fichier fait {size}, plus que cette page ne peut garder en mémoire ({limit}). Téléchargez-le plutôt — un téléchargement est écrit directement sur le disque.', + 'download.too_large_for_memory': '« {name} » fait {size}. Ce navigateur ne peut enregistrer un fichier de cette taille qu\'en l\'écrivant au fil de l\'eau sur le disque, ce qu\'il ne peut pas faire ici — il devrait garder le fichier entier en mémoire. Utilisez l\'application de bureau, ou Chrome ou Edge.', 'group.upload_indexing': 'indexation…', 'video.err_transport': 'Transport non connecté', 'video.err_mse': 'Codec non pris en charge pour la diffusion : {codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js index c61b8e5..fe76b9e 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js @@ -215,6 +215,8 @@ export default { 'video.close': 'Chiudi (Esc)', 'preview.pdf_fallback': 'Questo browser non mostrerà il PDF nella pagina. Lo scarichi ' + 'invece — in ogni caso è stato decifrato qui.', + 'preview.too_large': 'Questo file è di {size}, più di quanto questa pagina possa tenere in memoria ({limit}). Lo scarichi invece — un download viene scritto direttamente su disco.', + 'download.too_large_for_memory': '«{name}» è di {size}. Questo browser può salvare un file di queste dimensioni solo scrivendolo su disco man mano, e qui non può farlo — dovrebbe tenere l’intero file in memoria. Usi l’applicazione desktop, oppure Chrome o Edge.', 'group.upload_indexing': 'indicizzazione…', 'video.err_transport': 'Trasporto non connesso', 'video.err_mse': 'Codec non supportato per lo streaming: {codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js index 20c5fd1..272be73 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js @@ -212,6 +212,8 @@ export default { 'video.close': '閉じる（Esc）', 'preview.pdf_fallback': 'このブラウザーはページ内に PDF を表示しません。' + 'ダウンロードしてご覧ください。いずれにせよ復号はここで行われています。', + 'preview.too_large': 'このファイルは {size} で、このページがメモリに保持できる上限（{limit}）を超えています。代わりにダウンロードしてください。ダウンロードはディスクに直接書き込まれます。', + 'download.too_large_for_memory': '「{name}」は {size} です。このブラウザーでこの大きさのファイルを保存するにはディスクへ逐次書き出すしかありませんが、ここではそれができません — ファイル全体をメモリに保持することになります。デスクトップアプリ、または Chrome か Edge をお使いください。', 'group.upload_indexing': 'インデックスを作成中…', 'video.err_transport': 'トランスポートが接続されていません', 'video.err_mse': 'ストリーミング再生に対応していないコーデックです：{codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js index 4b992ae..76f3586 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js @@ -216,6 +216,8 @@ export default { 'video.close': 'Sluiten (Esc)', 'preview.pdf_fallback': 'Deze browser toont de PDF niet in de pagina zelf. Download ' + 'hem in plaats daarvan — ontsleuteld werd hij hoe dan ook hier.', + 'preview.too_large': 'Dit bestand is {size}, meer dan deze pagina in het geheugen kan houden ({limit}). Download het in plaats daarvan — een download wordt rechtstreeks naar schijf geschreven.', + 'download.too_large_for_memory': '“{name}” is {size}. Deze browser kan een bestand van die omvang alleen opslaan door het meteen naar schijf te schrijven, en dat kan hier niet — het hele bestand zou in het geheugen moeten. Gebruik de desktop-app, of Chrome of Edge.', 'group.upload_indexing': 'indexeren…', 'video.err_transport': 'Transport niet verbonden', 'video.err_mse': 'Codec wordt niet ondersteund voor streamen: {codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js index 5389220..dd05487 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js @@ -221,6 +221,8 @@ export default { 'video.close': 'Zamknij (Esc)', 'preview.pdf_fallback': 'Ta przeglądarka nie wyświetli pliku PDF na stronie. Proszę ' + 'go pobrać — i tak został odszyfrowany tutaj.', + 'preview.too_large': 'Ten plik ma {size}, więcej niż ta strona może utrzymać w pamięci ({limit}). Proszę go zamiast tego pobrać — pobieranie jest zapisywane wprost na dysk.', + 'download.too_large_for_memory': '„{name}” ma {size}. Ta przeglądarka może zapisać plik tej wielkości tylko strumieniowo na dysk, a tutaj nie ma takiej możliwości — musiałaby utrzymać cały plik w pamięci. Proszę użyć aplikacji desktopowej albo przeglądarki Chrome lub Edge.', 'group.upload_indexing': 'indeksowanie…', 'video.err_transport': 'Transport nie jest połączony', 'video.err_mse': 'Kodek nieobsługiwany przy odtwarzaniu strumieniowym: {codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js index d72a6e7..4632d36 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js @@ -216,6 +216,8 @@ export default { 'video.close': 'Fechar (Esc)', 'preview.pdf_fallback': 'Este navegador não exibirá o PDF na própria página. Baixe ' + 'o arquivo — de todo modo ele foi descriptografado aqui.', + 'preview.too_large': 'Este arquivo tem {size}, mais do que esta página consegue manter na memória ({limit}). Baixe-o em vez disso — um download é gravado direto no disco.', + 'download.too_large_for_memory': '"{name}" tem {size}. Este navegador só consegue salvar um arquivo desse tamanho gravando-o direto no disco, e aqui ele não tem como — precisaria manter o arquivo inteiro na memória. Use o aplicativo para computador, ou Chrome ou Edge.', 'group.upload_indexing': 'indexando…', 'video.err_transport': 'Transporte não conectado', 'video.err_mse': 'Codec sem suporte para transmissão: {codec}', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js index c672a13..b6ff3c9 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js @@ -209,6 +209,8 @@ export default { 'video.from_start': "从头开始播放", 'video.close': '关闭（Esc）', 'preview.pdf_fallback': '此浏览器不会在页面内显示该 PDF。请改为下载——无论如何它都已在本地解密。', + 'preview.too_large': '该文件为 {size}，超出本页面可在内存中保存的上限（{limit}）。请改为下载——下载会直接写入磁盘。', + 'download.too_large_for_memory': '“{name}”为 {size}。此浏览器只能通过边下边写入磁盘来保存这么大的文件，而这里无法做到——它将不得不把整个文件放在内存中。请使用桌面应用，或 Chrome、Edge。', 'group.upload_indexing': '建立索引中…', 'video.err_transport': '传输未连接', 'video.err_mse': '该编解码器不支持流式播放：{codec}', diff --git a/packages/meshbay-hub/tests/test_memory_ceiling.py b/packages/meshbay-hub/tests/test_memory_ceiling.py new file mode 100644 index 0000000..9966e48 --- /dev/null +++ b/packages/meshbay-hub/tests/test_memory_ceiling.py @@ -0,0 +1,209 @@ +""" +No download above the ceiling is ever collected in the page. + +`pipelinedDownload` with no `writable` allocates `new Array(totalChunks)` and +keeps every decrypted chunk, so whatever `_openDownloadTarget` returns `null` +for is a file held whole in RAM. That floor had no upper bound: the +`!window.showSaveFilePicker` branch returned `null` at any size, so on a browser +without the File System Access API a 20 GB film went to memory whenever the +service-worker path did not answer — which happens for ordinary reasons. The +symptom was the tab dying, with nothing in the source to lead back here. + +The real `_openDownloadTarget` is lifted out of `file-utils.js` **as text** and +executed against stubbed browsers, on the rule this repo already follows for the +video player: model the environment, never the code under test. A test that +transcribed the decision tree would agree with a broken version of it by +construction. + +`test_no_unguarded_memory_floor` is the one that outlives today's branches: it +reads the function and fails if a `return null` appears in it that does not go +through the guard — which is what a fourth fallback added in a hurry would look +like. +""" + +import json +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +STATIC = Path(__file__).resolve().parents[1] / "src" / "meshbay_hub" / "static" +FILE_UTILS = STATIC / "file-utils.js" + +pytestmark = pytest.mark.skipif( + shutil.which("node") is None or not FILE_UTILS.exists(), + reason="node or the SPA sources are not available") + +CEILING = 100 * 1024 * 1024 +GB = 1024 * 1024 * 1024 + + +def _lift(name, source): + """The text of one top-level declaration, from its opening line to the + column-0 brace that closes it. Nothing is re-typed into this test.""" + start = source.index(name) + end = source.index("\n}\n", start) + len("\n}\n") + return source[start:end] + + +@pytest.fixture(scope="module") +def target_fn(): + """The ceiling, its error and the real function — read, never re-typed.""" + src = FILE_UTILS.read_text() + ceiling = re.search(r"^const MEMORY_CEILING = .*?;$", src, re.M) + assert ceiling, "MEMORY_CEILING is gone from file-utils.js" + # The test's own CEILING constant must agree with the source's, or every + # boundary case below is asserting against a number nothing uses. + assert str(CEILING) in ceiling.group(0).replace(" ", "") or \ + eval(ceiling.group(0).split("=")[1].strip(" ;")) == CEILING + return "\n".join([ + ceiling.group(0), + _lift("class TooLargeForMemoryError", src), + _lift("async function _openDownloadTarget", src), + ]) + + +def _run(target_fn, tmp_path, *, size, native=False, granted=False, + streamed=False, picker=False, mode="auto"): + """Drive the real function against one browser shape.""" + script = tmp_path / "case.mjs" + script.write_text(f""" +// Stubs for everything the lifted function reaches. `formatSize` and `t` only +// build the message; the assertions are about which branch was taken. +const formatSize = (n) => `${{n}} B`; +const t = (key, vars) => key + ' ' + JSON.stringify(vars); +const platform = {{ + capabilities: {{ nativeSave: {json.dumps(native)} }}, + nativeSave: async () => ({{ name: 'n', writable: {{}} }}), + bridgeMessage: (e) => String(e), +}}; +const downloads = {{ + BLOB_LIMIT: 512 * 1024 * 1024, + // Called by the refusal to name why the streamed path declined -- absent + // from this stub, the error constructor threw TypeError and the test saw the + // wrong failure entirely. + lastStreamFailure: () => 'stubbed: no streamed target in this harness', + getMode: () => {json.dumps(mode)}, + openTarget: async () => ({json.dumps(granted)} ? {{ name: 'g', writable: {{}} }} : null), + openStreamedDownload: async () => + ({json.dumps(streamed)} ? {{ name: 's', writable: {{}} }} : null), +}}; +globalThis.window = {{}}; +if ({json.dumps(picker)}) {{ + window.showSaveFilePicker = async () => ({{ + name: 'p', createWritable: async () => ({{}}), + }}); +}} + +{target_fn} + +let outcome; +try {{ + const r = await _openDownloadTarget('film.mkv', {size}); + outcome = r === null ? {{ kind: 'memory' }} + : r === false ? {{ kind: 'cancelled' }} + : {{ kind: 'stream', name: r.name }}; +}} catch (err) {{ + outcome = {{ kind: 'refused', name: err.name, message: err.message }}; +}} +console.log(JSON.stringify(outcome)); +""") + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +# ── The hole this was written for ─────────────────────────────────────────── + +def test_a_film_is_refused_rather_than_collected_in_memory(target_fn, tmp_path): + """Firefox/Safari shape: no picker, no granted folder, the worker did not + answer. This returned null — 20 GB into a tab.""" + out = _run(target_fn, tmp_path, size=20 * GB) + assert out["kind"] == "refused", out + assert out["name"] == "TooLargeForMemoryError" + + +def test_the_refusal_says_how_big_and_what_the_limit_is(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=20 * GB) + assert "download.too_large_for_memory" in out["message"] + assert str(20 * GB) in out["message"] + assert str(CEILING) in out["message"] + + +def test_the_same_browser_in_ask_mode_is_refused_too(target_fn, tmp_path): + """'ask' skips the service-worker block entirely, so it reached the + unguarded branch without even trying to stream.""" + out = _run(target_fn, tmp_path, size=20 * GB, mode="ask") + assert out["kind"] == "refused", out + + +# ── What must keep working ────────────────────────────────────────────────── + +def test_something_small_still_uses_the_memory_floor(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=4 * 1024 * 1024) + assert out["kind"] == "memory", out + + +def test_the_boundary_is_the_ceiling_itself(target_fn, tmp_path): + assert _run(target_fn, tmp_path, size=CEILING)["kind"] == "memory" + assert _run(target_fn, tmp_path, size=CEILING + 1)["kind"] == "refused" + + +def test_a_granted_folder_streams_whatever_the_size(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=20 * GB, granted=True) + assert out == {"kind": "stream", "name": "g"} + + +def test_the_service_worker_streams_whatever_the_size(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=20 * GB, streamed=True) + assert out == {"kind": "stream", "name": "s"} + + +def test_the_desktop_app_streams_whatever_the_size(target_fn, tmp_path): + out = _run(target_fn, tmp_path, size=20 * GB, native=True) + assert out == {"kind": "stream", "name": "n"} + + +def test_a_browser_with_a_picker_is_offered_one_instead_of_being_refused( + target_fn, tmp_path): + """Chrome/Edge: the file is large, nothing streamed yet, but Save As does. + A refusal here would be this fix breaking a path that was never broken.""" + out = _run(target_fn, tmp_path, size=20 * GB, picker=True) + assert out == {"kind": "stream", "name": "p"} + + +# ── The one that outlives today's branches ────────────────────────────────── + +def test_no_unguarded_memory_floor(target_fn): + """Every `return null` in the function goes through the guard. + + A fourth fallback appended to the chain — which is exactly how the third one + got here — is caught by this even though no case above covers it. + """ + body = target_fn[target_fn.index("async function _openDownloadTarget"):] + lines = body.splitlines() + # The guard's own `return null` is the one legitimate instance, so cut its + # definition out before looking. Comments go too — the branch that used to + # be the bug is now described in one, and a test that reads prose is the + # mistake already recorded in CLAUDE.md for the packaged systemd unit. + start = next(n for n, l in enumerate(lines) if "const _memoryFloor" in l) + end = next(n for n in range(start, len(lines)) if lines[n].strip() == "};") + rest = lines[:start] + lines[end + 1:] + code = [re.sub(r"//.*$", "", l) for l in rest] + bare = [l.strip() for l in code if re.search(r"\breturn null\b", l)] + assert bare == [], ( + "an unguarded in-memory fallback was added to _openDownloadTarget; " + "return _memoryFloor() instead: " + "; ".join(bare)) + + +def test_the_guard_is_what_the_preview_uses_too(target_fn): + """`FilePreview` decrypts a whole entry with no writable at all, so it needs + the same ceiling — and must import it rather than keep a second number.""" + files_app = (STATIC / "files-app.js").read_text() + assert "MEMORY_CEILING" in files_app + assert re.search(r"entry\.size\s*>\s*MEMORY_CEILING", files_app), ( + "the preview modal must refuse an oversized entry before fetching it") + assert not re.search(r"100\s*\*\s*1024\s*\*\s*1024", files_app), ( + "the ceiling is defined once, in file-utils.js") diff --git a/packages/meshbay-hub/tests/test_zip_size_limit.py b/packages/meshbay-hub/tests/test_zip_size_limit.py index 203c10c..44ad59e 100644 --- a/packages/meshbay-hub/tests/test_zip_size_limit.py +++ b/packages/meshbay-hub/tests/test_zip_size_limit.py @@ -6,11 +6,16 @@ folder as a zip" button — Files' single folder, Files' multi-folder selection, and the Photos album button (docs/photos.md §3) — so the limit is checked once, there, and holds for all of them. -Two things are worth pinning. That an oversized folder is refused *before* +Three things are worth pinning. That an oversized folder is refused *before* `_openDownloadTarget`, because a save dialog for an archive that will never be -written is worse than no dialog at all. And that a folder at exactly the limit +written is worse than no dialog at all. That a folder at exactly the limit still goes through, since an off-by-one here silently costs a whole megabyte -of allowance and nobody would ever notice. +of allowance and nobody would ever notice. And that the two limits in play do +not contradict each other: ZIP_MAX_BYTES (512 MB) bounds the archive, while +MEMORY_CEILING (100 MB, test_memory_ceiling.py) bounds what may be built in the +page — so a 400 MB zip is allowed when there is somewhere to stream it and +refused when the only route left is memory. The `confirm()` that offers the +build-in-memory path therefore only ever appears below the ceiling. """ import json @@ -30,7 +35,7 @@ pytestmark = pytest.mark.skipif( MIB = 1024 * 1024 -def _run(total_bytes, tmp_path): +def _run(total_bytes, tmp_path, picker=False): """ Call downloadDirectory over one folder holding `total_bytes`, and report what it did: the errors it set, how many times it put a question to the @@ -44,6 +49,7 @@ def _run(total_bytes, tmp_path): (tmp_path / "package.json").write_text('{"type":"module"}') script = tmp_path / "case.mjs" + picker_js = "true" if picker else "false" script.write_text(f""" const store = new Map(); globalThis.localStorage = {{ @@ -60,6 +66,17 @@ const out = {{ errors: [], started: 0, asked: 0 }}; // asks first. Answering yes is what lets the at-the-limit case get as far as // starting a transfer, and `asked` is how the refusal proves it never did. globalThis.confirm = () => {{ out.asked += 1; return true; }}; +// With `picker`, the browser can stream to a file the person chooses, which is +// the only legal route for an archive over MEMORY_CEILING. Never exercised — +// the stubbed `transfers.start` below does not run the job — it just has to be +// a target rather than null. +if ({picker_js}) {{ + window.showSaveFilePicker = async () => ({{ + name: 'album.zip', + createWritable: async () => ({{ write: async () => {{}}, close: async () => {{}}, + abort: async () => {{}} }}), + }}); +}} const M = await import('{(sandbox / "file-utils.js").as_posix()}'); @@ -101,7 +118,37 @@ def test_an_oversized_folder_is_refused_before_anything_opens(tmp_path): def test_a_folder_exactly_at_the_limit_still_downloads(tmp_path): - """The bound is inclusive: `> ZIP_MAX_BYTES`, not `>=`.""" - result = _run(512 * MIB, tmp_path) + """The bound is inclusive: `> ZIP_MAX_BYTES`, not `>=`. + + Given somewhere to stream to, because 512 MB is five times MEMORY_CEILING + and building it in the page is no longer a route this code will take. That + is what the next test is about; this one is still only about the off-by-one. + """ + result = _run(512 * MIB, tmp_path, picker=True) assert result["errors"] == [] assert result["started"] == 1 + assert result["asked"] == 0, "nothing is built in memory when it can stream" + + +def test_a_zip_over_the_memory_ceiling_is_refused_when_nothing_streams(tmp_path): + """ + Between the two limits — larger than the page may hold, smaller than the + archive limit — and no way to stream it. Before the ceiling existed this + asked "build it in memory?" and, on yes, held 400 MB in the tab. + + The refusal names the memory ceiling, not the zip limit: quoting 512 MB at + someone whose folder is under 512 MB would be a message about the wrong + rule. + """ + result = _run(400 * MIB, tmp_path) + assert result["started"] == 0 + assert result["asked"] == 0, ( + "the person must not be offered a build-in-memory path above the ceiling") + assert result["errors"] and "group.zip_too_large" not in result["errors"][0] + + +def test_a_small_folder_may_still_be_built_in_memory(tmp_path): + """The floor is intact below the ceiling — that is what it is for.""" + result = _run(4 * MIB, tmp_path) + assert result["errors"] == [] + assert result["asked"] == 1 and result["started"] == 1 -- cgit v1.2.3 From 4b94468d24913c3071b48eeefb43367f4f5cd523 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Tue, 8 Sep 2026 14:53:03 +0200 Subject: feat(node): make the transfer caps settable, node-wide and per group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3 of ~/next/improve-downloads.md. Step 2 built the pools with constants; this gives them to the operator, in the two scopes they belong to. **The pools are the machine's.** `[node] max_concurrent_downloads` and `max_concurrent_uploads`, default 8, on the §2.11 pattern: node.toml for a fresh install, a roster.db override for immediate effect, editable from the Node page and from `meshbay-node transfers show|set`, applied live through the one `set_capacity` step 1 fixed. **The per-member cap is a group's.** How many transfers one member may run at once here — on the node like every other group setting (not the hub, which would have authority over someone else's disk; not node.toml, which is hand-written and needs a restart), changed by a signed operator instruction (`OP_TRANSFER_LIMITS`, subject "d=2,u=2" so what is signed names the outcome), broadcast to the group, and read live by the pools. That was the one thing step 2's shape could not express: `per_member` was a single node-wide number. `group_limits` and `member_cap(kind, member)` make it a lookup — the group's own value if it has one, the node's default otherwise — and it is deliberately the only dimension that is not node-wide. Three refusals, each with a test: - **absent means the default (2), never "unlimited".** A group that predates the setting coming back unlimited would leave the node-wide pool as the only control, which is the situation slots exist to end; - **zero is not "unlimited"**, and is not "this member may not transfer" either: the floor is one everywhere, and the CLI says to revoke the member instead; - **an unreadable row reads as unset**, not as zero — the same discipline the sealed messages follow, where a payload that does not open must never become a default state on its own. `handshake_ack` carries this member's own caps for this group, so the interface can say "2 of your 2 slots are busy" instead of drawing a bare spinner. Absent reads as "no limit known" and the hint is not drawn — never as "unlimited", which would have the interface contradicting the node. 1164 node, 793 hub, 0 failed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HCGdheDLxGReuKHga3BtST --- .../meshbay-common/src/meshbay_common/adminop.py | 6 + .../meshbay-common/src/meshbay_common/protocol.py | 2 + .../src/meshbay_hub/static/locales/de.js | 2 + .../src/meshbay_hub/static/locales/en.js | 2 + .../src/meshbay_hub/static/locales/es.js | 2 + .../src/meshbay_hub/static/locales/fr.js | 2 + .../src/meshbay_hub/static/locales/it.js | 2 + .../src/meshbay_hub/static/locales/ja.js | 2 + .../src/meshbay_hub/static/locales/nl.js | 2 + .../src/meshbay_hub/static/locales/pl.js | 2 + .../src/meshbay_hub/static/locales/pt-BR.js | 2 + .../src/meshbay_hub/static/locales/zh-CN.js | 2 + .../src/meshbay_hub/static/node-page.js | 14 ++ packages/meshbay-node/src/meshbay_node/config.py | 21 +++ packages/meshbay-node/src/meshbay_node/daemon.py | 69 +++++++++- packages/meshbay-node/src/meshbay_node/ops.py | 28 ++++ packages/meshbay-node/src/meshbay_node/roster.py | 33 +++++ .../meshbay-node/src/meshbay_node/transfers.py | 34 ++++- .../src/meshbay_node/transport/webrtc_server.py | 89 ++++++++++++ packages/meshbay-node/tests/test_cli_dispatch.py | 5 + .../meshbay-node/tests/test_transfer_settings.py | 152 +++++++++++++++++++++ 21 files changed, 470 insertions(+), 3 deletions(-) create mode 100644 packages/meshbay-node/tests/test_transfer_settings.py (limited to 'packages/meshbay-hub/src/meshbay_hub') diff --git a/packages/meshbay-common/src/meshbay_common/adminop.py b/packages/meshbay-common/src/meshbay_common/adminop.py index ed6e940..5c7345b 100644 --- a/packages/meshbay-common/src/meshbay_common/adminop.py +++ b/packages/meshbay-common/src/meshbay_common/adminop.py @@ -61,6 +61,12 @@ OP_APPS_ENABLED = "apps_enabled" # security property in itself, but the pattern (every operator setting is # signed) is what keeps the authorization model simple to reason about. OP_SET_SCAN_SETTINGS = "set_scan_settings" +# How many transfers one member may run at once in this group. Signed like the +# rest: an unsigned cap is one any member can raise for themselves, which makes +# the control a suggestion. The subject is "d=2,u=2" so what the operator is +# shown before signing names the outcome and not the operation -- the same rule +# member_upload's on/off subject follows. +OP_TRANSFER_LIMITS = "transfer_limits" # Whether the node uses the operator's own API token/language instead of the # shipped default — node-wide (docs/mediacenter.md §5.5), one credential # shared by every group. Signed like the rest: it turns on outbound diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index 4890d3b..8a521bb 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -152,6 +152,8 @@ class MNP: MEMBER_UPLOAD_ACK = "member_upload_ack" APPS_ENABLED = "apps_enabled" # operator → node: which group apps to show APPS_ENABLED_ACK = "apps_enabled_ack" + TRANSFER_LIMITS = "transfer_limits" # operator → node: per-member caps for this group + TRANSFER_LIMITS_ACK = "transfer_limits_ack" # node → this group: the new caps SET_SCAN_SETTINGS = "set_scan_settings" # operator → node: reconcile/debounce timing SET_SCAN_SETTINGS_ACK = "set_scan_settings_ack" MEDIA_META_REQ = "media_meta_req" # client → node: TMDB metadata for a path diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js index 1ebab8b..8ed1ef0 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js @@ -739,6 +739,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': 'Max. gleichzeitige Downloads', + 'node.setting_max_uploads': 'Max. gleichzeitige Uploads', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js index d85f51b..285ff7a 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js @@ -890,6 +890,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': 'Max concurrent downloads', + 'node.setting_max_uploads': 'Max concurrent uploads', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js index bfe4112..f74c763 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js @@ -734,6 +734,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': 'Descargas simultáneas máximas', + 'node.setting_max_uploads': 'Subidas simultáneas máximas', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js index 9addec5..0c5103f 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js @@ -737,6 +737,8 @@ export default { 'node.setting_pair_ttl': 'Durée du code d\'appairage', 'node.setting_device_ttl': 'Durée des demandes d\'appareil', 'node.setting_max_streams': 'Flux vidéo simultanés max', + 'node.setting_max_downloads': 'Téléchargements simultanés max', + 'node.setting_max_uploads': 'Téléversements simultanés max', 'node.setting_transcode': 'Transcoder les vidéos incompatibles', 'node.setting_unit_hours': 'heures', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js index fe76b9e..ebac541 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js @@ -736,6 +736,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': 'Download simultanei massimi', + 'node.setting_max_uploads': 'Caricamenti simultanei massimi', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js index 272be73..1b49ddb 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js @@ -724,6 +724,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': '同時ダウンロードの上限', + 'node.setting_max_uploads': '同時アップロードの上限', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js index 76f3586..7cf4cd4 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js @@ -738,6 +738,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': 'Max. gelijktijdige downloads', + 'node.setting_max_uploads': 'Max. gelijktijdige uploads', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js index dd05487..6edffd5 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js @@ -756,6 +756,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': 'Maks. równoczesnych pobierań', + 'node.setting_max_uploads': 'Maks. równoczesnych wysyłek', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js index 4632d36..3a6fa22 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js @@ -735,6 +735,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': 'Máximo de downloads simultâneos', + 'node.setting_max_uploads': 'Máximo de envios simultâneos', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js index b6ff3c9..f43ef72 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js @@ -711,6 +711,8 @@ export default { 'node.setting_pair_ttl': 'Pairing code TTL', 'node.setting_device_ttl': 'Device request TTL', 'node.setting_max_streams': 'Max concurrent streams', + 'node.setting_max_downloads': '最大同时下载数', + 'node.setting_max_uploads': '最大同时上传数', 'node.setting_transcode': 'Transcode incompatible video', 'node.setting_unit_hours': 'hours', 'node.setting_unit_minutes': 'min', diff --git a/packages/meshbay-hub/src/meshbay_hub/static/node-page.js b/packages/meshbay-hub/src/meshbay_hub/static/node-page.js index 2e216a9..c1d57f9 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/node-page.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/node-page.js @@ -1043,6 +1043,20 @@ export function NodePage({ groups }) { onInput=${e => setEditSettings(s => ({...s, max_concurrent_streams: parseInt(e.target.value) || 1}))} /> + +