diff options
13 files changed, 162 insertions, 6 deletions
diff --git a/docs/USERGUIDE.md b/docs/USERGUIDE.md index 32f31dd..3c0e30c 100644 --- a/docs/USERGUIDE.md +++ b/docs/USERGUIDE.md @@ -618,8 +618,13 @@ bell, with a progress bar, the current rate, and a cancel button each: Any member can take a whole folder: **⋮ → Download as zip** on the folder's row. The archive is built in the browser as the files arrive and written straight to -disk, so a 40 GB folder costs 40 GB of disk and a few megabytes of memory. +disk, so it costs disk space and only a few megabytes of memory. +- **A folder has to be 512 MB or smaller.** Larger than that and the button + refuses, naming the folder's size — take a subfolder at a time, or the files + individually. This is a deliberate cap, not a technical one: the writer would + happily stream a hundred gigabytes. Selecting several folders at once applies + it to each of them separately, so one oversized folder does not stop the rest. - Nothing is compressed. Group content is video, images and archives — already compressed — so deflating would spend CPU on every byte to save nothing, in the same thread that is decrypting. diff --git a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js index 09ee6c9..241d761 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js @@ -15,6 +15,14 @@ function formatSize(bytes) { return (bytes / (1024 * 1024 * 1024)).toFixed(2) + ' GB'; } +// An arbitrary ceiling on one directory zip. Not a technical limit — the +// writer streams and holds one chunk plus a record per file, so it would +// happily produce a hundred gigabytes — but a deliberate one: past this +// size the honest answer is a subfolder at a time, or the files +// individually. Counted in the same 1024-based units formatSize prints, so +// the number in the refusal is the number in this constant. +const ZIP_MAX_BYTES = 512 * 1024 * 1024; + function formatDate(ts) { return new Date(ts * 1000).toLocaleDateString(undefined, { year: 'numeric', month: 'short', day: 'numeric', @@ -240,10 +248,10 @@ async function downloadEntry(transfers, transport, gek, entry) { /** * Download a directory as a zip, written straight to disk. * - * An archive of a group directory is routinely tens of gigabytes, so it is - * never held anywhere: each file is fetched chunk by chunk, decrypted, and - * handed to the zip writer, which hands it to the file the browser opened. - * Peak memory is one chunk plus one small record per file. + * Nothing is held anywhere: each file is fetched chunk by chunk, decrypted, + * and handed to the zip writer, which hands it to the file the browser + * opened. Peak memory is one chunk plus one small record per file — which is + * why ZIP_MAX_BYTES below is a policy, not a constraint this code has. * * Without the File System Access API there is nowhere to stream to, and the * only alternative is to build the whole thing in memory — so that path is @@ -266,6 +274,21 @@ async function downloadDirectory(transfers, transport, gek, entries, dir, { setE const totalBytes = files.reduce((n, f) => n + (f.entry.size || 0), 0); const suggested = (dir.split('/').pop() || 'files') + '.zip'; + // Checked here rather than by disabling the button: Files zips a whole + // multi-directory selection in one click (`for (const d of selectedDirs)`), + // so the answer is per directory and has to be given where each one is + // actually started — an oversized folder is refused and its siblings still + // download. Before _openDownloadTarget, so nothing opens a save dialog for + // an archive that is not going to be written. + if (totalBytes > ZIP_MAX_BYTES) { + setError(t('group.zip_too_large', { + name: dir.split('/').pop() || dir, + size: formatSize(totalBytes), + limit: formatSize(ZIP_MAX_BYTES), + })); + return; + } + // totalBytes decides how this is delivered, but it is not the archive's // size — headers and the central directory come on top — so it is not // announced as a Content-Length that the download would then miss. @@ -327,7 +350,7 @@ async function downloadDirectory(transfers, transport, gek, entries, dir, { setE export { FILE_ICONS, - formatSize, formatDate, PREVIEWABLE_TEXT, canPreview, CHUNK_SIZE, + formatSize, formatDate, PREVIEWABLE_TEXT, canPreview, CHUNK_SIZE, ZIP_MAX_BYTES, _openDownloadTarget, _saveBlob, pipelinedDownload, downloadEntry, downloadDirectory, }; diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js index e5febde..97a835d 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/de.js @@ -593,6 +593,9 @@ export default { 'group.zip_no_stream': 'Dieser Browser kann einen Download nicht direkt auf die ' + 'Festplatte schreiben, daher muss {name} ({size}) zuerst im Arbeitsspeicher ' + 'aufgebaut werden. Bei einem großen Archiv kann das fehlschlagen. Fortfahren?', + 'group.zip_too_large': '„{name}“ ist {size} groß, und ein Ordner lässt sich nur bis {limit} als ' + + 'Zip herunterladen. Nehmen Sie einen Unterordner nach dem anderen oder die ' + + 'Dateien einzeln.', 'group.rmdir': 'Ordner löschen', 'group.rmdir_confirm': 'Den Ordner „{name}“ löschen? Er muss leer sein.', 'group.rmdir_not_empty': 'Dieser Ordner ist nicht leer. Löschen Sie zuerst seinen ' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js index a171304..c55a702 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/en.js @@ -709,6 +709,8 @@ export default { 'group.zip_no_stream': 'This browser cannot write a download straight to disk, ' + 'so {name} ({size}) has to be built in memory first. On a large archive ' + 'that may fail. Continue?', + 'group.zip_too_large': '"{name}" is {size}, and a folder can only be downloaded as a zip up to ' + + '{limit}. Take a subfolder at a time, or the files individually.', 'group.rmdir': 'Delete folder', 'group.rmdir_confirm': 'Delete the folder "{name}"? It must be empty.', 'group.rmdir_not_empty': 'That folder is not empty. Delete what is in it first — ' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js index 3b081cf..9ce3900 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/es.js @@ -589,6 +589,8 @@ export default { 'group.zip_no_stream': 'Este navegador no puede escribir una descarga directamente ' + 'en el disco, así que {name} ({size}) tiene que construirse antes en memoria. Con ' + 'un archivo comprimido grande eso puede fallar. ¿Continuar?', + 'group.zip_too_large': '«{name}» ocupa {size}, y una carpeta solo puede descargarse en zip hasta ' + + '{limit}. Descargue una subcarpeta cada vez, o los archivos por separado.', 'group.rmdir': 'Eliminar la carpeta', 'group.rmdir_confirm': '¿Eliminar la carpeta «{name}»? Debe estar vacía.', 'group.rmdir_not_empty': 'Esa carpeta no está vacía. Elimine antes lo que contiene — ' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js index 7ecd4a2..d612f71 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/fr.js @@ -592,6 +592,8 @@ export default { 'group.zip_no_stream': 'Ce navigateur ne sait pas écrire un téléchargement ' + 'directement sur le disque : {name} ({size}) doit donc être assemblé en mémoire ' + 'au préalable. Sur une archive volumineuse, cela peut échouer. Continuer ?', + 'group.zip_too_large': '« {name} » fait {size}, or un dossier ne peut être téléchargé en zip ' + + 'que jusqu’à {limit}. Prenez un sous-dossier à la fois, ou les fichiers un par un.', 'group.rmdir': 'Supprimer le dossier', 'group.rmdir_confirm': 'Supprimer le dossier « {name} » ? Il doit être vide.', 'group.rmdir_not_empty': 'Ce dossier n’est pas vide. Supprimez d’abord ce qu’il ' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js index 6ae92cc..c61b8e5 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/it.js @@ -592,6 +592,8 @@ export default { 'group.zip_no_stream': 'Questo browser non sa scrivere un download direttamente su ' + 'disco, quindi {name} ({size}) deve prima essere costruito in memoria. Con un ' + 'archivio grande l’operazione può fallire. Continuare?', + 'group.zip_too_large': '«{name}» occupa {size} e una cartella può essere scaricata in zip solo fino ' + + 'a {limit}. Prendi una sottocartella alla volta, o i file singolarmente.', 'group.rmdir': 'Elimina la cartella', 'group.rmdir_confirm': 'Eliminare la cartella «{name}»? Deve essere vuota.', 'group.rmdir_not_empty': 'Quella cartella non è vuota. Elimini prima ciò che ' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js index e5a4a21..20c5fd1 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/ja.js @@ -582,6 +582,8 @@ export default { 'group.zip_no_stream': 'このブラウザーはダウンロードをディスクへ直接書き込めないため、' + '{name}（{size}）をいったんメモリー上で組み立てる必要があります。' + '大きなアーカイブでは失敗することがあります。続けますか？', + 'group.zip_too_large': '「{name}」は {size} あり、フォルダーを zip でダウンロードできるのは ' + + '{limit} までです。サブフォルダーを一つずつ、またはファイルを個別にダウンロードしてください。', 'group.rmdir': 'フォルダーを削除', 'group.rmdir_confirm': 'フォルダー「{name}」を削除しますか？空である必要があります。', 'group.rmdir_not_empty': 'そのフォルダーは空ではありません。先に中身を削除してください。' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js index 7bee011..4b992ae 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/nl.js @@ -593,6 +593,8 @@ export default { 'group.zip_no_stream': 'Deze browser kan een download niet rechtstreeks naar schijf ' + 'schrijven, dus {name} ({size}) moet eerst in het geheugen worden opgebouwd. Bij ' + 'een groot archief kan dat mislukken. Doorgaan?', + 'group.zip_too_large': '"{name}" is {size} en een map kan alleen tot {limit} als zip worden ' + + 'gedownload. Neem één submap per keer, of de bestanden afzonderlijk.', 'group.rmdir': 'Map verwijderen', 'group.rmdir_confirm': 'De map "{name}" verwijderen? Ze moet leeg zijn.', 'group.rmdir_not_empty': 'Die map is niet leeg. Verwijder eerst wat erin staat — ' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js index ddcf9bc..5389220 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pl.js @@ -609,6 +609,8 @@ export default { 'group.zip_no_stream': 'Ta przeglądarka nie potrafi zapisywać pobieranych danych ' + 'wprost na dysk, więc {name} ({size}) trzeba najpierw zbudować w pamięci. Przy ' + 'dużym archiwum może się to nie udać. Kontynuować?', + 'group.zip_too_large': '„{name}” ma {size}, a folder można pobrać jako zip tylko do {limit}. ' + + 'Pobieraj po jednym podfolderze albo pliki pojedynczo.', 'group.rmdir': 'Usuń folder', 'group.rmdir_confirm': 'Usunąć folder „{name}”? Musi być pusty.', 'group.rmdir_not_empty': 'Ten folder nie jest pusty. Proszę najpierw usunąć jego ' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js index ce31383..d72a6e7 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/pt-BR.js @@ -591,6 +591,8 @@ export default { 'group.zip_no_stream': 'Este navegador não consegue escrever um download direto no ' + 'disco, então {name} ({size}) precisa ser montado antes na memória. Em um arquivo ' + 'grande isso pode falhar. Continuar?', + 'group.zip_too_large': '"{name}" tem {size}, e uma pasta só pode ser baixada em zip até {limit}. ' + + 'Baixe uma subpasta por vez, ou os arquivos individualmente.', 'group.rmdir': 'Excluir a pasta', 'group.rmdir_confirm': 'Excluir a pasta "{name}"? Ela precisa estar vazia.', 'group.rmdir_not_empty': 'Essa pasta não está vazia. Exclua antes o que está dentro ' diff --git a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js index 260d11c..c672a13 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/locales/zh-CN.js @@ -569,6 +569,8 @@ export default { 'group.zip_empty': '该文件夹里没有可下载的内容。', 'group.zip_no_stream': '此浏览器无法把下载内容直接写入磁盘，' + '因此 {name}（{size}）必须先在内存中构建。若压缩包很大，这可能会失败。是否继续？', + 'group.zip_too_large': '“{name}” 为 {size}，而文件夹打包为 zip 下载的上限是 {limit}。' + + '请逐个下载子文件夹，或单独下载文件。', 'group.rmdir': '删除文件夹', 'group.rmdir_confirm': '删除文件夹“{name}”？它必须是空的。', 'group.rmdir_not_empty': '该文件夹不是空的。请先删除其中的内容——' diff --git a/packages/meshbay-hub/tests/test_zip_size_limit.py b/packages/meshbay-hub/tests/test_zip_size_limit.py new file mode 100644 index 0000000..203c10c --- /dev/null +++ b/packages/meshbay-hub/tests/test_zip_size_limit.py @@ -0,0 +1,107 @@ +""" +The arbitrary ceiling on a directory zip. + +`downloadDirectory` is the one implementation behind every "download this +folder as a zip" button — Files' single folder, Files' multi-folder selection, +and the Photos album button (docs/photos.md §3) — so the limit is checked +once, there, and holds for all of them. + +Two things are worth pinning. That an oversized folder is refused *before* +`_openDownloadTarget`, because a save dialog for an archive that will never be +written is worse than no dialog at all. And that a folder at exactly the limit +still goes through, since an off-by-one here silently costs a whole megabyte +of allowance and nobody would ever notice. +""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +STATIC = Path(__file__).resolve().parents[1] / "src" / "meshbay_hub" / "static" +FILE_UTILS = STATIC / "file-utils.js" + +pytestmark = pytest.mark.skipif( + shutil.which("node") is None or not FILE_UTILS.exists(), + reason="node or the SPA sources are not available") + +MIB = 1024 * 1024 + + +def _run(total_bytes, tmp_path): + """ + Call downloadDirectory over one folder holding `total_bytes`, and report + what it did: the errors it set, how many times it put a question to the + person, and how many transfers it started. + """ + sandbox = tmp_path / "static" + sandbox.mkdir() + for src in STATIC.glob("*.js"): + (sandbox / src.name).write_text(src.read_text(encoding="utf-8"), + encoding="utf-8") + (tmp_path / "package.json").write_text('{"type":"module"}') + + script = tmp_path / "case.mjs" + script.write_text(f""" +const store = new Map(); +globalThis.localStorage = {{ + getItem: k => (store.has(k) ? store.get(k) : null), + setItem: (k, v) => store.set(k, String(v)), + removeItem: k => store.delete(k), +}}; +// Node 22 defines `navigator` itself, so it is left alone; `window` is what +// platform.js reaches for to decide it is not running in the desktop app. +globalThis.window = globalThis; +const out = {{ errors: [], started: 0, asked: 0 }}; +// Reached only once the size check has passed: with no File System Access API +// under Node, downloadDirectory falls through to its build-in-memory path and +// asks first. Answering yes is what lets the at-the-limit case get as far as +// starting a transfer, and `asked` is how the refusal proves it never did. +globalThis.confirm = () => {{ out.asked += 1; return true; }}; + +const M = await import('{(sandbox / "file-utils.js").as_posix()}'); + +const transfers = {{ start: () => {{ out.started += 1; }} }}; +const transport = {{ connected: true }}; +// One file, in the folder itself — entriesUnder keys on `path`. +const entries = [{{ id: 'f1', name: 'big.bin', path: 'album', + size: {total_bytes}, added_at: 0 }}]; + +await M.downloadDirectory(transfers, transport, null, entries, 'album', {{ + setError: (m) => out.errors.push(m), +}}); + +out.limit = M.ZIP_MAX_BYTES; +console.log(JSON.stringify(out)); +""", encoding="utf-8") + + proc = subprocess.run(["node", str(script)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +def test_the_limit_is_512_mib(tmp_path): + """The number the refusal quotes is the number the module enforces.""" + assert _run(0, tmp_path)["limit"] == 512 * MIB + + +def test_an_oversized_folder_is_refused_before_anything_opens(tmp_path): + """ + One byte over. Nothing is started, and the person is told why — a silent + return would read as a dead button. + """ + result = _run(512 * MIB + 1, tmp_path) + assert result["started"] == 0, "no transfer may begin" + assert result["asked"] == 0, "nor may a save dialog have been put up first" + assert result["errors"] == ["group.zip_too_large"], ( + "the refusal must name its own key; t() falls back to the key with no " + "catalogue loaded, and test_locales.py holds the ten translations of it") + + +def test_a_folder_exactly_at_the_limit_still_downloads(tmp_path): + """The bound is inclusive: `> ZIP_MAX_BYTES`, not `>=`.""" + result = _run(512 * MIB, tmp_path) + assert result["errors"] == [] + assert result["started"] == 1 |