Merge branch 'fix/large-download-paths'

Concurrent-transfer limits, with the queue, the pause and the flag day.

A node now caps how many transfers it runs at once (8 downloads, 8 uploads,
node-wide) and how many one member may run in one group (2 by default,
operator-signed). Beyond that the node answers "queued" and the client waits its
turn, visibly, in the transfers panel — and a slot that frees starts whatever is
next, skipping past a member who is at their own cap rather than letting them
stall everyone behind them.

Browsing is never subject to a slot: not the poster grid, not the covers, not
opening a photo to look at it. That is structural — a transfer is what the
transfers widget shows — and the exemption is bounded rather than open, at two
files in flight per session, because an exemption with no bound is a leaseless
branch under another name.

Transfers can be cancelled, and now paused and resumed. A paused one holds
nothing: its slot goes back at once and resuming rejoins the queue at the tail.
Uploads survive the connection that started them and resume where the node
stopped, asked for inside the seal rather than on a clear message. What they
leave behind when they are abandoned is reaped, which closes a disk leak that
predates this work.

MNP 3.0 makes the lease compulsory and refuses 2.x at the handshake, with the
desktop client checking `client.minimum` before connecting so an un-updated one
says "update" instead of failing every connection in a protocol vocabulary.

Fourteen defects were found on the way, eight of them by a person clicking
Download and pasting a console — none of which 2075 tests could reach. Section
12 of ~/next/improve-downloads.md is that report, including the three this work
introduced itself and the one that turned out to be caused by an instruction to
hard-reload after each deployment.

Node suite 1209 passed, hub suite 866 passed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HCGdheDLxGReuKHga3BtST

1 files changed, 309 insertions, 45 deletions

diff --git a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js
index 241d761..99d9f9a 100644
--- a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js
+++ b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js
@@ -47,15 +47,64 @@ const CHUNK_SIZE = 1024 * 1024;
 // that hesitates, and looks like a hang while it is quiet.
 const PIPELINE_WINDOW = 8;
 
+// The most this code will ever collect in the page.
+//
+// Below every streaming target there is a floor: `pipelinedDownload` with no
+// `writable` allocates `new Array(totalChunks)` and keeps every decrypted
+// chunk, and `_saveBlob` hands the lot to the browser. That floor is fine for
+// something small and is a dead tab for a film. It had **no upper bound**: the
+// `!window.showSaveFilePicker` branch below returned null at any size, so on a
+// browser without the File System Access API (Firefox, Safari) a 20 GB film
+// went to RAM whenever the service-worker path did not answer — which happens
+// for ordinary reasons (an uncontrolled page, a stream that cannot be
+// transferred, the 8 s timeout). Nothing logged, nothing refused; the symptom
+// was the tab dying, with no error attributable to this code.
+//
+// So: above this, there is no floor. A refusal naming what happened is
+// recoverable and a dead tab is not. `CLAUDE.md`'s standing lesson is that a
+// fallback chain reaches its floor silently — this is that floor being given a
+// bottom.
+const MEMORY_CEILING = 100 * 1024 * 1024;
+
+/**
+ * Thrown instead of falling through to the in-memory floor.
+ *
+ * This should now be unreachable in ordinary use: the streamed path is primed
+ * at application start and retried on demand, so a browser with a service
+ * worker has somewhere to write whatever the size. If it is ever raised, the
+ * reason the streamed path declined is appended — untranslated, because it is a
+ * diagnostic and a vague failure is what made the original bug invisible.
+ */
+class TooLargeForMemoryError extends Error {
+  constructor(filename, size) {
+    const why = downloads.lastStreamFailure();
+    super(t('download.too_large_for_memory', {
+      name: filename, size: formatSize(size), limit: formatSize(MEMORY_CEILING),
+    }) + (why ? ` (${why})` : ''));
+    this.name = 'TooLargeForMemoryError';
+  }
+}
+
 /**
  * Open somewhere to write, honouring the user's download setting.
  *
  * Returns a target ({writable, name}), null for "no stream available — collect
  * it and hand the browser a blob", or false for "the person dismissed the
  * dialog", which is not an error and must not start a transfer.
+ *
+ * **Never returns null above MEMORY_CEILING.** Every `return null` below is
+ * guarded by `_memoryFloor`, which throws instead. A fourth fallback added
+ * later must go through it too — `test_memory_ceiling.py` fails the build if a
+ * bare `return null` appears in this function.
  */
 async function _openDownloadTarget(filename, size = 0, pickerOpts = {},
-                                   swSize = size) {
+                                   swSize = size, { batched = false } = {}) {
+  // "Collect it in the page", or a refusal when that would be too much.
+  const _memoryFloor = () => {
+    if (size > MEMORY_CEILING) throw new TooLargeForMemoryError(filename, size);
+    return null;
+  };
+
   // On a desktop build this is the whole answer, and it comes first.
   //
   // The two browser paths below are both unavailable there — `showDirectoryPicker`
@@ -69,7 +118,10 @@ async function _openDownloadTarget(filename, size = 0, pickerOpts = {},
         filename, { auto: downloads.getMode() === 'auto' });
       // Null means the person dismissed the dialog, which is not an error and
       // must not start a transfer.
-      return native || false;
+      //
+      // The desktop sink is an open file stream in the main process: not
+      // writing to it for a while costs nothing and loses nothing.
+      return native ? { ...native, pausable: true } : false;
     } catch (err) {
       console.warn('[MeshBay] native save failed:', platform.bridgeMessage(err));
       return false;
@@ -87,25 +139,143 @@ async function _openDownloadTarget(filename, size = 0, pickerOpts = {},
   // write, which is how this works at all in Firefox: the alternative there is
   // to collect gigabytes in a tab. It goes to the browser's own download
   // folder, without a dialog, which is what "save automatically" meant.
-  if (downloads.getMode() === 'auto') {
+  //
+  // Tried in "ask" mode too when the file is large and this browser has no Save
+  // As of its own. The mode is about whether to show a dialog; it was never
+  // meant to decide whether a 20 GB film can be downloaded at all, and on
+  // Firefox and Safari — where `showSaveFilePicker` does not exist — skipping
+  // this block left nothing but the in-memory floor. A preference must not cost
+  // a capability.
+  const canPick = typeof window.showSaveFilePicker === 'function';
+  // `batched` means this is not the first download of a batch, and it makes the
+  // streamed path preferred whatever the mode.
+  //
+  // "Ask where to save" asks per file, which is right for one file and wrong
+  // for four: a browser grants one picker per user gesture, so the second
+  // dialog has no gesture behind it and the third and fourth wait behind a
+  // dialog that waits for a human — reported from Chrome as three downloads
+  // frozen. There is no gesture left to spend, so there is nothing to lose by
+  // streaming instead: the file still lands on disk, in the browser's own
+  // download folder. Only the choice of folder goes, and it was not on offer.
+  if (downloads.getMode() === 'auto' || batched
+      || (size > MEMORY_CEILING && !canPick)) {
     const streamed = await downloads.openStreamedDownload(filename, swSize);
     if (streamed) return streamed;
-    // Nothing to stream to: small enough for memory, and no dialog.
-    if (size < downloads.BLOB_LIMIT) return null;
+    // Nothing to stream to: small enough for memory, and no dialog. The old
+    // comparison here was against downloads.BLOB_LIMIT (512 MB), five times
+    // this ceiling — and it was the *only* size test in the whole chain, with
+    // the branch below it unguarded.
+    if (size <= MEMORY_CEILING) return _memoryFloor();
   }
 
-  if (!window.showSaveFilePicker) return null;
+  // No File System Access API — Firefox, Safari. This is the branch that used
+  // to return null at any size.
+  if (!canPick) return _memoryFloor();
+  // A dialog is the one outcome nobody can diagnose after the fact: it looks
+  // the same whether it was asked for, or fallen back to because the worker
+  // did not answer. Say which, once per download, so the next report from a
+  // browser we do not have does not need a second round trip.
+  console.info('[MeshBay] asking where to save %s — mode=%s batched=%s stream=%s',
+               filename, downloads.getMode(), batched,
+               downloads.lastStreamFailure() || 'not attempted');
   try {
     const handle = await window.showSaveFilePicker({
       suggestedName: filename, ...pickerOpts,
     });
-    return { writable: await handle.createWritable(), name: handle.name || filename };
+    return { writable: await handle.createWritable(),
+             name: handle.name || filename, pausable: true };
   } catch (err) {
     if (err.name === 'AbortError') return false;
+    // "Must be handling a user gesture to show a file picker."
+    //
+    // A browser grants one picker per gesture, and downloading three files is
+    // one gesture. So the second and third throw this, and the person sees a
+    // failed transfer with a message from Chrome about gestures, for having
+    // done something entirely reasonable.
+    //
+    // The streamed path needs no gesture at all, which makes it the right
+    // answer here rather than a consolation: the file still lands on disk, in
+    // the browser's own download folder, written as it arrives. Only the choice
+    // of folder is lost, and it was already lost — there was no picker to make
+    // it in.
+    if (err.name === 'SecurityError' || /user gesture/i.test(err.message || '')) {
+      console.warn('[MeshBay] no gesture left for a save dialog; streaming '
+                   + 'this one to the download folder instead');
+      const streamed = await downloads.openStreamedDownload(filename, swSize);
+      if (streamed) return streamed;
+      if (size <= MEMORY_CEILING) return _memoryFloor();
+    }
     throw err;
   }
 }
 
+// Target openings run one at a time, across every download on the page.
+//
+// A browser shows one file picker at a time and grants one per user gesture, so
+// four downloads asking at once get one dialog and three failures. That used to
+// be prevented by accident: `downloadEntry` awaited the target inline, and
+// files-app.js's `for (const e of selected) await downloadFile(e)` serialised
+// them. Opening the target inside `prepare` — so the row appears at the click
+// instead of tens of seconds later — removed that accident, and four pickers
+// raced. Chrome showed one, prompted for a second, and the rest timed out;
+// Firefox and Electron never noticed, because neither opens a picker at all.
+//
+// So the queue is explicit now, and it is the *targets* that queue, not the
+// rows: every download still appears the moment it is asked for.
+//
+// Two things keep the queue from becoming the problem it was meant to solve.
+// It only ever holds openings that could actually put a dialog on screen, and
+// no opening waits behind another for longer than a budget.
+let _targetQueue = Promise.resolve();
+
+let _targetsInFlight = 0;
+
+// How long an opening waits for the one ahead of it before going anyway.
+//
+// A queue with no bound is a way for one stuck opening to freeze every later
+// download for the life of the page, since `_targetQueue` is never reset. That
+// is what turned a slow first download into four rows stuck at "preparing" on
+// Firefox. Generous, because a dialog legitimately waits for a person and
+// cutting in front of one would be worse than waiting; finite, because the
+// alternative is a download panel that never recovers.
+//
+// Going anyway is safe: whatever was ahead is still the only unbatched opening,
+// so the one released here takes the streamed path and opens no second dialog.
+const TARGET_QUEUE_BUDGET_MS = 90000;
+
+function _openTargetInTurn(filename, size, pickerOpts, swSize) {
+  // Only an opening that could show a dialog has any reason to wait. Firefox
+  // and Safari have no `showSaveFilePicker` at all, so nothing there can race
+  // anything, and queueing them bought nothing while costing everything: four
+  // downloads that used to open their targets at the same time became four
+  // that waited on the slowest.
+  const canPick = typeof window !== 'undefined'
+    && typeof window.showSaveFilePicker === 'function';
+  if (!canPick) return _openDownloadTarget(filename, size, pickerOpts, swSize);
+
+  // Anything that has to wait its turn is, by definition, not the first of the
+  // batch — so it will not be the one holding the user's gesture.
+  const batched = _targetsInFlight > 0;
+  _targetsInFlight += 1;
+  const mine = _waitBriefly(_targetQueue, TARGET_QUEUE_BUDGET_MS)
+    .then(() => _openDownloadTarget(filename, size, pickerOpts, swSize,
+                                    { batched }))
+    .finally(() => { _targetsInFlight -= 1; });
+  // The chain must not break on a rejection, or one refused download stops
+  // every later one from ever opening a target.
+  _targetQueue = mine.catch(() => {});
+  return mine;
+}
+
+/** Settles with `promise`, or after `ms`, whichever comes first. */
+function _waitBriefly(promise, ms) {
+  return new Promise((resolve) => {
+    const timer = setTimeout(resolve, ms);
+    promise.then(() => { clearTimeout(timer); resolve(); },
+                 () => { clearTimeout(timer); resolve(); });
+  });
+}
+
 /** The download of last resort, for browsers with no way to stream to disk. */
 function _saveBlob(blob, filename) {
   const url = URL.createObjectURL(blob);
@@ -131,17 +301,47 @@ function _saveBlob(blob, filename) {
 const CHUNK_RETRY_ATTEMPTS = 6;
 const CHUNK_RETRY_DELAY_MS = 1500;
 
+// How long one megabyte may take to reach the disk before we call it stuck.
+//
+// Every other await on this path is bounded and says so when it expires:
+// `_sendAndWait` logs a Response timeout, `_fetchChunkResilient` retries and
+// then throws. `writable.write()` was the exception — a sink that stops
+// consuming (a service-worker stream the browser has stopped reading, a file
+// handle that has gone away) leaves it pending for ever. It never rejects, so
+// there is no error, no log and no failed transfer: the progress bar simply
+// stops, the console stays empty, and the node is perfectly healthy the whole
+// time, which is what made this invisible.
+//
+// Generous on purpose. A megabyte takes milliseconds on any working sink; a
+// minute means the sink is gone, not slow.
+const WRITE_STALL_MS = 60000;
+
+/** `writable.write`, but it fails instead of hanging for ever. */
+async function _writeOrStall(writable, bytes, at) {
+  let timer = 0;
+  const stalled = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(
+      t('group.download_write_stalled', { seconds: WRITE_STALL_MS / 1000 })
+      + ` (chunk ${at})`)), WRITE_STALL_MS);
+  });
+  try {
+    await Promise.race([writable.write(bytes), stalled]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 function _isRetryableTransportError(err) {
   return err.name === 'TransportLostError'
     || err.message === 'Response timeout'
     || (err.message || '').startsWith('DataChannel not open');
 }
 
-async function _fetchChunkResilient(transport, fileId, index) {
+async function _fetchChunkResilient(transport, fileId, index, tr = '') {
   let lastErr;
   for (let attempt = 0; attempt < CHUNK_RETRY_ATTEMPTS; attempt++) {
     try {
-      return await transport.fetchChunk(fileId, index);
+      return await transport.fetchChunk(fileId, index, tr);
     } catch (err) {
       if (!_isRetryableTransportError(err)) throw err;
       lastErr = err;
@@ -154,14 +354,15 @@ async function _fetchChunkResilient(transport, fileId, index) {
 }
 
 async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk,
-                                 writable, signal) {
-  const results = writable ? null : new Array(totalChunks);
-  let nextSend = 0, nextRecv = 0;
+                                 writable, signal, tr = '', fromChunk = 0,
+                                 results = null) {
+  if (!writable && !results) results = new Array(totalChunks);
+  let nextSend = fromChunk, nextRecv = fromChunk;
   const inflight = new Array(totalChunks);
 
   const fire = () => {
     while (nextSend < totalChunks && nextSend - nextRecv < PIPELINE_WINDOW) {
-      inflight[nextSend] = _fetchChunkResilient(transport, fileId, nextSend);
+      inflight[nextSend] = _fetchChunkResilient(transport, fileId, nextSend, tr);
       nextSend++;
     }
   };
@@ -173,6 +374,21 @@ async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk
       err.name = 'AbortError';
       throw err;
     }
+    // Between two chunks, never inside one. Everything written so far is a
+    // whole number of chunks, which is what makes resuming exact rather than
+    // approximate — `fromChunk` is a position, not an estimate, and a resumed
+    // file is never appended to at an offset nobody checked.
+    //
+    // The chunks already in flight past this point are abandoned and asked for
+    // again on resume: at most one pipeline window of duplicated traffic, in
+    // exchange for not having to hold a half-received window across a pause of
+    // unknown length.
+    if (signal && signal.paused) {
+      signal.resumeFrom = nextRecv;
+      const err = new Error('Paused');
+      err.name = 'PausedError';
+      throw err;
+    }
     const chunkMsg = await inflight[nextRecv];
     // One shape, and a refusal for anything else. There used to be two fallbacks
     // below this: a base64 `ct_b64` chunk, which was the real wire format until
@@ -192,7 +408,7 @@ async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk
     const plaintext = await window.MeshBayCrypto.decryptChunkBin(
       gekKey, fileId, nextRecv, chunkMsg.nonce, chunkMsg.ct);
     if (writable) {
-      await writable.write(plaintext);
+      await _writeOrStall(writable, plaintext, nextRecv);
     } else {
       results[nextRecv] = plaintext;
     }
@@ -210,33 +426,67 @@ async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk
  * download button — both just want "get this entry to disk".
  */
 async function downloadEntry(transfers, transport, gek, entry) {
-  const target = await _openDownloadTarget(entry.name, entry.size);
-  if (target === false) return;          // the picker was dismissed
-
+  const totalChunks = Math.ceil(entry.size / CHUNK_SIZE);
   const openRef = { url: null };
+  let target = null;
+  // The in-memory fallback's accumulator, held out here so a pause does not
+  // discard what has already been decrypted.
+  const memoryChunks = new Array(totalChunks);
+
   transfers.start({
-    kind: 'download', name: (target && target.name) || entry.name,
-    total: entry.size, transport,
-    open: target
-      ? (target.open || null)
-      : () => { if (openRef.url) window.open(openRef.url, '_blank'); },
-    run: async ({ signal, onProgress }) => {
-      const totalChunks = Math.ceil(entry.size / CHUNK_SIZE);
-      let done = 0;
+    kind: 'download', name: entry.name, total: entry.size, transport,
+
+    // The row exists from the click. Opening a target is what takes the time —
+    // the streamed path waits for the worker (twice), a Save As dialog waits
+    // for a person — and doing it before the row meant three clicks produced no
+    // panel at all and then several rows at once.
+    prepare: async () => {
+      target = await _openTargetInTurn(entry.name, entry.size);
+      // Dismissed: nothing was started, so nothing is left on screen.
+      if (target === false) return false;
+      // `pausable` travels with the target, because only the target knows. The
+      // in-memory fallback (a null target) is just an array and pauses fine.
+      return target ? { name: target.name, pausable: !!target.pausable }
+                    : { pausable: true };
+    },
+
+    // After the target, never before: a granted slot has to be taken up within
+    // the node's deadline, and opening a target can outlast it. See §8.1 of
+    // ~/next/improve-downloads.md — the other order was tried and cost two of
+    // three downloads.
+    makeLease: () => transport.openTransfer({
+      kind: 'download', bytes: entry.size, chunks: totalChunks }),
+
+    open: () => (target && target.open) ? target.open()
+      : (openRef.url ? window.open(openRef.url, '_blank') : undefined),
+
+    // Kept across a pause: the chunks collected so far on the in-memory path.
+    // A resumed run fills in from where it stopped rather than starting a
+    // second array and throwing the first away.
+    run: async ({ signal, onProgress, lease, from = 0 }) => {
+      let done = from * CHUNK_SIZE;
       const onChunk = (bytes) => { done += bytes; onProgress(done, entry.size); };
 
       if (target) {
         try {
           await pipelinedDownload(transport, gek, entry.id, totalChunks,
-                                  onChunk, target.writable, signal);
+                                  onChunk, target.writable, signal,
+                                  lease && lease.tr, from);
           await target.writable.close();
         } catch (err) {
-          await target.writable.abort().catch(() => {});
+          // A pause is not a failure, and the target must survive it: aborting
+          // here would delete the `.part` (Electron) or the file just created
+          // in the granted folder, and resuming would then have nothing to
+          // continue. Only a real end tears the target down.
+          if (err.name !== 'PausedError') {
+            await target.writable.abort().catch(() => {});
+          }
           throw err;
         }
       } else {
         const chunks = await pipelinedDownload(
-          transport, gek, entry.id, totalChunks, onChunk, null, signal);
+          transport, gek, entry.id, totalChunks, onChunk, null, signal,
+          lease && lease.tr, from, memoryChunks);
         const blob = new Blob(chunks);
         _saveBlob(blob, entry.name);
         openRef.url = URL.createObjectURL(blob);
@@ -292,25 +542,37 @@ async function downloadDirectory(transfers, transport, gek, entries, dir, { setE
   // totalBytes decides how this is delivered, but it is not the archive's
   // size — headers and the central directory come on top — so it is not
   // announced as a Content-Length that the download would then miss.
-  const target = await _openDownloadTarget(suggested, totalBytes, {
-    types: [{ description: 'ZIP archive',
-              accept: { 'application/zip': ['.zip'] } }],
-  }, 0);
-  if (target === false) return;
-  if (!target && !confirm(t('group.zip_no_stream', {
-    size: formatSize(totalBytes), name: suggested,
-  }))) {
-    return;
-  }
   const zipOpenRef = { url: null };
+  let target = null;
 
   transfers.start({
-    kind: 'download', name: (target && target.name) || suggested,
-    total: totalBytes, transport,
-    open: target
-      ? (target.open || null)
-      : () => { if (zipOpenRef.url) window.open(zipOpenRef.url, '_blank'); },
-    run: async ({ signal, onProgress }) => {
+    kind: 'download', name: suggested, total: totalBytes, transport,
+
+    // Same order as downloadEntry: the row first, then the target, then the
+    // slot. A folder of forty files is exactly where the wait is longest.
+    prepare: async () => {
+      target = await _openTargetInTurn(suggested, totalBytes, {
+        types: [{ description: 'ZIP archive',
+                  accept: { 'application/zip': ['.zip'] } }],
+      }, 0);
+      if (target === false) return false;
+      if (!target && !confirm(t('group.zip_no_stream', {
+        size: formatSize(totalBytes), name: suggested,
+      }))) {
+        return false;
+      }
+      return target ? { name: target.name } : true;
+    },
+
+    // **One** lease for the archive, not one per file. Dozens of leases for a
+    // folder would deadlock against the member's own cap: the job cannot finish
+    // until it holds them all, and it can never hold more than two.
+    makeLease: () => transport.openTransfer({
+      kind: 'download', bytes: totalBytes, chunks: files.length }),
+
+    open: () => (target && target.open) ? target.open()
+      : (zipOpenRef.url ? window.open(zipOpenRef.url, '_blank') : undefined),
+    run: async ({ signal, onProgress, lease }) => {
       const writable = target ? target.writable : null;
       const parts = writable ? null : [];
       let written = 0;
@@ -330,7 +592,8 @@ async function downloadDirectory(transfers, transport, gek, entries, dir, { setE
             transport, gek, entry.id, totalChunks,
             (bytes) => { written += bytes; onProgress(written, totalBytes); },
             // pipelinedDownload writes in order, which the archive needs.
-            { write: (plaintext) => zip.write(plaintext) }, signal);
+            { write: (plaintext) => zip.write(plaintext) }, signal,
+            lease && lease.tr);
           await zip.end();
         }
         await zip.finish();
@@ -351,6 +614,7 @@ async function downloadDirectory(transfers, transport, gek, entries, dir, { setE
 export {
   FILE_ICONS,
   formatSize, formatDate, PREVIEWABLE_TEXT, canPreview, CHUNK_SIZE, ZIP_MAX_BYTES,
+  MEMORY_CEILING, TooLargeForMemoryError,
   _openDownloadTarget, _saveBlob, pipelinedDownload, downloadEntry,
   downloadDirectory,
 };