path: root/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js

import * as downloads from './downloads.js';
import * as platform from './platform.js';
import { t } from './i18n.js';
import { ZipStream, entriesUnder } from './zipstream.js';

const FILE_ICONS = {
  video: '\u{1F3AC}', audio: '\u{1F3B5}', image: '\u{1F5BC}',
  document: '\u{1F4C4}', archive: '\u{1F4E6}', other: '\u{1F4CE}',
};

function formatSize(bytes) {
  if (bytes < 1024) return bytes + ' B';
  if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
  if (bytes < 1024 * 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
  return (bytes / (1024 * 1024 * 1024)).toFixed(2) + ' GB';
}

// An arbitrary ceiling on one directory zip. Not a technical limit — the
// writer streams and holds one chunk plus a record per file, so it would
// happily produce a hundred gigabytes — but a deliberate one: past this
// size the honest answer is a subfolder at a time, or the files
// individually. Counted in the same 1024-based units formatSize prints, so
// the number in the refusal is the number in this constant.
const ZIP_MAX_BYTES = 512 * 1024 * 1024;

function formatDate(ts) {
  return new Date(ts * 1000).toLocaleDateString(undefined, {
    year: 'numeric', month: 'short', day: 'numeric',
  });
}

const PREVIEWABLE_TEXT =
  /\.(txt|md|json|csv|log|xml|yaml|yml|ini|conf|py|js|html|css|sh|c|h|java|rs|go|rb|toml)$/i;

function canPreview(e) {
  return ['image', 'video', 'audio', 'document'].includes(e.type)
    || PREVIEWABLE_TEXT.test(e.name);
}

const CHUNK_SIZE = 1024 * 1024;
// Segments allowed in flight while there is room to put them. This is a window,
// topped up as segments land, and not a debt released in one go: accumulating a
// credit per append and handing the lot over when the buffer finally had room
// sent 6 MB in a burst, overshot the target by a minute of film, and then said
// nothing for the next forty-six seconds. Measured in Chrome against real
// fragmented MP4. A stream that arrives in gulps has no margin for a network
// that hesitates, and looks like a hang while it is quiet.
const PIPELINE_WINDOW = 8;

// The most this code will ever collect in the page.
//
// Below every streaming target there is a floor: `pipelinedDownload` with no
// `writable` allocates `new Array(totalChunks)` and keeps every decrypted
// chunk, and `_saveBlob` hands the lot to the browser. That floor is fine for
// something small and is a dead tab for a film. It had **no upper bound**: the
// `!window.showSaveFilePicker` branch below returned null at any size, so on a
// browser without the File System Access API (Firefox, Safari) a 20 GB film
// went to RAM whenever the service-worker path did not answer — which happens
// for ordinary reasons (an uncontrolled page, a stream that cannot be
// transferred, the 8 s timeout). Nothing logged, nothing refused; the symptom
// was the tab dying, with no error attributable to this code.
//
// So: above this, there is no floor. A refusal naming what happened is
// recoverable and a dead tab is not. `CLAUDE.md`'s standing lesson is that a
// fallback chain reaches its floor silently — this is that floor being given a
// bottom.
const MEMORY_CEILING = 100 * 1024 * 1024;

/**
 * Thrown instead of falling through to the in-memory floor.
 *
 * This should now be unreachable in ordinary use: the streamed path is primed
 * at application start and retried on demand, so a browser with a service
 * worker has somewhere to write whatever the size. If it is ever raised, the
 * reason the streamed path declined is appended — untranslated, because it is a
 * diagnostic and a vague failure is what made the original bug invisible.
 */
class TooLargeForMemoryError extends Error {
  constructor(filename, size) {
    const why = downloads.lastStreamFailure();
    super(t('download.too_large_for_memory', {
      name: filename, size: formatSize(size), limit: formatSize(MEMORY_CEILING),
    }) + (why ? ` (${why})` : ''));
    this.name = 'TooLargeForMemoryError';
  }
}

/**
 * Open somewhere to write, honouring the user's download setting.
 *
 * Returns a target ({writable, name}), null for "no stream available — collect
 * it and hand the browser a blob", or false for "the person dismissed the
 * dialog", which is not an error and must not start a transfer.
 *
 * **Never returns null above MEMORY_CEILING.** Every `return null` below is
 * guarded by `_memoryFloor`, which throws instead. A fourth fallback added
 * later must go through it too — `test_memory_ceiling.py` fails the build if a
 * bare `return null` appears in this function.
 */
async function _openDownloadTarget(filename, size = 0, pickerOpts = {},
                                   swSize = size, { batched = false } = {}) {
  // "Collect it in the page", or a refusal when that would be too much.
  const _memoryFloor = () => {
    if (size > MEMORY_CEILING) throw new TooLargeForMemoryError(filename, size);
    return null;
  };

  // On a desktop build this is the whole answer, and it comes first.
  //
  // The two browser paths below are both unavailable there — `showDirectoryPicker`
  // does not exist, and Chromium refuses a service worker on a custom scheme —
  // so without this the chain fell all the way through to its floor, which
  // collects the file in the page and hands the browser a blob. A gigabyte of
  // film meant a gigabyte of RAM, and a Save As dialog at the *end*.
  if (platform.capabilities.nativeSave) {
    try {
      const native = await platform.nativeSave(
        filename, { auto: downloads.getMode() === 'auto' });
      // Null means the person dismissed the dialog, which is not an error and
      // must not start a transfer.
      return native || false;
    } catch (err) {
      console.warn('[MeshBay] native save failed:', platform.bridgeMessage(err));
      return false;
    }
  }

  try {
    const target = await downloads.openTarget(filename);
    if (target) return target;
  } catch (err) {
    console.warn('[MeshBay] download folder unusable:', err.message);
  }

  // No granted folder. A service worker can still hand the browser a stream to
  // write, which is how this works at all in Firefox: the alternative there is
  // to collect gigabytes in a tab. It goes to the browser's own download
  // folder, without a dialog, which is what "save automatically" meant.
  //
  // Tried in "ask" mode too when the file is large and this browser has no Save
  // As of its own. The mode is about whether to show a dialog; it was never
  // meant to decide whether a 20 GB film can be downloaded at all, and on
  // Firefox and Safari — where `showSaveFilePicker` does not exist — skipping
  // this block left nothing but the in-memory floor. A preference must not cost
  // a capability.
  const canPick = typeof window.showSaveFilePicker === 'function';
  // `batched` means this is not the first download of a batch, and it makes the
  // streamed path preferred whatever the mode.
  //
  // "Ask where to save" asks per file, which is right for one file and wrong
  // for four: a browser grants one picker per user gesture, so the second
  // dialog has no gesture behind it and the third and fourth wait behind a
  // dialog that waits for a human — reported from Chrome as three downloads
  // frozen. There is no gesture left to spend, so there is nothing to lose by
  // streaming instead: the file still lands on disk, in the browser's own
  // download folder. Only the choice of folder goes, and it was not on offer.
  if (downloads.getMode() === 'auto' || batched
      || (size > MEMORY_CEILING && !canPick)) {
    const streamed = await downloads.openStreamedDownload(filename, swSize);
    if (streamed) return streamed;
    // Nothing to stream to: small enough for memory, and no dialog. The old
    // comparison here was against downloads.BLOB_LIMIT (512 MB), five times
    // this ceiling — and it was the *only* size test in the whole chain, with
    // the branch below it unguarded.
    if (size <= MEMORY_CEILING) return _memoryFloor();
  }

  // No File System Access API — Firefox, Safari. This is the branch that used
  // to return null at any size.
  if (!canPick) return _memoryFloor();
  // A dialog is the one outcome nobody can diagnose after the fact: it looks
  // the same whether it was asked for, or fallen back to because the worker
  // did not answer. Say which, once per download, so the next report from a
  // browser we do not have does not need a second round trip.
  console.info('[MeshBay] asking where to save %s — mode=%s batched=%s stream=%s',
               filename, downloads.getMode(), batched,
               downloads.lastStreamFailure() || 'not attempted');
  try {
    const handle = await window.showSaveFilePicker({
      suggestedName: filename, ...pickerOpts,
    });
    return { writable: await handle.createWritable(), name: handle.name || filename };
  } catch (err) {
    if (err.name === 'AbortError') return false;
    // "Must be handling a user gesture to show a file picker."
    //
    // A browser grants one picker per gesture, and downloading three files is
    // one gesture. So the second and third throw this, and the person sees a
    // failed transfer with a message from Chrome about gestures, for having
    // done something entirely reasonable.
    //
    // The streamed path needs no gesture at all, which makes it the right
    // answer here rather than a consolation: the file still lands on disk, in
    // the browser's own download folder, written as it arrives. Only the choice
    // of folder is lost, and it was already lost — there was no picker to make
    // it in.
    if (err.name === 'SecurityError' || /user gesture/i.test(err.message || '')) {
      console.warn('[MeshBay] no gesture left for a save dialog; streaming '
                   + 'this one to the download folder instead');
      const streamed = await downloads.openStreamedDownload(filename, swSize);
      if (streamed) return streamed;
      if (size <= MEMORY_CEILING) return _memoryFloor();
    }
    throw err;
  }
}

// Target openings run one at a time, across every download on the page.
//
// A browser shows one file picker at a time and grants one per user gesture, so
// four downloads asking at once get one dialog and three failures. That used to
// be prevented by accident: `downloadEntry` awaited the target inline, and
// files-app.js's `for (const e of selected) await downloadFile(e)` serialised
// them. Opening the target inside `prepare` — so the row appears at the click
// instead of tens of seconds later — removed that accident, and four pickers
// raced. Chrome showed one, prompted for a second, and the rest timed out;
// Firefox and Electron never noticed, because neither opens a picker at all.
//
// So the queue is explicit now, and it is the *targets* that queue, not the
// rows: every download still appears the moment it is asked for.
//
// Two things keep the queue from becoming the problem it was meant to solve.
// It only ever holds openings that could actually put a dialog on screen, and
// no opening waits behind another for longer than a budget.
let _targetQueue = Promise.resolve();

let _targetsInFlight = 0;

// How long an opening waits for the one ahead of it before going anyway.
//
// A queue with no bound is a way for one stuck opening to freeze every later
// download for the life of the page, since `_targetQueue` is never reset. That
// is what turned a slow first download into four rows stuck at "preparing" on
// Firefox. Generous, because a dialog legitimately waits for a person and
// cutting in front of one would be worse than waiting; finite, because the
// alternative is a download panel that never recovers.
//
// Going anyway is safe: whatever was ahead is still the only unbatched opening,
// so the one released here takes the streamed path and opens no second dialog.
const TARGET_QUEUE_BUDGET_MS = 90000;

function _openTargetInTurn(filename, size, pickerOpts, swSize) {
  // Only an opening that could show a dialog has any reason to wait. Firefox
  // and Safari have no `showSaveFilePicker` at all, so nothing there can race
  // anything, and queueing them bought nothing while costing everything: four
  // downloads that used to open their targets at the same time became four
  // that waited on the slowest.
  const canPick = typeof window !== 'undefined'
    && typeof window.showSaveFilePicker === 'function';
  if (!canPick) return _openDownloadTarget(filename, size, pickerOpts, swSize);

  // Anything that has to wait its turn is, by definition, not the first of the
  // batch — so it will not be the one holding the user's gesture.
  const batched = _targetsInFlight > 0;
  _targetsInFlight += 1;
  const mine = _waitBriefly(_targetQueue, TARGET_QUEUE_BUDGET_MS)
    .then(() => _openDownloadTarget(filename, size, pickerOpts, swSize,
                                    { batched }))
    .finally(() => { _targetsInFlight -= 1; });
  // The chain must not break on a rejection, or one refused download stops
  // every later one from ever opening a target.
  _targetQueue = mine.catch(() => {});
  return mine;
}

/** Settles with `promise`, or after `ms`, whichever comes first. */
function _waitBriefly(promise, ms) {
  return new Promise((resolve) => {
    const timer = setTimeout(resolve, ms);
    promise.then(() => { clearTimeout(timer); resolve(); },
                 () => { clearTimeout(timer); resolve(); });
  });
}

/** The download of last resort, for browsers with no way to stream to disk. */
function _saveBlob(blob, filename) {
  const url = URL.createObjectURL(blob);
  const a = document.createElement('a');
  a.href = url;
  a.download = filename;
  document.body.appendChild(a);
  a.click();
  document.body.removeChild(a);
  URL.revokeObjectURL(url);
}

// A dead transport (screen-lock WebRTC failure, see transport.js's
// _reconnectLoop) surfaces here as a rejected fetchChunk — TransportLostError
// when the pending request was killed outright, a plain timeout if it was
// still waiting when this ran. Either way the chunk itself was never the
// problem, and the file already on disk (writable has real bytes in it by
// now) is worth more than an all-or-nothing download: retry the same chunk
// instead of letting one bad moment abort the whole transfer. Each retry
// re-enters transport.fetchChunk, whose own _sendAndWait waits out an
// in-flight reconnect before trying again, so this loop is mostly just
// giving that reconnect the time and the attempts to land.
const CHUNK_RETRY_ATTEMPTS = 6;
const CHUNK_RETRY_DELAY_MS = 1500;

// How long one megabyte may take to reach the disk before we call it stuck.
//
// Every other await on this path is bounded and says so when it expires:
// `_sendAndWait` logs a Response timeout, `_fetchChunkResilient` retries and
// then throws. `writable.write()` was the exception — a sink that stops
// consuming (a service-worker stream the browser has stopped reading, a file
// handle that has gone away) leaves it pending for ever. It never rejects, so
// there is no error, no log and no failed transfer: the progress bar simply
// stops, the console stays empty, and the node is perfectly healthy the whole
// time, which is what made this invisible.
//
// Generous on purpose. A megabyte takes milliseconds on any working sink; a
// minute means the sink is gone, not slow.
const WRITE_STALL_MS = 60000;

/** `writable.write`, but it fails instead of hanging for ever. */
async function _writeOrStall(writable, bytes, at) {
  let timer = 0;
  const stalled = new Promise((_, reject) => {
    timer = setTimeout(() => reject(new Error(
      t('group.download_write_stalled', { seconds: WRITE_STALL_MS / 1000 })
      + ` (chunk ${at})`)), WRITE_STALL_MS);
  });
  try {
    await Promise.race([writable.write(bytes), stalled]);
  } finally {
    clearTimeout(timer);
  }
}

function _isRetryableTransportError(err) {
  return err.name === 'TransportLostError'
    || err.message === 'Response timeout'
    || (err.message || '').startsWith('DataChannel not open');
}

async function _fetchChunkResilient(transport, fileId, index, tr = '') {
  let lastErr;
  for (let attempt = 0; attempt < CHUNK_RETRY_ATTEMPTS; attempt++) {
    try {
      return await transport.fetchChunk(fileId, index, tr);
    } catch (err) {
      if (!_isRetryableTransportError(err)) throw err;
      lastErr = err;
      if (attempt < CHUNK_RETRY_ATTEMPTS - 1) {
        await new Promise((r) => setTimeout(r, CHUNK_RETRY_DELAY_MS));
      }
    }
  }
  throw lastErr;
}

async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk,
                                 writable, signal, tr = '') {
  const results = writable ? null : new Array(totalChunks);
  let nextSend = 0, nextRecv = 0;
  const inflight = new Array(totalChunks);

  const fire = () => {
    while (nextSend < totalChunks && nextSend - nextRecv < PIPELINE_WINDOW) {
      inflight[nextSend] = _fetchChunkResilient(transport, fileId, nextSend, tr);
      nextSend++;
    }
  };

  fire();
  while (nextRecv < totalChunks) {
    if (signal && signal.aborted) {
      const err = new Error('Cancelled');
      err.name = 'AbortError';
      throw err;
    }
    const chunkMsg = await inflight[nextRecv];
    // One shape, and a refusal for anything else. There used to be two fallbacks
    // below this: a base64 `ct_b64` chunk, which was the real wire format until
    // the binary switch in Phase 9.15 and which no node has sent since, and a
    // plaintext branch that base64-decoded `undefined` when neither field was
    // there. That last one is why this now throws: a chunk we cannot decrypt has
    // to stop the download, not write whatever it decoded into the file the user
    // is saving. MNP 0.15 removed the last producer of the base64 shape (it was
    // still emitted on the QUIC transport), so a node that sends one is a node
    // older than the SPA serving this page.
    if (!gekKey || !chunkMsg.ct) {
      throw new Error(
        `Chunk ${nextRecv} of ${fileId.slice(0, 8)} cannot be decrypted `
        + `(${gekKey ? 'unexpected chunk format' : 'no group key'}) — `
        + 'the node may be running an older version.');
    }
    const plaintext = await window.MeshBayCrypto.decryptChunkBin(
      gekKey, fileId, nextRecv, chunkMsg.nonce, chunkMsg.ct);
    if (writable) {
      await _writeOrStall(writable, plaintext, nextRecv);
    } else {
      results[nextRecv] = plaintext;
    }
    nextRecv++;
    fire();
    if (onChunk) onChunk(plaintext.byteLength, nextRecv, totalChunks);
  }
  return results;
}

/**
 * Download one file through the transfers widget: picks a target, streams
 * and decrypts it, and falls back to a blob when there is nowhere to stream
 * to. Shared by the Files table/toolbar and the video/preview modals' own
 * download button — both just want "get this entry to disk".
 */
async function downloadEntry(transfers, transport, gek, entry) {
  const totalChunks = Math.ceil(entry.size / CHUNK_SIZE);
  const openRef = { url: null };
  let target = null;

  transfers.start({
    kind: 'download', name: entry.name, total: entry.size, transport,

    // The row exists from the click. Opening a target is what takes the time —
    // the streamed path waits for the worker (twice), a Save As dialog waits
    // for a person — and doing it before the row meant three clicks produced no
    // panel at all and then several rows at once.
    prepare: async () => {
      target = await _openTargetInTurn(entry.name, entry.size);
      // Dismissed: nothing was started, so nothing is left on screen.
      if (target === false) return false;
      return target ? { name: target.name } : true;
    },

    // After the target, never before: a granted slot has to be taken up within
    // the node's deadline, and opening a target can outlast it. See §8.1 of
    // ~/next/improve-downloads.md — the other order was tried and cost two of
    // three downloads.
    makeLease: () => transport.openTransfer({
      kind: 'download', bytes: entry.size, chunks: totalChunks }),

    open: () => (target && target.open) ? target.open()
      : (openRef.url ? window.open(openRef.url, '_blank') : undefined),

    run: async ({ signal, onProgress, lease }) => {
      let done = 0;
      const onChunk = (bytes) => { done += bytes; onProgress(done, entry.size); };

      if (target) {
        try {
          await pipelinedDownload(transport, gek, entry.id, totalChunks,
                                  onChunk, target.writable, signal,
                                  lease && lease.tr);
          await target.writable.close();
        } catch (err) {
          await target.writable.abort().catch(() => {});
          throw err;
        }
      } else {
        const chunks = await pipelinedDownload(
          transport, gek, entry.id, totalChunks, onChunk, null, signal,
          lease && lease.tr);
        const blob = new Blob(chunks);
        _saveBlob(blob, entry.name);
        openRef.url = URL.createObjectURL(blob);
      }
    },
  });
}

/**
 * Download a directory as a zip, written straight to disk.
 *
 * Nothing is held anywhere: each file is fetched chunk by chunk, decrypted,
 * and handed to the zip writer, which hands it to the file the browser
 * opened. Peak memory is one chunk plus one small record per file — which is
 * why ZIP_MAX_BYTES below is a policy, not a constraint this code has.
 *
 * Without the File System Access API there is nowhere to stream to, and the
 * only alternative is to build the whole thing in memory — so that path is
 * offered but says what it costs first.
 *
 * Lifted out of files-app.js (docs/photos.md §3) so photos-app.js's own
 * "zip this album" button calls the same implementation rather than a
 * second one — nothing here is Files-specific once `entries`/`transport`/
 * `gek`/`setError` are passed in, the same shared-context shape every app
 * already receives (apps.md §2).
 */
async function downloadDirectory(transfers, transport, gek, entries, dir, { setError }) {
  if (!transport || !transport.connected) return;

  const files = entriesUnder(entries, dir);
  if (!files.length) {
    setError(t('group.zip_empty'));
    return;
  }
  const totalBytes = files.reduce((n, f) => n + (f.entry.size || 0), 0);
  const suggested = (dir.split('/').pop() || 'files') + '.zip';

  // Checked here rather than by disabling the button: Files zips a whole
  // multi-directory selection in one click (`for (const d of selectedDirs)`),
  // so the answer is per directory and has to be given where each one is
  // actually started — an oversized folder is refused and its siblings still
  // download. Before _openDownloadTarget, so nothing opens a save dialog for
  // an archive that is not going to be written.
  if (totalBytes > ZIP_MAX_BYTES) {
    setError(t('group.zip_too_large', {
      name: dir.split('/').pop() || dir,
      size: formatSize(totalBytes),
      limit: formatSize(ZIP_MAX_BYTES),
    }));
    return;
  }

  // totalBytes decides how this is delivered, but it is not the archive's
  // size — headers and the central directory come on top — so it is not
  // announced as a Content-Length that the download would then miss.
  const zipOpenRef = { url: null };
  let target = null;

  transfers.start({
    kind: 'download', name: suggested, total: totalBytes, transport,

    // Same order as downloadEntry: the row first, then the target, then the
    // slot. A folder of forty files is exactly where the wait is longest.
    prepare: async () => {
      target = await _openTargetInTurn(suggested, totalBytes, {
        types: [{ description: 'ZIP archive',
                  accept: { 'application/zip': ['.zip'] } }],
      }, 0);
      if (target === false) return false;
      if (!target && !confirm(t('group.zip_no_stream', {
        size: formatSize(totalBytes), name: suggested,
      }))) {
        return false;
      }
      return target ? { name: target.name } : true;
    },

    // **One** lease for the archive, not one per file. Dozens of leases for a
    // folder would deadlock against the member's own cap: the job cannot finish
    // until it holds them all, and it can never hold more than two.
    makeLease: () => transport.openTransfer({
      kind: 'download', bytes: totalBytes, chunks: files.length }),

    open: () => (target && target.open) ? target.open()
      : (zipOpenRef.url ? window.open(zipOpenRef.url, '_blank') : undefined),
    run: async ({ signal, onProgress, lease }) => {
      const writable = target ? target.writable : null;
      const parts = writable ? null : [];
      let written = 0;
      try {
        const zip = new ZipStream(async (bytes) => {
          if (writable) await writable.write(bytes);
          else parts.push(bytes.slice());
        });

        for (const { entry, name } of files) {
          await zip.begin(name, entry.size,
                          new Date((entry.added_at || 0) * 1000));
          // A zero-byte file has no chunk to ask for; the header and an empty
          // descriptor are the whole entry.
          const totalChunks = Math.ceil(entry.size / CHUNK_SIZE);
          if (totalChunks > 0) await pipelinedDownload(
            transport, gek, entry.id, totalChunks,
            (bytes) => { written += bytes; onProgress(written, totalBytes); },
            // pipelinedDownload writes in order, which the archive needs.
            { write: (plaintext) => zip.write(plaintext) }, signal,
            lease && lease.tr);
          await zip.end();
        }
        await zip.finish();
        if (writable) await writable.close();
        else {
          const blob = new Blob(parts, { type: 'application/zip' });
          _saveBlob(blob, suggested);
          zipOpenRef.url = URL.createObjectURL(blob);
        }
      } catch (err) {
        if (writable) await writable.abort().catch(() => {});
        throw err;
      }
    },
  });
}

export {
  FILE_ICONS,
  formatSize, formatDate, PREVIEWABLE_TEXT, canPreview, CHUNK_SIZE, ZIP_MAX_BYTES,
  MEMORY_CEILING, TooLargeForMemoryError,
  _openDownloadTarget, _saveBlob, pipelinedDownload, downloadEntry,
  downloadDirectory,
};