fix(transport): auto-reconnect after WebRTC failure, without dropping in-flight streams/downloads

Confirmed live (client trace + node logs, mobile screen-lock ~5min): ICE
goes disconnected -> failed within ~10s on both ends, but the DataChannel's
readyState stays "open" throughout, so nothing failed fast — every request
just sat out its own 8s/30s timeout, matching the reported symptom (poster
spinners, blocked chat, dead new streams).

transport.js: on connectionState "failed", reject pending requests
immediately (TransportLostError) and start a self-contained reconnect loop
(capped exponential backoff, redoes the full signaling handshake — the node
already discards the old session on its own "failed"/"closed", so there is
nothing lower-level to resume). New hooks: onNeedToken (fetch a fresh JWT,
since the captured one may have expired during the outage) and
onReconnected (let a consumer resume something that was mid-flight).

file-utils.js: pipelinedDownload retries a lost chunk instead of aborting
the whole transfer — covers Files downloads, poster/thumbnail fetches, and
music-player.js's blob-based track download, all of which go through it.

video-player.js: onReconnected reissues the existing seek-to-current-time
path, which already knows how to land a new stream_init on the live
SourceBuffer without resetting playback.

Playing audio is unaffected either way — musicbay.md's design downloads a
track to a blob before playing it, so a dead transport was never a network
dependency for what is already playing.

Stays on this branch until confirmed by real-device testing.

4 files changed, 224 insertions, 8 deletions

diff --git a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js
index b44d105..ba76ac9 100644
--- a/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js
+++ b/packages/meshbay-hub/src/meshbay_hub/static/file-utils.js
@@ -117,6 +117,41 @@ function _b64ToU8(b64) {
   return arr;
 }
 
+// A dead transport (screen-lock WebRTC failure, see transport.js's
+// _reconnectLoop) surfaces here as a rejected fetchChunk — TransportLostError
+// when the pending request was killed outright, a plain timeout if it was
+// still waiting when this ran. Either way the chunk itself was never the
+// problem, and the file already on disk (writable has real bytes in it by
+// now) is worth more than an all-or-nothing download: retry the same chunk
+// instead of letting one bad moment abort the whole transfer. Each retry
+// re-enters transport.fetchChunk, whose own _sendAndWait waits out an
+// in-flight reconnect before trying again, so this loop is mostly just
+// giving that reconnect the time and the attempts to land.
+const CHUNK_RETRY_ATTEMPTS = 6;
+const CHUNK_RETRY_DELAY_MS = 1500;
+
+function _isRetryableTransportError(err) {
+  return err.name === 'TransportLostError'
+    || err.message === 'Response timeout'
+    || (err.message || '').startsWith('DataChannel not open');
+}
+
+async function _fetchChunkResilient(transport, fileId, index) {
+  let lastErr;
+  for (let attempt = 0; attempt < CHUNK_RETRY_ATTEMPTS; attempt++) {
+    try {
+      return await transport.fetchChunk(fileId, index);
+    } catch (err) {
+      if (!_isRetryableTransportError(err)) throw err;
+      lastErr = err;
+      if (attempt < CHUNK_RETRY_ATTEMPTS - 1) {
+        await new Promise((r) => setTimeout(r, CHUNK_RETRY_DELAY_MS));
+      }
+    }
+  }
+  throw lastErr;
+}
+
 async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk,
                                  writable, signal) {
   const results = writable ? null : new Array(totalChunks);
@@ -125,7 +160,7 @@ async function pipelinedDownload(transport, gekKey, fileId, totalChunks, onChunk
 
   const fire = () => {
     while (nextSend < totalChunks && nextSend - nextRecv < PIPELINE_WINDOW) {
-      inflight[nextSend] = transport.fetchChunk(fileId, nextSend);
+      inflight[nextSend] = _fetchChunkResilient(transport, fileId, nextSend);
       nextSend++;
     }
   };
diff --git a/packages/meshbay-hub/src/meshbay_hub/static/group-page.js b/packages/meshbay-hub/src/meshbay_hub/static/group-page.js
index cdbe612..8683a70 100644
--- a/packages/meshbay-hub/src/meshbay_hub/static/group-page.js
+++ b/packages/meshbay-hub/src/meshbay_hub/static/group-page.js
@@ -226,6 +226,11 @@ function GroupPage({ groupId, group, token, username, userId, userPrefs,
         // any other, and two sources for one address is how they drift.
         const transport = new window.MeshBayTransport(HUB, live);
         transportRef.current = transport;
+        // Consulted only by the automatic reconnect after a WebRTC failure
+        // (transport.js's _reconnectLoop) — the token captured by this
+        // connect() call can be stale by then, since the whole point is that
+        // some real time (screen lock, a dead NAT mapping) passed unnoticed.
+        transport.onNeedToken = async () => (await ensureFreshToken()) || token;
 
         const ack = await transport.connect(
           nodeId, live, groupId, null, sessionKeys, session.bundleKey, username,
diff --git a/packages/meshbay-hub/src/meshbay_hub/static/transport.js b/packages/meshbay-hub/src/meshbay_hub/static/transport.js
index 6c038f6..56ec2c7 100644
--- a/packages/meshbay-hub/src/meshbay_hub/static/transport.js
+++ b/packages/meshbay-hub/src/meshbay_hub/static/transport.js
@@ -214,6 +214,29 @@ class MeshBayTransport {
     // several uploads may be in flight at once and their acks interleave; the
     // node names the file in every one.
     this._uploaders = new Map();
+    // Set once close() runs — stops the automatic reconnect from firing on a
+    // connection the caller tore down on purpose (leaving the group, page
+    // unload), which would otherwise race back in right as everything else
+    // is being torn down.
+    this._closed = false;
+    // The arguments connect() was last given, minus the token (refreshed at
+    // reconnect time — see onNeedToken) and sessionKeys (kept live on `this`,
+    // since a reconnect must reuse the identity connect() settled on, not
+    // whatever the very first caller passed in — see _reconnectLoop).
+    this._connectArgs = null;
+    this._lastToken = null;
+    this._reconnectPromise = null;
+    this._reconnectAttempts = 0;
+    // True only for the duration of the connect() call _reconnectLoop makes
+    // to actually retry — as opposed to the backoff delay around it, which
+    // is most of _reconnectPromise's lifetime. Needed because that connect()
+    // call sends its own handshake through _sendAndWait, which would
+    // otherwise see the very _reconnectPromise it is running inside of as
+    // "a reconnect to wait for" and stall every handshake step for the full
+    // 6s gate below before ever sending it.
+    this._inReconnectAttempt = false;
+    this._onReconnected = null;
+    this._onNeedToken = null;
   }
 
   get connected() { return this._connected; }
@@ -235,6 +258,17 @@ class MeshBayTransport {
   set onMusicbrainzConfig(fn) { this._onMusicbrainzConfig = fn; }
   set onMusicbrainzEnabled(fn) { this._onMusicbrainzEnabled = fn; }
   set onIndexProgress(fn) { this._onIndexProgress = fn; }
+  // Fired once an automatic reconnect (see _reconnectLoop) lands a fresh
+  // handshake, so a consumer with something mid-flight on the old channel —
+  // today only the video player — can pick back up rather than sit dead.
+  set onReconnected(fn) { this._onReconnected = fn; }
+  // Reconnecting redoes the handshake, which needs a JWT that may have gone
+  // stale while the connection was down for minutes. Without this the
+  // reconnect resends whatever token the original connect() call captured,
+  // which the node's clock-skew check (stale_request) or plain expiry can
+  // by then have already invalidated. Set to whatever the caller uses to
+  // refresh the hub session token (see group-page.js's ensureFreshToken).
+  set onNeedToken(fn) { this._onNeedToken = fn; }
 
   get sessionKeys() { return this._sessionKeys; }
 
@@ -244,6 +278,12 @@ class MeshBayTransport {
 
   async connect(nodeId, jwtToken, groupId, gekRaw, sessionKeys, bundleKey, username,
                 userId, joinCode) {
+    // Remembered for _reconnectLoop, which calls connect() again with these
+    // same values (plus a freshly-fetched token and the identity connect()
+    // itself settles on below) after the WebRTC connection is declared
+    // "failed" — see the pc.onconnectionstatechange handler further down.
+    this._connectArgs = { nodeId, groupId, gekRaw, bundleKey, username, userId, joinCode };
+    this._lastToken = jwtToken;
     this._gekRaw = gekRaw || null;
     this._sessionKeys = sessionKeys || null;
     this._bundleKey = bundleKey || null;
@@ -292,13 +332,35 @@ class MeshBayTransport {
       if (channelReject) channelReject(new Error('DataChannel error'));
     };
 
-    this._pc.onconnectionstatechange = () => {
-      console.log('[MeshBay] PC state:', this._pc.connectionState);
-      trace('pc_state', { state: this._pc.connectionState });
+    // Captured locally rather than read back through `this._pc`: once a
+    // reconnect replaces it, a late event from this (by then orphaned) pc
+    // must still be judged against the pc it actually came from, not
+    // whatever is current — the `pc === this._pc` check below is what that
+    // buys.
+    const pc = this._pc;
+    pc.onconnectionstatechange = () => {
+      console.log('[MeshBay] PC state:', pc.connectionState);
+      trace('pc_state', { state: pc.connectionState });
+      // "failed" is ICE's own verdict that nothing here will recover on its
+      // own (unlike a transient "disconnected", which often clears itself) —
+      // confirmed live: mobile screen lock for several minutes reliably
+      // produces disconnected → failed about 10s apart, on both ends, and
+      // nothing today ever moves past that without a full page reload.
+      // `channel.readyState` is no help distinguishing this: it was observed
+      // staying "open" throughout, so every send from here on would simply
+      // sit out its own timeout instead of failing fast.
+      if (pc.connectionState === 'failed' && pc === this._pc && !this._closed) {
+        this._connected = false;
+        this._reconnect();
+        const err = new Error('WebRTC connection lost');
+        err.name = 'TransportLostError';
+        for (const [, p] of this._pending) p.reject(err);
+        this._pending.clear();
+      }
     };
-    this._pc.oniceconnectionstatechange = () => {
-      console.log('[MeshBay] ICE state:', this._pc.iceConnectionState);
-      trace('ice_state', { state: this._pc.iceConnectionState });
+    pc.oniceconnectionstatechange = () => {
+      console.log('[MeshBay] ICE state:', pc.iceConnectionState);
+      trace('ice_state', { state: pc.iceConnectionState });
     };
 
     // Diagnostic-only: a periodic health ping and a resume-triggered one, so
@@ -576,6 +638,82 @@ class MeshBayTransport {
   }
 
   /**
+   * Kick off (or join, if one is already running) the automatic reconnect
+   * after the WebRTC connection is declared unrecoverable. Idempotent: every
+   * caller racing to reconnect at once — the connectionstatechange handler,
+   * and any request that lands in the gap _sendAndWait waits out below —
+   * shares the one attempt instead of piling up parallel handshakes against
+   * the node.
+   */
+  _reconnect() {
+    if (this._closed) return Promise.resolve();
+    if (!this._reconnectPromise) {
+      this._reconnectPromise = this._reconnectLoop().finally(() => {
+        this._reconnectPromise = null;
+      });
+    }
+    return this._reconnectPromise;
+  }
+
+  /**
+   * Redo the signaling handshake from scratch — the only thing that works
+   * once aiortc has declared a connection "failed": the node discards that
+   * session the moment it sees the same state (webrtc_server.py's
+   * on_state_change), so there is no lower-level session left to resume, only
+   * a fresh one to negotiate. Retries with capped exponential backoff
+   * (1s, 2s, 4s ... 30s) rather than a fixed number of attempts, because the
+   * two real causes seen so far — a mobile carrier dropping the NAT mapping
+   * during screen lock, and the node's own machine being briefly unreachable
+   * — both resolve on their own eventually, and there is no good moment to
+   * decide the user would rather see a dead app than keep waiting.
+   */
+  async _reconnectLoop() {
+    this._reconnectAttempts = 0;
+    while (!this._closed) {
+      this._reconnectAttempts += 1;
+      const delayMs = Math.min(30000, 1000 * 2 ** (this._reconnectAttempts - 1));
+      trace('reconnect_wait', { attempt: this._reconnectAttempts, delay_ms: delayMs });
+      await new Promise((r) => setTimeout(r, delayMs));
+      if (this._closed) return;
+      try {
+        // Best-effort: these are already unusable, but leaving them wired up
+        // risks a stray late event from the old pc doing something once a
+        // new one is in `this._pc` — the `pc === this._pc` guard above closes
+        // most of that gap, this closes the rest.
+        try { this._channel && this._channel.close(); } catch { /* already gone */ }
+        try { this._pc && this._pc.close(); } catch { /* already gone */ }
+        const args = this._connectArgs;
+        const token = this._onNeedToken ? await this._onNeedToken() : this._lastToken;
+        trace('reconnect_attempt', { attempt: this._reconnectAttempts });
+        this._inReconnectAttempt = true;
+        try {
+          await this.connect(args.nodeId, token, args.groupId, args.gekRaw,
+                              this._sessionKeys, args.bundleKey, args.username,
+                              args.userId, args.joinCode);
+        } finally {
+          this._inReconnectAttempt = false;
+        }
+        trace('reconnect_ok', { attempt: this._reconnectAttempts });
+        console.log('[MeshBay] Reconnected after', this._reconnectAttempts, 'attempt(s)');
+        if (this._onReconnected) {
+          try { this._onReconnected(); } catch (e) {
+            console.error('[MeshBay] onReconnected handler threw:', e);
+          }
+        }
+        return;
+      } catch (e) {
+        trace('reconnect_attempt_failed', {
+          attempt: this._reconnectAttempts, error: String(e && e.message || e),
+        });
+        console.warn('[MeshBay] Reconnect attempt', this._reconnectAttempts,
+                     'failed:', e.message);
+        // Loop again with a longer backoff — closing over `args`/`token`
+        // freshly next time, in case the token was the actual problem.
+      }
+    }
+  }
+
+  /**
    * Pair this browser with the node using a one-time code (M3, and the same
    * substitution as H3).
    *
@@ -1590,6 +1728,11 @@ class MeshBayTransport {
   get gekRaw() { return this._gekRaw; }
 
   close() {
+    // Must be set before pc.close() below: that close() itself can drive the
+    // pc to "closed" synchronously, and the connectionstatechange handler
+    // only skips reconnecting because of this flag, not because "closed" is
+    // absent from its own trigger condition.
+    this._closed = true;
     if (this._diagCleanup) { this._diagCleanup(); this._diagCleanup = null; }
     if (this._channel) this._channel.close();
     if (this._pc) this._pc.close();
@@ -1600,7 +1743,21 @@ class MeshBayTransport {
 
   // ── Internal ──────────────────────────────────────────────────────────────
 
-  _sendAndWait(obj, timeoutMs = 30000) {
+  async _sendAndWait(obj, timeoutMs = 30000) {
+    // A reconnect already in flight (see _reconnectLoop) means the channel
+    // this would send on is the one just declared dead. Waiting here, bounded
+    // rather than open-ended — the loop can be backing off for up to 30s
+    // between attempts, and a caller (in particular file-utils.js's
+    // pipelinedDownload, which retries a lost chunk itself) should get its
+    // own timeout rather than sit through someone else's backoff — gives a
+    // fresh handshake a real chance to land before this request is even
+    // attempted, instead of guaranteeing it dies with the old one.
+    if (this._reconnectPromise && !this._inReconnectAttempt) {
+      await Promise.race([
+        this._reconnectPromise.catch(() => {}),
+        new Promise((r) => setTimeout(r, 6000)),
+      ]);
+    }
     return new Promise((resolve, reject) => {
       const id = this._seqId++;
       const timeout = setTimeout(() => {
diff --git a/packages/meshbay-hub/src/meshbay_hub/static/video-player.js b/packages/meshbay-hub/src/meshbay_hub/static/video-player.js
index 82e1116..78760d8 100644
--- a/packages/meshbay-hub/src/meshbay_hub/static/video-player.js
+++ b/packages/meshbay-hub/src/meshbay_hub/static/video-player.js
@@ -529,6 +529,24 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) {
         setPhase('error');
       };
 
+      // The old stream died with the connection (the node retires it the
+      // moment its session goes away — see webrtc_server.py's
+      // on_state_change), so there is nothing to resume on the wire, only a
+      // reason to ask again. requestSeek already knows how to land a new
+      // stream_init on the live SourceBuffer without resetting playback —
+      // exactly what dragging the scrubber does — so reusing it here means a
+      // screen-lock reconnect looks like a seek to where the film already
+      // was, not a reload.
+      transport.onReconnected = () => {
+        if (cancelled) return;
+        const v = videoRef.current;
+        const seek = requestSeekRef.current;
+        if (!v || !seek) return;
+        console.log('[MeshBay] transport reconnected — resuming stream at',
+                    v.currentTime.toFixed(1));
+        seek(v.currentTime);
+      };
+
       transport.onStreamInit = (msg) => {
         if (cancelled) return;
         if (msg.file_id && msg.file_id !== entry.id) return;
@@ -849,6 +867,7 @@ function VideoPlayer({ entry, transportRef, gekRef, onClose, onDownload }) {
         transport.onStreamData = null;
         transport.onStreamEnd = null;
         transport.onStreamError = null;
+        transport.onReconnected = null;
       }
       // The queue can hold several megabytes of decrypted video.
       queueRef.current = [];