fix(hub): a group whose node is down is no longer something the reader waits for

A node is a machine in somebody's house, so with a handful of groups one is
always off. Search treated that as the exception and charged the reader for it.

Measured on twelve groups against a virtual clock, a live group answering in
200 ms and a dead one taking the full deadline: one node down in the first batch
put a blank page and a progress bar in front of the reader for **ten seconds**,
while two groups of that same batch had answered in two hundred milliseconds and
nine others had not been dialled at all. Four down, spread out — the shape a real
reader has — was ten seconds to the first result and **forty** to the last.

Two causes, neither of them the connection deadline. Results were drawn once a
batch was complete rather than as each arrived, so an index already in hand waited
on a node that was not answering. And the batches were sequential, so a dead group
did not merely cost its own deadline, it postponed every group behind it.

So the three is a ceiling on concurrency and never a batch — `inFlight` starts the
next group the moment one ends — and each index is drawn when it lands.

That alone is not enough, and the tests say where it stops: a ceiling still lets
silent nodes hold every place at once, and with four of twelve down the last three
live groups still waited out a deadline. So the browser remembers which groups
were silent and dials them last, which puts all eight on screen in 600 ms. The
list is advisory and rewritten from what each sweep saw: a private window, storage
that refuses, a first visit or a node that has come back all fall through to the
hub's own order, cost one sweep, and correct themselves.

The "n groups unreachable" line also waited for the sweep to finish, which is the
one moment it is no longer needed. It now appears as they are found.

First result, before and after, twelve groups: one down 10 s → 200 ms; four down
10 s → 200 ms with every reachable group on screen by 600 ms; three down and
listed first 10 s → 200 ms from the second visit on.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

1 files changed, 95 insertions, 33 deletions

diff --git a/packages/meshbay-hub/src/meshbay_hub/static/search-page.js b/packages/meshbay-hub/src/meshbay_hub/static/search-page.js
index 679b3f9..26d4e4c 100644
--- a/packages/meshbay-hub/src/meshbay_hub/static/search-page.js
+++ b/packages/meshbay-hub/src/meshbay_hub/static/search-page.js
@@ -16,7 +16,10 @@ import { transfers } from './transfers.js';
 import { mergeUnitEntries } from './source-merge.js';
 import { useStickyBand } from './sticky.js';
 
-const BATCH_SIZE = 3;
+// How many groups Search dials at once — a ceiling on concurrency, never a
+// batch. See `inFlight` below for why the difference is the whole of what a
+// reader waits for when a node is down.
+const MAX_IN_FLIGHT = 3;
 // One WebRTC peer connection per group the search view touches. The cap bounds
 // how many a busy account (many groups on the hub) keeps open at once; groups
 // past it connect lazily when a tile of theirs scrolls into view (video-app.js's
@@ -214,40 +217,102 @@ async function fetchGroupIndex(groupId, token, bundleKey, username, userId) {
   }
 }
 
-async function fetchAllIndexes(groups, token, username, userId, onProgress, onBatch) {
+/**
+ * Run `fn` over `items`, `n` at a time, starting the next the moment one ends.
+ *
+ * Deliberately not batches. A batch waits for its slowest member before the
+ * next one starts, and a group whose node is down is always the slowest member
+ * there is: it costs the full connection deadline. Measured on twelve groups
+ * with four of them down — the shape a reader really has — batches of three
+ * showed the first result after ten seconds and finished after forty, because
+ * every dead group stalled its own batch and postponed all the ones behind it.
+ * A dead group here occupies one of `n` places and holds up nothing else.
+ */
+async function inFlight(items, n, fn) {
+  let next = 0;
+  const worker = async () => {
+    while (next < items.length) {
+      const i = next;
+      next += 1;
+      await fn(items[i], i);
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(n, items.length) }, worker));
+}
+
+// Which groups did not answer last time, so they can be dialled last.
+//
+// A ceiling of three still leaves one bad case: if the first three groups the
+// hub lists are all down, all three places are held for the full deadline and
+// nothing else starts. A node that is off tends to stay off, so the browser
+// remembers and puts the known-silent ones at the back — where their deadline
+// runs beside results that are already on screen instead of in front of them.
+//
+// Per browser and advisory: unreadable storage, a private window, a first visit
+// or a stale list all degrade to the hub's own order, which is the behaviour
+// without this. It is rewritten after every sweep, so a node coming back costs
+// one sweep of being last and then returns to its place.
+const DOWN_KEY = 'meshbay.search.down';
+
+function lastKnownDown() {
+  try {
+    return new Set(JSON.parse(localStorage.getItem(DOWN_KEY) || '[]'));
+  } catch {
+    return new Set();
+  }
+}
+
+function rememberDown(ids) {
+  try {
+    localStorage.setItem(DOWN_KEY, JSON.stringify([...ids]));
+  } catch { /* private window, or storage refused: the order is advisory */ }
+}
+
+async function fetchAllIndexes(groups, token, username, userId, onProgress, onResult) {
   const bundleKey = session.bundleKey || await _loadBundleKey();
   if (bundleKey) session.bundleKey = bundleKey;
 
   const total = groups.length;
   let done = 0;
   const unreachable = [];
+  const downNow = new Set();
   const results = new Map();
 
-  for (let i = 0; i < groups.length; i += BATCH_SIZE) {
-    const batch = groups.slice(i, i + BATCH_SIZE);
-    await Promise.all(batch.map(async (g) => {
-      try {
-        const result = await fetchGroupIndex(g.id, token, bundleKey, username, userId);
-        if (result && result.unlisted) {
-          // Left out of Search by its operator: not shown, and not down either.
-        } else if (result) {
-          results.set(g.id, {
-            ...result,
-            groupName: g.name,
-            groupOwner: g.owner_username,
-          });
-          cacheGroupIndex(g.id, g.name, g.owner_username, result.entries, result.roots);
-        } else {
-          unreachable.push(g.name || g.id);
-        }
-      } catch {
+  // Stable, so groups that answered keep the hub's order among themselves.
+  const wasDown = lastKnownDown();
+  const queue = [...groups].sort(
+    (a, b) => (wasDown.has(a.id) ? 1 : 0) - (wasDown.has(b.id) ? 1 : 0));
+
+  await inFlight(queue, MAX_IN_FLIGHT, async (g) => {
+    try {
+      const result = await fetchGroupIndex(g.id, token, bundleKey, username, userId);
+      if (result && result.unlisted) {
+        // Left out of Search by its operator: not shown, and not down either.
+      } else if (result) {
+        results.set(g.id, {
+          ...result,
+          groupName: g.name,
+          groupOwner: g.owner_username,
+        });
+        cacheGroupIndex(g.id, g.name, g.owner_username, result.entries, result.roots);
+        // Drawn now, not when this group's neighbours are done. Its index is
+        // already in hand; holding it back until a group that is not answering
+        // has finished not answering is ten seconds of blank page for work that
+        // completed in two hundred milliseconds.
+        onResult(results);
+      } else {
         unreachable.push(g.name || g.id);
+        downNow.add(g.id);
       }
-      done++;
-      onProgress({ done, total, unreachable: [...unreachable] });
-    }));
-    onBatch(new Map(results));
-  }
+    } catch {
+      unreachable.push(g.name || g.id);
+      downNow.add(g.id);
+    }
+    done++;
+    onProgress({ done, total, unreachable: [...unreachable] });
+  });
+  rememberDown(downNow);
   return { results, unreachable };
 }
 
@@ -478,13 +543,10 @@ function SearchPage({ token, username, userId, groups, onPlayQueue, userPrefs })
 
     (async () => {
       const groupIds = [...indexedGroups.keys()].slice(0, MAX_POOL_SIZE);
-      for (let i = 0; i < groupIds.length; i += BATCH_SIZE) {
-        if (cancelled) break;
-        const batch = groupIds.slice(i, i + BATCH_SIZE);
-        await Promise.all(batch.map(async (gid) => {
-          try { await connectGroup(gid); } catch { /* skip */ }
-        }));
-      }
+      await inFlight(groupIds, MAX_IN_FLIGHT, async (gid) => {
+        if (cancelled) return;
+        try { await connectGroup(gid); } catch { /* skip */ }
+      });
     })();
 
     return () => { cancelled = true; };
@@ -762,7 +824,7 @@ function SearchPage({ token, username, userId, groups, onPlayQueue, userPrefs })
         </div>
       `}
 
-      ${!fetching && progress.unreachable.length > 0 && html`
+      ${progress.unreachable.length > 0 && html`
         <p class="search-unreachable">
           ${t('search.unreachable', { n: progress.unreachable.length })}
         </p>