fix(hub): a group whose node is down is no longer something the reader waits for

A node is a machine in somebody's house, so with a handful of groups one is
always off. Search treated that as the exception and charged the reader for it.

Measured on twelve groups against a virtual clock, a live group answering in
200 ms and a dead one taking the full deadline: one node down in the first batch
put a blank page and a progress bar in front of the reader for **ten seconds**,
while two groups of that same batch had answered in two hundred milliseconds and
nine others had not been dialled at all. Four down, spread out — the shape a real
reader has — was ten seconds to the first result and **forty** to the last.

Two causes, neither of them the connection deadline. Results were drawn once a
batch was complete rather than as each arrived, so an index already in hand waited
on a node that was not answering. And the batches were sequential, so a dead group
did not merely cost its own deadline, it postponed every group behind it.

So the three is a ceiling on concurrency and never a batch — `inFlight` starts the
next group the moment one ends — and each index is drawn when it lands.

That alone is not enough, and the tests say where it stops: a ceiling still lets
silent nodes hold every place at once, and with four of twelve down the last three
live groups still waited out a deadline. So the browser remembers which groups
were silent and dials them last, which puts all eight on screen in 600 ms. The
list is advisory and rewritten from what each sweep saw: a private window, storage
that refuses, a first visit or a node that has come back all fall through to the
hub's own order, cost one sweep, and correct themselves.

The "n groups unreachable" line also waited for the sweep to finish, which is the
one moment it is no longer needed. It now appears as they are found.

First result, before and after, twelve groups: one down 10 s → 200 ms; four down
10 s → 200 ms with every reachable group on screen by 600 ms; three down and
listed first 10 s → 200 ms from the second visit on.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

1 files changed, 144 insertions, 0 deletions

diff --git a/packages/meshbay-hub/tests/harness/search_fanout_harness.mjs b/packages/meshbay-hub/tests/harness/search_fanout_harness.mjs
new file mode 100644
index 0000000..74b4bb3
--- /dev/null
+++ b/packages/meshbay-hub/tests/harness/search_fanout_harness.mjs
@@ -0,0 +1,144 @@
+// Run the SHIPPED `fetchAllIndexes` against a fake clock and report *when the
+// reader sees something*.
+//
+// The question this answers is not how long the whole sweep takes. It is how
+// long a group whose node is down makes everybody else wait — which is a
+// property of the fan-out, not of any one connection's deadline.
+//
+// `fetchAllIndexes`, `inFlight` and the concurrency ceiling are lifted out of
+// search-page.js as text
+// and executed; what is modelled is the clock and one group's reachability.
+//
+// Usage: node search_fanout_harness.mjs <path to search-page.js> <json config>
+import { readFileSync } from 'fs';
+
+const src = readFileSync(process.argv[2], 'utf8');
+const cfg = JSON.parse(process.argv[3] || '{}');
+
+const {
+  groups = 12,
+  // Indexes (0-based, in the hub's listing order) of the groups whose node is
+  // down. They take `deadMs`; every other group answers in `liveMs`.
+  dead = [],
+  liveMs = 200,
+  deadMs = 10000,
+  // Group indexes the browser remembers as silent from a previous sweep.
+  knownDown = null,
+  runUntil = 600000,
+} = cfg;
+
+// The browser's own store, modelled: a first visit has nothing, and a private
+// window throws on both halves.
+const refuses = knownDown === 'refuses';
+const stored = (knownDown === null || refuses)
+  ? null
+  : JSON.stringify(knownDown.map((n) => `g-${n}`));
+let written = null;
+globalThis.localStorage = {
+  getItem: () => {
+    if (refuses) throw new Error('storage disabled');
+    return stored;
+  },
+  setItem: (_k, v) => {
+    if (refuses) throw new Error('storage disabled');
+    written = v;
+  },
+};
+
+// ── The clock ────────────────────────────────────────────────────────────────
+
+let now = 0;
+let nextId = 1;
+let timers = [];
+globalThis.setTimeout = (fn, ms) => {
+  const t = { at: now + (ms || 0), fn, id: nextId++ };
+  timers.push(t);
+  return t.id;
+};
+globalThis.clearTimeout = (id) => { timers = timers.filter((t) => t.id !== id); };
+
+const flush = async () => { for (let i = 0; i < 50; i++) await Promise.resolve(); };
+
+async function run(until) {
+  await flush();
+  while (timers.length) {
+    const due = timers.reduce((a, b) => (b.at < a.at ? b : a));
+    if (due.at > until) break;
+    timers = timers.filter((t) => t !== due);
+    now = due.at;
+    due.fn();
+    await flush();
+  }
+  now = until;
+}
+
+// ── The environment ──────────────────────────────────────────────────────────
+
+const shown = [];   // [{ at, groups }] — one entry per render the reader gets
+
+const session = { bundleKey: 'k' };
+const _loadBundleKey = async () => 'k';
+const cacheGroupIndex = () => {};
+
+// One group's index, answered on the clock rather than over a network.
+const fetchGroupIndex = (groupId) => new Promise((resolve, reject) => {
+  const n = Number(String(groupId).split('-')[1]);
+  const isDead = dead.includes(n);
+  setTimeout(
+    () => (isDead ? reject(new Error('Connection timeout'))
+                  : resolve({ entries: [{ id: `e${n}` }], roots: {} })),
+    isDead ? deadMs : liveMs,
+  );
+});
+
+// ── The code under test, as text ─────────────────────────────────────────────
+
+const ceiling = /^const MAX_IN_FLIGHT = (\d+);/m.exec(src);
+if (!ceiling) throw new Error('MAX_IN_FLIGHT is gone from search-page.js');
+
+// Both functions, because the sweep is the two of them: `inFlight` decides what
+// runs when, and lifting only its caller would leave the harness measuring a
+// concurrency it had supplied itself.
+const lift = (signature) => {
+  const start = src.indexOf(signature);
+  if (start < 0) throw new Error(`${signature} is gone from search-page.js`);
+  return src.slice(start, src.indexOf('\n}\n', start) + 3);
+};
+
+const make = new Function(
+  'session', '_loadBundleKey', 'cacheGroupIndex', 'fetchGroupIndex', 'localStorage',
+  `const MAX_IN_FLIGHT = ${ceiling[1]};
+   const DOWN_KEY = 'harness';
+   ${lift('function lastKnownDown(')}
+   ${lift('function rememberDown(')}
+   ${lift('async function inFlight(')}
+   ${lift('async function fetchAllIndexes(')}
+   return fetchAllIndexes;`,
+);
+const fetchAllIndexes = make(
+  session, _loadBundleKey, cacheGroupIndex, fetchGroupIndex, globalThis.localStorage);
+
+// ── The scenario ─────────────────────────────────────────────────────────────
+
+const list = Array.from({ length: groups }, (_, i) => ({ id: `g-${i}`, name: `G${i}` }));
+
+let finishedAt = null;
+fetchAllIndexes(
+  list, 'tok', 'alice', 'u1',
+  () => {},                                            // progress counters only
+  (results) => { shown.push({ at: now, groups: results.size }); },
+).then(() => { finishedAt = now; });
+
+await run(runUntil);
+
+console.log(JSON.stringify({
+  // When the reader first sees any result at all, and how many.
+  firstPaintAt: shown.length ? shown[0].at : null,
+  firstPaintGroups: shown.length ? shown[0].groups : 0,
+  // Every render, so a test can see whether live groups waited on dead ones.
+  renders: shown,
+  finishedAt,
+  inFlight: Number(ceiling[1]),
+  // What the browser will remember for next time.
+  remembered: written === null ? null : JSON.parse(written),
+}));