feat(node): persistent index cache, visible scan progress, adaptive reconcile, and delta sync

Indexer performance work, in four parts:

- Persistent (path, size, mtime) -> hash cache (indexer/cache.py) so a node
  restart no longer re-hashes every file — measured at 23 minutes for a
  114 GB library on a slow disk before this, near-instant after. Hashing
  is deliberately kept sequential (max_workers=1): it was never actually
  concurrent despite the pool size, and two interleaved reads seek-thrash
  a spinning disk instead of going faster.

- Byte-based scan progress (IndexProgress), surfaced via the loopback
  index-status route, the handshake ack, and a periodic INDEX_PROGRESS
  push to connected peers — drives a progress bar in the Create Group
  wizard and "add a directory" in Settings, and an animated presence dot.
  Guaranteed to settle back to idle via try/finally and a final push on
  the scanning->false transition.

- The reconcile backstop's directory walks now run in the executor
  instead of blocking the daemon's event loop; its interval defaults to
  10 min (was 60s) with adaptive backoff to 2h when nothing changes,
  reset on a real change or a peer connecting, and is now a per-group
  operator setting (signed op + group Settings UI).

- INDEX_DELTA wired up (protocol support existed, nothing called it):
  _on_index_change now sends additions/deletions instead of rebuilding
  the full entries list, coalesced over a short window so a burst of
  file events produces one push, and the hub swarm registration for
  public groups only (re-)registers newly added hashes.

Also fixes several bugs found while testing the above against real
libraries (a 114 GB and a 100+ GB group on a USB HDD):

- /api/reload blocked until the reload — including a brand-new group's
  full initial scan — finished, which the Electron bridge's fixed 30s
  call timeout turned into a hard failure on any real library. The route
  now fires the reload without waiting (ops.start_reload), matching
  add_root/remove_root's existing pattern; the wizard's own step order
  was fixed to wait for the group to actually appear hosted before the
  steps that need it (extra roots, GEK), with retries for the residual
  race between that and the daemon's own bookkeeping.
- transport.js's hand-rolled msgpack codec had no case for uint64/int64
  (0xcf/0xd3) and crashed decoding any message containing one — hit by
  IndexProgress.scanned_bytes/total_bytes for any group over ~4.3 GB.
  Verified against real msgpack-encoded bytes from the Python side.
- chat_hist_resp, and this change's own index_progress and
  set_scan_settings_ack pushes, were not routed by message type and
  could be handed to an unrelated pending request by the transport's
  "oldest pending" fallback, stalling it until its own 30s timeout and
  corrupting whatever received the wrong reply in its place.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016SF6RKNBKg9qejmoMJ9ybA

1 files changed, 80 insertions, 11 deletions

diff --git a/packages/meshbay-hub/src/meshbay_hub/static/app.js b/packages/meshbay-hub/src/meshbay_hub/static/app.js
index d18d9a4..044457b 100644
--- a/packages/meshbay-hub/src/meshbay_hub/static/app.js
+++ b/packages/meshbay-hub/src/meshbay_hub/static/app.js
@@ -298,7 +298,7 @@ function Nav({ user, theme, onThemeChange, onLogout, onMenuToggle, unreadCount,
 
 // ── Sidebar ──────────────────────────────────────────────────────────────────
 
-function Sidebar({ groups, presence, route, menuOpen, role, hasNodeKey }) {
+function Sidebar({ groups, presence, indexProgressPct, route, menuOpen, role, hasNodeKey }) {
   const isStaff = role === 'moderator' || role === 'admin';
   return html`
     <aside class="sidebar ${menuOpen ? 'open' : ''}">
@@ -339,13 +339,16 @@ function Sidebar({ groups, presence, route, menuOpen, role, hasNodeKey }) {
             // actually cares about. Anything else is "not known yet".
             const state = presence[g.id] ?? (g.node_online === true ? 'online'
               : g.node_online === false ? 'offline' : 'unknown');
+            const label = state === 'indexing'
+              ? t('presence.indexing', { pct: indexProgressPct[g.id] ?? 0 })
+              : t('presence.' + state);
             return html`
               <a key=${g.id}
                  class="sidebar-item ${route === '/group/' + g.id ? 'active' : ''}"
                  href="#/group/${g.id}">
                 <span class="presence presence-${state}"
-                      title="${t('presence.' + state)}"
-                      aria-label="${t('presence.' + state)}"></span>
+                      title="${label}"
+                      aria-label="${label}"></span>
                 <span class="sidebar-item-name">${g.name}</span>
               </a>
             `;
@@ -840,6 +843,9 @@ function CreateGroupWizard({ token, username, onCreated }) {
   const [setupSteps, setSetupSteps] = useState([]);
   const [setupError, setSetupError] = useState('');
   const [groupId, setGroupId] = useState('');
+  // Bytes-based, not file-count-based: one 20 GB file finishing last must
+  // not read as "8 of 9 done" while it is still the only thing running.
+  const [indexProgress, setIndexProgress] = useState(null);
 
   const linkNodeKey = useCallback(async (pk) => {
     if (!pk) return;
@@ -908,12 +914,14 @@ function CreateGroupWizard({ token, username, onCreated }) {
     const steps = [
       { label: t('wizard.step_create_hub'), status: 'pending' },
       { label: t('wizard.step_attach'), status: 'pending' },
+      { label: t('wizard.step_index'), status: 'pending' },
     ];
     if (roots.length > 1)
       steps.push({ label: t('wizard.step_add_roots'), status: 'pending' });
     steps.push({ label: t('wizard.step_gek'), status: 'pending' });
     steps.push({ label: t('wizard.step_pair'), status: 'pending' });
     setSetupSteps([...steps]);
+    setIndexProgress(null);
 
     let si = 0;
     const update = (status) => {
@@ -922,6 +930,27 @@ function CreateGroupWizard({ token, username, onCreated }) {
     };
     const advance = () => { si++; };
 
+    // Every step from here on is scoped to the group the node just attached.
+    // The node hot-loads a brand-new group synchronously — scan included —
+    // before it is added to groups_ctx or its own in-memory config
+    // (daemon.py _reload_config_inner: the config swap is the *last* thing
+    // that function does, a beat after the scan, not atomic with it) — so a
+    // call that lands in that beat gets refused even though the wait above
+    // already reported the scan as done. A handful of short retries absorbs
+    // that gap without a real cross-process synchronization primitive.
+    const withRetry = async (fn, attempts = 5, delayMs = 400) => {
+      for (let i = 0; i < attempts; i++) {
+        try {
+          return await fn();
+        } catch (err) {
+          const msg = String((err && err.message) || '');
+          const notHostedYet = /not configured on this node|not hosted on this node/i.test(msg);
+          if (!notHostedYet || i === attempts - 1) throw err;
+          await new Promise((r) => setTimeout(r, delayMs));
+        }
+      }
+    };
+
     try {
       // 1. Create group on hub
       update('running');
@@ -942,32 +971,48 @@ function CreateGroupWizard({ token, username, onCreated }) {
         attachBody.upload_dir = mainRoot.path;
       }
       await platform.node.call('POST', '/api/groups/attach', attachBody);
+      // Fire-and-forget on the node's side (ui/app.py) — this call itself
+      // returns immediately, well before the scan below finishes. It used
+      // to be the thing the wizard waited on, which is exactly what made
+      // "Attaching to node" time out on a real library (see ops.start_reload).
       await platform.node.call('POST', '/api/reload');
       update('done');
       advance();
 
-      // 3. Add extra roots (if >1)
+      // 3. Wait for the node's own initial scan of this group to finish —
+      // the group is not usable for anything below (extra roots, GEK) until
+      // this finishes, so nobody lands on a page that looks broken, or hits
+      // a "not configured" error from racing ahead of it. Can take tens of
+      // minutes on a slow disk (see the StarWars benchmark) — the node
+      // keeps scanning on its own either way (test_hot_reload_survives_
+      // client_close.py); this step is only about not lying about it.
+      update('running');
+      await platform.waitForGroupHosted(gid, setIndexProgress);
+      update('done');
+      advance();
+
+      // 4. Add extra roots (if >1)
       if (roots.length > 1) {
         update('running');
         for (let i = 0; i < roots.length; i++) {
           if (i === (uploadIdx < roots.length ? uploadIdx : 0)) continue;
           const r = roots[i];
-          await platform.node.call('POST', `/api/groups/${gid}/roots`, {
+          await withRetry(() => platform.node.call('POST', `/api/groups/${gid}/roots`, {
             path: r.path, name: r.name,
             upload: i === uploadIdx,
-          });
+          }));
         }
         update('done');
         advance();
       }
 
-      // 4. GEK init
+      // 5. GEK init
       update('running');
-      await platform.node.call('POST', `/api/groups/${gid}/gek`);
+      await withRetry(() => platform.node.call('POST', `/api/groups/${gid}/gek`));
       update('done');
       advance();
 
-      // 5. Generate pairing code
+      // 6. Generate pairing code
       update('running');
       const pairResult = await platform.node.call('POST', '/api/operator/pair');
       if (pairResult && pairResult.code) {
@@ -976,7 +1021,7 @@ function CreateGroupWizard({ token, username, onCreated }) {
       }
       update('done');
 
-      // Reload node config so it picks up the new group
+      // Reload once more so any roots added at step 4 are picked up.
       try { await platform.node.call('POST', '/api/reload'); } catch { /* best effort */ }
 
       setStep(3);
@@ -1105,6 +1150,9 @@ function CreateGroupWizard({ token, username, onCreated }) {
 
   // Step 2: Automatic setup progress
   if (step === 2) {
+    const pct = indexProgress && indexProgress.total_bytes
+      ? Math.min(100, Math.round(100 * indexProgress.scanned_bytes / indexProgress.total_bytes))
+      : 0;
     return html`<div class="page-content">
       <h2>${t('wizard.title')}</h2>
       <p class="page-message">${t('wizard.setting_up')}</p>
@@ -1120,6 +1168,19 @@ function CreateGroupWizard({ token, username, onCreated }) {
           </div>
         `)}
       </div>
+      ${indexProgress && indexProgress.scanning && html`
+        <div class="index-progress" style="margin-top:12px">
+          <div class="index-progress-bar">
+            <div class="index-progress-fill" style="width:${pct}%"></div>
+          </div>
+          <div class="index-progress-label">${t('wizard.indexing_progress', { pct })}</div>
+          ${indexProgress.current_dir && html`
+            <div class="index-progress-dir">
+              ${t('wizard.indexing_current_dir', { dir: indexProgress.current_dir })}
+            </div>
+          `}
+        </div>
+      `}
       ${setupError && html`
         <div class="error-msg" style="margin-top:16px">${setupError}</div>
         <div style="display:flex;gap:8px;margin-top:8px">
@@ -2835,8 +2896,15 @@ function App() {
   // for the session only: it is a cache of observations, not a source of truth,
   // and a reload should go back to asking.
   const [presence, setPresence] = useState({});
-  const notePresence = useCallback((gid, state) => {
+  // Percentage alongside 'indexing' presence — kept separate from `presence`
+  // itself so a changing % does not require treating every tick as a new
+  // presence state (see the Sidebar dot's title/aria-label).
+  const [indexProgressPct, setIndexProgressPct] = useState({});
+  const notePresence = useCallback((gid, state, pct) => {
     setPresence(prev => (prev[gid] === state ? prev : { ...prev, [gid]: state }));
+    if (pct !== undefined) {
+      setIndexProgressPct(prev => (prev[gid] === pct ? prev : { ...prev, [gid]: pct }));
+    }
   }, []);
 
   const handleLeftGroup = useCallback((gid) => {
@@ -3084,6 +3152,7 @@ function App() {
         ${user && html`<${Sidebar}
           groups=${groups}
           presence=${presence}
+          indexProgressPct=${indexProgressPct}
           route=${route}
           menuOpen=${menuOpen}
           role=${user.role}