From 059eb0318daf27d98bd1c9532705ff406c0a10f8 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Thu, 24 Sep 2026 00:10:41 +0200 Subject: fix(hub): Search reaches each group with one offer, and a busy hub is not a dead node The other half of the 4G failure. The page negotiated every group twice: the sweep opened a connection, read the index and closed it, then the warm-up opened the same group again. The sweep, the warm-up and the tiles each had a concurrency ceiling of their own, and together they went past what the hub admits per account. Whatever the hub refused was then reported as "node unreachable" and remembered as down, which put that group last next time. - Every connection goes through ConnectionPool, which holds the page's one ceiling (six at once, sized for twenty groups on a phone) and keeps what the sweep opened for the tiles. A visit costs one offer per group. A refresh costs none for a connection that answers a four-second ping, and a connection that died while the phone slept is replaced, not waited on. - A connection whose index is being read is held against eviction. With more groups than the pool keeps, it was otherwise the least recently used one. - Negotiations still under way when the page closes close what they get, and a sweep cut short that way remembers nobody as down. - transport.js sends an offer again on 429, 502 or 503, honouring Retry-After, with jittered waits of about twenty seconds at worst. Search counts each retry as progress. A 404, 403 or 504 still fails at once, so a dead node costs no time. The fan-out tests assumed a ceiling of three and were re-measured: four dead groups of twelve now hold nothing back, even on a first visit. The pool and the retry run as shipped code, lifted as text, against a fake clock. Each guard was checked by removing it and seeing its test fail. Co-Authored-By: Claude Opus 5.5 --- .../src/meshbay_hub/static/transport.js | 87 ++++++++++++++++++---- 1 file changed, 71 insertions(+), 16 deletions(-) (limited to 'packages/meshbay-hub/src/meshbay_hub/static/transport.js') diff --git a/packages/meshbay-hub/src/meshbay_hub/static/transport.js b/packages/meshbay-hub/src/meshbay_hub/static/transport.js index f6adaac..79cd8e1 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/transport.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/transport.js @@ -59,6 +59,57 @@ const UPLOAD_BUFFER_HIGH = 1024 * 1024; // that never answers costs a pause rather than the whole attempt. const ICE_GATHER_TIMEOUT_MS = 4000; +// Answers to an offer that are about the hub's load, never about the node: 429 +// is one of the hub's per-account ceilings (docs/MESHBAY_DESIGN.md §7.2), and +// 502/503 is the hub restarting behind its proxy. An offer refused for either is +// sent again, so it is not reported as a node that cannot be reached — which is +// what a single 429 used to become in Search, for a group whose node was +// answering every other offer in under a second. Anything else — 404 for a node +// that is not connected, 403, 504 for a node that did not answer — fails at once: +// retrying it would make a dead node cost time instead of costing nothing. +const OFFER_RETRY_STATUSES = new Set([429, 502, 503]); +// Worst case about twenty seconds of waiting, all of it on a hub that is +// answering. Search's own deadline treats each retry as progress and is still +// bounded by its ceiling. +const OFFER_RETRY_DELAYS_MS = [500, 1000, 2000, 4000, 8000]; +const OFFER_RETRY_AFTER_MAX_MS = 10000; + +/** + * POST an offer, sending it again while the hub refuses it for load. + * + * `Retry-After` is honoured when the hub gives one, and every delay is jittered: + * the refusals this exists for come in bursts — several groups dialled at once, + * or every connection of a phone reconnecting as it wakes — and retries that + * all land on the same millisecond would be refused together again. + * + * `isClosed` is asked after each wait, so a caller that gave up (Search's + * deadline, a page that unmounted) sends no further offer on its behalf. + */ +async function postOffer(call, url, init, { isClosed, onRetry } = {}) { + for (let attempt = 0; ; attempt += 1) { + const resp = await call(url, init); + if (resp.ok) return resp; + if (!OFFER_RETRY_STATUSES.has(resp.status) || attempt >= OFFER_RETRY_DELAYS_MS.length) { + const detail = await resp.json().catch(() => ({})); + const err = new Error(`Signaling failed: ${resp.status} ${detail.detail || ''}`); + err.status = resp.status; + throw err; + } + const after = Number(resp.headers && resp.headers.get && resp.headers.get('Retry-After')); + const base = after > 0 + ? Math.min(after * 1000, OFFER_RETRY_AFTER_MAX_MS) + : OFFER_RETRY_DELAYS_MS[attempt]; + const delay = Math.round(base * (0.75 + Math.random() * 0.5)); + if (onRetry) onRetry(resp.status, delay, attempt + 1); + await new Promise((resolve) => setTimeout(resolve, delay)); + if (isClosed && isClosed()) { + const err = new Error('Transport closed while the hub was busy'); + err.status = resp.status; + throw err; + } + } +} + const STREAM_CREDITS = 24; function _aborted() { @@ -823,23 +874,27 @@ class MeshBayTransport { // app:// origin is refused by CORS. const call = (window.MeshBayPlatform && window.MeshBayPlatform.apiFetch) || fetch; - const resp = await call( + const resp = await postOffer(call, `${this._hubUrl}/v1/nodes/${nodeId}/webrtc/offer`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${this._accessToken}`, - }, - body: JSON.stringify({ - sdp: this._pc.localDescription.sdp, - ice_candidates: [], - }), - }); - - if (!resp.ok) { - const detail = await resp.json().catch(() => ({})); - throw new Error(`Signaling failed: ${resp.status} ${detail.detail || ''}`); - } + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${this._accessToken}`, + }, + body: JSON.stringify({ + sdp: this._pc.localDescription.sdp, + ice_candidates: [], + }), + }, { + isClosed: () => this._closed, + onRetry: (status, delayMs, attempt) => { + console.warn('[MeshBay] Hub refused the offer with', status, + '— sending it again in', delayMs, 'ms (attempt', attempt, ')'); + trace('offer_retry', { status, delay_ms: delayMs, attempt }); + // The hub answered: a busy hub, not a silent node. + this._noteConnectProgress('offer_retry'); + }, + }); const answer = await resp.json(); this._rawAnswerSdp = answer.sdp; -- cgit v1.2.3