diff options
9 files changed, 777 insertions, 9 deletions
diff --git a/packages/meshbay-common/src/meshbay_common/__init__.py b/packages/meshbay-common/src/meshbay_common/__init__.py index 0ca0005..1f82f88 100644 --- a/packages/meshbay-common/src/meshbay_common/__init__.py +++ b/packages/meshbay-common/src/meshbay_common/__init__.py @@ -49,5 +49,8 @@ __version__ = "0.7.0" # *set*, replaced whole in one signed op — a photo library is routinely # scattered across several folders, not one. Additive: an older client # never sends the op and never expects either field. -MNP_VERSION = "0.11" +# 0.12: added `link_preview_req`/`link_preview_resp` — the node unfurls a URL +# pasted in chat into an OpenGraph card. Additive: an older node logs "unknown +# type" and the client just shows the bare link, as it always did. +MNP_VERSION = "0.12" MHP_VERSION = "0.1" diff --git a/packages/meshbay-common/src/meshbay_common/protocol.py b/packages/meshbay-common/src/meshbay_common/protocol.py index dd377e7..d4e3ccd 100644 --- a/packages/meshbay-common/src/meshbay_common/protocol.py +++ b/packages/meshbay-common/src/meshbay_common/protocol.py @@ -37,6 +37,11 @@ class MNP: CHAT_ATTACHMENT = "chat_attach" # attachment metadata CHAT_HISTORY = "chat_hist" # request message history (newest, or before a cursor) CHAT_HISTORY_RESPONSE = "chat_hist_resp" # history response with messages + # Link unfurl: the node fetches a URL a member pasted and returns an + # OpenGraph card. Additive (0.12) — an older node just logs "unknown type" + # and the client shows the bare link, exactly as before. + LINK_PREVIEW_REQ = "link_preview_req" # client → node: unfurl this URL + LINK_PREVIEW_RESP = "link_preview_resp" # node → client: card fields, or ok:false # Liveness on an *already open* channel. A peer that goes away without # closing leaves a DataChannel that still reads as connected until the next # real request hangs, and there was no way to ask. This is not a discovery diff --git a/packages/meshbay-hub/src/meshbay_hub/static/chat-app.js b/packages/meshbay-hub/src/meshbay_hub/static/chat-app.js index 0babf51..3d518b3 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/chat-app.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/chat-app.js @@ -15,15 +15,20 @@ import { formatSize, CHUNK_SIZE, pipelinedDownload } from './file-utils.js'; */ const URL_RE = /\bhttps?:\/\/[^\s<>"']+/gi; +function _trimUrl(url) { + let tail = ''; + while (/[.,;:!?)\]]$/.test(url)) { tail = url.slice(-1) + tail; url = url.slice(0, -1); } + return url; +} + function linkify(text) { const out = []; let last = 0; for (const m of String(text).matchAll(URL_RE)) { if (m.index > last) out.push(text.slice(last, m.index)); // Trailing punctuation is almost never part of the address. - let url = m[0]; - let tail = ''; - while (/[.,;:!?)\]]$/.test(url)) { tail = url.slice(-1) + tail; url = url.slice(0, -1); } + const url = _trimUrl(m[0]); + const tail = m[0].slice(url.length); out.push(html`<a href=${url} target="_blank" rel="noopener noreferrer" class="chat-link">${url}</a>`); if (tail) out.push(tail); @@ -33,6 +38,79 @@ function linkify(text) { return out; } +/** The first http(s) URL in a message, address only, or null. */ +function firstUrl(text) { + const m = String(text).match(URL_RE); + return m ? _trimUrl(m[0]) : null; +} + +// Unfurled cards and their images, per session. The node keeps nothing +// durable (draft-v6 §2.7) and re-answers on request; this stops the same +// message re-asking on every scroll/rerender. `_previewCache` values are the +// node's response object, or the string 'pending' / 'failed'. +const _previewCache = new Map(); +const _previewImgCache = new Map(); + +function LinkPreview({ url, transportRef, gekRef }) { + const [data, setData] = useState(() => { + const c = _previewCache.get(url); + return c && typeof c === 'object' ? c : null; + }); + const [imgUrl, setImgUrl] = useState(() => null); + + useEffect(() => { + let cancelled = false; + const cached = _previewCache.get(url); + if (cached === 'failed') return; + if (cached && typeof cached === 'object') { setData(cached); return; } + if (cached === 'pending') return; + + const transport = transportRef.current; + if (!transport || !transport.connected) return; + _previewCache.set(url, 'pending'); + transport.fetchLinkPreview(url) + .then(resp => { + const value = resp && resp.ok ? resp : 'failed'; + _previewCache.set(url, value); + if (!cancelled && typeof value === 'object') setData(value); + }) + .catch(() => { _previewCache.set(url, 'failed'); }); + return () => { cancelled = true; }; + }, [url]); + + const thumbHash = data && data.image_thumb_hash; + useEffect(() => { + if (!thumbHash) return; + const cached = _previewImgCache.get(thumbHash); + if (cached) { setImgUrl(cached); return; } + let cancelled = false; + (async () => { + const transport = transportRef.current; + if (!transport || !transport.connected) return; + try { + const chunks = await pipelinedDownload(transport, gekRef.current, thumbHash, 1); + if (cancelled) return; + const blobUrl = URL.createObjectURL(new Blob(chunks, { type: 'image/jpeg' })); + _previewImgCache.set(thumbHash, blobUrl); + setImgUrl(blobUrl); + } catch { /* card renders without the image */ } + })(); + return () => { cancelled = true; }; + }, [thumbHash]); + + if (!data || !data.ok) return null; + return html` + <a class="chat-link-preview" href=${url} target="_blank" rel="noopener noreferrer"> + ${imgUrl && html`<img class="clp-img" src=${imgUrl} alt="" />`} + <span class="clp-body"> + ${data.site_name && html`<span class="clp-site">${data.site_name}</span>`} + ${data.title && html`<span class="clp-title">${data.title}</span>`} + ${data.description && html`<span class="clp-desc">${data.description}</span>`} + </span> + </a> + `; +} + function formatTime(ts) { const d = new Date(ts * 1000); const now = new Date(); @@ -217,6 +295,27 @@ function ChatPanel({ transportRef, username, entries, gekRef, onRefreshIndex, if (atBottomRef.current) list.scrollTop = list.scrollHeight; }, [messages]); + // Keep the view pinned to the newest message while the reader is at the + // bottom, through everything that grows the content *after* the initial + // paint: attachment thumbnails and link-preview cards fetched over the + // network, late layout, and the panel resizing itself with `fit()` below. + // Without this, opening the Chat tab reliably lands a screen or two above + // the last message — the layout effect above ran when the list was still + // short. Does nothing once the reader scrolls up (`atBottomRef` is false), + // so it never fights reading back or the "load older" anchor. + useEffect(() => { + const list = listRef.current; + if (!list || typeof ResizeObserver === 'undefined') return; + const stick = () => { + if (atBottomRef.current) list.scrollTop = list.scrollHeight; + }; + const ro = new ResizeObserver(stick); + ro.observe(list); + for (const child of list.children) ro.observe(child); + stick(); + return () => ro.disconnect(); + }, [messages]); + // The panel was `calc(100vh - 220px)`: a guess at how much sits above it. On a // phone the group header — title, description, edit link, delete button, tabs // — is closer to 430px, so the panel ran past the fold and the composer ended @@ -374,6 +473,8 @@ function ChatPanel({ transportRef, username, entries, gekRef, onRefreshIndex, ? _dayLabel(m.timestamp) : null; const parsed = _parsePayload(m.payload); const att = parsed && parsed.attachment; + const msgText = parsed && typeof parsed.text === 'string' ? parsed.text : m.payload; + const msgUrl = att ? null : firstUrl(msgText); return html` ${daySep && html` <div class="chat-day" key=${'d' + m.id}><span>${daySep}</span></div> @@ -403,10 +504,9 @@ function ChatPanel({ transportRef, username, entries, gekRef, onRefreshIndex, <div class="chat-att-size">${formatSize(att.size)}</div> </div> ` : html` - <span class="chat-text"> - ${linkify(parsed && typeof parsed.text === 'string' - ? parsed.text : m.payload)} - </span> + <span class="chat-text">${linkify(msgText)}</span> + ${msgUrl && html`<${LinkPreview} url=${msgUrl} + transportRef=${transportRef} gekRef=${gekRef} />`} `} <span class="chat-time">${formatTime(m.timestamp)}</span> </div> diff --git a/packages/meshbay-hub/src/meshbay_hub/static/style.css b/packages/meshbay-hub/src/meshbay_hub/static/style.css index 975f47c..61a5e3b 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/style.css +++ b/packages/meshbay-hub/src/meshbay_hub/static/style.css @@ -581,6 +581,58 @@ button:disabled { opacity: 0.5; cursor: not-allowed; } .chat-text { white-space: pre-wrap; } +/* Link unfurl card. Sits on its own line inside the bubble and carries its + own surface colours so it stays readable inside an own-message bubble too. */ +.chat-link-preview { + flex: 0 0 100%; + display: flex; + gap: 10px; + margin-top: 4px; + padding: 8px; + max-width: 380px; + border: 1px solid var(--border); + border-left: 3px solid var(--accent); + border-radius: 8px; + background: var(--bg-base); + color: var(--text); + text-decoration: none; + overflow: hidden; +} +.chat-link-preview:hover { background: var(--bg-surface); } +.clp-img { + width: 72px; + height: 72px; + flex-shrink: 0; + object-fit: cover; + border-radius: 6px; + background: var(--bg-surface); +} +.clp-body { display: flex; flex-direction: column; gap: 2px; min-width: 0; } +.clp-site { + font-size: 0.7em; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.03em; +} +.clp-title { + font-weight: 600; + font-size: 0.9em; + line-height: 1.3; + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} +.clp-desc { + font-size: 0.82em; + color: var(--text-secondary); + line-height: 1.35; + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + .chat-time { font-size: 0.65em; color: var(--text-dim); diff --git a/packages/meshbay-hub/src/meshbay_hub/static/transport.js b/packages/meshbay-hub/src/meshbay_hub/static/transport.js index 9f85ee1..1a146ce 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/transport.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/transport.js @@ -847,6 +847,20 @@ class MeshBayTransport { } /** + * Unfurl a URL pasted in chat. The node fetches it (the browser cannot — + * CSP and CORS — and would leak every reader's IP), parses an OpenGraph + * card, and caches any image in its thumb store; `image_thumb_hash` then + * rides the normal file_req path like a poster. `ok: false` means "no + * preview" (blocked, unreachable, not HTML) — the caller just shows the + * bare link. Keyed by url: a message with several links fires one each. + */ + async fetchLinkPreview(url) { + const msg = await this._sendAndWait({ type: 'link_preview_req', v: '0.6', url }); + if (msg.type === 'error') throw new Error(msg.detail); + return msg; + } + + /** * One season's own overview/air_date/poster (docs/mediacenter.md §5.4's * per-season view) — a show's own tmdb_meta is one static field that does * not necessarily describe every season alike, found live: a 3-season @@ -1854,6 +1868,9 @@ class MeshBayTransport { ? `chunk:${obj.file_id}:${obj.chunk_index}` : obj.type === 'ping' ? `ping:${obj.token}` : obj.type === 'media_meta_req' ? `media_meta:${obj.file_id}` + // One chat message can carry several links, each unfurled on its + // own; matching by arrival order would swap two cards. + : obj.type === 'link_preview_req' ? `link_preview:${obj.url}` // Same reordering hazard as media_meta_req: an album grid fires // one music_meta_req per visible tile, several at a time. : obj.type === 'music_meta_req' ? `music_meta:${obj.file_id}` @@ -2171,6 +2188,16 @@ class MeshBayTransport { return; } + // Same reasoning as media_meta_resp: keyed by url, and "nobody's waiting" + // must not fall through. + if (msg.type === 'link_preview_resp') { + const key = `link_preview:${msg.url}`; + for (const [, handler] of this._pending) { + if (handler._key === key) { handler.resolve(msg); return; } + } + return; + } + // Same reasoning as media_meta_resp: keyed, not arrival-order, and // "nobody's waiting any more" must not fall through either. if (msg.type === 'music_meta_resp') { diff --git a/packages/meshbay-node/src/meshbay_node/linkpreview.py b/packages/meshbay-node/src/meshbay_node/linkpreview.py new file mode 100644 index 0000000..64067d1 --- /dev/null +++ b/packages/meshbay-node/src/meshbay_node/linkpreview.py @@ -0,0 +1,251 @@ +""" +Link unfurling for chat — fetch a URL a member pasted and pull an +OpenGraph-style card out of it (title, description, site name, image). + +Who does the fetch matters. It is **the node**, not the browser and not the +hub: + + * the browser cannot — a strict `img-src`/`connect-src` and CORS block it, + and a direct fetch would leak every reader's IP to the linked host on + every render; + * the hub must not — it never touches group content (draft-v6 §2.5); + * the node already fetches third-party metadata for the Videos and Music + apps (`_fetch_and_cache_poster`), over the same authorised path. + +Because the node makes an outbound request to an address a *member* chose, +this is an SSRF surface. `safe_url()` is the gate: http(s) only, no +credentials, and the resolved address must be globally routable — no +loopback, private, link-local, multicast or reserved range. Redirects are +followed by hand so every hop is re-checked. Residual: a DNS name that +resolves clean here and to something internal microseconds later at connect +time (rebinding) — narrow, and closed properly by pinning the checked IP, +which is a follow-up. + +Nothing is stored durably: the caller keeps an in-memory TTL cache and the +OG image rides the existing `media_cache` thumb store (same as a poster). +""" + +from __future__ import annotations + +import ipaddress +import logging +import socket +from html.parser import HTMLParser +from io import BytesIO +from urllib.parse import urljoin, urlsplit + +import httpx + +log = logging.getLogger(__name__) + +_TIMEOUT = 5.0 +_MAX_REDIRECTS = 3 +_MAX_HTML_BYTES = 512 * 1024 +_MAX_IMAGE_BYTES = 2 * 1024 * 1024 +_IMAGE_MAX_DIM = 600 +_UA = "MeshBayBot/1.0 (+https://meshbay.org; link preview)" + + +class UnsafeURL(ValueError): + """The URL points somewhere the node must not fetch from.""" + + +def _addr_is_public(ip: str) -> bool: + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return False + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None: + addr = addr.ipv4_mapped + return not ( + addr.is_private or addr.is_loopback or addr.is_link_local + or addr.is_multicast or addr.is_reserved or addr.is_unspecified + ) + + +def safe_url(url: str) -> str: + """Return the URL unchanged if it is safe to fetch, else raise UnsafeURL.""" + if not isinstance(url, str) or len(url) > 2048: + raise UnsafeURL("missing or oversized") + parts = urlsplit(url) + if parts.scheme not in ("http", "https"): + raise UnsafeURL(f"scheme {parts.scheme!r}") + if parts.username or parts.password: + raise UnsafeURL("credentials in URL") + host = parts.hostname + if not host: + raise UnsafeURL("no host") + # An IP literal is checked directly; a name is resolved and every answer + # must be public — a hostname with one public and one 127.0.0.1 record + # would otherwise be a way in. + try: + infos = socket.getaddrinfo(host, parts.port or (443 if parts.scheme == "https" else 80), + proto=socket.IPPROTO_TCP) + except socket.gaierror as e: + raise UnsafeURL(f"cannot resolve: {e}") + resolved = {info[4][0] for info in infos} + if not resolved: + raise UnsafeURL("resolves to nothing") + bad = [ip for ip in resolved if not _addr_is_public(ip)] + if bad: + raise UnsafeURL(f"non-public address {bad[0]}") + return url + + +class _HeadParser(HTMLParser): + """Collects <title> text and name/property→content from <meta> in <head>. + + Stops caring once <body> starts: everything a card needs is in the head, + and a 512 KB page of body is not worth walking. + """ + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.metas: dict[str, str] = {} + self.title: str | None = None + self._in_title = False + self.done = False + + def handle_starttag(self, tag, attrs): + if tag == "body": + self.done = True + elif tag == "title": + self._in_title = True + elif tag == "meta": + a = {k.lower(): (v or "") for k, v in attrs} + key = (a.get("property") or a.get("name") or "").lower().strip() + if key and "content" in a and key not in self.metas: + self.metas[key] = a["content"].strip() + + def handle_endtag(self, tag): + if tag == "title": + self._in_title = False + + def handle_data(self, data): + if self._in_title and self.title is None: + text = data.strip() + if text: + self.title = text + + +def _first(metas: dict[str, str], *keys: str) -> str | None: + for k in keys: + v = metas.get(k) + if v: + return v + return None + + +async def _get(client: httpx.AsyncClient, url: str) -> httpx.Response: + """One GET with manual, re-validated redirects.""" + current = safe_url(url) + for _ in range(_MAX_REDIRECTS + 1): + resp = await client.get(current, headers={"User-Agent": _UA}, + follow_redirects=False) + if resp.is_redirect and "location" in resp.headers: + current = safe_url(urljoin(current, resp.headers["location"])) + continue + return resp + raise UnsafeURL("too many redirects") + + +async def fetch_preview(url: str, *, client: httpx.AsyncClient | None = None) -> dict | None: + """ + Return {url, title, description, site_name, image_url} for a URL, or None + if it cannot be unfurled (unreachable, not HTML, nothing worth showing). + Never raises for an ordinary failure — the caller treats "no preview" as + the common case, exactly like a TMDB miss. + """ + own = client is None + if own: + client = httpx.AsyncClient(timeout=_TIMEOUT, max_redirects=0) + try: + safe_url(url) + resp = await _get(client, url) + ctype = resp.headers.get("content-type", "").split(";")[0].strip().lower() + if resp.status_code != 200 or ctype not in ("text/html", "application/xhtml+xml"): + return None + + body = b"" + async for chunk in resp.aiter_bytes(): + body += chunk + if len(body) >= _MAX_HTML_BYTES: + break + final_url = str(resp.url) + + parser = _HeadParser() + try: + parser.feed(body.decode(resp.encoding or "utf-8", errors="replace")) + except Exception: + pass + m = parser.metas + + title = _first(m, "og:title", "twitter:title") or parser.title + description = _first(m, "og:description", "twitter:description", "description") + site_name = _first(m, "og:site_name") or urlsplit(final_url).hostname + image = _first(m, "og:image", "og:image:url", "og:image:secure_url", + "twitter:image", "twitter:image:src") + if image: + image = urljoin(final_url, image) + try: + safe_url(image) + except UnsafeURL: + image = None + + if not title and not description: + return None + + return { + "url": url, + "title": (title or "")[:300] or None, + "description": (description or "")[:600] or None, + "site_name": (site_name or "")[:120] or None, + "image_url": image, + } + except (httpx.HTTPError, UnsafeURL) as e: + log.debug("link preview for %s: %s", url[:80], e) + return None + finally: + if own: + await client.aclose() + + +async def fetch_image(url: str, *, client: httpx.AsyncClient | None = None) -> bytes | None: + """Fetch and re-encode an OG image to a small JPEG. None on any failure.""" + own = client is None + if own: + client = httpx.AsyncClient(timeout=_TIMEOUT, max_redirects=0) + try: + safe_url(url) + resp = await _get(client, url) + ctype = resp.headers.get("content-type", "").split(";")[0].strip().lower() + if resp.status_code != 200 or not ctype.startswith("image/"): + return None + raw = b"" + async for chunk in resp.aiter_bytes(): + raw += chunk + if len(raw) > _MAX_IMAGE_BYTES: + return None + return _downscale(raw) + except (httpx.HTTPError, UnsafeURL) as e: + log.debug("link preview image %s: %s", url[:80], e) + return None + finally: + if own: + await client.aclose() + + +def _downscale(raw: bytes) -> bytes | None: + try: + from PIL import Image + except ImportError: + return None + try: + with Image.open(BytesIO(raw)) as im: + im = im.convert("RGB") + im.thumbnail((_IMAGE_MAX_DIM, _IMAGE_MAX_DIM)) + out = BytesIO() + im.save(out, format="JPEG", quality=80) + return out.getvalue() + except Exception: + return None diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py index 252e202..4958b92 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py @@ -97,7 +97,7 @@ from meshbay_common.webcrypto import chunk_key_aes, encrypt_chunk_aes from meshbay_common.protocol import MNP, index_entry_wire from meshbay_node.indexer import GroupIndex from meshbay_node.indexer.indexer import DirectoryIndexer -from meshbay_node import ops +from meshbay_node import linkpreview, ops # Re-imported under its original name: every call site and existing test in # this module still refers to it as `_probe_video`. The implementation lives # in media_probe.py so the indexer package (imported just above) can call it @@ -115,6 +115,34 @@ log = logging.getLogger(__name__) CHUNK_SIZE = 1024 * 1024 MAX_MSG = 64 * 1024 * 1024 +# Chat link-preview results, kept in memory only (draft-v6 §2.7: the node +# produces enrichment on demand and keeps nothing durable — the asking device +# caches). Bounded and time-limited so a busy group cannot grow it without end +# and a page that changed its card is picked up within the hour. +_LINK_PREVIEW_TTL = 3600 +_LINK_PREVIEW_MAX = 256 +_link_preview_cache: dict[str, tuple[float, dict]] = {} + + +def _link_preview_cache_get(url: str) -> dict | None: + hit = _link_preview_cache.get(url) + if hit is None: + return None + ts, value = hit + if time.time() - ts > _LINK_PREVIEW_TTL: + _link_preview_cache.pop(url, None) + return None + return value + + +def _link_preview_cache_put(url: str, value: dict) -> None: + if not url: + return + if len(_link_preview_cache) >= _LINK_PREVIEW_MAX: + oldest = min(_link_preview_cache, key=lambda k: _link_preview_cache[k][0]) + _link_preview_cache.pop(oldest, None) + _link_preview_cache[url] = (time.time(), value) + # Upload limits (finding C5a). Uploads used to land directly in the shared root under # a name the client chose, overwriting whatever was already there — which both violated # node sovereignty and defeated the delete authorization (overwrite a file, become its @@ -396,6 +424,8 @@ class WebRTCPeerSession: self._do_chat_message(msg) elif mtype == MNP.CHAT_HISTORY: self._do_chat_history(msg) + elif mtype == MNP.LINK_PREVIEW_REQ: + self._spawn(self._do_link_preview_request(msg)) elif mtype == MNP.PING: self._do_ping(msg) elif mtype == MNP.FILE_UPLOAD: @@ -3409,6 +3439,53 @@ class WebRTCPeerSession: ], }) + async def _do_link_preview_request(self, msg: dict) -> None: + """ + Unfurl a URL a member pasted into chat (draft-v6 §2.7 enrichment rule: + the client asks, the node produces on demand, the asking device + caches — nothing durable here). + + `linkpreview.safe_url` is the SSRF gate: the URL a *member* chose + decides an outbound request from the operator's machine, so http(s) + only and the resolved address must be globally routable. Failure of + any kind — blocked, unreachable, not HTML, nothing worth showing — + comes back as `ok: false`, the way a TMDB miss does; the client then + just shows the bare link. + """ + url = msg.get("url") + key = url if isinstance(url, str) else "" + cached = _link_preview_cache_get(key) + if cached is not None: + self._send({**cached, "type": MNP.LINK_PREVIEW_RESP, "v": MNP_VERSION}) + return + + resp: dict = {"type": MNP.LINK_PREVIEW_RESP, "v": MNP_VERSION, + "url": key, "ok": False} + try: + meta = await linkpreview.fetch_preview(url) + if meta is not None: + resp.update(ok=True, title=meta["title"], + description=meta["description"], + site_name=meta["site_name"]) + image_url = meta.get("image_url") + media_cache = self._ctx.get("media_cache") + if image_url and media_cache is not None: + synthetic_id = f"linkpreview:{image_url}" + thumb_hash = await media_cache.get_thumb_hash_by_file_id(synthetic_id) + if thumb_hash is None: + jpeg = await linkpreview.fetch_image(image_url) + if jpeg: + thumb_hash = blake3.blake3(jpeg).hexdigest() + await media_cache.put_thumb(thumb_hash, synthetic_id, jpeg) + if thumb_hash: + resp["image_thumb_hash"] = thumb_hash + except Exception as e: + log.debug("link_preview_req %s: %s", key[:80], e) + + _link_preview_cache_put(key, {k: v for k, v in resp.items() + if k not in ("type", "v")}) + self._send(resp) + def _do_file_upload(self, msg: dict) -> None: ctx = self._group_ctx() filename = msg.get("filename", "") diff --git a/packages/meshbay-node/tests/test_link_preview_request.py b/packages/meshbay-node/tests/test_link_preview_request.py new file mode 100644 index 0000000..d975f91 --- /dev/null +++ b/packages/meshbay-node/tests/test_link_preview_request.py @@ -0,0 +1,98 @@ +""" +`_do_link_preview_request` — routing, the media_cache image round-trip, and +the in-memory result cache. + +The fetch itself (`linkpreview.fetch_preview` / `fetch_image`) is covered by +test_linkpreview.py and stubbed here, so this is purely about what the +handler does with a result: reply shape, storing the OG image under its +blake3 in the thumb store, and not re-fetching a URL it has already seen. +""" + +import blake3 +import pytest +from meshbay_common.protocol import MNP +from meshbay_node import linkpreview +from meshbay_node.media_cache import MediaCache +from meshbay_node.transport import webrtc_server +from meshbay_node.transport.webrtc_server import WebRTCPeerSession + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def media_cache(tmp_path): + c = MediaCache(db_path=tmp_path / "media_cache.db") + await c.open() + yield c + await c.close() + + +@pytest.fixture(autouse=True) +def _clear_cache(): + webrtc_server._link_preview_cache.clear() + yield + webrtc_server._link_preview_cache.clear() + + +def _session(media_cache): + s = WebRTCPeerSession.__new__(WebRTCPeerSession) + s._ctx = {"media_cache": media_cache} + s.sent = [] + s._send = s.sent.append + return s + + +async def test_reply_carries_the_card_and_caches_the_image(media_cache, monkeypatch): + async def fake_preview(url, **k): + return {"url": url, "title": "Hello", "description": "World", + "site_name": "Example", "image_url": "https://example.com/c.png"} + + async def fake_image(url, **k): + return b"jpeg-bytes" + monkeypatch.setattr(linkpreview, "fetch_preview", fake_preview) + monkeypatch.setattr(linkpreview, "fetch_image", fake_image) + + s = _session(media_cache) + await s._do_link_preview_request({"url": "https://example.com/p"}) + + (resp,) = s.sent + assert resp["type"] == MNP.LINK_PREVIEW_RESP + assert resp["ok"] is True + assert resp["title"] == "Hello" + assert resp["site_name"] == "Example" + want_hash = blake3.blake3(b"jpeg-bytes").hexdigest() + assert resp["image_thumb_hash"] == want_hash + # And the bytes are in the thumb store, servable via the normal file_req path. + assert await media_cache.get_thumb(want_hash) == b"jpeg-bytes" + + +async def test_unfurlable_failure_is_ok_false(media_cache, monkeypatch): + async def none_preview(url, **k): + return None + monkeypatch.setattr(linkpreview, "fetch_preview", none_preview) + + s = _session(media_cache) + await s._do_link_preview_request({"url": "http://169.254.169.254/"}) + + (resp,) = s.sent + assert resp["ok"] is False + assert "image_thumb_hash" not in resp + + +async def test_second_request_for_the_same_url_is_served_from_cache(media_cache, monkeypatch): + calls = {"n": 0} + + async def counting_preview(url, **k): + calls["n"] += 1 + return {"url": url, "title": "Once", "description": None, + "site_name": None, "image_url": None} + monkeypatch.setattr(linkpreview, "fetch_preview", counting_preview) + + s = _session(media_cache) + await s._do_link_preview_request({"url": "https://example.com/a"}) + await s._do_link_preview_request({"url": "https://example.com/a"}) + + assert calls["n"] == 1 + assert len(s.sent) == 2 + assert s.sent[0]["title"] == s.sent[1]["title"] == "Once" + assert all(r["type"] == MNP.LINK_PREVIEW_RESP for r in s.sent) diff --git a/packages/meshbay-node/tests/test_linkpreview.py b/packages/meshbay-node/tests/test_linkpreview.py new file mode 100644 index 0000000..0dd951a --- /dev/null +++ b/packages/meshbay-node/tests/test_linkpreview.py @@ -0,0 +1,155 @@ +""" +`linkpreview` — the SSRF gate and the OpenGraph parse. + +The gate is the part with teeth: the URL is chosen by a *member*, and it +decides an outbound request from the operator's machine. Anything that is not +a public http(s) address must be refused before a socket opens. +""" + +import socket + +import httpx +import pytest +from meshbay_node import linkpreview +from meshbay_node.linkpreview import UnsafeURL, safe_url + +PUBLIC_IP = "93.184.216.34" # example.com, historically + + +@pytest.fixture +def resolves_public(monkeypatch): + """Every hostname resolves to one public address.""" + def fake_getaddrinfo(host, port, *a, **k): + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", + (PUBLIC_IP, port or 80))] + monkeypatch.setattr(linkpreview.socket, "getaddrinfo", fake_getaddrinfo) + + +# ── safe_url ──────────────────────────────────────────────────────────────── + +@pytest.mark.parametrize("url", [ + "http://127.0.0.1/x", + "http://localhost/x", # resolves to loopback on any box + "http://169.254.169.254/latest/meta-data/", # cloud metadata + "http://[::1]/x", + "http://10.1.2.3/x", + "http://192.168.0.1/x", + "http://172.16.0.1/x", + "http://0.0.0.0/x", + "http://[::ffff:127.0.0.1]/x", # v4-mapped loopback + "ftp://example.com/x", + "file:///etc/passwd", + "http://user:pass@example.com/x", + "javascript:alert(1)", + "not a url", +]) +def test_safe_url_refuses(url): + with pytest.raises(UnsafeURL): + safe_url(url) + + +def test_safe_url_accepts_a_public_host(resolves_public): + assert safe_url("https://example.com/some/page") == "https://example.com/some/page" + + +def test_safe_url_refuses_a_host_with_any_private_record(monkeypatch): + def mixed(host, port, *a, **k): + return [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (PUBLIC_IP, port)), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)), + ] + monkeypatch.setattr(linkpreview.socket, "getaddrinfo", mixed) + with pytest.raises(UnsafeURL): + safe_url("https://sneaky.example/x") + + +# ── fetch_preview ────────────────────────────────────────────────────────── + +_HTML = """ +<!doctype html><html><head> + <title>Fallback Title</title> + <meta property="og:title" content="The Real Title"> + <meta property="og:description" content="A short summary of the page."> + <meta property="og:site_name" content="Example"> + <meta property="og:image" content="/card.png"> + <meta name="description" content="ignored, og wins"> +</head><body>...body we should not need...</body></html> +""" + + +def _client(handler): + return httpx.AsyncClient(transport=httpx.MockTransport(handler), + timeout=5.0, max_redirects=0) + + +async def test_fetch_preview_reads_opengraph(resolves_public): + def handler(request): + return httpx.Response(200, headers={"content-type": "text/html; charset=utf-8"}, + text=_HTML) + async with _client(handler) as c: + meta = await linkpreview.fetch_preview("https://example.com/article", client=c) + assert meta["title"] == "The Real Title" + assert meta["description"] == "A short summary of the page." + assert meta["site_name"] == "Example" + assert meta["image_url"] == "https://example.com/card.png" # absolutised + + +async def test_fetch_preview_falls_back_to_title_tag(resolves_public): + def handler(request): + return httpx.Response(200, headers={"content-type": "text/html"}, + text="<html><head><title>Just A Title</title></head></html>") + async with _client(handler) as c: + meta = await linkpreview.fetch_preview("https://example.com/", client=c) + assert meta["title"] == "Just A Title" + assert meta["description"] is None + + +async def test_fetch_preview_gives_up_on_non_html(resolves_public): + def handler(request): + return httpx.Response(200, headers={"content-type": "application/pdf"}, + content=b"%PDF-1.4") + async with _client(handler) as c: + assert await linkpreview.fetch_preview("https://example.com/x.pdf", client=c) is None + + +async def test_fetch_preview_gives_up_when_nothing_worth_showing(resolves_public): + def handler(request): + return httpx.Response(200, headers={"content-type": "text/html"}, + text="<html><head></head><body>hi</body></html>") + async with _client(handler) as c: + assert await linkpreview.fetch_preview("https://example.com/", client=c) is None + + +async def test_fetch_preview_revalidates_redirects(monkeypatch): + # First host is public; it 302s to a loopback address. + calls = {"n": 0} + + def resolve(host, port, *a, **k): + ip = PUBLIC_IP if host == "ok.example" else "127.0.0.1" + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (ip, port or 80))] + monkeypatch.setattr(linkpreview.socket, "getaddrinfo", resolve) + + def handler(request): + calls["n"] += 1 + return httpx.Response(302, headers={"location": "http://internal.example/secret"}) + async with _client(handler) as c: + meta = await linkpreview.fetch_preview("https://ok.example/start", client=c) + assert meta is None + assert calls["n"] == 1 # stopped at the redirect, never fetched internal + + +async def test_fetch_image_downscales(resolves_public): + from io import BytesIO + + from PIL import Image + buf = BytesIO() + Image.new("RGB", (2000, 1500), (10, 20, 30)).save(buf, format="PNG") + big_png = buf.getvalue() + + def handler(request): + return httpx.Response(200, headers={"content-type": "image/png"}, content=big_png) + async with _client(handler) as c: + jpeg = await linkpreview.fetch_image("https://example.com/card.png", client=c) + assert jpeg and jpeg[:2] == b"\xff\xd8" # JPEG SOI + with Image.open(BytesIO(jpeg)) as im: + assert max(im.size) <= linkpreview._IMAGE_MAX_DIM |