diff options
Diffstat (limited to 'packages/meshbay-node')
4 files changed, 582 insertions, 1 deletions
diff --git a/packages/meshbay-node/src/meshbay_node/linkpreview.py b/packages/meshbay-node/src/meshbay_node/linkpreview.py new file mode 100644 index 0000000..64067d1 --- /dev/null +++ b/packages/meshbay-node/src/meshbay_node/linkpreview.py @@ -0,0 +1,251 @@ +""" +Link unfurling for chat — fetch a URL a member pasted and pull an +OpenGraph-style card out of it (title, description, site name, image). + +Who does the fetch matters. It is **the node**, not the browser and not the +hub: + + * the browser cannot — a strict `img-src`/`connect-src` and CORS block it, + and a direct fetch would leak every reader's IP to the linked host on + every render; + * the hub must not — it never touches group content (draft-v6 §2.5); + * the node already fetches third-party metadata for the Videos and Music + apps (`_fetch_and_cache_poster`), over the same authorised path. + +Because the node makes an outbound request to an address a *member* chose, +this is an SSRF surface. `safe_url()` is the gate: http(s) only, no +credentials, and the resolved address must be globally routable — no +loopback, private, link-local, multicast or reserved range. Redirects are +followed by hand so every hop is re-checked. Residual: a DNS name that +resolves clean here and to something internal microseconds later at connect +time (rebinding) — narrow, and closed properly by pinning the checked IP, +which is a follow-up. + +Nothing is stored durably: the caller keeps an in-memory TTL cache and the +OG image rides the existing `media_cache` thumb store (same as a poster). +""" + +from __future__ import annotations + +import ipaddress +import logging +import socket +from html.parser import HTMLParser +from io import BytesIO +from urllib.parse import urljoin, urlsplit + +import httpx + +log = logging.getLogger(__name__) + +_TIMEOUT = 5.0 +_MAX_REDIRECTS = 3 +_MAX_HTML_BYTES = 512 * 1024 +_MAX_IMAGE_BYTES = 2 * 1024 * 1024 +_IMAGE_MAX_DIM = 600 +_UA = "MeshBayBot/1.0 (+https://meshbay.org; link preview)" + + +class UnsafeURL(ValueError): + """The URL points somewhere the node must not fetch from.""" + + +def _addr_is_public(ip: str) -> bool: + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return False + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None: + addr = addr.ipv4_mapped + return not ( + addr.is_private or addr.is_loopback or addr.is_link_local + or addr.is_multicast or addr.is_reserved or addr.is_unspecified + ) + + +def safe_url(url: str) -> str: + """Return the URL unchanged if it is safe to fetch, else raise UnsafeURL.""" + if not isinstance(url, str) or len(url) > 2048: + raise UnsafeURL("missing or oversized") + parts = urlsplit(url) + if parts.scheme not in ("http", "https"): + raise UnsafeURL(f"scheme {parts.scheme!r}") + if parts.username or parts.password: + raise UnsafeURL("credentials in URL") + host = parts.hostname + if not host: + raise UnsafeURL("no host") + # An IP literal is checked directly; a name is resolved and every answer + # must be public — a hostname with one public and one 127.0.0.1 record + # would otherwise be a way in. + try: + infos = socket.getaddrinfo(host, parts.port or (443 if parts.scheme == "https" else 80), + proto=socket.IPPROTO_TCP) + except socket.gaierror as e: + raise UnsafeURL(f"cannot resolve: {e}") + resolved = {info[4][0] for info in infos} + if not resolved: + raise UnsafeURL("resolves to nothing") + bad = [ip for ip in resolved if not _addr_is_public(ip)] + if bad: + raise UnsafeURL(f"non-public address {bad[0]}") + return url + + +class _HeadParser(HTMLParser): + """Collects <title> text and name/property→content from <meta> in <head>. + + Stops caring once <body> starts: everything a card needs is in the head, + and a 512 KB page of body is not worth walking. + """ + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.metas: dict[str, str] = {} + self.title: str | None = None + self._in_title = False + self.done = False + + def handle_starttag(self, tag, attrs): + if tag == "body": + self.done = True + elif tag == "title": + self._in_title = True + elif tag == "meta": + a = {k.lower(): (v or "") for k, v in attrs} + key = (a.get("property") or a.get("name") or "").lower().strip() + if key and "content" in a and key not in self.metas: + self.metas[key] = a["content"].strip() + + def handle_endtag(self, tag): + if tag == "title": + self._in_title = False + + def handle_data(self, data): + if self._in_title and self.title is None: + text = data.strip() + if text: + self.title = text + + +def _first(metas: dict[str, str], *keys: str) -> str | None: + for k in keys: + v = metas.get(k) + if v: + return v + return None + + +async def _get(client: httpx.AsyncClient, url: str) -> httpx.Response: + """One GET with manual, re-validated redirects.""" + current = safe_url(url) + for _ in range(_MAX_REDIRECTS + 1): + resp = await client.get(current, headers={"User-Agent": _UA}, + follow_redirects=False) + if resp.is_redirect and "location" in resp.headers: + current = safe_url(urljoin(current, resp.headers["location"])) + continue + return resp + raise UnsafeURL("too many redirects") + + +async def fetch_preview(url: str, *, client: httpx.AsyncClient | None = None) -> dict | None: + """ + Return {url, title, description, site_name, image_url} for a URL, or None + if it cannot be unfurled (unreachable, not HTML, nothing worth showing). + Never raises for an ordinary failure — the caller treats "no preview" as + the common case, exactly like a TMDB miss. + """ + own = client is None + if own: + client = httpx.AsyncClient(timeout=_TIMEOUT, max_redirects=0) + try: + safe_url(url) + resp = await _get(client, url) + ctype = resp.headers.get("content-type", "").split(";")[0].strip().lower() + if resp.status_code != 200 or ctype not in ("text/html", "application/xhtml+xml"): + return None + + body = b"" + async for chunk in resp.aiter_bytes(): + body += chunk + if len(body) >= _MAX_HTML_BYTES: + break + final_url = str(resp.url) + + parser = _HeadParser() + try: + parser.feed(body.decode(resp.encoding or "utf-8", errors="replace")) + except Exception: + pass + m = parser.metas + + title = _first(m, "og:title", "twitter:title") or parser.title + description = _first(m, "og:description", "twitter:description", "description") + site_name = _first(m, "og:site_name") or urlsplit(final_url).hostname + image = _first(m, "og:image", "og:image:url", "og:image:secure_url", + "twitter:image", "twitter:image:src") + if image: + image = urljoin(final_url, image) + try: + safe_url(image) + except UnsafeURL: + image = None + + if not title and not description: + return None + + return { + "url": url, + "title": (title or "")[:300] or None, + "description": (description or "")[:600] or None, + "site_name": (site_name or "")[:120] or None, + "image_url": image, + } + except (httpx.HTTPError, UnsafeURL) as e: + log.debug("link preview for %s: %s", url[:80], e) + return None + finally: + if own: + await client.aclose() + + +async def fetch_image(url: str, *, client: httpx.AsyncClient | None = None) -> bytes | None: + """Fetch and re-encode an OG image to a small JPEG. None on any failure.""" + own = client is None + if own: + client = httpx.AsyncClient(timeout=_TIMEOUT, max_redirects=0) + try: + safe_url(url) + resp = await _get(client, url) + ctype = resp.headers.get("content-type", "").split(";")[0].strip().lower() + if resp.status_code != 200 or not ctype.startswith("image/"): + return None + raw = b"" + async for chunk in resp.aiter_bytes(): + raw += chunk + if len(raw) > _MAX_IMAGE_BYTES: + return None + return _downscale(raw) + except (httpx.HTTPError, UnsafeURL) as e: + log.debug("link preview image %s: %s", url[:80], e) + return None + finally: + if own: + await client.aclose() + + +def _downscale(raw: bytes) -> bytes | None: + try: + from PIL import Image + except ImportError: + return None + try: + with Image.open(BytesIO(raw)) as im: + im = im.convert("RGB") + im.thumbnail((_IMAGE_MAX_DIM, _IMAGE_MAX_DIM)) + out = BytesIO() + im.save(out, format="JPEG", quality=80) + return out.getvalue() + except Exception: + return None diff --git a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py index 252e202..4958b92 100644 --- a/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py +++ b/packages/meshbay-node/src/meshbay_node/transport/webrtc_server.py @@ -97,7 +97,7 @@ from meshbay_common.webcrypto import chunk_key_aes, encrypt_chunk_aes from meshbay_common.protocol import MNP, index_entry_wire from meshbay_node.indexer import GroupIndex from meshbay_node.indexer.indexer import DirectoryIndexer -from meshbay_node import ops +from meshbay_node import linkpreview, ops # Re-imported under its original name: every call site and existing test in # this module still refers to it as `_probe_video`. The implementation lives # in media_probe.py so the indexer package (imported just above) can call it @@ -115,6 +115,34 @@ log = logging.getLogger(__name__) CHUNK_SIZE = 1024 * 1024 MAX_MSG = 64 * 1024 * 1024 +# Chat link-preview results, kept in memory only (draft-v6 §2.7: the node +# produces enrichment on demand and keeps nothing durable — the asking device +# caches). Bounded and time-limited so a busy group cannot grow it without end +# and a page that changed its card is picked up within the hour. +_LINK_PREVIEW_TTL = 3600 +_LINK_PREVIEW_MAX = 256 +_link_preview_cache: dict[str, tuple[float, dict]] = {} + + +def _link_preview_cache_get(url: str) -> dict | None: + hit = _link_preview_cache.get(url) + if hit is None: + return None + ts, value = hit + if time.time() - ts > _LINK_PREVIEW_TTL: + _link_preview_cache.pop(url, None) + return None + return value + + +def _link_preview_cache_put(url: str, value: dict) -> None: + if not url: + return + if len(_link_preview_cache) >= _LINK_PREVIEW_MAX: + oldest = min(_link_preview_cache, key=lambda k: _link_preview_cache[k][0]) + _link_preview_cache.pop(oldest, None) + _link_preview_cache[url] = (time.time(), value) + # Upload limits (finding C5a). Uploads used to land directly in the shared root under # a name the client chose, overwriting whatever was already there — which both violated # node sovereignty and defeated the delete authorization (overwrite a file, become its @@ -396,6 +424,8 @@ class WebRTCPeerSession: self._do_chat_message(msg) elif mtype == MNP.CHAT_HISTORY: self._do_chat_history(msg) + elif mtype == MNP.LINK_PREVIEW_REQ: + self._spawn(self._do_link_preview_request(msg)) elif mtype == MNP.PING: self._do_ping(msg) elif mtype == MNP.FILE_UPLOAD: @@ -3409,6 +3439,53 @@ class WebRTCPeerSession: ], }) + async def _do_link_preview_request(self, msg: dict) -> None: + """ + Unfurl a URL a member pasted into chat (draft-v6 §2.7 enrichment rule: + the client asks, the node produces on demand, the asking device + caches — nothing durable here). + + `linkpreview.safe_url` is the SSRF gate: the URL a *member* chose + decides an outbound request from the operator's machine, so http(s) + only and the resolved address must be globally routable. Failure of + any kind — blocked, unreachable, not HTML, nothing worth showing — + comes back as `ok: false`, the way a TMDB miss does; the client then + just shows the bare link. + """ + url = msg.get("url") + key = url if isinstance(url, str) else "" + cached = _link_preview_cache_get(key) + if cached is not None: + self._send({**cached, "type": MNP.LINK_PREVIEW_RESP, "v": MNP_VERSION}) + return + + resp: dict = {"type": MNP.LINK_PREVIEW_RESP, "v": MNP_VERSION, + "url": key, "ok": False} + try: + meta = await linkpreview.fetch_preview(url) + if meta is not None: + resp.update(ok=True, title=meta["title"], + description=meta["description"], + site_name=meta["site_name"]) + image_url = meta.get("image_url") + media_cache = self._ctx.get("media_cache") + if image_url and media_cache is not None: + synthetic_id = f"linkpreview:{image_url}" + thumb_hash = await media_cache.get_thumb_hash_by_file_id(synthetic_id) + if thumb_hash is None: + jpeg = await linkpreview.fetch_image(image_url) + if jpeg: + thumb_hash = blake3.blake3(jpeg).hexdigest() + await media_cache.put_thumb(thumb_hash, synthetic_id, jpeg) + if thumb_hash: + resp["image_thumb_hash"] = thumb_hash + except Exception as e: + log.debug("link_preview_req %s: %s", key[:80], e) + + _link_preview_cache_put(key, {k: v for k, v in resp.items() + if k not in ("type", "v")}) + self._send(resp) + def _do_file_upload(self, msg: dict) -> None: ctx = self._group_ctx() filename = msg.get("filename", "") diff --git a/packages/meshbay-node/tests/test_link_preview_request.py b/packages/meshbay-node/tests/test_link_preview_request.py new file mode 100644 index 0000000..d975f91 --- /dev/null +++ b/packages/meshbay-node/tests/test_link_preview_request.py @@ -0,0 +1,98 @@ +""" +`_do_link_preview_request` — routing, the media_cache image round-trip, and +the in-memory result cache. + +The fetch itself (`linkpreview.fetch_preview` / `fetch_image`) is covered by +test_linkpreview.py and stubbed here, so this is purely about what the +handler does with a result: reply shape, storing the OG image under its +blake3 in the thumb store, and not re-fetching a URL it has already seen. +""" + +import blake3 +import pytest +from meshbay_common.protocol import MNP +from meshbay_node import linkpreview +from meshbay_node.media_cache import MediaCache +from meshbay_node.transport import webrtc_server +from meshbay_node.transport.webrtc_server import WebRTCPeerSession + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def media_cache(tmp_path): + c = MediaCache(db_path=tmp_path / "media_cache.db") + await c.open() + yield c + await c.close() + + +@pytest.fixture(autouse=True) +def _clear_cache(): + webrtc_server._link_preview_cache.clear() + yield + webrtc_server._link_preview_cache.clear() + + +def _session(media_cache): + s = WebRTCPeerSession.__new__(WebRTCPeerSession) + s._ctx = {"media_cache": media_cache} + s.sent = [] + s._send = s.sent.append + return s + + +async def test_reply_carries_the_card_and_caches_the_image(media_cache, monkeypatch): + async def fake_preview(url, **k): + return {"url": url, "title": "Hello", "description": "World", + "site_name": "Example", "image_url": "https://example.com/c.png"} + + async def fake_image(url, **k): + return b"jpeg-bytes" + monkeypatch.setattr(linkpreview, "fetch_preview", fake_preview) + monkeypatch.setattr(linkpreview, "fetch_image", fake_image) + + s = _session(media_cache) + await s._do_link_preview_request({"url": "https://example.com/p"}) + + (resp,) = s.sent + assert resp["type"] == MNP.LINK_PREVIEW_RESP + assert resp["ok"] is True + assert resp["title"] == "Hello" + assert resp["site_name"] == "Example" + want_hash = blake3.blake3(b"jpeg-bytes").hexdigest() + assert resp["image_thumb_hash"] == want_hash + # And the bytes are in the thumb store, servable via the normal file_req path. + assert await media_cache.get_thumb(want_hash) == b"jpeg-bytes" + + +async def test_unfurlable_failure_is_ok_false(media_cache, monkeypatch): + async def none_preview(url, **k): + return None + monkeypatch.setattr(linkpreview, "fetch_preview", none_preview) + + s = _session(media_cache) + await s._do_link_preview_request({"url": "http://169.254.169.254/"}) + + (resp,) = s.sent + assert resp["ok"] is False + assert "image_thumb_hash" not in resp + + +async def test_second_request_for_the_same_url_is_served_from_cache(media_cache, monkeypatch): + calls = {"n": 0} + + async def counting_preview(url, **k): + calls["n"] += 1 + return {"url": url, "title": "Once", "description": None, + "site_name": None, "image_url": None} + monkeypatch.setattr(linkpreview, "fetch_preview", counting_preview) + + s = _session(media_cache) + await s._do_link_preview_request({"url": "https://example.com/a"}) + await s._do_link_preview_request({"url": "https://example.com/a"}) + + assert calls["n"] == 1 + assert len(s.sent) == 2 + assert s.sent[0]["title"] == s.sent[1]["title"] == "Once" + assert all(r["type"] == MNP.LINK_PREVIEW_RESP for r in s.sent) diff --git a/packages/meshbay-node/tests/test_linkpreview.py b/packages/meshbay-node/tests/test_linkpreview.py new file mode 100644 index 0000000..0dd951a --- /dev/null +++ b/packages/meshbay-node/tests/test_linkpreview.py @@ -0,0 +1,155 @@ +""" +`linkpreview` — the SSRF gate and the OpenGraph parse. + +The gate is the part with teeth: the URL is chosen by a *member*, and it +decides an outbound request from the operator's machine. Anything that is not +a public http(s) address must be refused before a socket opens. +""" + +import socket + +import httpx +import pytest +from meshbay_node import linkpreview +from meshbay_node.linkpreview import UnsafeURL, safe_url + +PUBLIC_IP = "93.184.216.34" # example.com, historically + + +@pytest.fixture +def resolves_public(monkeypatch): + """Every hostname resolves to one public address.""" + def fake_getaddrinfo(host, port, *a, **k): + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", + (PUBLIC_IP, port or 80))] + monkeypatch.setattr(linkpreview.socket, "getaddrinfo", fake_getaddrinfo) + + +# ── safe_url ──────────────────────────────────────────────────────────────── + +@pytest.mark.parametrize("url", [ + "http://127.0.0.1/x", + "http://localhost/x", # resolves to loopback on any box + "http://169.254.169.254/latest/meta-data/", # cloud metadata + "http://[::1]/x", + "http://10.1.2.3/x", + "http://192.168.0.1/x", + "http://172.16.0.1/x", + "http://0.0.0.0/x", + "http://[::ffff:127.0.0.1]/x", # v4-mapped loopback + "ftp://example.com/x", + "file:///etc/passwd", + "http://user:pass@example.com/x", + "javascript:alert(1)", + "not a url", +]) +def test_safe_url_refuses(url): + with pytest.raises(UnsafeURL): + safe_url(url) + + +def test_safe_url_accepts_a_public_host(resolves_public): + assert safe_url("https://example.com/some/page") == "https://example.com/some/page" + + +def test_safe_url_refuses_a_host_with_any_private_record(monkeypatch): + def mixed(host, port, *a, **k): + return [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (PUBLIC_IP, port)), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)), + ] + monkeypatch.setattr(linkpreview.socket, "getaddrinfo", mixed) + with pytest.raises(UnsafeURL): + safe_url("https://sneaky.example/x") + + +# ── fetch_preview ────────────────────────────────────────────────────────── + +_HTML = """ +<!doctype html><html><head> + <title>Fallback Title</title> + <meta property="og:title" content="The Real Title"> + <meta property="og:description" content="A short summary of the page."> + <meta property="og:site_name" content="Example"> + <meta property="og:image" content="/card.png"> + <meta name="description" content="ignored, og wins"> +</head><body>...body we should not need...</body></html> +""" + + +def _client(handler): + return httpx.AsyncClient(transport=httpx.MockTransport(handler), + timeout=5.0, max_redirects=0) + + +async def test_fetch_preview_reads_opengraph(resolves_public): + def handler(request): + return httpx.Response(200, headers={"content-type": "text/html; charset=utf-8"}, + text=_HTML) + async with _client(handler) as c: + meta = await linkpreview.fetch_preview("https://example.com/article", client=c) + assert meta["title"] == "The Real Title" + assert meta["description"] == "A short summary of the page." + assert meta["site_name"] == "Example" + assert meta["image_url"] == "https://example.com/card.png" # absolutised + + +async def test_fetch_preview_falls_back_to_title_tag(resolves_public): + def handler(request): + return httpx.Response(200, headers={"content-type": "text/html"}, + text="<html><head><title>Just A Title</title></head></html>") + async with _client(handler) as c: + meta = await linkpreview.fetch_preview("https://example.com/", client=c) + assert meta["title"] == "Just A Title" + assert meta["description"] is None + + +async def test_fetch_preview_gives_up_on_non_html(resolves_public): + def handler(request): + return httpx.Response(200, headers={"content-type": "application/pdf"}, + content=b"%PDF-1.4") + async with _client(handler) as c: + assert await linkpreview.fetch_preview("https://example.com/x.pdf", client=c) is None + + +async def test_fetch_preview_gives_up_when_nothing_worth_showing(resolves_public): + def handler(request): + return httpx.Response(200, headers={"content-type": "text/html"}, + text="<html><head></head><body>hi</body></html>") + async with _client(handler) as c: + assert await linkpreview.fetch_preview("https://example.com/", client=c) is None + + +async def test_fetch_preview_revalidates_redirects(monkeypatch): + # First host is public; it 302s to a loopback address. + calls = {"n": 0} + + def resolve(host, port, *a, **k): + ip = PUBLIC_IP if host == "ok.example" else "127.0.0.1" + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (ip, port or 80))] + monkeypatch.setattr(linkpreview.socket, "getaddrinfo", resolve) + + def handler(request): + calls["n"] += 1 + return httpx.Response(302, headers={"location": "http://internal.example/secret"}) + async with _client(handler) as c: + meta = await linkpreview.fetch_preview("https://ok.example/start", client=c) + assert meta is None + assert calls["n"] == 1 # stopped at the redirect, never fetched internal + + +async def test_fetch_image_downscales(resolves_public): + from io import BytesIO + + from PIL import Image + buf = BytesIO() + Image.new("RGB", (2000, 1500), (10, 20, 30)).save(buf, format="PNG") + big_png = buf.getvalue() + + def handler(request): + return httpx.Response(200, headers={"content-type": "image/png"}, content=big_png) + async with _client(handler) as c: + jpeg = await linkpreview.fetch_image("https://example.com/card.png", client=c) + assert jpeg and jpeg[:2] == b"\xff\xd8" # JPEG SOI + with Image.open(BytesIO(jpeg)) as im: + assert max(im.size) <= linkpreview._IMAGE_MAX_DIM |