1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
|
"""
Link unfurling for chat — fetch a URL a member pasted and pull an
OpenGraph-style card out of it (title, description, site name, image).
Who does the fetch matters. It is **the node**, not the browser and not the
hub:
* the browser cannot — a strict `img-src`/`connect-src` and CORS block it,
and a direct fetch would leak every reader's IP to the linked host on
every render;
* the hub must not — it never touches group content (draft-v6 §2.5);
* the node already fetches third-party metadata for the Videos and Music
apps (`_fetch_and_cache_poster`), over the same authorised path.
Because the node makes an outbound request to an address a *member* chose,
this is an SSRF surface. `safe_url()` is the gate: http(s) only, no
credentials, and the resolved address must be globally routable — no
loopback, private, link-local, multicast or reserved range. Redirects are
followed by hand so every hop is re-checked. Residual: a DNS name that
resolves clean here and to something internal microseconds later at connect
time (rebinding) — narrow, and closed properly by pinning the checked IP,
which is a follow-up.
Nothing is stored durably: the caller keeps an in-memory TTL cache and the
OG image rides the existing `media_cache` thumb store (same as a poster).
"""
from __future__ import annotations
import ipaddress
import logging
import socket
from html.parser import HTMLParser
from io import BytesIO
from urllib.parse import urljoin, urlsplit
import httpx
log = logging.getLogger(__name__)
_TIMEOUT = 5.0
_MAX_REDIRECTS = 3
_MAX_HTML_BYTES = 512 * 1024
_MAX_IMAGE_BYTES = 2 * 1024 * 1024
_IMAGE_MAX_DIM = 600
_UA = "MeshBayBot/1.0 (+https://meshbay.org; link preview)"
class UnsafeURL(ValueError):
"""The URL points somewhere the node must not fetch from."""
def _addr_is_public(ip: str) -> bool:
try:
addr = ipaddress.ip_address(ip)
except ValueError:
return False
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
addr = addr.ipv4_mapped
return not (
addr.is_private or addr.is_loopback or addr.is_link_local
or addr.is_multicast or addr.is_reserved or addr.is_unspecified
)
def safe_url(url: str) -> str:
"""Return the URL unchanged if it is safe to fetch, else raise UnsafeURL."""
if not isinstance(url, str) or len(url) > 2048:
raise UnsafeURL("missing or oversized")
parts = urlsplit(url)
if parts.scheme not in ("http", "https"):
raise UnsafeURL(f"scheme {parts.scheme!r}")
if parts.username or parts.password:
raise UnsafeURL("credentials in URL")
host = parts.hostname
if not host:
raise UnsafeURL("no host")
# An IP literal is checked directly; a name is resolved and every answer
# must be public — a hostname with one public and one 127.0.0.1 record
# would otherwise be a way in.
try:
infos = socket.getaddrinfo(host, parts.port or (443 if parts.scheme == "https" else 80),
proto=socket.IPPROTO_TCP)
except socket.gaierror as e:
raise UnsafeURL(f"cannot resolve: {e}")
resolved = {info[4][0] for info in infos}
if not resolved:
raise UnsafeURL("resolves to nothing")
bad = [ip for ip in resolved if not _addr_is_public(ip)]
if bad:
raise UnsafeURL(f"non-public address {bad[0]}")
return url
class _HeadParser(HTMLParser):
"""Collects <title> text and name/property→content from <meta> in <head>.
Stops caring once <body> starts: everything a card needs is in the head,
and a 512 KB page of body is not worth walking.
"""
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.metas: dict[str, str] = {}
self.title: str | None = None
self._in_title = False
self.done = False
def handle_starttag(self, tag, attrs):
if tag == "body":
self.done = True
elif tag == "title":
self._in_title = True
elif tag == "meta":
a = {k.lower(): (v or "") for k, v in attrs}
key = (a.get("property") or a.get("name") or "").lower().strip()
if key and "content" in a and key not in self.metas:
self.metas[key] = a["content"].strip()
def handle_endtag(self, tag):
if tag == "title":
self._in_title = False
def handle_data(self, data):
if self._in_title and self.title is None:
text = data.strip()
if text:
self.title = text
def _first(metas: dict[str, str], *keys: str) -> str | None:
for k in keys:
v = metas.get(k)
if v:
return v
return None
async def _get(client: httpx.AsyncClient, url: str) -> httpx.Response:
"""One GET with manual, re-validated redirects."""
current = safe_url(url)
for _ in range(_MAX_REDIRECTS + 1):
resp = await client.get(current, headers={"User-Agent": _UA},
follow_redirects=False)
if resp.is_redirect and "location" in resp.headers:
current = safe_url(urljoin(current, resp.headers["location"]))
continue
return resp
raise UnsafeURL("too many redirects")
async def fetch_preview(url: str, *, client: httpx.AsyncClient | None = None) -> dict | None:
"""
Return {url, title, description, site_name, image_url} for a URL, or None
if it cannot be unfurled (unreachable, not HTML, nothing worth showing).
Never raises for an ordinary failure — the caller treats "no preview" as
the common case, exactly like a TMDB miss.
"""
own = client is None
if own:
client = httpx.AsyncClient(timeout=_TIMEOUT, max_redirects=0)
try:
safe_url(url)
resp = await _get(client, url)
ctype = resp.headers.get("content-type", "").split(";")[0].strip().lower()
if resp.status_code != 200 or ctype not in ("text/html", "application/xhtml+xml"):
return None
body = b""
async for chunk in resp.aiter_bytes():
body += chunk
if len(body) >= _MAX_HTML_BYTES:
break
final_url = str(resp.url)
parser = _HeadParser()
try:
parser.feed(body.decode(resp.encoding or "utf-8", errors="replace"))
except Exception:
pass
m = parser.metas
title = _first(m, "og:title", "twitter:title") or parser.title
description = _first(m, "og:description", "twitter:description", "description")
site_name = _first(m, "og:site_name") or urlsplit(final_url).hostname
image = _first(m, "og:image", "og:image:url", "og:image:secure_url",
"twitter:image", "twitter:image:src")
if image:
image = urljoin(final_url, image)
try:
safe_url(image)
except UnsafeURL:
image = None
if not title and not description:
return None
return {
"url": url,
"title": (title or "")[:300] or None,
"description": (description or "")[:600] or None,
"site_name": (site_name or "")[:120] or None,
"image_url": image,
}
except (httpx.HTTPError, UnsafeURL) as e:
log.debug("link preview for %s: %s", url[:80], e)
return None
finally:
if own:
await client.aclose()
async def fetch_image(url: str, *, client: httpx.AsyncClient | None = None) -> bytes | None:
"""Fetch and re-encode an OG image to a small JPEG. None on any failure."""
own = client is None
if own:
client = httpx.AsyncClient(timeout=_TIMEOUT, max_redirects=0)
try:
safe_url(url)
resp = await _get(client, url)
ctype = resp.headers.get("content-type", "").split(";")[0].strip().lower()
if resp.status_code != 200 or not ctype.startswith("image/"):
return None
raw = b""
async for chunk in resp.aiter_bytes():
raw += chunk
if len(raw) > _MAX_IMAGE_BYTES:
return None
return _downscale(raw)
except (httpx.HTTPError, UnsafeURL) as e:
log.debug("link preview image %s: %s", url[:80], e)
return None
finally:
if own:
await client.aclose()
def _downscale(raw: bytes) -> bytes | None:
try:
from PIL import Image
except ImportError:
return None
try:
with Image.open(BytesIO(raw)) as im:
im = im.convert("RGB")
im.thumbnail((_IMAGE_MAX_DIM, _IMAGE_MAX_DIM))
out = BytesIO()
im.save(out, format="JPEG", quality=80)
return out.getvalue()
except Exception:
return None
|