1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
|
"""
`linkpreview` — the SSRF gate and the OpenGraph parse.
The gate is the part with teeth: the URL is chosen by a *member*, and it
decides an outbound request from the operator's machine. Anything that is not
a public http(s) address must be refused before a socket opens.
"""
import socket
import httpx
import pytest
from meshbay_node import linkpreview
from meshbay_node.linkpreview import UnsafeURL, safe_url
PUBLIC_IP = "93.184.216.34" # example.com, historically
@pytest.fixture
def resolves_public(monkeypatch):
"""Every hostname resolves to one public address."""
def fake_getaddrinfo(host, port, *a, **k):
return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "",
(PUBLIC_IP, port or 80))]
monkeypatch.setattr(linkpreview.socket, "getaddrinfo", fake_getaddrinfo)
# ── safe_url ────────────────────────────────────────────────────────────────
@pytest.mark.parametrize("url", [
"http://127.0.0.1/x",
"http://localhost/x", # resolves to loopback on any box
"http://169.254.169.254/latest/meta-data/", # cloud metadata
"http://[::1]/x",
"http://10.1.2.3/x",
"http://192.168.0.1/x",
"http://172.16.0.1/x",
"http://0.0.0.0/x",
"http://[::ffff:127.0.0.1]/x", # v4-mapped loopback
"ftp://example.com/x",
"file:///etc/passwd",
"http://user:pass@example.com/x",
"javascript:alert(1)",
"not a url",
])
def test_safe_url_refuses(url):
with pytest.raises(UnsafeURL):
safe_url(url)
@pytest.mark.parametrize("url", [
"http://example.com:22/x", # SSH
"http://example.com:3306/x", # MySQL
"http://example.com:6379/x", # Redis
"http://example.com:9200/x", # Elasticsearch
"http://example.com:5000/x", # a common internal admin port
])
def test_safe_url_refuses_non_web_ports(url, resolves_public):
with pytest.raises(UnsafeURL):
safe_url(url)
@pytest.mark.parametrize("url", [
"http://example.com/x", # implicit 80
"https://example.com/x", # implicit 443
"http://example.com:80/x",
"https://example.com:443/x",
"http://example.com:8080/x",
"https://example.com:8443/x",
])
def test_safe_url_allows_the_web_ports(url, resolves_public):
assert safe_url(url) == url
def test_safe_url_accepts_a_public_host(resolves_public):
assert safe_url("https://example.com/some/page") == "https://example.com/some/page"
def test_safe_url_refuses_a_host_with_any_private_record(monkeypatch):
def mixed(host, port, *a, **k):
return [
(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (PUBLIC_IP, port)),
(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)),
]
monkeypatch.setattr(linkpreview.socket, "getaddrinfo", mixed)
with pytest.raises(UnsafeURL):
safe_url("https://sneaky.example/x")
# ── fetch_preview ──────────────────────────────────────────────────────────
_HTML = """
<!doctype html><html><head>
<title>Fallback Title</title>
<meta property="og:title" content="The Real Title">
<meta property="og:description" content="A short summary of the page.">
<meta property="og:site_name" content="Example">
<meta property="og:image" content="/card.png">
<meta name="description" content="ignored, og wins">
</head><body>...body we should not need...</body></html>
"""
def _client(handler):
return httpx.AsyncClient(transport=httpx.MockTransport(handler),
timeout=5.0, max_redirects=0)
async def test_fetch_preview_reads_opengraph(resolves_public):
def handler(request):
return httpx.Response(200, headers={"content-type": "text/html; charset=utf-8"},
text=_HTML)
async with _client(handler) as c:
meta = await linkpreview.fetch_preview("https://example.com/article", client=c)
assert meta["title"] == "The Real Title"
assert meta["description"] == "A short summary of the page."
assert meta["site_name"] == "Example"
assert meta["image_url"] == "https://example.com/card.png" # absolutised
async def test_fetch_preview_falls_back_to_title_tag(resolves_public):
def handler(request):
return httpx.Response(200, headers={"content-type": "text/html"},
text="<html><head><title>Just A Title</title></head></html>")
async with _client(handler) as c:
meta = await linkpreview.fetch_preview("https://example.com/", client=c)
assert meta["title"] == "Just A Title"
assert meta["description"] is None
async def test_fetch_preview_gives_up_on_non_html(resolves_public):
def handler(request):
return httpx.Response(200, headers={"content-type": "application/pdf"},
content=b"%PDF-1.4")
async with _client(handler) as c:
assert await linkpreview.fetch_preview("https://example.com/x.pdf", client=c) is None
async def test_fetch_preview_gives_up_when_nothing_worth_showing(resolves_public):
def handler(request):
return httpx.Response(200, headers={"content-type": "text/html"},
text="<html><head></head><body>hi</body></html>")
async with _client(handler) as c:
assert await linkpreview.fetch_preview("https://example.com/", client=c) is None
async def test_fetch_preview_revalidates_redirects(monkeypatch):
# First host is public; it 302s to a loopback address.
calls = {"n": 0}
def resolve(host, port, *a, **k):
ip = PUBLIC_IP if host == "ok.example" else "127.0.0.1"
return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (ip, port or 80))]
monkeypatch.setattr(linkpreview.socket, "getaddrinfo", resolve)
def handler(request):
calls["n"] += 1
return httpx.Response(302, headers={"location": "http://internal.example/secret"})
async with _client(handler) as c:
meta = await linkpreview.fetch_preview("https://ok.example/start", client=c)
assert meta is None
assert calls["n"] == 1 # stopped at the redirect, never fetched internal
async def test_fetch_image_downscales(resolves_public):
from io import BytesIO
from PIL import Image
buf = BytesIO()
Image.new("RGB", (2000, 1500), (10, 20, 30)).save(buf, format="PNG")
big_png = buf.getvalue()
def handler(request):
return httpx.Response(200, headers={"content-type": "image/png"}, content=big_png)
async with _client(handler) as c:
jpeg = await linkpreview.fetch_image("https://example.com/card.png", client=c)
assert jpeg and jpeg[:2] == b"\xff\xd8" # JPEG SOI
with Image.open(BytesIO(jpeg)) as im:
assert max(im.size) <= linkpreview._IMAGE_MAX_DIM
async def test_fetch_image_refuses_a_decompression_bomb(resolves_public, monkeypatch):
from io import BytesIO
from PIL import Image
# A tiny file that reports enormous dimensions from its header alone.
monkeypatch.setattr(linkpreview, "_MAX_IMAGE_PIXELS", 1_000_000)
buf = BytesIO()
Image.new("RGB", (2000, 2000), (0, 0, 0)).save(buf, format="PNG") # 4 MP > cap
bomb = buf.getvalue()
def handler(request):
return httpx.Response(200, headers={"content-type": "image/png"}, content=bomb)
async with _client(handler) as c:
assert await linkpreview.fetch_image("https://example.com/x.png", client=c) is None
# ── Size caps bind while reading, not after ─────────────────────────────────
#
# `client.get` read and decoded the whole body before the caps looked at it, so
# a link posted in chat could make the node hold any amount of data. These
# count what the server actually had to hand over.
_CHUNK = 64 * 1024
def _endless(counter, head=b""):
async def body():
if head:
counter["sent"] += len(head)
yield head
for _ in range(2000): # 125 MiB if nobody stops
counter["sent"] += _CHUNK
yield b"x" * _CHUNK
return body()
async def test_a_huge_page_is_not_read_past_the_cap(resolves_public):
counter = {"sent": 0}
head = b"<html><head><title>T</title></head><body>"
def handler(request):
return httpx.Response(200, headers={"content-type": "text/html"},
content=_endless(counter, head))
async with _client(handler) as c:
meta = await linkpreview.fetch_preview("https://example.com/", client=c)
assert meta is not None and meta["title"] == "T"
assert counter["sent"] <= linkpreview._MAX_HTML_BYTES + 2 * _CHUNK
async def test_a_huge_image_is_refused_without_being_read(resolves_public):
counter = {"sent": 0}
def handler(request):
return httpx.Response(200, headers={"content-type": "image/png"},
content=_endless(counter))
async with _client(handler) as c:
assert await linkpreview.fetch_image("https://example.com/x.png", client=c) is None
assert counter["sent"] <= linkpreview._MAX_IMAGE_BYTES + 2 * _CHUNK
async def test_a_compressed_body_is_capped_after_decoding(resolves_public):
import zlib
comp = zlib.compressobj(9, zlib.DEFLATED, 31) # gzip
counter = {"inflated": 0}
async def body():
yield comp.compress(b"<html><head><title>T</title></head><body>")
for _ in range(2000): # 125 MiB inflated
counter["inflated"] += _CHUNK
# Flushed per chunk, so what is counted is what went on the wire.
yield comp.compress(b"x" * _CHUNK) + comp.flush(zlib.Z_SYNC_FLUSH)
yield comp.flush()
def handler(request):
return httpx.Response(200, headers={"content-type": "text/html",
"content-encoding": "gzip"},
content=body())
async with _client(handler) as c:
meta = await linkpreview.fetch_preview("https://example.com/", client=c)
assert meta is not None
assert counter["inflated"] < 50 * _CHUNK # stopped early, not at 125 MiB
async def test_a_declared_oversized_image_is_not_read(resolves_public):
counter = {"sent": 0}
def handler(request):
return httpx.Response(
200, headers={"content-type": "image/png",
"content-length": str(linkpreview._MAX_IMAGE_BYTES + 1)},
content=_endless(counter))
async with _client(handler) as c:
assert await linkpreview.fetch_image("https://example.com/x.png", client=c) is None
assert counter["sent"] <= _CHUNK
|