1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
"""
Name rules that have to give the same answer everywhere.
Most people share from an external drive formatted exFAT or NTFS, on Windows.
Those filesystems are case-insensitive and case-preserving, store no POSIX
permissions and no symlinks, and Windows refuses a set of names outright. So two
names that are plainly different on ext4 can be the same file somewhere else, or
no file at all — and an index built on one machine is read on another.
"Are these two names the same?" therefore has one answer, and it lives here.
What this module deliberately does **not** do: rewrite names. A name is stored as
the filesystem gave it, because that is the string that opens the file. Folding
and normalization exist for *comparison*, never for storage.
"""
from __future__ import annotations
import os
import re
import unicodedata
from pathlib import Path, PurePath
# Windows refuses these as a basename, with or without an extension: `AUX.txt`
# is as impossible as `AUX`. A group indexed on Linux can hold them, and a
# Windows client then cannot write the file it just downloaded.
WINDOWS_RESERVED = frozenset({
"CON", "PRN", "AUX", "NUL",
*(f"COM{i}" for i in range(1, 10)),
*(f"LPT{i}" for i in range(1, 10)),
})
# Reserved on Windows; `/` is reserved everywhere. Control characters go too.
_RESERVED_CHARS = set('<>:"/\\|?*') | {chr(c) for c in range(32)}
# Windows without long-path support. A deep media library reaches this.
MAX_PATH_WINDOWS = 260
def nfc(text: str) -> str:
"""
Canonical form for comparison.
`Café.mkv` written on macOS (NFD: `e` + combining acute) and on Windows
(NFC: precomposed `é`) are different byte strings that name the same file to
a human, and to most filesystems. For French names this is routine.
"""
return unicodedata.normalize("NFC", text)
def fold(text: str) -> str:
"""
The form in which two names are "the same file".
NFC first, then `casefold` — which is not `lower()`: it handles the cases
`lower()` misses, and those are the ones that show up as a bug report rather
than a test failure.
"""
return nfc(text).casefold()
def fold_path(rel: str) -> str:
"""`fold` applied per segment, so separators survive."""
return "/".join(fold(seg) for seg in rel.split("/"))
def portable_name_problem(name: str) -> str | None:
"""
Why `name` cannot be written on some supported platform, or None.
Used to warn, not to refuse: a file already on the operator's disk is a fact,
and the answer is to tell whoever downloads it that it was renamed — not to
pretend it is not there.
"""
if not name:
return "empty name"
if name in (".", ".."):
return "reserved name"
bad = sorted(set(name) & _RESERVED_CHARS)
if bad:
printable = "".join(c if c.isprintable() else "?" for c in bad)
return f"contains reserved characters: {printable}"
# Windows strips these silently, so `file .` and `file` become the same
# thing after a round trip.
if name[-1] in " .":
return "ends with a space or a dot"
stem = name.split(".", 1)[0]
if stem.upper() in WINDOWS_RESERVED:
return f"reserved on Windows ({stem.upper()})"
return None
def is_portable_name(name: str) -> bool:
return portable_name_problem(name) is None
def long_path(path: Path | str) -> str:
"""
A path string safe to hand to the OS.
On Windows, prefix with `\\\\?\\` so `MAX_PATH` does not truncate a deep
library. The prefix requires an absolute, already-normalized path, and it is
a no-op everywhere else.
"""
text = str(path)
if os.name != "nt" or text.startswith("\\\\?\\"):
return text
resolved = str(PurePath(text))
if len(resolved) < MAX_PATH_WINDOWS and not text.startswith("\\\\"):
return text
if text.startswith("\\\\"): # UNC: \\server\share → \\?\UNC\server\share
return "\\\\?\\UNC" + text[1:]
return "\\\\?\\" + text
def find_fold_collisions(names: list[str]) -> dict[str, list[str]]:
"""
Names that collide once folded, keyed by the folded form.
Two entries that fold together cannot both exist on a case-insensitive
filesystem. On ext4 they can, which is how an index becomes unrepresentable
for the person who downloads it — so this is reported to the operator rather
than resolved silently: only they know which file they meant.
"""
seen: dict[str, list[str]] = {}
for name in names:
seen.setdefault(fold(name), []).append(name)
return {k: v for k, v in seen.items() if len(v) > 1}
_TRAILING = re.compile(r"[ .]+$")
def sanitize_for_download(name: str, *, replacement: str = "_") -> str:
"""
A name that can be written on the running platform, from one that may not be.
For the client saving a file, never for the node storing one. Returns the
name unchanged when it is already portable, so the common case is identity
and the caller can tell whether it renamed anything by comparing.
"""
if is_portable_name(name):
return name
out = "".join(replacement if c in _RESERVED_CHARS else c for c in name)
out = _TRAILING.sub("", out)
stem, dot, ext = out.partition(".")
if stem.upper() in WINDOWS_RESERVED:
stem = f"{stem}{replacement}"
out = f"{stem}{dot}{ext}"
return out or "unnamed"
|