diff options
| author | Christophe Besson <cbesson@gmail.com> | 2026-10-15 14:56:35 +0200 |
|---|---|---|
| committer | Christophe Besson <cbesson@gmail.com> | 2026-10-15 14:56:35 +0200 |
| commit | 7fc2fa04d7d88a1cdf0d290ddd2de3815a7d52a5 (patch) | |
| tree | c2ca93040bb95f5bcb667846a2fa91ddf0dd99bb /packaging | |
| parent | 9dd6d1003cf92c0e4e7f23414565314691aa5516 (diff) | |
| download | meshbay-7fc2fa04d7d88a1cdf0d290ddd2de3815a7d52a5.tar.gz | |
build: audit the shipped Python dependencies with pip-audit
No lock file pins the dependencies, so each build installs the newest
releases and what ships is only known once it has. packaging/audit_deps.py
runs pip-audit then: build-common.sh on the venv before stripping it,
build-node-runtime.ps1 on the environment before PyInstaller freezes it.
A known vulnerability stops the build, unless its id is listed in
packaging/pip-audit-ignore.txt with why it does not apply. No network only
skips the audit with a warning, as does a pip-audit that cannot be installed
or writes no report. pip-audit runs from a venv of its own, never from the
one that ships. Our packages are not audited (not on PyPI), nor pip,
setuptools and wheel, which do not ship.
Checked with the real build-common.sh (79 packages, none known), with
PyJWT 2.14 (the build stops on its two advisories, and goes on once they are
listed) and offline (a warning, the build goes on). The PowerShell step was
not run here.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Diffstat (limited to 'packaging')
| -rw-r--r-- | packaging/README.md | 12 | ||||
| -rw-r--r-- | packaging/audit_deps.py | 131 | ||||
| -rwxr-xr-x | packaging/build/build-common.sh | 7 | ||||
| -rw-r--r-- | packaging/pip-audit-ignore.txt | 6 | ||||
| -rw-r--r-- | packaging/win/build-node-runtime.ps1 | 9 |
5 files changed, 165 insertions, 0 deletions
diff --git a/packaging/README.md b/packaging/README.md index 55cc39a..8156ca0 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -99,6 +99,18 @@ Its architecture is the machine's, not `all`: argon2-cffi's bindings are compiled code. Its licence notices are in `/opt/meshbay-cli/THIRD-PARTY-NOTICES.txt`. +### Known vulnerabilities + +No lock file pins the Python dependencies: every build installs the newest +releases, so what ships is only known once a build has installed it. That is +where [`audit_deps.py`](audit_deps.py) runs pip-audit — `build-common.sh` on the +venv before stripping it, `build-node-runtime.ps1` on the environment before +PyInstaller freezes it. A known vulnerability stops the build, unless its id is +in [`pip-audit-ignore.txt`](pip-audit-ignore.txt) with why it does not apply. +No network only skips the audit, with a warning. pip-audit itself runs from a +venv of its own (`<staging>/pip-audit`, `packages/meshbay-client/build/_pip-audit` +on Windows), never from the one that ships. + The man page, `meshbay-cli(1)`, is generated at build time from the client's own argument parser (`python -m meshbay_cli.manpage`), so it lists the options the program has; `tests/test_manpage.py` fails when an action or option misses diff --git a/packaging/audit_deps.py b/packaging/audit_deps.py new file mode 100644 index 0000000..513b1dd --- /dev/null +++ b/packaging/audit_deps.py @@ -0,0 +1,131 @@ +""" +Audit the Python environment a build is about to ship against known +vulnerabilities, with pip-audit. + + python audit_deps.py [--tool-dir DIR] + +Run with the interpreter of the environment being shipped: what it lists is +what is installed beside it. The builds resolve the newest releases when they +run (there is no lock file), so this is the one moment the shipped set is known. + +- A known vulnerability stops the build (exit 1), unless its id is listed in + packaging/pip-audit-ignore.txt with the reason it does not apply. +- No network is not a reason to stop: the audit is skipped with a warning and + the build goes on (exit 0). So is a pip-audit that cannot be installed or + produces no report, which is what a network failing halfway looks like. + +pip-audit runs from a venv of its own (--tool-dir, kept between builds), never +from the shipped environment: nothing it pulls in may end up in a package. +Our own packages are left out — they are not on PyPI — and so are pip, +setuptools and wheel, which install the environment and do not ship in it: +build-common.sh strips them from the venv and PyInstaller does not freeze them. +""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import urllib.request +from importlib import metadata +from pathlib import Path + +PIP_AUDIT = "pip-audit==2.10.1" +# What pip-audit asks by default; reachable means the audit can run. +PROBE = "https://pypi.org/pypi/pip-audit/json" +IGNORE_FILE = Path(__file__).resolve().parent / "pip-audit-ignore.txt" +NOT_SHIPPED = {"pip", "setuptools", "wheel"} + + +def warn(msg: str) -> None: + print(f"!! pip-audit skipped: {msg}", file=sys.stderr) + + +def shipped() -> list[str]: + seen: dict[str, str] = {} + for dist in metadata.distributions(): + name = (dist.metadata["Name"] or "").strip() + if name and not name.lower().startswith("meshbay") and name.lower() not in NOT_SHIPPED: + seen.setdefault(name.lower(), f"{name}=={dist.version}") + return sorted(seen.values()) + + +def ignored() -> list[str]: + if not IGNORE_FILE.exists(): + return [] + ids = [] + for line in IGNORE_FILE.read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if line: + ids.append(line.split()[0]) + return ids + + +def tool(tool_dir: Path) -> Path | None: + exe = tool_dir / ("Scripts/pip-audit.exe" if os.name == "nt" else "bin/pip-audit") + py = tool_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + if not py.exists(): + if subprocess.run([sys.executable, "-m", "venv", "--clear", str(tool_dir)], + capture_output=True).returncode != 0: + return None + r = subprocess.run([str(py), "-m", "pip", "install", "--quiet", + "--disable-pip-version-check", PIP_AUDIT], + capture_output=True, text=True) + return exe if r.returncode == 0 and exe.exists() else None + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--tool-dir", type=Path, + default=Path(tempfile.gettempdir()) / "meshbay-pip-audit") + args = ap.parse_args() + + try: + urllib.request.urlopen(PROBE, timeout=10).close() + except Exception as e: + warn(f"no network ({e.__class__.__name__}: {e})") + return 0 + + exe = tool(args.tool_dir) + if exe is None: + warn(f"could not install {PIP_AUDIT}") + return 0 + + with tempfile.TemporaryDirectory() as tmp: + req = Path(tmp) / "shipped.txt" + req.write_text("\n".join(shipped()) + "\n", encoding="utf-8") + report = Path(tmp) / "report.json" + cmd = [str(exe), "-r", str(req), "--no-deps", "--disable-pip", + "--progress-spinner", "off", "-f", "json", "-o", str(report)] + for vid in ignored(): + cmd += ["--ignore-vuln", vid] + run = subprocess.run(cmd, capture_output=True, text=True) + try: + data = json.loads(report.read_text(encoding="utf-8")) + except (OSError, ValueError): + tail = (run.stderr or run.stdout).strip().splitlines()[-3:] + warn("no report — " + " | ".join(tail)) + return 0 + + # The same id can come back twice, from two advisory sources. + found: dict[tuple[str, str, str], str] = {} + for d in data.get("dependencies", []): + for v in d.get("vulns", []): + found.setdefault((d["name"], d["version"], v["id"]), + ", ".join(v.get("fix_versions") or []) or "none") + audited = sum(1 for d in data.get("dependencies", []) if "skip_reason" not in d) + if not found: + print(f" pip-audit: {audited} packages, no known vulnerability") + return 0 + print(f"!! pip-audit: {len(found)} known vulnerabilit{'y' if len(found) == 1 else 'ies'}:", + file=sys.stderr) + for (name, version, vid), fix in found.items(): + print(f"!! {name} {version} {vid} (fixed in: {fix})", file=sys.stderr) + print(f"!! Upgrade, or list the id in {IGNORE_FILE.name} with why it does not apply.", + file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packaging/build/build-common.sh b/packaging/build/build-common.sh index 2010d1b..ca5bd1b 100755 --- a/packaging/build/build-common.sh +++ b/packaging/build/build-common.sh @@ -60,6 +60,13 @@ echo " writing THIRD-PARTY-NOTICES.txt" "$VENV_BUILD/bin/python" "$REPO/packaging/third_party_notices.py" \ -o "$ROOT/opt/meshbay-common/THIRD-PARTY-NOTICES.txt" meshbay-hub meshbay-node +# --- Known vulnerabilities in what ships ----------------------------------- +# One stops the build unless packaging/pip-audit-ignore.txt says why it does not +# apply; no network only skips the audit, with a warning. pip-audit runs from a +# venv of its own under the staging dir, never from the one shipped. +echo " auditing dependencies (pip-audit)" +"$VENV_BUILD/bin/python" "$REPO/packaging/audit_deps.py" --tool-dir "$STAGING/pip-audit" + # --- Strip build tools from the venv (not needed at runtime) --------------- echo " stripping build tools" "$VENV_BUILD/bin/pip" uninstall -y pip setuptools wheel 2>&1 | tail -1 diff --git a/packaging/pip-audit-ignore.txt b/packaging/pip-audit-ignore.txt new file mode 100644 index 0000000..8aec984 --- /dev/null +++ b/packaging/pip-audit-ignore.txt @@ -0,0 +1,6 @@ +# Vulnerabilities packaging/audit_deps.py lets through, one per line: +# +# <id> # why it does not apply to MeshBay, and when to look again +# +# An id is what pip-audit prints (PYSEC-…, GHSA-…, CVE-…). Remove a line once +# the fixed version ships. diff --git a/packaging/win/build-node-runtime.ps1 b/packaging/win/build-node-runtime.ps1 index 3d26ec2..24baa0b 100644 --- a/packaging/win/build-node-runtime.ps1 +++ b/packaging/win/build-node-runtime.ps1 @@ -91,6 +91,15 @@ Step "installing meshbay-common, meshbay-node, meshbay-cli and PyInstaller" # 'Hidden import "tzdata" not found' when it is absent. if ($LASTEXITCODE -ne 0) { throw "pip install failed" } +# Known vulnerabilities in what is about to be frozen: one stops the build +# unless packaging\pip-audit-ignore.txt says why it does not apply; no network +# only skips the audit, with a warning. pip-audit runs from a venv of its own, +# never from the one PyInstaller freezes. +Step "auditing dependencies (pip-audit)" +& $Python (Join-Path $Repo "packaging\audit_deps.py") ` + --tool-dir (Join-Path $Client "build\_pip-audit") +if ($LASTEXITCODE -ne 0) { throw "pip-audit found known vulnerabilities" } + # --- 3. freeze -------------------------------------------------------- if (Test-Path $OutDir) { Remove-Item -Recurse -Force $OutDir } $pyiWork = Join-Path $Client "build\_pyinstaller" |