SPS · log · files · refs · tags · releases · people · languages · GitHub · commit
3878b88
Add maintainer tools for auditing and drafting recipes
DistroDeveloperParents
Stat
.github/workflows/audit-recipes.yml | 70 ++++ Makefile | 26 +- tests/test_audit_recipes.sh | 255 +++++++++++++ tools/audit-recipes | 586 +++++++++++++++++++++++++++++ tools/new-recipe | 709 ++++++++++++++++++++++++++++++++++++ 5 files changed, 1643 insertions(+), 3 deletions(-)
Diff
diff --git a/.github/workflows/audit-recipes.yml b/.github/workflows/audit-recipes.yml
new file mode 100644
index 0000000..d02412b
--- /dev/null
+++ b/.github/workflows/audit-recipes.yml
@@ -0,0 +1,70 @@
+name: audit-recipes
+
+# A recipe whose upstream tarball moved, was re-rolled, or started answering
+# with an HTML sign-in page is the most common way a source distribution stops
+# installing, and nothing in `src update` can see it. This job finds it before
+# a user's build does.
+
+on:
+ schedule:
+ # Weekly, early Monday UTC.
+ - cron: '17 4 * * 1'
+ workflow_dispatch:
+ inputs:
+ net:
+ description: 'Download every source and verify its SHA-256'
+ type: boolean
+ default: true
+
+permissions:
+ contents: read
+
+jobs:
+ audit:
+ runs-on: ubuntu-latest
+ timeout-minutes: 180
+ steps:
+ - name: Check out SPS
+ uses: actions/checkout@v4
+
+ - name: Check out sps-core
+ uses: actions/checkout@v4
+ with:
+ repository: RobertFlexx/sps-core
+ path: trees/core
+
+ - name: Check out sps-extra
+ uses: actions/checkout@v4
+ with:
+ repository: RobertFlexx/sps-extra
+ path: trees/extra
+
+ - name: Offline audit
+ run: |
+ python3 tools/audit-recipes \
+ --trees "$PWD/trees/core:$PWD/trees/extra"
+
+ - name: Verify every source hash
+ if: github.event_name == 'schedule' || inputs.net
+ run: |
+ python3 tools/audit-recipes \
+ --trees "$PWD/trees/core:$PWD/trees/extra" \
+ --net --jobs 16 --timeout 900 --json >audit.json || status=$?
+ python3 - <<'PY'
+ import json
+ doc = json.load(open("audit.json"))
+ bad = [f for f in doc["findings"]
+ if f["code"] in ("HASH-MISMATCH", "SOURCE-UNREACHABLE")]
+ for f in bad:
+ print(f"{f['code']} {f['package']}\n {f['detail']}")
+ print(f"\n{doc['recipes']} recipes, {len(bad)} unusable source(s)")
+ raise SystemExit(1 if bad else 0)
+ PY
+
+ - name: Upload findings
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: audit-findings
+ path: audit.json
+ if-no-files-found: ignore
diff --git a/Makefile b/Makefile
index a52bb83..51a5e14 100644
--- a/Makefile
+++ b/Makefile
@@ -6,11 +6,12 @@ MANDIR = $(PREFIX)/share/man
SYSCONFDIR = /etc/sps
DESTDIR =
-PROGRAMS = src sget mkpkg pkin pkdel pkstat pkcheck pkmark setup mkiso
+PROGRAMS = src sget mkpkg pkin pkdel pkstat pkcheck pkmark pkconf setup mkiso
AWK_LIBS = common.awk config.awk db.awk deps.awk index.awk package.awk recipe.awk repository.awk relations.awk version.awk
.PHONY: all check lint check-version source-archive release-check \
- install install-config clean iso iso-slim iso-plasma
+ install install-config clean iso iso-slim iso-plasma \
+ audit-recipes audit-recipes-net
all:
@printf '%s\n' "SPS is interpreted; run 'make check' or 'make install'."
@@ -34,7 +35,8 @@ lint:
[ -f $$script ] || continue; \
$(POSIX_SHELL) -n $$script; \
done
- @python3 -m py_compile tools/upgrade-recipes tools/write-stock-services.py
+ @python3 -m py_compile tools/upgrade-recipes tools/write-stock-services.py \
+ tools/audit-recipes
@set -e; for script in lib/mkiso/initrd-init lib/mkiso/live-rc \
lib/mkiso/live-plasma lib/mkiso/live-plasma-session \
lib/mkiso/live-init lib/mkiso/udhcpc-script; do \
@@ -49,6 +51,24 @@ lint:
@awk -f lib/common.awk -f lib/config.awk </dev/null >/dev/null
@awk -f lib/version.awk </dev/null >/dev/null
+# Recipe-tree audits. SPS_RECIPE_TREES, or RECIPE_TREES on the command line,
+# selects the trees; the default is the usual /usr/src/sps checkouts.
+#
+# make audit-recipes RECIPE_TREES=/usr/src/sps/core:/usr/src/sps/extra
+#
+# audit-recipes is offline and fast enough for every commit. audit-recipes-net
+# downloads every declared source and verifies its SHA-256, which is the only
+# way to catch an upstream tarball that moved or was re-rolled. Run it on a
+# schedule, not on every push.
+RECIPE_TREES =
+
+audit-recipes:
+ @python3 tools/audit-recipes $(if $(RECIPE_TREES),--trees $(RECIPE_TREES))
+
+audit-recipes-net:
+ @python3 tools/audit-recipes --net \
+ $(if $(RECIPE_TREES),--trees $(RECIPE_TREES))
+
check-version:
@LC_ALL=C awk '\
NR == 1 && $$0 ~ /^[0-9]+\.[0-9]+\.[0-9]+$$/ { valid = 1 } \
diff --git a/tests/test_audit_recipes.sh b/tests/test_audit_recipes.sh
new file mode 100755
index 0000000..5bf48c9
--- /dev/null
+++ b/tests/test_audit_recipes.sh
@@ -0,0 +1,255 @@
+#!/bin/sh
+set -eu
+
+case $0 in */*) test_dir=${0%/*} ;; *) test_dir=. ;; esac
+project_dir=$(CDPATH= cd "$test_dir/.." 2>/dev/null && pwd) || exit 1
+fail() { printf 'test_audit_recipes: %s\n' "$*" >&2; exit 1; }
+
+tool=$project_dir/tools/audit-recipes
+[ -f "$tool" ] || fail "missing $tool"
+command -v python3 >/dev/null 2>&1 || fail 'python3 is required for audit-recipes'
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/sps-audit-test.XXXXXX") || fail mktemp
+trap 'rm -rf "$tmp"' 0 HUP INT TERM
+
+# audit-recipes never reaches the network unless --net is passed, so every
+# case below is offline and deterministic.
+audit() { python3 "$tool" --trees "$1" --json 2>/dev/null; }
+
+codes() {
+ python3 -c '
+import json, sys
+doc = json.load(sys.stdin)
+print(" ".join(sorted({f["code"] for f in doc["findings"]})))
+'
+}
+
+# A clean tree must produce no findings and exit 0.
+mkdir -p "$tmp/clean/base/alpha" "$tmp/clean/base/beta"
+cat >"$tmp/clean/base/alpha/recipe" <<'EOF'
+name alpha
+version 1.0
+release 1
+arch any
+description alpha: clean fixture
+
+source https://example.org/alpha-1.0.tar.gz
+hash sha256:8ed3f6ad685b959ead7022518e1af76cd816f8e8ec7ccdda1ed4018e8f2223f8
+
+depend beta
+install mkdir -p "$PKG/usr/share/alpha"
+EOF
+cat >"$tmp/clean/base/beta/recipe" <<'EOF'
+name beta
+version 2.0
+release 1
+arch any
+description beta: clean fixture
+
+source https://example.org/beta-2.0.tar.gz
+hash sha256:f44e64e75f3948e9f73f8dfa94721c4ce8cbb4f265c4790c702b2d41cfbf2753
+
+install mkdir -p "$PKG/usr/share/beta"
+EOF
+if ! python3 "$tool" --trees "$tmp/clean" >/dev/null 2>&1; then
+ fail 'a clean tree must exit 0'
+fi
+[ -z "$(audit "$tmp/clean" | codes)" ] ||
+ fail "clean tree reported findings: $(audit "$tmp/clean" | codes)"
+
+# A dependency on a package no recipe provides halts a real install, so it
+# must be an error rather than a warning.
+mkdir -p "$tmp/missing/base/alpha"
+cat >"$tmp/missing/base/alpha/recipe" <<'EOF'
+name alpha
+version 1.0
+release 1
+arch any
+description alpha: missing dependency fixture
+
+depend nosuchpackage
+install mkdir -p "$PKG/usr/share/alpha"
+EOF
+audit "$tmp/missing" | codes | grep -q MISSING-DEPENDENCY ||
+ fail 'a dependency with no providing recipe must be reported'
+if python3 "$tool" --trees "$tmp/missing" >/dev/null 2>&1; then
+ fail 'a missing dependency must make audit-recipes exit nonzero'
+fi
+
+# sget rejects a dependency cycle at install time; the audit must find it in
+# the tree first.
+mkdir -p "$tmp/cycle/base/one" "$tmp/cycle/base/two"
+cat >"$tmp/cycle/base/one/recipe" <<'EOF'
+name one
+version 1.0
+release 1
+arch any
+description one: cycle fixture
+
+depend two
+install mkdir -p "$PKG/usr/share/one"
+EOF
+cat >"$tmp/cycle/base/two/recipe" <<'EOF'
+name two
+version 1.0
+release 1
+arch any
+description two: cycle fixture
+
+depend one
+install mkdir -p "$PKG/usr/share/two"
+EOF
+audit "$tmp/cycle" | codes | grep -q DEPENDENCY-CYCLE ||
+ fail 'a runtime dependency cycle must be reported'
+
+# Two recipes for one name in a single repository is the duplicate src(1)
+# rejects; the same name in two trees only shadows, so it is a warning.
+mkdir -p "$tmp/dup/base/thing" "$tmp/dup/other/thing"
+for dir in "$tmp/dup/base/thing" "$tmp/dup/other/thing"; do
+ cat >"$dir/recipe" <<'EOF'
+name thing
+version 1.0
+release 1
+arch any
+description thing: duplicate fixture
+
+install mkdir -p "$PKG/usr/share/thing"
+EOF
+done
+audit "$tmp/dup" | codes | grep -q DUPLICATE-IN-REPOSITORY ||
+ fail 'a duplicate package name in one tree must be reported'
+
+# Malformed metadata.
+mkdir -p "$tmp/bad/base/broken"
+cat >"$tmp/bad/base/broken/recipe" <<'EOF'
+name broken
+version 1.0
+release 1
+arch any
+description broken: malformed fixture
+homepage https://example.org/
+
+source https://example.org/broken-1.0.tar.gz
+source https://example.org/extra.patch
+hash sha256:nothexatall
+
+depend broken
+install make install
+EOF
+bad_codes=$(audit "$tmp/bad" | codes)
+for expected in UNKNOWN-KEY SOURCE-HASH-COUNT BAD-HASH-FORMAT SELF-DEPENDENCY \
+ INSTALL-OUTSIDE-PKG; do
+ case " $bad_codes " in
+ *" $expected "*) ;;
+ *) fail "expected $expected in: $bad_codes" ;;
+ esac
+done
+
+# A recipe whose name does not match its directory still indexes, but the
+# maintainer tooling and `cd extra/CATEGORY/NAME` both key off the directory.
+mkdir -p "$tmp/mismatch/base/oldname"
+cat >"$tmp/mismatch/base/oldname/recipe" <<'EOF'
+name newname
+version 1.0
+release 1
+arch any
+description newname: directory mismatch fixture
+
+install mkdir -p "$PKG/usr/share/newname"
+EOF
+audit "$tmp/mismatch" | codes | grep -q NAME-DIR-MISMATCH ||
+ fail 'a recipe whose name differs from its directory must be reported'
+
+# A recipe that reuses another package's digest for a different URL is a
+# copy-paste mistake that silently builds the wrong source.
+mkdir -p "$tmp/reuse/base/aaa" "$tmp/reuse/base/bbb"
+cat >"$tmp/reuse/base/aaa/recipe" <<'EOF'
+name aaa
+version 1.0
+release 1
+arch any
+description aaa: hash reuse fixture
+
+source https://example.org/aaa-1.0.tar.gz
+hash sha256:a4d26868017c0ccffe2efe50944ef4211834660cca834c6e9f86dec6a88246fa
+
+install mkdir -p "$PKG/usr/share/aaa"
+EOF
+cat >"$tmp/reuse/base/bbb/recipe" <<'EOF'
+name bbb
+version 1.0
+release 1
+arch any
+description bbb: hash reuse fixture
+
+source https://example.org/bbb-1.0.tar.gz
+hash sha256:a4d26868017c0ccffe2efe50944ef4211834660cca834c6e9f86dec6a88246fa
+
+install mkdir -p "$PKG/usr/share/bbb"
+EOF
+audit "$tmp/reuse" | codes | grep -q HASH-REUSED-DIFFERENT-SOURCE ||
+ fail 'one digest shared by two different URLs must be reported'
+
+# The sudo package configures --with-vardir=/var/db/sudo. That is not
+# privilege escalation and must not be flagged.
+mkdir -p "$tmp/sudo/base/sudo"
+cat >"$tmp/sudo/base/sudo/recipe" <<'EOF'
+name sudo
+version 1.0
+release 1
+arch any
+description sudo: false-positive fixture
+
+configure ./configure --prefix=/usr --with-vardir=/var/db/sudo
+install make install DESTDIR="$PKG"
+EOF
+if audit "$tmp/sudo" | codes | grep -q PHASE-USES-SUDO; then
+ fail 'the sudo package must not be flagged as using sudo'
+fi
+mkdir -p "$tmp/realsudo/base/thing"
+cat >"$tmp/realsudo/base/thing/recipe" <<'EOF'
+name thing
+version 1.0
+release 1
+arch any
+description thing: real escalation fixture
+
+install sudo make install DESTDIR="$PKG"
+EOF
+audit "$tmp/realsudo" | codes | grep -q PHASE-USES-SUDO ||
+ fail 'a phase that actually runs sudo must be reported'
+
+# SPS builds CPython with --with-platlibdir=lib64. A module staged under
+# /usr/lib/pythonX.Y is unimportable, but naming that path in order to
+# relocate it into lib64 is exactly right and must not be flagged.
+mkdir -p "$tmp/pylib/base/python-bad"
+cat >"$tmp/pylib/base/python-bad/recipe" <<'EOF'
+name python-bad
+version 1.0
+release 1
+arch any
+description python-bad: wrong site-packages fixture
+
+install mkdir -p "$PKG/usr/lib/python3.14/site-packages"
+EOF
+audit "$tmp/pylib" | codes | grep -q PYTHON-SITE-PACKAGES-LIB ||
+ fail 'a module staged under /usr/lib/pythonX.Y must be reported'
+
+mkdir -p "$tmp/pyok/base/python-good"
+cat >"$tmp/pyok/base/python-good/recipe" <<'EOF'
+name python-good
+version 1.0
+release 1
+arch any
+description python-good: relocating fixture
+
+install python3 -m pip install --root="$PKG" --prefix=/usr .
+install if [ -d "$PKG/usr/lib/python3.14" ]; then \
+ cp -a "$PKG/usr/lib/python3.14/." "$PKG/usr/lib64/python3.14/"; \
+ rm -rf "$PKG/usr/lib/python3.14"; fi
+EOF
+if audit "$tmp/pyok" | codes | grep -q PYTHON-SITE-PACKAGES-LIB; then
+ fail 'a recipe that relocates into lib64 must not be flagged'
+fi
+
+printf '%s\n' 'audit-recipes tests passed'
diff --git a/tools/audit-recipes b/tools/audit-recipes
new file mode 100755
index 0000000..370eb0b
--- /dev/null
+++ b/tools/audit-recipes
@@ -0,0 +1,586 @@
+#!/usr/bin/env python3
+"""Audit SPS recipe trees for defects that break an install.
+
+src(1) already validates recipe syntax, same-repository uniqueness, and the
+selected dependency graph when it builds an index. This tool covers what an
+index build cannot: cross-tree consistency, copy-paste mistakes, and whether
+every declared source still downloads and still matches its recorded SHA-256.
+
+A recipe whose upstream tarball moved, was re-rolled, or silently changed is
+the single most common way a source distribution stops installing. Nothing
+notices until a user runs sget and mkpkg aborts on a checksum. Run this in CI
+and before a release.
+
+Offline checks (default):
+
+ python3 tools/audit-recipes --trees /usr/src/sps/core:/usr/src/sps/extra
+
+Online checks (downloads every source, streams it through sha256, keeps
+nothing on disk):
+
+ python3 tools/audit-recipes --trees ... --net --jobs 16
+
+Exit status is 0 when no finding is at or above --fail-level.
+"""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import hashlib
+import json
+import os
+import re
+import ssl
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+# Do NOT send a browser User-Agent. SourceForge and several other hosts serve
+# an HTML interstitial to browser agents and the real archive to curl/wget, so
+# a browser UA turns every SourceForge recipe into a bogus hash mismatch.
+UA = "curl/8.5.0"
+
+# Keys accepted by the SPS 1.0 recipe parser. Anything else is an error.
+SINGLE_KEYS = {"name", "version", "release", "arch", "description"}
+MULTI_KEYS = {
+ "source", "hash", "depend", "builddep", "optional", "conflict",
+ "prepare", "configure", "build", "install",
+}
+ALL_KEYS = SINGLE_KEYS | MULTI_KEYS
+
+NAME_RE = re.compile(r"^[0-9A-Za-z][0-9A-Za-z+_.-]*$")
+VERSION_RE = re.compile(r"^[0-9A-Za-z][0-9A-Za-z+_.~-]*$")
+HASH_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
+CONSTRAINT_RE = re.compile(r"^([0-9A-Za-z][0-9A-Za-z+_.-]*)(?:(<=|>=|=|<|>)(.+))?$")
+
+# Severity ordering, low to high.
+LEVELS = ["info", "warn", "error"]
+
+
+def level_at_least(level: str, floor: str) -> bool:
+ return LEVELS.index(level) >= LEVELS.index(floor)
+
+
+class Finding:
+ __slots__ = ("level", "code", "package", "path", "detail")
+
+ def __init__(self, level: str, code: str, package: str, path: Path, detail: str):
+ self.level = level
+ self.code = code
+ self.package = package
+ self.path = path
+ self.detail = detail
+
+ def as_dict(self) -> dict:
+ return {
+ "level": self.level,
+ "code": self.code,
+ "package": self.package,
+ "path": str(self.path),
+ "detail": self.detail,
+ }
+
+ def __str__(self) -> str:
+ head = f"{self.level:<5} {self.code:<22} {self.package or '-'}"
+ if self.detail:
+ return f"{head}\n {self.detail}\n {self.path}"
+ return f"{head}\n {self.path}"
+
+
+def parse_recipe(path: Path) -> dict:
+ """Parse one recipe the way lib/recipe.awk does.
+
+ Returns single-valued keys as strings and repeatable keys as lists. Keys
+ the parser does not know are collected in "unknown" so the audit can
+ report them rather than silently dropping them.
+ """
+ rec: dict = {
+ "path": path,
+ "name": "",
+ "version": "",
+ "release": "",
+ "arch": "",
+ "description": "",
+ "unknown": [],
+ "duplicate_single": [],
+ "malformed": [],
+ }
+ for key in MULTI_KEYS:
+ rec[key] = []
+ try:
+ raw = path.read_text(encoding="utf-8", errors="replace")
+ except OSError as exc:
+ rec["malformed"].append(f"unreadable: {exc}")
+ return rec
+ rec["raw"] = raw
+
+ pending = ""
+ for physical in raw.splitlines():
+ line = pending + physical if pending else physical
+ pending = ""
+ if line.endswith("\\"):
+ pending = line[:-1]
+ continue
+ stripped = line.strip()
+ if not stripped or stripped.startswith("#"):
+ continue
+ parts = stripped.split(None, 1)
+ key = parts[0]
+ value = parts[1].strip() if len(parts) > 1 else ""
+ if key not in ALL_KEYS:
+ rec["unknown"].append(key)
+ continue
+ if not value:
+ rec["malformed"].append(f"empty value for '{key}'")
+ continue
+ if key in SINGLE_KEYS:
+ if rec[key]:
+ rec["duplicate_single"].append(key)
+ rec[key] = value
+ else:
+ rec[key].append(value)
+ if pending:
+ rec["malformed"].append("recipe ends with a line continuation")
+ return rec
+
+
+def expand(value: str, rec: dict) -> str:
+ """Expand the four variables the recipe parser defines."""
+ out = value
+ for var, key in (("name", "name"), ("version", "version"),
+ ("release", "release"), ("arch", "arch")):
+ out = out.replace("${" + var + "}", rec.get(key) or "")
+ return out
+
+
+def split_specs(records: list[str]) -> list[str]:
+ """A depend/builddep record may hold several whitespace/comma specs."""
+ out: list[str] = []
+ for record in records:
+ for tok in re.split(r"[,\s]+", record.strip()):
+ if tok:
+ out.append(tok)
+ return out
+
+
+def collect(trees: list[Path]) -> list[dict]:
+ recipes: list[dict] = []
+ for tree in trees:
+ for path in sorted(tree.rglob("recipe")):
+ if not path.is_file():
+ continue
+ rec = parse_recipe(path)
+ rec["tree"] = tree
+ recipes.append(rec)
+ return recipes
+
+
+def static_checks(recipes: list[dict], universe: list[dict] | None = None) -> list[Finding]:
+ """Check `recipes`, resolving dependencies against `universe`.
+
+ With --only the caller audits a handful of packages but their dependencies
+ still live in the rest of the tree, so the name universe must stay whole or
+ every external dependency looks missing.
+ """
+ if universe is None:
+ universe = recipes
+ findings: list[Finding] = []
+
+ def add(level, code, rec, detail=""):
+ findings.append(Finding(level, code, rec.get("name") or "", rec["path"], detail))
+
+ by_name: dict[str, list[dict]] = {}
+ for rec in universe:
+ if rec["name"]:
+ by_name.setdefault(rec["name"], []).append(rec)
+
+ hash_owner: dict[str, str] = {}
+ url_owner: dict[str, str] = {}
+
+ for rec in recipes:
+ path = rec["path"]
+ pkgdir = path.parent.name
+ name = rec["name"]
+
+ for problem in rec["malformed"]:
+ add("error", "MALFORMED", rec, problem)
+ for key in rec["unknown"]:
+ add("error", "UNKNOWN-KEY", rec, f"'{key}' is not a recipe key")
+ for key in rec["duplicate_single"]:
+ add("error", "DUPLICATE-KEY", rec, f"'{key}' may appear only once")
+
+ # Required metadata.
+ if not name:
+ add("error", "MISSING-NAME", rec)
+ elif not NAME_RE.match(name):
+ add("error", "BAD-NAME", rec, name)
+ if not rec["version"]:
+ add("error", "MISSING-VERSION", rec)
+ elif not VERSION_RE.match(rec["version"]):
+ add("error", "BAD-VERSION", rec, rec["version"])
+ if not rec["release"]:
+ add("error", "MISSING-RELEASE", rec)
+ elif not rec["release"].isdigit():
+ add("warn", "NONNUMERIC-RELEASE", rec, rec["release"])
+ if not rec["install"]:
+ add("error", "NO-INSTALL-PHASE", rec)
+ if not rec["description"]:
+ add("warn", "NO-DESCRIPTION", rec)
+ if rec["arch"] and rec["arch"] == "noarch":
+ add("info", "NOARCH-SPELLING", rec, "official recipes use 'any'")
+
+ # The package directory should be the package name. src(1) does not
+ # require it (categories are not identity) but a mismatch means
+ # `cd extra/libs/foo` does not get you package foo, and the upgrade
+ # tooling keys off directory names.
+ if name and name != pkgdir:
+ add("warn", "NAME-DIR-MISMATCH", rec, f"name '{name}' in directory '{pkgdir}'")
+
+ # Sources and hashes pair up by declaration order.
+ nsrc, nhash = len(rec["source"]), len(rec["hash"])
+ if nsrc != nhash:
+ add("error", "SOURCE-HASH-COUNT", rec,
+ f"{nsrc} source record(s) but {nhash} hash record(s)")
+ for digest in rec["hash"]:
+ if not HASH_RE.match(digest):
+ add("error", "BAD-HASH-FORMAT", rec, digest)
+ continue
+ body = digest[7:]
+ if len(set(body)) <= 2:
+ add("error", "PLACEHOLDER-HASH", rec, digest)
+
+ for index, source in enumerate(rec["source"]):
+ url = expand(source, rec)
+ if re.search(r"\$\{[a-z]+\}", url):
+ add("error", "UNEXPANDED-VARIABLE", rec, source)
+ if any(c.isspace() for c in url):
+ add("error", "SOURCE-WHITESPACE", rec, source)
+ remote = url.startswith(("http://", "https://", "ftp://"))
+ if url.startswith("http://"):
+ add("warn", "PLAINTEXT-SOURCE", rec, url)
+ if remote:
+ # A remote URL that never mentions the version will silently
+ # keep serving an old tarball after a version bump, or (worse)
+ # serve a moving target that breaks the recorded hash.
+ if rec["version"] and rec["version"] not in url and \
+ "${version}" not in source:
+ add("warn", "URL-WITHOUT-VERSION", rec, url)
+ if url in url_owner and url_owner[url] != name:
+ add("info", "SHARED-SOURCE-URL", rec,
+ f"{url} is also used by {url_owner[url]}")
+ else:
+ url_owner[url] = name
+ elif url.startswith("file://"):
+ if not url.startswith("file:///"):
+ add("error", "RELATIVE-FILE-URL", rec, url)
+ else:
+ # Bare relative path, resolved from the recipe directory.
+ if url.startswith("/"):
+ add("warn", "ABSOLUTE-LOCAL-SOURCE", rec, url)
+ elif not (path.parent / url).exists():
+ add("error", "MISSING-LOCAL-SOURCE", rec, url)
+
+ if index < len(rec["hash"]) and HASH_RE.match(rec["hash"][index]):
+ body = rec["hash"][index][7:]
+ owner = f"{name}:{url}"
+ if body in hash_owner and hash_owner[body] != owner:
+ prior = hash_owner[body]
+ # Identical URL in two packages legitimately shares a
+ # digest (the kernel recipes all build the same tarball).
+ if prior.split(":", 1)[1] != url:
+ add("error", "HASH-REUSED-DIFFERENT-SOURCE", rec,
+ f"{body} is also the hash of {prior}")
+ else:
+ hash_owner.setdefault(body, owner)
+
+ if rec["source"] and not rec["hash"]:
+ add("error", "SOURCE-WITHOUT-HASH", rec)
+
+ # Dependency hygiene.
+ deps = split_specs(rec["depend"])
+ bdeps = split_specs(rec["builddep"])
+ for group, label in ((deps, "depend"), (bdeps, "builddep"),
+ (split_specs(rec["optional"]), "optional"),
+ (split_specs(rec["conflict"]), "conflict")):
+ for spec in group:
+ m = CONSTRAINT_RE.match(spec)
+ if not m:
+ add("error", "BAD-DEPENDENCY-SPEC", rec, f"{label} {spec}")
+ continue
+ if m.group(1) == name:
+ add("error", "SELF-DEPENDENCY", rec, f"{label} {spec}")
+ seen_dep: set[str] = set()
+ for spec in deps:
+ base = spec.split("<")[0].split(">")[0].split("=")[0]
+ if base in seen_dep:
+ add("warn", "DUPLICATE-DEPEND", rec, base)
+ seen_dep.add(base)
+ seen_build: set[str] = set()
+ for spec in bdeps:
+ base = spec.split("<")[0].split(">")[0].split("=")[0]
+ if base in seen_build:
+ add("warn", "DUPLICATE-BUILDDEP", rec, base)
+ seen_build.add(base)
+ if base in seen_dep:
+ # A runtime dependency is always installed before the build,
+ # so naming it again as a build dependency is redundant.
+ add("info", "DEPEND-AND-BUILDDEP", rec, base)
+ for spec in split_specs(rec["conflict"]):
+ base = spec.split("<")[0].split(">")[0].split("=")[0]
+ if base in seen_dep:
+ add("error", "DEPEND-AND-CONFLICT", rec, base)
+
+ # A phase that writes outside $PKG corrupts the build host.
+ for phase in ("prepare", "configure", "build", "install"):
+ for command in rec[phase]:
+ # Only a leading command word counts. The sudo package's own
+ # --with-vardir=/var/db/sudo is not privilege escalation.
+ if re.search(r"(?:^|[;&|]\s*|\$\(\s*)(?:sudo|doas)\s", command):
+ add("error", "PHASE-USES-SUDO", rec, f"{phase} {command}")
+ if re.search(r"(^|\s)(make|ninja|meson|cmake)\s+install(\s|$)", command) \
+ and "$PKG" not in command and "DESTDIR" not in command:
+ add("error", "INSTALL-OUTSIDE-PKG", rec, f"{phase} {command}")
+ if re.search(r"--libdir[= ]/?usr/lib(?![0-9])", command) or \
+ re.search(r"--libdir[= ]lib(?![0-9])(\s|$)", command):
+ add("warn", "LIBDIR-NOT-LIB64", rec, f"{phase} {command}")
+
+ # SPS builds CPython with --with-platlibdir=lib64, so a module staged
+ # under /usr/lib/pythonX.Y is somewhere the interpreter never looks.
+ # Judge the recipe as a whole: mentioning that path is fine when the
+ # recipe also relocates it into lib64.
+ phase_text = " ".join(
+ command for phase in ("prepare", "configure", "build", "install")
+ for command in rec[phase])
+ if "/usr/lib/python" in phase_text and "lib64/python" not in phase_text:
+ add("error", "PYTHON-SITE-PACKAGES-LIB", rec,
+ "stages a module under /usr/lib/pythonX.Y and never moves it "
+ "to /usr/lib64")
+
+ # Cross-tree duplicate names. src(1) only rejects duplicates inside one
+ # repository; the same name in core and extra resolves by priority and
+ # quietly shadows one of them.
+ selected = {id(rec) for rec in recipes}
+ for name, group in sorted(by_name.items()):
+ if len(group) < 2:
+ continue
+ trees = {str(r["tree"]) for r in group}
+ level = "error" if len(trees) == 1 else "warn"
+ code = "DUPLICATE-IN-REPOSITORY" if len(trees) == 1 else "SHADOWED-ACROSS-REPOSITORIES"
+ for rec in group:
+ if id(rec) not in selected:
+ continue
+ others = [str(r["path"]) for r in group if r is not rec]
+ findings.append(Finding(level, code, name, rec["path"],
+ "also defined at " + ", ".join(others)))
+
+ findings.extend(graph_checks(recipes, by_name))
+ return findings
+
+
+def graph_checks(recipes: list[dict], by_name: dict[str, list[dict]]) -> list[Finding]:
+ """Missing dependencies and dependency cycles across the whole tree."""
+ findings: list[Finding] = []
+ known = set(by_name)
+
+ runtime: dict[str, set[str]] = {}
+ for rec in recipes:
+ name = rec["name"]
+ if not name:
+ continue
+ edges: set[str] = set()
+ for label, records in (("depend", rec["depend"]), ("builddep", rec["builddep"])):
+ for spec in split_specs(records):
+ base = CONSTRAINT_RE.match(spec)
+ if not base:
+ continue
+ dep = base.group(1)
+ if dep not in known:
+ findings.append(Finding(
+ "error", "MISSING-DEPENDENCY", name, rec["path"],
+ f"{label} {dep} is not provided by any recipe in the tree"))
+ elif label == "depend":
+ edges.add(dep)
+ runtime.setdefault(name, set()).update(edges)
+
+ # Iterative DFS so a deep tree cannot blow the recursion limit.
+ WHITE, GREY, BLACK = 0, 1, 2
+ colour = dict.fromkeys(runtime, WHITE)
+ reported: set[tuple[str, ...]] = set()
+ for root in sorted(runtime):
+ if colour[root] != WHITE:
+ continue
+ stack = [(root, iter(sorted(runtime.get(root, ()))))]
+ path = [root]
+ colour[root] = GREY
+ while stack:
+ node, children = stack[-1]
+ advanced = False
+ for child in children:
+ if child not in colour:
+ continue
+ if colour[child] == GREY:
+ cycle = path[path.index(child):] + [child]
+ key = tuple(sorted(set(cycle)))
+ if key not in reported:
+ reported.add(key)
+ rec = by_name[child][0]
+ findings.append(Finding(
+ "error", "DEPENDENCY-CYCLE", child, rec["path"],
+ " -> ".join(cycle)))
+ continue
+ if colour[child] == WHITE:
+ colour[child] = GREY
+ path.append(child)
+ stack.append((child, iter(sorted(runtime.get(child, ())))))
+ advanced = True
+ break
+ if not advanced:
+ colour[node] = BLACK
+ stack.pop()
+ if path:
+ path.pop()
+ return findings
+
+
+def fetch_sha256(url: str, timeout: int) -> tuple[str, int]:
+ """Stream a URL through sha256 without keeping it on disk."""
+ req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "*/*"})
+ ctx = ssl.create_default_context()
+ digest = hashlib.sha256()
+ total = 0
+ with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
+ while True:
+ chunk = resp.read(1 << 20)
+ if not chunk:
+ break
+ digest.update(chunk)
+ total += len(chunk)
+ return digest.hexdigest(), total
+
+
+def net_checks(recipes: list[dict], jobs: int, timeout: int,
+ progress: bool) -> list[Finding]:
+ tasks: list[tuple[dict, str, str]] = []
+ for rec in recipes:
+ for index, source in enumerate(rec["source"]):
+ if index >= len(rec["hash"]):
+ continue
+ url = expand(source, rec)
+ if not url.startswith(("http://", "https://", "ftp://")):
+ continue
+ want = rec["hash"][index]
+ if not HASH_RE.match(want):
+ continue
+ tasks.append((rec, url, want[7:]))
+
+ findings: list[Finding] = []
+ done = 0
+ total = len(tasks)
+
+ def run(task):
+ rec, url, want = task
+ try:
+ got, size = fetch_sha256(url, timeout)
+ except (urllib.error.HTTPError, urllib.error.URLError, OSError,
+ ssl.SSLError, ValueError) as exc:
+ return Finding("error", "SOURCE-UNREACHABLE", rec["name"], rec["path"],
+ f"{url}\n {exc}")
+ if got != want:
+ return Finding("error", "HASH-MISMATCH", rec["name"], rec["path"],
+ f"{url}\n recorded sha256:{want}\n"
+ f" actual sha256:{got} ({size} bytes)")
+ return None
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
+ for result in pool.map(run, tasks):
+ done += 1
+ if progress and (done % 25 == 0 or done == total):
+ print(f" verified {done}/{total} sources", file=sys.stderr, flush=True)
+ if result is not None:
+ findings.append(result)
+ return findings
+
+
+def default_trees() -> list[Path]:
+ env = os.environ.get("SPS_RECIPE_TREES")
+ if env:
+ return [Path(p) for p in env.split(":") if p]
+ candidates = [Path("/usr/src/sps/core"), Path("/usr/src/sps/extra")]
+ return [p for p in candidates if p.is_dir()]
+
+
+def main(argv: list[str] | None = None) -> int:
+ ap = argparse.ArgumentParser(
+ prog="audit-recipes",
+ description="Audit SPS recipe trees for install-breaking defects.")
+ ap.add_argument("--trees", help="colon-separated recipe tree roots")
+ ap.add_argument("--only", help="comma-separated package names to audit")
+ ap.add_argument("--net", action="store_true",
+ help="download every source and verify its SHA-256")
+ ap.add_argument("--jobs", type=int, default=12, help="parallel downloads")
+ ap.add_argument("--timeout", type=int, default=600,
+ help="per-source download timeout in seconds")
+ ap.add_argument("--fail-level", choices=LEVELS, default="error",
+ help="lowest severity that makes this command fail")
+ ap.add_argument("--json", action="store_true", help="machine-readable output")
+ ap.add_argument("--quiet", action="store_true", help="only print findings")
+ args = ap.parse_args(argv)
+
+ trees = [Path(p) for p in args.trees.split(":") if p] if args.trees else default_trees()
+ if not trees:
+ print("audit-recipes: no recipe trees; pass --trees or set SPS_RECIPE_TREES",
+ file=sys.stderr)
+ return 2
+ for tree in trees:
+ if not tree.is_dir():
+ print(f"audit-recipes: not a directory: {tree}", file=sys.stderr)
+ return 2
+
+ universe = collect(trees)
+ recipes = universe
+ if args.only:
+ wanted = {n.strip() for n in args.only.split(",") if n.strip()}
+ recipes = [r for r in universe if r["name"] in wanted]
+ unknown = wanted - {r["name"] for r in recipes}
+ for name in sorted(unknown):
+ print(f"audit-recipes: no recipe named {name}", file=sys.stderr)
+ if not recipes:
+ print("audit-recipes: no recipes found", file=sys.stderr)
+ return 2
+
+ if not args.quiet:
+ print(f"audit-recipes: {len(recipes)} recipes in "
+ f"{', '.join(str(t) for t in trees)}", file=sys.stderr)
+
+ findings = static_checks(recipes, universe)
+ if args.net:
+ if not args.quiet:
+ print("audit-recipes: verifying sources over the network",
+ file=sys.stderr)
+ findings.extend(net_checks(recipes, args.jobs, args.timeout,
+ not args.quiet))
+
+ findings.sort(key=lambda f: (-LEVELS.index(f.level), f.code, f.package))
+
+ if args.json:
+ json.dump({"recipes": len(recipes),
+ "findings": [f.as_dict() for f in findings]},
+ sys.stdout, indent=2)
+ sys.stdout.write("\n")
+ else:
+ for finding in findings:
+ print(finding)
+ counts = {level: 0 for level in LEVELS}
+ for finding in findings:
+ counts[finding.level] += 1
+ if not args.quiet or findings:
+ print(f"\n{len(recipes)} recipes: "
+ f"{counts['error']} error, {counts['warn']} warn, "
+ f"{counts['info']} info", file=sys.stderr)
+
+ return 1 if any(level_at_least(f.level, args.fail_level) for f in findings) else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/new-recipe b/tools/new-recipe
new file mode 100755
index 0000000..1c1ee80
--- /dev/null
+++ b/tools/new-recipe
@@ -0,0 +1,709 @@
+#!/usr/bin/env python3
+"""Draft SPS recipes for upstream projects.
+
+Writing a recipe by hand means finding the newest release, downloading it,
+computing a SHA-256, working out the build system, and translating upstream's
+dependency names into SPS package names. This tool does that from upstream
+metadata so a maintainer reviews a draft instead of typing one.
+
+Every recipe it writes carries a real digest. Nothing is guessed: a source
+that cannot be resolved and hashed is reported and skipped.
+
+ tools/new-recipe --tree /usr/src/sps/extra pypi:requests
+ tools/new-recipe --tree /usr/src/sps/extra --category devel pypi:pytest
+ tools/new-recipe --tree /usr/src/sps/extra --from-file wanted.txt
+ tools/new-recipe --tree /usr/src/sps/extra url:https://x.org/foo-1.2.tar.xz
+
+A drafted recipe is a starting point, not a finished package. Build it before
+you commit it, and run tools/audit-recipes over the tree afterwards.
+"""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import hashlib
+import io
+import json
+import os
+import re
+import ssl
+import sys
+import tarfile
+import urllib.error
+import urllib.parse
+import urllib.request
+import zipfile
+from pathlib import Path
+
+# See tools/audit-recipes: a browser User-Agent makes SourceForge and friends
+# answer with an HTML interstitial instead of the archive.
+UA = "curl/8.5.0"
+
+PYPI = "https://pypi.org/pypi/{}/json"
+CRATES = "https://crates.io/api/v1/crates/{}"
+METACPAN = "https://fastapi.metacpan.org/v1/release/{}"
+GITHUB_API = "https://api.github.com"
+
+# PyPI build backends that SPS packages, so --no-build-isolation works.
+BACKEND_PACKAGES = {
+ "setuptools.build_meta": "python-setuptools",
+ "setuptools.build_meta:__legacy__": "python-setuptools",
+ "flit_core.buildapi": "python-flit-core",
+ "flit.buildapi": "python-flit-core",
+ "hatchling.build": "python-hatchling",
+ "hatchling.ouroboros": "python-hatchling",
+ "poetry.core.masonry.api": "python-poetry-core",
+ "poetry.masonry.api": "python-poetry-core",
+ "pdm.backend": "python-pdm-backend",
+ "maturin": "python-maturin",
+ "scikit_build_core.build": "python-scikit-build-core",
+ "mesonpy": "python-meson-python",
+}
+
+# Upstream pkg-config / distribution names that do not match the SPS name.
+NAME_OVERRIDES = {
+ "pillow": "python-pillow",
+ "pyyaml": "python-yaml",
+ "beautifulsoup4": "python-beautifulsoup4",
+}
+
+
+# Staged Python modules must end up under /usr/lib64, which is where a CPython
+# configured with --with-platlibdir=lib64 looks. Written as one logical recipe
+# line with continuations so it stays a single `install` record.
+PY_LIB64_FIXUP = (
+ 'pyver=$(python3 -c \'import sys; print("%d.%d" % sys.version_info[:2])\'); \\\n'
+ ' if [ -d "$PKG/usr/lib/python$pyver" ]; then \\\n'
+ ' mkdir -p "$PKG/usr/lib64"; \\\n'
+ ' cp -a "$PKG/usr/lib/python$pyver/." "$PKG/usr/lib64/python$pyver/"; \\\n'
+ ' rm -rf "$PKG/usr/lib/python$pyver"; \\\n'
+ ' rmdir "$PKG/usr/lib" 2>/dev/null || :; fi'
+)
+
+
+class ResolveError(RuntimeError):
+ pass
+
+
+def http_bytes(url: str, timeout: int = 60, accept: str = "") -> bytes:
+ headers = {"User-Agent": UA}
+ if accept:
+ headers["Accept"] = accept
+ token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+ if token and "api.github.com" in url:
+ headers["Authorization"] = f"Bearer {token}"
+ req = urllib.request.Request(url, headers=headers)
+ ctx = ssl.create_default_context()
+ try:
+ with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
+ return resp.read()
+ except (urllib.error.HTTPError, urllib.error.URLError, OSError) as exc:
+ raise ResolveError(f"{url}: {exc}") from exc
+
+
+def http_json(url: str, timeout: int = 60) -> dict:
+ try:
+ return json.loads(http_bytes(url, timeout, "application/json"))
+ except json.JSONDecodeError as exc:
+ raise ResolveError(f"{url}: invalid JSON: {exc}") from exc
+
+
+def fetch_and_hash(url: str, timeout: int = 900) -> tuple[bytes, str]:
+ data = http_bytes(url, timeout)
+ return data, hashlib.sha256(data).hexdigest()
+
+
+def sps_python_name(dist: str) -> str:
+ """Translate a PyPI distribution name into an SPS package name."""
+ slug = re.sub(r"[_.]+", "-", dist.strip().lower())
+ if slug in NAME_OVERRIDES:
+ return NAME_OVERRIDES[slug]
+ if slug.startswith("python-"):
+ return slug
+ return f"python-{slug}"
+
+
+def parse_requires_dist(entries: list[str] | None) -> list[str]:
+ """Runtime dependencies only: drop extras and non-matching markers.
+
+ A requirement guarded by `extra == "..."` is optional by definition, and
+ one guarded by a python_version or sys_platform marker that cannot apply
+ here would install a package nothing imports.
+ """
+ out: list[str] = []
+ for entry in entries or []:
+ spec = entry.split(";", 1)
+ name = re.split(r"[\s(\[<>=!~]", spec[0].strip(), maxsplit=1)[0].strip()
+ if not name:
+ continue
+ if len(spec) > 1:
+ marker = spec[1]
+ if "extra" in marker:
+ continue
+ if "sys_platform" in marker and "linux" not in marker:
+ continue
+ if re.search(r"platform_system\s*==\s*[\"'](Windows|Darwin)", marker):
+ continue
+ mapped = sps_python_name(name)
+ if mapped not in out:
+ out.append(mapped)
+ return out
+
+
+def read_member(data: bytes, filename: str, wanted: str) -> str | None:
+ """Read one file out of an sdist without unpacking it to disk."""
+ try:
+ if filename.endswith(".zip"):
+ with zipfile.ZipFile(io.BytesIO(data)) as zf:
+ for name in zf.namelist():
+ if name.split("/", 1)[-1] == wanted:
+ return zf.read(name).decode("utf-8", "replace")
+ return None
+ with tarfile.open(fileobj=io.BytesIO(data), mode="r:*") as tf:
+ for member in tf.getmembers():
+ if not member.isfile():
+ continue
+ parts = member.name.split("/", 1)
+ if len(parts) == 2 and parts[1] == wanted:
+ handle = tf.extractfile(member)
+ if handle is None:
+ return None
+ return handle.read().decode("utf-8", "replace")
+ except (tarfile.TarError, zipfile.BadZipFile, OSError, EOFError):
+ return None
+ return None
+
+
+def detect_pypi_backend(data: bytes, filename: str) -> str | None:
+ """Return the SPS package providing this sdist's build backend.
+
+ None means no build dependency is needed. That is the case for a
+ self-hosting backend: setuptools and flit_core declare `backend-path`, so
+ pip imports the backend out of the unpacked sdist itself and the package
+ can be built before any backend is installed. Without that, the bootstrap
+ would be circular.
+ """
+ pyproject = read_member(data, filename, "pyproject.toml")
+ if pyproject:
+ if re.search(r"^\s*backend-path\s*=", pyproject, re.M):
+ return None
+ match = re.search(r'build-backend\s*=\s*["\']([^"\']+)["\']', pyproject)
+ if match:
+ backend = match.group(1).strip()
+ if backend in BACKEND_PACKAGES:
+ return BACKEND_PACKAGES[backend]
+ root = backend.split(".", 1)[0].split(":", 1)[0]
+ return f"python-{root.replace('_', '-')}"
+ # A pyproject.toml with no build-backend defaults to setuptools.
+ return "python-setuptools"
+ return "python-setuptools"
+
+
+def resolve_pypi(spec: str) -> dict:
+ dist = spec
+ doc = http_json(PYPI.format(urllib.parse.quote(dist)))
+ info = doc["info"]
+ version = info["version"]
+ sdist = None
+ for entry in doc.get("urls", []):
+ if entry.get("packagetype") == "sdist":
+ sdist = entry
+ break
+ if sdist is None:
+ raise ResolveError(f"pypi:{dist} {version} publishes no source distribution")
+ url = sdist["url"]
+ digest = sdist["digests"]["sha256"]
+ filename = sdist["filename"]
+
+ # The sdist is needed only to read pyproject.toml. Verify it against the
+ # digest PyPI published rather than trusting the transfer.
+ backend: str | None = "python-setuptools"
+ try:
+ data = http_bytes(url, 300)
+ if hashlib.sha256(data).hexdigest() != digest:
+ raise ResolveError(f"pypi:{dist}: sdist digest does not match PyPI metadata")
+ backend = detect_pypi_backend(data, filename)
+ except ResolveError:
+ raise
+ except Exception: # noqa: BLE001 - backend detection is best effort
+ pass
+
+ summary = (info.get("summary") or f"{dist} Python module").strip()
+ summary = summary.rstrip(".")
+ name = sps_python_name(dist)
+ depends = ["filesystem", "python"] + parse_requires_dist(info.get("requires_dist"))
+ # python is built --with-ensurepip=install, so pip needs no package.
+ builddeps = [backend] if backend else []
+
+ # Rebuild the URL from ${version} where the filename embeds it, so a later
+ # version bump only has to change one line.
+ source = url
+ if f"-{version}." in url:
+ source = url.replace(f"-{version}.", "-${version}.")
+
+ return {
+ "name": name,
+ "version": version,
+ "description": f"{name}: {summary}",
+ "arch": "any",
+ "source": source,
+ "hash": digest,
+ "depend": depends,
+ "builddep": [d for d in builddeps if d != name],
+ "phases": [
+ ("install", 'python3 -m pip install --no-deps --no-build-isolation '
+ '--no-compile --root="$PKG" --prefix=/usr .'),
+ # pip follows the running interpreter's install scheme. SPS builds
+ # CPython with --with-platlibdir=lib64, but a build driven by a
+ # host interpreter that uses plain "lib" would stage the module
+ # where the target python never looks. Normalize it either way.
+ ("install", PY_LIB64_FIXUP),
+ ],
+ }
+
+
+def resolve_crate(spec: str) -> dict:
+ doc = http_json(CRATES.format(urllib.parse.quote(spec)))
+ crate = doc["crate"]
+ version = crate["max_stable_version"]
+ checksum = None
+ for entry in doc.get("versions", []):
+ if entry.get("num") == version:
+ checksum = entry.get("checksum")
+ break
+ if not checksum:
+ raise ResolveError(f"crate:{spec}: no checksum for {version}")
+ summary = (crate.get("description") or f"{spec} Rust crate").strip().rstrip(".")
+ return {
+ "name": spec,
+ "version": version,
+ "description": f"{spec}: {summary}",
+ "arch": "",
+ "source": f"https://static.crates.io/crates/{spec}/{spec}-${{version}}.crate",
+ "hash": checksum,
+ "depend": ["filesystem"],
+ "builddep": ["rust"],
+ "phases": [
+ ("build", 'cargo build --release --locked'),
+ ("install", 'install -Dm755 "target/release/%s" "$PKG/usr/bin/%s"' % (spec, spec)),
+ ],
+ }
+
+
+# Dual-life distributions that ship inside perl itself. They exist on CPAN, so
+# nothing stops a resolver from drafting them, but packaging them separately
+# would shadow files the perl package already owns. Depending on them is also
+# pointless: installing perl already satisfies the requirement.
+CORE_PERL_DISTS = {
+ "archive-tar", "attribute-handlers", "autodie", "autouse", "base",
+ "bignum", "carp", "compress-raw-bzip2", "compress-raw-lzma",
+ "compress-raw-zlib", "config-perl-v", "constant", "cpan", "data-dumper",
+ "db_file", "devel-ppport", "digest", "digest-md5", "digest-sha",
+ "dumpvalue", "encode", "encoding-warnings", "env", "errno",
+ "experimental", "exporter", "extutils-cbuilder", "extutils-constant",
+ "extutils-install", "extutils-makemaker", "extutils-manifest",
+ "extutils-parsexs", "file-fetch", "file-path", "file-temp",
+ "filter-simple", "filter-util-call", "getopt-long", "i18n-langtags",
+ "if", "io", "io-compress", "io-socket-ip", "io-zlib", "ipc-cmd",
+ "ipc-sysv", "json-pp", "libnet", "locale-maketext",
+ "locale-maketext-simple", "math-bigint", "math-bigint-fastcalc",
+ "math-bigrat", "math-complex", "memoize", "mime-base64",
+ "module-corelist", "module-load", "module-load-conditional",
+ "module-loaded", "module-metadata", "net-ping", "params-check",
+ "parent", "pathtools", "perl-ostype", "perlfaq", "perlio-via-quotedprint",
+ "pod-checker", "pod-escapes", "pod-perldoc", "pod-simple", "pod-usage",
+ "podlators", "safe", "scalar-list-utils", "search-dict",
+ "selfloader", "socket", "storable", "sys-syslog", "term-ansicolor",
+ "term-cap", "term-complete", "term-readline", "test", "test-harness",
+ "test-simple", "text-abbrev", "text-balanced", "text-parsewords",
+ "text-tabs+wrap", "thread-queue", "thread-semaphore", "tie-file",
+ "tie-refhash", "time-hires", "time-local", "time-piece",
+ "unicode-collate", "unicode-normalize", "version", "xsloader",
+}
+
+_CPAN_DIST_CACHE: dict[str, str | None] = {}
+
+
+def cpan_distribution_of(module: str) -> str | None:
+ """Map a Perl module name to its CPAN distribution.
+
+ Returns None for modules that ship with perl itself, which must not become
+ packages: there is no separate release to build, and depending on one
+ would be an unsatisfiable dependency.
+ """
+ if module in _CPAN_DIST_CACHE:
+ return _CPAN_DIST_CACHE[module]
+ result: str | None = None
+ try:
+ doc = http_json(
+ "https://fastapi.metacpan.org/v1/module/"
+ f"{urllib.parse.quote(module)}?fields=distribution", 30)
+ dist = doc.get("distribution")
+ if dist and dist != "perl" and dist.lower() not in CORE_PERL_DISTS:
+ result = dist
+ except ResolveError:
+ result = None
+ _CPAN_DIST_CACHE[module] = result
+ return result
+
+
+def cpan_release(module: str) -> dict:
+ """Fetch a release document, tolerating the wrong case.
+
+ The /release/NAME endpoint matches the distribution name exactly, so
+ "html-treebuilder" 404s where "HTML-Tree" succeeds. Fall back to a search
+ that is case-insensitive and also resolves a module name to its release.
+ """
+ try:
+ return http_json(METACPAN.format(urllib.parse.quote(module)))
+ except ResolveError:
+ pass
+ query = urllib.parse.quote(module.replace("-", " "))
+ url = ("https://fastapi.metacpan.org/v1/release/_search"
+ f"?q=distribution:{query}%20AND%20status:latest&size=5")
+ doc = http_json(url)
+ hits = doc.get("hits", {}).get("hits", [])
+ for hit in hits:
+ source = hit.get("_source", {})
+ if (source.get("distribution", "")).lower() == module.lower():
+ return source
+ # A module name rather than a distribution name: ask which release owns it.
+ owner = http_json(
+ "https://fastapi.metacpan.org/v1/module/"
+ f"{urllib.parse.quote(module.replace('-', '::'))}?fields=distribution", 30)
+ dist = owner.get("distribution")
+ if not dist:
+ raise ResolveError(f"cpan:{module}: no release found")
+ return http_json(METACPAN.format(urllib.parse.quote(dist)))
+
+
+def cpan_runtime_depends(doc: dict) -> list[str]:
+ modules: list[str] = []
+ for entry in doc.get("dependency") or []:
+ if entry.get("phase") != "runtime":
+ continue
+ if entry.get("relationship") != "requires":
+ continue
+ module = entry.get("module", "")
+ if not module or module == "perl":
+ continue
+ modules.append(module)
+ out: list[str] = []
+ for module in dict.fromkeys(modules):
+ dist = cpan_distribution_of(module)
+ if not dist:
+ continue
+ name = "perl-" + dist.lower()
+ if name not in out:
+ out.append(name)
+ return out
+
+
+def resolve_cpan(spec: str) -> dict:
+ module = spec.replace("::", "-")
+ doc = cpan_release(module)
+ version = str(doc["version"])
+ url = doc["download_url"]
+ _data, digest = fetch_and_hash(url)
+ summary = (doc.get("abstract") or f"{spec} Perl module").strip().rstrip(".")
+ dist = doc.get("distribution") or module
+ name = "perl-" + dist.lower()
+ source = url.replace(f"-{version}.", "-${version}.") \
+ if f"-{version}." in url else url
+
+ uses_build_pl = doc.get("metadata", {}).get("generated_by", "")
+ phases = [
+ ("configure", 'perl Makefile.PL INSTALLDIRS=vendor'),
+ ("build", 'make -j"$MAKEJOBS"'),
+ ("install", 'make install DESTDIR="$PKG"'),
+ ]
+ if "Module::Build" in uses_build_pl:
+ phases = [
+ ("configure", 'perl Build.PL installdirs=vendor'),
+ ("build", './Build'),
+ ("install", './Build install destdir="$PKG"'),
+ ]
+ depends = ["filesystem", "perl"]
+ for dep in cpan_runtime_depends(doc):
+ if dep != name and dep not in depends:
+ depends.append(dep)
+ return {
+ "name": name,
+ "version": version,
+ "description": f"{name}: {summary}",
+ "arch": "",
+ "source": source,
+ "hash": digest,
+ "depend": depends,
+ "builddep": [],
+ "phases": phases,
+ }
+
+
+def resolve_github(spec: str) -> dict:
+ if "/" not in spec:
+ raise ResolveError(f"github:{spec} must be owner/repo")
+ owner, repo = spec.split("/", 1)
+ tags = http_json(f"{GITHUB_API}/repos/{owner}/{repo}/tags?per_page=50")
+ chosen = None
+ for tag in tags:
+ name = tag.get("name", "")
+ if re.search(r"(alpha|beta|rc|pre|dev|snapshot)", name, re.I):
+ continue
+ chosen = name
+ break
+ if not chosen:
+ raise ResolveError(f"github:{spec}: no stable tag found")
+ version = chosen.lstrip("vV")
+ # Always the refs/tags spelling: a repository with both a tag and a branch
+ # of the same name rejects the short /archive/NAME.tar.gz form.
+ url = f"https://github.com/{owner}/{repo}/archive/refs/tags/{chosen}.tar.gz"
+ data, digest = fetch_and_hash(url)
+ source = url.replace(f"/{chosen}.tar.gz", "/${version}.tar.gz") \
+ if chosen == version else url
+ meta = {}
+ try:
+ meta = http_json(f"{GITHUB_API}/repos/{owner}/{repo}")
+ except ResolveError:
+ pass
+ summary = (meta.get("description") or f"{repo}").strip().rstrip(".")
+ phases = detect_c_build(data)
+ return {
+ "name": repo.lower(),
+ "version": version,
+ "description": f"{repo.lower()}: {summary}",
+ "arch": "",
+ "source": source,
+ "hash": digest,
+ "depend": ["filesystem"],
+ "builddep": phases[1],
+ "phases": phases[0],
+ }
+
+
+def resolve_url(spec: str) -> dict:
+ url = spec
+ data, digest = fetch_and_hash(url)
+ leaf = url.rsplit("/", 1)[-1]
+ match = re.match(r"^(?P<name>[A-Za-z0-9_+.-]+?)[-_](?P<version>[0-9][0-9A-Za-z.]*)"
+ r"(?:\.orig)?\.(?:tar\.(?:gz|xz|bz2|zst)|tgz|txz|tbz2|zip)$", leaf)
+ if not match:
+ raise ResolveError(f"url:{url}: cannot infer name and version from '{leaf}'")
+ name = match.group("name").lower()
+ version = match.group("version").rstrip(".")
+ source = url.replace(version, "${version}", 1) if version in url else url
+ phases = detect_c_build(data)
+ return {
+ "name": name,
+ "version": version,
+ "description": f"{name}: TODO write a one-line description",
+ "arch": "",
+ "source": source,
+ "hash": digest,
+ "depend": ["filesystem"],
+ "builddep": phases[1],
+ "phases": phases[0],
+ }
+
+
+def detect_c_build(data: bytes) -> tuple[list[tuple[str, str]], list[str]]:
+ """Pick build phases by looking at the archive's top-level files."""
+ top: set[str] = set()
+ try:
+ with tarfile.open(fileobj=io.BytesIO(data), mode="r:*") as tf:
+ for member in tf.getmembers()[:4000]:
+ parts = member.name.split("/")
+ if len(parts) == 2 and parts[1]:
+ top.add(parts[1])
+ except (tarfile.TarError, OSError, EOFError):
+ pass
+
+ if "meson.build" in top:
+ return ([
+ ("configure", 'meson setup build --prefix=/usr --libdir=lib64 '
+ '--buildtype=release'),
+ ("build", 'meson compile -C build -j "$MAKEJOBS"'),
+ ("install", 'DESTDIR="$PKG" meson install -C build'),
+ ], ["meson", "ninja", "pkgconf"])
+ if "CMakeLists.txt" in top:
+ return ([
+ ("configure", 'cmake -S . -B build -G Ninja -DCMAKE_INSTALL_PREFIX=/usr '
+ '-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib64'),
+ ("build", 'cmake --build build --parallel "$MAKEJOBS"'),
+ ("install", 'DESTDIR="$PKG" cmake --install build'),
+ ], ["cmake", "ninja", "pkgconf"])
+ if "configure" in top:
+ return ([
+ ("configure", './configure --prefix=/usr --libdir=/usr/lib64 --disable-static'),
+ ("build", 'make -j"$MAKEJOBS"'),
+ ("install", 'make install DESTDIR="$PKG"'),
+ ], ["pkgconf"])
+ if "configure.ac" in top or "configure.in" in top or "autogen.sh" in top:
+ return ([
+ ("prepare", './autogen.sh || autoreconf -fi'),
+ ("configure", './configure --prefix=/usr --libdir=/usr/lib64 --disable-static'),
+ ("build", 'make -j"$MAKEJOBS"'),
+ ("install", 'make install DESTDIR="$PKG"'),
+ ], ["autoconf", "automake", "libtool", "pkgconf"])
+ if "Cargo.toml" in top:
+ return ([
+ ("build", 'cargo build --release --locked'),
+ ("install", 'TODO install the built binaries into "$PKG"'),
+ ], ["rust"])
+ if "go.mod" in top:
+ return ([
+ ("build", 'go build -trimpath -ldflags "-s -w" ./...'),
+ ("install", 'TODO install the built binaries into "$PKG"'),
+ ], ["go"])
+ if "Makefile" in top or "GNUmakefile" in top:
+ return ([
+ ("build", 'make -j"$MAKEJOBS"'),
+ ("install", 'make install PREFIX=/usr DESTDIR="$PKG"'),
+ ], [])
+ return ([
+ ("install", 'TODO write the install phase'),
+ ], [])
+
+
+RESOLVERS = {
+ "pypi": resolve_pypi,
+ "crate": resolve_crate,
+ "cpan": resolve_cpan,
+ "github": resolve_github,
+ "url": resolve_url,
+ "https": lambda s: resolve_url("https:" + s),
+ "http": lambda s: resolve_url("http:" + s),
+}
+
+
+def resolve(spec: str) -> dict:
+ if ":" not in spec:
+ raise ResolveError(f"'{spec}' must be SCHEME:VALUE (pypi:, cpan:, crate:, github:, url:)")
+ scheme, value = spec.split(":", 1)
+ scheme = scheme.lower()
+ if scheme not in RESOLVERS:
+ raise ResolveError(f"unknown scheme '{scheme}' in '{spec}'")
+ return RESOLVERS[scheme](value)
+
+
+def render(rec: dict) -> str:
+ lines = [
+ f"name {rec['name']}",
+ f"version {rec['version']}",
+ "release 1",
+ ]
+ if rec.get("arch"):
+ lines.append(f"arch {rec['arch']}")
+ lines.append(f"description {rec['description']}")
+ lines.append("")
+ lines.append(f"source {rec['source']}")
+ lines.append(f"hash sha256:{rec['hash']}")
+ lines.append("")
+ for dep in rec["depend"]:
+ lines.append(f"depend {dep}")
+ for dep in rec.get("builddep", []):
+ lines.append(f"builddep {dep}")
+ if rec["depend"] or rec.get("builddep"):
+ lines.append("")
+ for key, command in rec["phases"]:
+ lines.append(f"{key:<11} {command}")
+ return "\n".join(lines) + "\n"
+
+
+def default_category(rec: dict, override: str | None) -> str:
+ if override:
+ return override
+ name = rec["name"]
+ if name.startswith("python-"):
+ return "python"
+ if name.startswith("perl-"):
+ return "perl"
+ return "libs"
+
+
+def main(argv: list[str] | None = None) -> int:
+ ap = argparse.ArgumentParser(
+ prog="new-recipe",
+ description="Draft SPS recipes from upstream metadata.")
+ ap.add_argument("specs", nargs="*", metavar="SPEC",
+ help="pypi:NAME, cpan:Module::Name, crate:NAME, "
+ "github:owner/repo, or url:URL")
+ ap.add_argument("--tree", help="recipe tree to write into")
+ ap.add_argument("--category", help="category directory (default: by kind)")
+ ap.add_argument("--from-file", help="read one SPEC per line from this file")
+ ap.add_argument("--jobs", type=int, default=8, help="parallel resolutions")
+ ap.add_argument("--force", action="store_true",
+ help="overwrite an existing recipe")
+ ap.add_argument("--print", dest="print_only", action="store_true",
+ help="write nothing; print the drafted recipes")
+ args = ap.parse_args(argv)
+
+ specs = list(args.specs)
+ if args.from_file:
+ for line in Path(args.from_file).read_text(encoding="utf-8").splitlines():
+ line = line.split("#", 1)[0].strip()
+ if line:
+ specs.append(line)
+ if not specs:
+ ap.error("no specs given")
+ if not args.print_only and not args.tree:
+ ap.error("--tree is required unless --print is used")
+
+ tree = Path(args.tree) if args.tree else None
+ if tree is not None and not tree.is_dir():
+ print(f"new-recipe: not a directory: {tree}", file=sys.stderr)
+ return 2
+
+ existing: set[str] = set()
+ if tree is not None:
+ for path in tree.rglob("recipe"):
+ for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+ if line.startswith("name"):
+ parts = line.split(None, 1)
+ if len(parts) == 2:
+ existing.add(parts[1].strip())
+ break
+
+ written = 0
+ skipped = 0
+ failed = 0
+
+ def work(spec: str):
+ try:
+ return spec, resolve(spec), None
+ except ResolveError as exc:
+ return spec, None, str(exc)
+ except Exception as exc: # noqa: BLE001
+ return spec, None, f"{type(exc).__name__}: {exc}"
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool:
+ for spec, rec, error in pool.map(work, specs):
+ if error:
+ print(f"new-recipe: {spec}: {error}", file=sys.stderr)
+ failed += 1
+ continue
+ text = render(rec)
+ if args.print_only:
+ print(f"# ---- {spec} ----")
+ print(text)
+ written += 1
+ continue
+ if rec["name"] in existing and not args.force:
+ print(f"new-recipe: {rec['name']} already exists; skipping",
+ file=sys.stderr)
+ skipped += 1
+ continue
+ dest = tree / default_category(rec, args.category) / rec["name"]
+ dest.mkdir(parents=True, exist_ok=True)
+ (dest / "recipe").write_text(text, encoding="utf-8")
+ existing.add(rec["name"])
+ print(f"new-recipe: wrote {dest / 'recipe'}")
+ written += 1
+
+ print(f"\n{written} drafted, {skipped} skipped, {failed} failed",
+ file=sys.stderr)
+ return 1 if failed else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())