Files

369 lines
17 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Fetch and verify pinned source dependencies without executing their code.
Source archives are retained in ignored storage. Only manifest-selected regular
files are materialized, excluded payloads are never extracted, and import patches
must reproduce the exact reviewed file hashes. No Git repository is cloned.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import stat
import subprocess
import tarfile
import tempfile
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
import zipfile
ROOT = Path(__file__).resolve().parents[2]
MAX_ARCHIVE_BYTES = 128 * 1024 * 1024
MAX_FILE_BYTES = 64 * 1024 * 1024
MAX_EXPANDED_BYTES = 1024 * 1024 * 1024
MAX_MEMBERS = 100_000
HASH = re.compile(r"^[0-9a-f]{64}$")
def sha(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def relative(name):
if not isinstance(name, str) or not name or "\\" in name or "\x00" in name:
raise ValueError("Invalid dependency path: " + repr(name))
path = PurePosixPath(name)
if path.is_absolute() or ".." in path.parts or ".git" in path.parts or re.match(r"^[A-Za-z]:", name):
raise ValueError("Unsafe dependency path: " + name)
value = path.as_posix()
if value == ".":
return ""
return value
def validate_record(record):
if 'optional' in record and not isinstance(record['optional'], bool):
raise ValueError('Dependency optional flag must be boolean')
directory = relative(record["directory"])
if not directory.startswith("third_party/") or directory == "third_party/manifests":
raise ValueError("Dependency directory must be beneath third_party")
suffix = directory.removeprefix("third_party/")
if suffix.startswith("manifests/"):
raise ValueError("Dependency cannot overwrite metadata")
for key in ("upstreamFileSHA256", "vendoredFileSHA256"):
if not isinstance(record[key], dict) or not record[key]:
raise ValueError("Dependency requires nonempty file hash maps")
for name, digest in record[key].items():
if relative(name) != name or (not isinstance(digest, str) or not HASH.fullmatch(digest)):
raise ValueError("Invalid file fingerprint: " + name)
if set(record["upstreamFileSHA256"]) != set(record["vendoredFileSHA256"]):
raise ValueError("Import patches may change bytes, not the approved file inventory")
if record.get("archiveSHA256") is not None and (not isinstance(record["archiveSHA256"], str) or not HASH.fullmatch(record["archiveSHA256"])):
raise ValueError("Invalid archive fingerprint")
for excluded in record.get("excludedFiles", []):
excluded = relative(excluded).rstrip("/")
if any(p == excluded or p.startswith(excluded + "/") for p in record["upstreamFileSHA256"]):
raise ValueError("Selected dependency file overlaps an exclusion: " + excluded)
if record.get("archiveRootPrefix"):
if not relative(record["archiveRootPrefix"].rstrip("/")):
raise ValueError("Archive root prefix cannot be empty")
return suffix
def source_url(record):
"""Use a release URL or a full immutable GitHub revision archive."""
url = record["sourceURL"]
if record.get("sourceRevision"):
revision = record["sourceRevision"]
parts = urlsplit(url)
fields = parts.path.strip("/").removesuffix(".git").split("/")
if parts.scheme != "https" or parts.hostname != "github.com" or len(fields) != 2 or not re.fullmatch(r"[0-9a-f]{40}", revision):
raise ValueError("Unsupported pinned Git source")
return f"https://codeload.github.com/{fields[0]}/{fields[1]}/tar.gz/{revision}"
if not record.get("archiveSHA256"):
raise ValueError("Release archives require a recorded SHA-256")
if urlsplit(url).scheme not in ("http", "https"):
raise ValueError("Dependency source must use HTTP or HTTPS")
return url
class SafeRedirect(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
if urlsplit(newurl).scheme not in ("http", "https"):
raise ValueError("Unsupported dependency redirect")
return super().redirect_request(req, fp, code, msg, headers, newurl)
def archive_names(name, record):
filename = Path(urlsplit(record["sourceURL"]).path).name
if record.get("sourceRevision"):
filename = name + "-" + record["sourceRevision"] + ".tar.gz"
if not filename or filename in (".", ".."):
raise ValueError("Invalid source archive filename")
digest = record.get("archiveSHA256")
return ([digest + "-" + filename, filename, digest + ".tar.gz"] if digest else [filename])
def verify_archive(path, record):
if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_ARCHIVE_BYTES:
raise ValueError("Source archive must be a bounded regular file: " + str(path))
digest = sha(path)
if record.get("archiveSHA256") and digest != record["archiveSHA256"]:
raise ValueError("Source archive SHA-256 mismatch: " + path.name)
return digest
def require_unlinked_ancestors(path, description):
"""Reject redirected destinations, retaining only macOS's public aliases.
Do not resolve an arbitrary user path before examining its full ancestry:
doing so would erase precisely the linked parent this check must detect.
"""
aliases = {Path("/tmp"): Path("/private/tmp"), Path("/var"): Path("/private/var")}
absolute = Path(path).absolute()
for candidate in (absolute, *absolute.parents):
if candidate.is_symlink() and not (
candidate in aliases and candidate.resolve() == aliases[candidate]):
raise ValueError(description + " cannot traverse a symbolic link")
def obtain_archive(name, record, cache, offline=False, timeout=30):
require_unlinked_ancestors(cache, "Archive cache")
cache.mkdir(parents=True, exist_ok=True)
names = archive_names(name, record)
for filename in names:
cached = cache / filename
if cached.exists() or cached.is_symlink():
verify_archive(cached, record)
return cached
if offline:
raise ValueError("Offline source archive missing: " + names[0])
url = source_url(record)
request = Request(url, headers={"User-Agent": "VCCPlus-source-fetch/1"})
target = cache / names[0]
temporary = None
try:
with tempfile.NamedTemporaryFile(prefix=".download-", dir=cache, delete=False) as output:
temporary = Path(output.name)
with build_opener(SafeRedirect()).open(request, timeout=timeout) as response:
if urlsplit(response.geturl()).scheme not in ("http", "https"):
raise ValueError("Unsupported dependency response URL")
size = 0
while chunk := response.read(1024 * 1024):
size += len(chunk)
if size > MAX_ARCHIVE_BYTES:
raise ValueError("Dependency archive exceeds download limit")
output.write(chunk)
verify_archive(temporary, record)
require_unlinked_ancestors(cache, "Archive cache")
if target.exists() or target.is_symlink():
raise ValueError("Archive cache target changed during download")
os.replace(temporary, target)
return target
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
def extract_selected(archive, destination, expected, archive_prefix=""):
"""Stream only known regular members; never use extract/extractall."""
observed = set()
all_names = set()
prefix = None
expanded = 0
count = 0
def consume(name, size, mode, regular, opener):
nonlocal prefix, expanded, count
count += 1
if count > MAX_MEMBERS:
raise ValueError("Source archive has too many members")
if not 0 <= size <= MAX_FILE_BYTES:
raise ValueError("Source archive member exceeds size limit: " + name)
expanded += size
if expanded > MAX_EXPANDED_BYTES:
raise ValueError("Source archive exceeds expansion limit")
normalized = relative(name)
if normalized in all_names:
raise ValueError("Duplicate source archive member: " + normalized)
all_names.add(normalized)
parts = normalized.split("/")
candidates = [(normalized, "")]
if len(parts) > 1:
candidates.append(("/".join(parts[1:]), parts[0] + "/"))
archive_root = archive_prefix.rstrip("/") + "/" if archive_prefix else ""
matches = [(path[len(archive_root):], root) for path, root in candidates
if path.startswith(archive_root) and path[len(archive_root):] in expected]
if not matches:
return
if len(matches) != 1:
raise ValueError("Ambiguous source archive member: " + normalized)
path, root = matches[0]
if prefix is not None and root != prefix:
raise ValueError("Source archive has inconsistent roots")
prefix = root
if path in observed or not regular:
raise ValueError("Selected member is duplicate, linked or not a regular file: " + normalized)
with opener() as stream:
data = stream.read(MAX_FILE_BYTES + 1)
if len(data) != size or hashlib.sha256(data).hexdigest() != expected[path]:
raise ValueError("Upstream source file SHA-256 mismatch: " + path)
target = destination / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
target.chmod(0o755 if mode & 0o111 else 0o644)
observed.add(path)
if zipfile.is_zipfile(archive):
with zipfile.ZipFile(archive) as source:
for member in source.infolist():
mode = member.external_attr >> 16
regular = not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG) and not (member.flag_bits & 1)
consume(member.filename, member.file_size, mode, regular, lambda member=member: source.open(member))
else:
with tarfile.open(archive, mode="r|*") as source:
for member in source:
consume(member.name, member.size, member.mode, member.isfile(), lambda member=member: source.extractfile(member))
if observed != set(expected):
raise ValueError("Source archive is missing selected files: " + ", ".join(sorted(set(expected) - observed)[:12]))
def verify_tree(root, expected):
if root.is_symlink() or not root.is_dir():
raise ValueError("Dependency source must be an ordinary directory")
actual = {}
for path in root.rglob("*"):
if path.is_symlink():
raise ValueError("Dependency contains a symbolic link: " + str(path))
if path.is_file():
actual[path.relative_to(root).as_posix()] = sha(path)
elif not path.is_dir():
raise ValueError("Dependency contains a special file")
if actual != expected:
changed = sorted(p for p in actual.keys() | expected.keys() if actual.get(p) != expected.get(p))
raise ValueError("Dependency source differs from reviewed import: " + ", ".join(changed[:12]))
def apply_patches(root, record, metadata):
allowed = set(record["upstreamFileSHA256"])
for filename in record.get("patches", []):
if relative(filename) != filename or "/" in filename:
raise ValueError("Import patch must be a metadata filename")
path = metadata / filename
if path.is_symlink() or not path.is_file():
raise ValueError("Missing import patch: " + filename)
text = path.read_text()
targets = []
for line in text.splitlines():
if line.startswith(("--- ", "+++ ")):
name = line[4:].split("\t", 1)[0]
if name[:2] not in ("a/", "b/") or relative(name[2:]) not in allowed:
raise ValueError("Import patch touches an unapproved path")
targets.append(name[2:])
if not targets:
raise ValueError("Import patch contains no unified file headers")
subprocess.run(["/usr/bin/patch", "-u", "-t", "-F", "0", "-p1", "-i", str(path.resolve())],
cwd=root, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
# A byte-perfect patch may still leave editor backups on some patch versions.
# Do not remove them silently; verify_tree fails on any extra output.
def load_records(project_root=ROOT):
"""Read ordinary import records plus supplemental pinned-source metadata.
Supplemental records fill provenance gaps only; they cannot replace or
overlap the existing reviewed dependency inventory.
"""
ordinary = project_root / "third_party/manifests"
supplemental = project_root / "tools/dependencies/manifests"
records, locations = {}, {}
for metadata in (ordinary, supplemental):
if metadata.is_symlink():
raise ValueError("Dependency metadata cannot be a symbolic link")
for path in sorted(metadata.glob("*.json")):
if path.is_symlink() or not path.is_file():
raise ValueError("Dependency manifest must be an ordinary file")
name = path.stem
if name in records:
raise ValueError("Duplicate dependency manifest: " + name)
record = json.loads(path.read_text())
suffix = validate_record(record)
for existing in records.values():
other = validate_record(existing)
if suffix == other or suffix.startswith(other + "/") or other.startswith(suffix + "/"):
raise ValueError("Overlapping dependency source directories")
source_url(record) # Reject unpinned/unsupported sources before any writes.
records[name], locations[name] = record, metadata
return records, locations
def materialize(name, record, metadata, output, cache, offline=False, verify_only=False):
suffix = validate_record(record)
destination = output / suffix
require_unlinked_ancestors(destination, "Dependency destination")
if destination.exists():
verify_tree(destination, record["vendoredFileSHA256"])
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "verified-existing"}
if verify_only:
raise ValueError("Dependency source has not been fetched: " + name)
archive = obtain_archive(name, record, cache, offline)
destination.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="." + name + "-", dir=destination.parent) as temporary:
staged = Path(temporary) / "source"
staged.mkdir()
extract_selected(archive, staged, record["upstreamFileSHA256"], record.get("archiveRootPrefix", ""))
apply_patches(staged, record, metadata)
verify_tree(staged, record["vendoredFileSHA256"])
require_unlinked_ancestors(destination, "Dependency destination")
if destination.exists() or destination.is_symlink():
raise ValueError("Dependency destination changed during preparation")
staged.rename(destination)
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "fetched-verified",
"archive_sha256": sha(archive)}
def default_selection(records):
"""Optional firmware preparation is explicit, never part of ordinary fetches."""
return [name for name, record in records.items() if not record.get('optional', False)]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--only", nargs="+", help="Manifest names, for example: --only toolshed")
parser.add_argument("--archive-cache", type=Path, default=ROOT / ".build/dependencies/downloads")
parser.add_argument("--output", type=Path, default=ROOT / "third_party", help="Ignored dependency source root")
parser.add_argument("--offline", action="store_true", help="Use only already cached original source archives")
parser.add_argument("--verify-only", action="store_true", help="Verify existing source trees without fetching")
args = parser.parse_args()
records, locations = load_records()
selected = args.only or default_selection(records)
unknown = set(selected) - records.keys()
if unknown:
parser.error("Unknown dependency manifest: " + ", ".join(sorted(unknown)))
if not records:
parser.error("No dependency manifests found")
output = args.output.absolute()
canonical_output = output.resolve()
metadata_roots = {ROOT / "third_party/manifests", ROOT / "tools/dependencies/manifests"}
if canonical_output == ROOT or canonical_output in ROOT.parents or canonical_output in metadata_roots:
parser.error("Choose a dependency source root separate from project metadata")
for name in dict.fromkeys(selected):
result = materialize(name, records[name], locations[name], output, args.archive_cache, args.offline, args.verify_only)
print(json.dumps(result, sort_keys=True), flush=True)
if __name__ == "__main__":
try:
main()
except (ValueError, OSError, tarfile.TarError, zipfile.BadZipFile, subprocess.CalledProcessError) as error:
raise SystemExit(str(error))