369 lines
17 KiB
Python
369 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Fetch and verify pinned source dependencies without executing their code.
|
|
|
|
Source archives are retained in ignored storage. Only manifest-selected regular
|
|
files are materialized, excluded payloads are never extracted, and import patches
|
|
must reproduce the exact reviewed file hashes. No Git repository is cloned.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
from urllib.parse import urlsplit
|
|
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
|
import zipfile
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
MAX_ARCHIVE_BYTES = 128 * 1024 * 1024
|
|
MAX_FILE_BYTES = 64 * 1024 * 1024
|
|
MAX_EXPANDED_BYTES = 1024 * 1024 * 1024
|
|
MAX_MEMBERS = 100_000
|
|
HASH = re.compile(r"^[0-9a-f]{64}$")
|
|
|
|
|
|
def sha(path):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def relative(name):
|
|
if not isinstance(name, str) or not name or "\\" in name or "\x00" in name:
|
|
raise ValueError("Invalid dependency path: " + repr(name))
|
|
path = PurePosixPath(name)
|
|
if path.is_absolute() or ".." in path.parts or ".git" in path.parts or re.match(r"^[A-Za-z]:", name):
|
|
raise ValueError("Unsafe dependency path: " + name)
|
|
value = path.as_posix()
|
|
if value == ".":
|
|
return ""
|
|
return value
|
|
|
|
|
|
def validate_record(record):
|
|
if 'optional' in record and not isinstance(record['optional'], bool):
|
|
raise ValueError('Dependency optional flag must be boolean')
|
|
directory = relative(record["directory"])
|
|
if not directory.startswith("third_party/") or directory == "third_party/manifests":
|
|
raise ValueError("Dependency directory must be beneath third_party")
|
|
suffix = directory.removeprefix("third_party/")
|
|
if suffix.startswith("manifests/"):
|
|
raise ValueError("Dependency cannot overwrite metadata")
|
|
for key in ("upstreamFileSHA256", "vendoredFileSHA256"):
|
|
if not isinstance(record[key], dict) or not record[key]:
|
|
raise ValueError("Dependency requires nonempty file hash maps")
|
|
for name, digest in record[key].items():
|
|
if relative(name) != name or (not isinstance(digest, str) or not HASH.fullmatch(digest)):
|
|
raise ValueError("Invalid file fingerprint: " + name)
|
|
if set(record["upstreamFileSHA256"]) != set(record["vendoredFileSHA256"]):
|
|
raise ValueError("Import patches may change bytes, not the approved file inventory")
|
|
if record.get("archiveSHA256") is not None and (not isinstance(record["archiveSHA256"], str) or not HASH.fullmatch(record["archiveSHA256"])):
|
|
raise ValueError("Invalid archive fingerprint")
|
|
for excluded in record.get("excludedFiles", []):
|
|
excluded = relative(excluded).rstrip("/")
|
|
if any(p == excluded or p.startswith(excluded + "/") for p in record["upstreamFileSHA256"]):
|
|
raise ValueError("Selected dependency file overlaps an exclusion: " + excluded)
|
|
if record.get("archiveRootPrefix"):
|
|
if not relative(record["archiveRootPrefix"].rstrip("/")):
|
|
raise ValueError("Archive root prefix cannot be empty")
|
|
return suffix
|
|
|
|
|
|
def source_url(record):
|
|
"""Use a release URL or a full immutable GitHub revision archive."""
|
|
url = record["sourceURL"]
|
|
if record.get("sourceRevision"):
|
|
revision = record["sourceRevision"]
|
|
parts = urlsplit(url)
|
|
fields = parts.path.strip("/").removesuffix(".git").split("/")
|
|
if parts.scheme != "https" or parts.hostname != "github.com" or len(fields) != 2 or not re.fullmatch(r"[0-9a-f]{40}", revision):
|
|
raise ValueError("Unsupported pinned Git source")
|
|
return f"https://codeload.github.com/{fields[0]}/{fields[1]}/tar.gz/{revision}"
|
|
if not record.get("archiveSHA256"):
|
|
raise ValueError("Release archives require a recorded SHA-256")
|
|
if urlsplit(url).scheme not in ("http", "https"):
|
|
raise ValueError("Dependency source must use HTTP or HTTPS")
|
|
return url
|
|
|
|
|
|
class SafeRedirect(HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
if urlsplit(newurl).scheme not in ("http", "https"):
|
|
raise ValueError("Unsupported dependency redirect")
|
|
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
|
|
|
|
def archive_names(name, record):
|
|
filename = Path(urlsplit(record["sourceURL"]).path).name
|
|
if record.get("sourceRevision"):
|
|
filename = name + "-" + record["sourceRevision"] + ".tar.gz"
|
|
if not filename or filename in (".", ".."):
|
|
raise ValueError("Invalid source archive filename")
|
|
digest = record.get("archiveSHA256")
|
|
return ([digest + "-" + filename, filename, digest + ".tar.gz"] if digest else [filename])
|
|
|
|
|
|
def verify_archive(path, record):
|
|
if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_ARCHIVE_BYTES:
|
|
raise ValueError("Source archive must be a bounded regular file: " + str(path))
|
|
digest = sha(path)
|
|
if record.get("archiveSHA256") and digest != record["archiveSHA256"]:
|
|
raise ValueError("Source archive SHA-256 mismatch: " + path.name)
|
|
return digest
|
|
|
|
|
|
|
|
def require_unlinked_ancestors(path, description):
|
|
"""Reject redirected destinations, retaining only macOS's public aliases.
|
|
|
|
Do not resolve an arbitrary user path before examining its full ancestry:
|
|
doing so would erase precisely the linked parent this check must detect.
|
|
"""
|
|
aliases = {Path("/tmp"): Path("/private/tmp"), Path("/var"): Path("/private/var")}
|
|
absolute = Path(path).absolute()
|
|
for candidate in (absolute, *absolute.parents):
|
|
if candidate.is_symlink() and not (
|
|
candidate in aliases and candidate.resolve() == aliases[candidate]):
|
|
raise ValueError(description + " cannot traverse a symbolic link")
|
|
|
|
|
|
def obtain_archive(name, record, cache, offline=False, timeout=30):
|
|
require_unlinked_ancestors(cache, "Archive cache")
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
names = archive_names(name, record)
|
|
for filename in names:
|
|
cached = cache / filename
|
|
if cached.exists() or cached.is_symlink():
|
|
verify_archive(cached, record)
|
|
return cached
|
|
if offline:
|
|
raise ValueError("Offline source archive missing: " + names[0])
|
|
url = source_url(record)
|
|
request = Request(url, headers={"User-Agent": "VCCPlus-source-fetch/1"})
|
|
target = cache / names[0]
|
|
temporary = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(prefix=".download-", dir=cache, delete=False) as output:
|
|
temporary = Path(output.name)
|
|
with build_opener(SafeRedirect()).open(request, timeout=timeout) as response:
|
|
if urlsplit(response.geturl()).scheme not in ("http", "https"):
|
|
raise ValueError("Unsupported dependency response URL")
|
|
size = 0
|
|
while chunk := response.read(1024 * 1024):
|
|
size += len(chunk)
|
|
if size > MAX_ARCHIVE_BYTES:
|
|
raise ValueError("Dependency archive exceeds download limit")
|
|
output.write(chunk)
|
|
verify_archive(temporary, record)
|
|
require_unlinked_ancestors(cache, "Archive cache")
|
|
if target.exists() or target.is_symlink():
|
|
raise ValueError("Archive cache target changed during download")
|
|
os.replace(temporary, target)
|
|
return target
|
|
finally:
|
|
if temporary is not None:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def extract_selected(archive, destination, expected, archive_prefix=""):
|
|
"""Stream only known regular members; never use extract/extractall."""
|
|
observed = set()
|
|
all_names = set()
|
|
prefix = None
|
|
expanded = 0
|
|
count = 0
|
|
|
|
def consume(name, size, mode, regular, opener):
|
|
nonlocal prefix, expanded, count
|
|
count += 1
|
|
if count > MAX_MEMBERS:
|
|
raise ValueError("Source archive has too many members")
|
|
if not 0 <= size <= MAX_FILE_BYTES:
|
|
raise ValueError("Source archive member exceeds size limit: " + name)
|
|
expanded += size
|
|
if expanded > MAX_EXPANDED_BYTES:
|
|
raise ValueError("Source archive exceeds expansion limit")
|
|
normalized = relative(name)
|
|
if normalized in all_names:
|
|
raise ValueError("Duplicate source archive member: " + normalized)
|
|
all_names.add(normalized)
|
|
parts = normalized.split("/")
|
|
candidates = [(normalized, "")]
|
|
if len(parts) > 1:
|
|
candidates.append(("/".join(parts[1:]), parts[0] + "/"))
|
|
archive_root = archive_prefix.rstrip("/") + "/" if archive_prefix else ""
|
|
matches = [(path[len(archive_root):], root) for path, root in candidates
|
|
if path.startswith(archive_root) and path[len(archive_root):] in expected]
|
|
if not matches:
|
|
return
|
|
if len(matches) != 1:
|
|
raise ValueError("Ambiguous source archive member: " + normalized)
|
|
path, root = matches[0]
|
|
if prefix is not None and root != prefix:
|
|
raise ValueError("Source archive has inconsistent roots")
|
|
prefix = root
|
|
if path in observed or not regular:
|
|
raise ValueError("Selected member is duplicate, linked or not a regular file: " + normalized)
|
|
with opener() as stream:
|
|
data = stream.read(MAX_FILE_BYTES + 1)
|
|
if len(data) != size or hashlib.sha256(data).hexdigest() != expected[path]:
|
|
raise ValueError("Upstream source file SHA-256 mismatch: " + path)
|
|
target = destination / path
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(data)
|
|
target.chmod(0o755 if mode & 0o111 else 0o644)
|
|
observed.add(path)
|
|
|
|
if zipfile.is_zipfile(archive):
|
|
with zipfile.ZipFile(archive) as source:
|
|
for member in source.infolist():
|
|
mode = member.external_attr >> 16
|
|
regular = not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG) and not (member.flag_bits & 1)
|
|
consume(member.filename, member.file_size, mode, regular, lambda member=member: source.open(member))
|
|
else:
|
|
with tarfile.open(archive, mode="r|*") as source:
|
|
for member in source:
|
|
consume(member.name, member.size, member.mode, member.isfile(), lambda member=member: source.extractfile(member))
|
|
if observed != set(expected):
|
|
raise ValueError("Source archive is missing selected files: " + ", ".join(sorted(set(expected) - observed)[:12]))
|
|
|
|
|
|
def verify_tree(root, expected):
|
|
if root.is_symlink() or not root.is_dir():
|
|
raise ValueError("Dependency source must be an ordinary directory")
|
|
actual = {}
|
|
for path in root.rglob("*"):
|
|
if path.is_symlink():
|
|
raise ValueError("Dependency contains a symbolic link: " + str(path))
|
|
if path.is_file():
|
|
actual[path.relative_to(root).as_posix()] = sha(path)
|
|
elif not path.is_dir():
|
|
raise ValueError("Dependency contains a special file")
|
|
if actual != expected:
|
|
changed = sorted(p for p in actual.keys() | expected.keys() if actual.get(p) != expected.get(p))
|
|
raise ValueError("Dependency source differs from reviewed import: " + ", ".join(changed[:12]))
|
|
|
|
|
|
def apply_patches(root, record, metadata):
|
|
allowed = set(record["upstreamFileSHA256"])
|
|
for filename in record.get("patches", []):
|
|
if relative(filename) != filename or "/" in filename:
|
|
raise ValueError("Import patch must be a metadata filename")
|
|
path = metadata / filename
|
|
if path.is_symlink() or not path.is_file():
|
|
raise ValueError("Missing import patch: " + filename)
|
|
text = path.read_text()
|
|
targets = []
|
|
for line in text.splitlines():
|
|
if line.startswith(("--- ", "+++ ")):
|
|
name = line[4:].split("\t", 1)[0]
|
|
if name[:2] not in ("a/", "b/") or relative(name[2:]) not in allowed:
|
|
raise ValueError("Import patch touches an unapproved path")
|
|
targets.append(name[2:])
|
|
if not targets:
|
|
raise ValueError("Import patch contains no unified file headers")
|
|
subprocess.run(["/usr/bin/patch", "-u", "-t", "-F", "0", "-p1", "-i", str(path.resolve())],
|
|
cwd=root, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
# A byte-perfect patch may still leave editor backups on some patch versions.
|
|
# Do not remove them silently; verify_tree fails on any extra output.
|
|
|
|
|
|
def load_records(project_root=ROOT):
|
|
"""Read ordinary import records plus supplemental pinned-source metadata.
|
|
|
|
Supplemental records fill provenance gaps only; they cannot replace or
|
|
overlap the existing reviewed dependency inventory.
|
|
"""
|
|
ordinary = project_root / "third_party/manifests"
|
|
supplemental = project_root / "tools/dependencies/manifests"
|
|
records, locations = {}, {}
|
|
for metadata in (ordinary, supplemental):
|
|
if metadata.is_symlink():
|
|
raise ValueError("Dependency metadata cannot be a symbolic link")
|
|
for path in sorted(metadata.glob("*.json")):
|
|
if path.is_symlink() or not path.is_file():
|
|
raise ValueError("Dependency manifest must be an ordinary file")
|
|
name = path.stem
|
|
if name in records:
|
|
raise ValueError("Duplicate dependency manifest: " + name)
|
|
record = json.loads(path.read_text())
|
|
suffix = validate_record(record)
|
|
for existing in records.values():
|
|
other = validate_record(existing)
|
|
if suffix == other or suffix.startswith(other + "/") or other.startswith(suffix + "/"):
|
|
raise ValueError("Overlapping dependency source directories")
|
|
source_url(record) # Reject unpinned/unsupported sources before any writes.
|
|
records[name], locations[name] = record, metadata
|
|
return records, locations
|
|
|
|
|
|
def materialize(name, record, metadata, output, cache, offline=False, verify_only=False):
|
|
suffix = validate_record(record)
|
|
destination = output / suffix
|
|
require_unlinked_ancestors(destination, "Dependency destination")
|
|
if destination.exists():
|
|
verify_tree(destination, record["vendoredFileSHA256"])
|
|
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "verified-existing"}
|
|
if verify_only:
|
|
raise ValueError("Dependency source has not been fetched: " + name)
|
|
archive = obtain_archive(name, record, cache, offline)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
with tempfile.TemporaryDirectory(prefix="." + name + "-", dir=destination.parent) as temporary:
|
|
staged = Path(temporary) / "source"
|
|
staged.mkdir()
|
|
extract_selected(archive, staged, record["upstreamFileSHA256"], record.get("archiveRootPrefix", ""))
|
|
apply_patches(staged, record, metadata)
|
|
verify_tree(staged, record["vendoredFileSHA256"])
|
|
require_unlinked_ancestors(destination, "Dependency destination")
|
|
if destination.exists() or destination.is_symlink():
|
|
raise ValueError("Dependency destination changed during preparation")
|
|
staged.rename(destination)
|
|
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "fetched-verified",
|
|
"archive_sha256": sha(archive)}
|
|
|
|
|
|
def default_selection(records):
|
|
"""Optional firmware preparation is explicit, never part of ordinary fetches."""
|
|
return [name for name, record in records.items() if not record.get('optional', False)]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--only", nargs="+", help="Manifest names, for example: --only toolshed")
|
|
parser.add_argument("--archive-cache", type=Path, default=ROOT / ".build/dependencies/downloads")
|
|
parser.add_argument("--output", type=Path, default=ROOT / "third_party", help="Ignored dependency source root")
|
|
parser.add_argument("--offline", action="store_true", help="Use only already cached original source archives")
|
|
parser.add_argument("--verify-only", action="store_true", help="Verify existing source trees without fetching")
|
|
args = parser.parse_args()
|
|
records, locations = load_records()
|
|
selected = args.only or default_selection(records)
|
|
unknown = set(selected) - records.keys()
|
|
if unknown:
|
|
parser.error("Unknown dependency manifest: " + ", ".join(sorted(unknown)))
|
|
if not records:
|
|
parser.error("No dependency manifests found")
|
|
output = args.output.absolute()
|
|
canonical_output = output.resolve()
|
|
metadata_roots = {ROOT / "third_party/manifests", ROOT / "tools/dependencies/manifests"}
|
|
if canonical_output == ROOT or canonical_output in ROOT.parents or canonical_output in metadata_roots:
|
|
parser.error("Choose a dependency source root separate from project metadata")
|
|
for name in dict.fromkeys(selected):
|
|
result = materialize(name, records[name], locations[name], output, args.archive_cache, args.offline, args.verify_only)
|
|
print(json.dumps(result, sort_keys=True), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (ValueError, OSError, tarfile.TarError, zipfile.BadZipFile, subprocess.CalledProcessError) as error:
|
|
raise SystemExit(str(error))
|