Publish VCC Plus OSS source distribution for 3.6.2 build 60

This commit is contained in:
Brian Holman committed 2026-11-13 00:14:03 -07:00
commit c652a5276d
1192 files changed
+272368

No files matched your search

+368
View File
@@ -0,0 +1,368 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Fetch and verify pinned source dependencies without executing their code.
Source archives are retained in ignored storage. Only manifest-selected regular
files are materialized, excluded payloads are never extracted, and import patches
must reproduce the exact reviewed file hashes. No Git repository is cloned.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import stat
import subprocess
import tarfile
import tempfile
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
import zipfile
ROOT = Path(__file__).resolve().parents[2]
MAX_ARCHIVE_BYTES = 128 * 1024 * 1024
MAX_FILE_BYTES = 64 * 1024 * 1024
MAX_EXPANDED_BYTES = 1024 * 1024 * 1024
MAX_MEMBERS = 100_000
HASH = re.compile(r"^[0-9a-f]{64}$")
def sha(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def relative(name):
if not isinstance(name, str) or not name or "\\" in name or "\x00" in name:
raise ValueError("Invalid dependency path: " + repr(name))
path = PurePosixPath(name)
if path.is_absolute() or ".." in path.parts or ".git" in path.parts or re.match(r"^[A-Za-z]:", name):
raise ValueError("Unsafe dependency path: " + name)
value = path.as_posix()
if value == ".":
return ""
return value
def validate_record(record):
if 'optional' in record and not isinstance(record['optional'], bool):
raise ValueError('Dependency optional flag must be boolean')
directory = relative(record["directory"])
if not directory.startswith("third_party/") or directory == "third_party/manifests":
raise ValueError("Dependency directory must be beneath third_party")
suffix = directory.removeprefix("third_party/")
if suffix.startswith("manifests/"):
raise ValueError("Dependency cannot overwrite metadata")
for key in ("upstreamFileSHA256", "vendoredFileSHA256"):
if not isinstance(record[key], dict) or not record[key]:
raise ValueError("Dependency requires nonempty file hash maps")
for name, digest in record[key].items():
if relative(name) != name or (not isinstance(digest, str) or not HASH.fullmatch(digest)):
raise ValueError("Invalid file fingerprint: " + name)
if set(record["upstreamFileSHA256"]) != set(record["vendoredFileSHA256"]):
raise ValueError("Import patches may change bytes, not the approved file inventory")
if record.get("archiveSHA256") is not None and (not isinstance(record["archiveSHA256"], str) or not HASH.fullmatch(record["archiveSHA256"])):
raise ValueError("Invalid archive fingerprint")
for excluded in record.get("excludedFiles", []):
excluded = relative(excluded).rstrip("/")
if any(p == excluded or p.startswith(excluded + "/") for p in record["upstreamFileSHA256"]):
raise ValueError("Selected dependency file overlaps an exclusion: " + excluded)
if record.get("archiveRootPrefix"):
if not relative(record["archiveRootPrefix"].rstrip("/")):
raise ValueError("Archive root prefix cannot be empty")
return suffix
def source_url(record):
"""Use a release URL or a full immutable GitHub revision archive."""
url = record["sourceURL"]
if record.get("sourceRevision"):
revision = record["sourceRevision"]
parts = urlsplit(url)
fields = parts.path.strip("/").removesuffix(".git").split("/")
if parts.scheme != "https" or parts.hostname != "github.com" or len(fields) != 2 or not re.fullmatch(r"[0-9a-f]{40}", revision):
raise ValueError("Unsupported pinned Git source")
return f"https://codeload.github.com/{fields[0]}/{fields[1]}/tar.gz/{revision}"
if not record.get("archiveSHA256"):
raise ValueError("Release archives require a recorded SHA-256")
if urlsplit(url).scheme not in ("http", "https"):
raise ValueError("Dependency source must use HTTP or HTTPS")
return url
class SafeRedirect(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
if urlsplit(newurl).scheme not in ("http", "https"):
raise ValueError("Unsupported dependency redirect")
return super().redirect_request(req, fp, code, msg, headers, newurl)
def archive_names(name, record):
filename = Path(urlsplit(record["sourceURL"]).path).name
if record.get("sourceRevision"):
filename = name + "-" + record["sourceRevision"] + ".tar.gz"
if not filename or filename in (".", ".."):
raise ValueError("Invalid source archive filename")
digest = record.get("archiveSHA256")
return ([digest + "-" + filename, filename, digest + ".tar.gz"] if digest else [filename])
def verify_archive(path, record):
if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_ARCHIVE_BYTES:
raise ValueError("Source archive must be a bounded regular file: " + str(path))
digest = sha(path)
if record.get("archiveSHA256") and digest != record["archiveSHA256"]:
raise ValueError("Source archive SHA-256 mismatch: " + path.name)
return digest
def require_unlinked_ancestors(path, description):
"""Reject redirected destinations, retaining only macOS's public aliases.
Do not resolve an arbitrary user path before examining its full ancestry:
doing so would erase precisely the linked parent this check must detect.
"""
aliases = {Path("/tmp"): Path("/private/tmp"), Path("/var"): Path("/private/var")}
absolute = Path(path).absolute()
for candidate in (absolute, *absolute.parents):
if candidate.is_symlink() and not (
candidate in aliases and candidate.resolve() == aliases[candidate]):
raise ValueError(description + " cannot traverse a symbolic link")
def obtain_archive(name, record, cache, offline=False, timeout=30):
require_unlinked_ancestors(cache, "Archive cache")
cache.mkdir(parents=True, exist_ok=True)
names = archive_names(name, record)
for filename in names:
cached = cache / filename
if cached.exists() or cached.is_symlink():
verify_archive(cached, record)
return cached
if offline:
raise ValueError("Offline source archive missing: " + names[0])
url = source_url(record)
request = Request(url, headers={"User-Agent": "VCCPlus-source-fetch/1"})
target = cache / names[0]
temporary = None
try:
with tempfile.NamedTemporaryFile(prefix=".download-", dir=cache, delete=False) as output:
temporary = Path(output.name)
with build_opener(SafeRedirect()).open(request, timeout=timeout) as response:
if urlsplit(response.geturl()).scheme not in ("http", "https"):
raise ValueError("Unsupported dependency response URL")
size = 0
while chunk := response.read(1024 * 1024):
size += len(chunk)
if size > MAX_ARCHIVE_BYTES:
raise ValueError("Dependency archive exceeds download limit")
output.write(chunk)
verify_archive(temporary, record)
require_unlinked_ancestors(cache, "Archive cache")
if target.exists() or target.is_symlink():
raise ValueError("Archive cache target changed during download")
os.replace(temporary, target)
return target
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
def extract_selected(archive, destination, expected, archive_prefix=""):
"""Stream only known regular members; never use extract/extractall."""
observed = set()
all_names = set()
prefix = None
expanded = 0
count = 0
def consume(name, size, mode, regular, opener):
nonlocal prefix, expanded, count
count += 1
if count > MAX_MEMBERS:
raise ValueError("Source archive has too many members")
if not 0 <= size <= MAX_FILE_BYTES:
raise ValueError("Source archive member exceeds size limit: " + name)
expanded += size
if expanded > MAX_EXPANDED_BYTES:
raise ValueError("Source archive exceeds expansion limit")
normalized = relative(name)
if normalized in all_names:
raise ValueError("Duplicate source archive member: " + normalized)
all_names.add(normalized)
parts = normalized.split("/")
candidates = [(normalized, "")]
if len(parts) > 1:
candidates.append(("/".join(parts[1:]), parts[0] + "/"))
archive_root = archive_prefix.rstrip("/") + "/" if archive_prefix else ""
matches = [(path[len(archive_root):], root) for path, root in candidates
if path.startswith(archive_root) and path[len(archive_root):] in expected]
if not matches:
return
if len(matches) != 1:
raise ValueError("Ambiguous source archive member: " + normalized)
path, root = matches[0]
if prefix is not None and root != prefix:
raise ValueError("Source archive has inconsistent roots")
prefix = root
if path in observed or not regular:
raise ValueError("Selected member is duplicate, linked or not a regular file: " + normalized)
with opener() as stream:
data = stream.read(MAX_FILE_BYTES + 1)
if len(data) != size or hashlib.sha256(data).hexdigest() != expected[path]:
raise ValueError("Upstream source file SHA-256 mismatch: " + path)
target = destination / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
target.chmod(0o755 if mode & 0o111 else 0o644)
observed.add(path)
if zipfile.is_zipfile(archive):
with zipfile.ZipFile(archive) as source:
for member in source.infolist():
mode = member.external_attr >> 16
regular = not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG) and not (member.flag_bits & 1)
consume(member.filename, member.file_size, mode, regular, lambda member=member: source.open(member))
else:
with tarfile.open(archive, mode="r|*") as source:
for member in source:
consume(member.name, member.size, member.mode, member.isfile(), lambda member=member: source.extractfile(member))
if observed != set(expected):
raise ValueError("Source archive is missing selected files: " + ", ".join(sorted(set(expected) - observed)[:12]))
def verify_tree(root, expected):
if root.is_symlink() or not root.is_dir():
raise ValueError("Dependency source must be an ordinary directory")
actual = {}
for path in root.rglob("*"):
if path.is_symlink():
raise ValueError("Dependency contains a symbolic link: " + str(path))
if path.is_file():
actual[path.relative_to(root).as_posix()] = sha(path)
elif not path.is_dir():
raise ValueError("Dependency contains a special file")
if actual != expected:
changed = sorted(p for p in actual.keys() | expected.keys() if actual.get(p) != expected.get(p))
raise ValueError("Dependency source differs from reviewed import: " + ", ".join(changed[:12]))
def apply_patches(root, record, metadata):
allowed = set(record["upstreamFileSHA256"])
for filename in record.get("patches", []):
if relative(filename) != filename or "/" in filename:
raise ValueError("Import patch must be a metadata filename")
path = metadata / filename
if path.is_symlink() or not path.is_file():
raise ValueError("Missing import patch: " + filename)
text = path.read_text()
targets = []
for line in text.splitlines():
if line.startswith(("--- ", "+++ ")):
name = line[4:].split("\t", 1)[0]
if name[:2] not in ("a/", "b/") or relative(name[2:]) not in allowed:
raise ValueError("Import patch touches an unapproved path")
targets.append(name[2:])
if not targets:
raise ValueError("Import patch contains no unified file headers")
subprocess.run(["/usr/bin/patch", "-u", "-t", "-F", "0", "-p1", "-i", str(path.resolve())],
cwd=root, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
# A byte-perfect patch may still leave editor backups on some patch versions.
# Do not remove them silently; verify_tree fails on any extra output.
def load_records(project_root=ROOT):
"""Read ordinary import records plus supplemental pinned-source metadata.
Supplemental records fill provenance gaps only; they cannot replace or
overlap the existing reviewed dependency inventory.
"""
ordinary = project_root / "third_party/manifests"
supplemental = project_root / "tools/dependencies/manifests"
records, locations = {}, {}
for metadata in (ordinary, supplemental):
if metadata.is_symlink():
raise ValueError("Dependency metadata cannot be a symbolic link")
for path in sorted(metadata.glob("*.json")):
if path.is_symlink() or not path.is_file():
raise ValueError("Dependency manifest must be an ordinary file")
name = path.stem
if name in records:
raise ValueError("Duplicate dependency manifest: " + name)
record = json.loads(path.read_text())
suffix = validate_record(record)
for existing in records.values():
other = validate_record(existing)
if suffix == other or suffix.startswith(other + "/") or other.startswith(suffix + "/"):
raise ValueError("Overlapping dependency source directories")
source_url(record) # Reject unpinned/unsupported sources before any writes.
records[name], locations[name] = record, metadata
return records, locations
def materialize(name, record, metadata, output, cache, offline=False, verify_only=False):
suffix = validate_record(record)
destination = output / suffix
require_unlinked_ancestors(destination, "Dependency destination")
if destination.exists():
verify_tree(destination, record["vendoredFileSHA256"])
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "verified-existing"}
if verify_only:
raise ValueError("Dependency source has not been fetched: " + name)
archive = obtain_archive(name, record, cache, offline)
destination.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="." + name + "-", dir=destination.parent) as temporary:
staged = Path(temporary) / "source"
staged.mkdir()
extract_selected(archive, staged, record["upstreamFileSHA256"], record.get("archiveRootPrefix", ""))
apply_patches(staged, record, metadata)
verify_tree(staged, record["vendoredFileSHA256"])
require_unlinked_ancestors(destination, "Dependency destination")
if destination.exists() or destination.is_symlink():
raise ValueError("Dependency destination changed during preparation")
staged.rename(destination)
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "fetched-verified",
"archive_sha256": sha(archive)}
def default_selection(records):
"""Optional firmware preparation is explicit, never part of ordinary fetches."""
return [name for name, record in records.items() if not record.get('optional', False)]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--only", nargs="+", help="Manifest names, for example: --only toolshed")
parser.add_argument("--archive-cache", type=Path, default=ROOT / ".build/dependencies/downloads")
parser.add_argument("--output", type=Path, default=ROOT / "third_party", help="Ignored dependency source root")
parser.add_argument("--offline", action="store_true", help="Use only already cached original source archives")
parser.add_argument("--verify-only", action="store_true", help="Verify existing source trees without fetching")
args = parser.parse_args()
records, locations = load_records()
selected = args.only or default_selection(records)
unknown = set(selected) - records.keys()
if unknown:
parser.error("Unknown dependency manifest: " + ", ".join(sorted(unknown)))
if not records:
parser.error("No dependency manifests found")
output = args.output.absolute()
canonical_output = output.resolve()
metadata_roots = {ROOT / "third_party/manifests", ROOT / "tools/dependencies/manifests"}
if canonical_output == ROOT or canonical_output in ROOT.parents or canonical_output in metadata_roots:
parser.error("Choose a dependency source root separate from project metadata")
for name in dict.fromkeys(selected):
result = materialize(name, records[name], locations[name], output, args.archive_cache, args.offline, args.verify_only)
print(json.dumps(result, sort_keys=True), flush=True)
if __name__ == "__main__":
try:
main()
except (ValueError, OSError, tarfile.TarError, zipfile.BadZipFile, subprocess.CalledProcessError) as error:
raise SystemExit(str(error))
@@ -0,0 +1,56 @@
{
"archiveRootPrefix": "cocoroms/",
"directory": "third_party/cocoroms",
"excludedFiles": [
"bas13.rom",
"extbas11.rom",
"disk11.rom",
"coco3.rom"
],
"license": "File-specific Tandy, Microsoft and Microware notices retained in reconstructed firmware and assembly comments.",
"optional": true,
"patches": [],
"project": "toolshed-cocoroms",
"romOutputs": {
"bas13.rom": {
"bytes": 8192,
"sha256": "5375b1caef9df815b781d9cd6df81ecdaddd2642b06a01adac521e40f1329751"
},
"coco3.rom": {
"bytes": 32768,
"sha256": "ad358430568250eb8330a37f85361d9ce67e5ae6d5a28c17b31802cd3953a5d2"
},
"disk11.rom": {
"bytes": 8192,
"sha256": "1c40c5a74e72edaeecc2511fee1aab347eae2c3d5fc6778cb622e3df14d63157"
},
"extbas11.rom": {
"bytes": 8192,
"sha256": "a18f270d3495d82b9f515de049f46253a623a2d9b1bb226c9ad338508344c5ee"
}
},
"selection": "Assembly-only inputs, including the referenced authors-picture assembly data, for Color BASIC 1.3, Extended Color BASIC 1.1, Disk BASIC 1.1 and Super Color BASIC 2.0; no preassembled ROM or binary input.",
"sourceRevision": "3667a41c9605b7fcb150eace29cc7f42d460be1a",
"sourceURL": "https://github.com/nitros9project/toolshed",
"upstreamFileSHA256": {
"Makefile": "4a29262fc1ba0b6106887425f5b0a4edb8ab81fd343af548cb3a863bda3aea51",
"bas.asm": "ca2e0fba27a1c960f56648ab7be6571a89a4aff6c433b9a2649924e308e8bf24",
"coco3.asm": "01147f426baf1b48fc6d590f18ca5974bedc3dfac486020e694d9f4b890d74f2",
"disk.asm": "735443637ebaddc358f6a12856c495cc92e08f3aaa8bb68cf9a0494d7655faae",
"equates.asm": "099d4ae145ee5151aa1eb7e9805388d338b2613fd53e2ba666ab757734d94442",
"extbas.asm": "75ce20734d0272f636a961b318fb898c555061194da12d9bfc26782fc4b5e42b",
"md5sums.txt": "f0374ad4bdd32f9cd322cfa5eafe04a0b27d2e90698b536aa26c00d1ea7c8eab",
"mugateerspic.asm": "2878b1a9fbf7cc3bde25475f5e256f25910fdb44b1b2f8e6f01883bcf13548a1"
},
"vendoredFileSHA256": {
"Makefile": "4a29262fc1ba0b6106887425f5b0a4edb8ab81fd343af548cb3a863bda3aea51",
"bas.asm": "ca2e0fba27a1c960f56648ab7be6571a89a4aff6c433b9a2649924e308e8bf24",
"coco3.asm": "01147f426baf1b48fc6d590f18ca5974bedc3dfac486020e694d9f4b890d74f2",
"disk.asm": "735443637ebaddc358f6a12856c495cc92e08f3aaa8bb68cf9a0494d7655faae",
"equates.asm": "099d4ae145ee5151aa1eb7e9805388d338b2613fd53e2ba666ab757734d94442",
"extbas.asm": "75ce20734d0272f636a961b318fb898c555061194da12d9bfc26782fc4b5e42b",
"md5sums.txt": "f0374ad4bdd32f9cd322cfa5eafe04a0b27d2e90698b536aa26c00d1ea7c8eab",
"mugateerspic.asm": "2878b1a9fbf7cc3bde25475f5e256f25910fdb44b1b2f8e6f01883bcf13548a1"
},
"version": "2.6.1-3667a41"
}
@@ -0,0 +1,29 @@
{
"project": "legacy-graphics",
"version": "74c571ecf055",
"sourceURL": "https://github.com/VCCE/VCC",
"sourceRevision": "74c571ecf055e843b48e1e91fd8ae2303e5bca69",
"archiveRootPrefix": "3rdparty/",
"directory": "third_party/legacy_graphics",
"license": "Original file-specific Khronos/OpenGL/stb notices retained",
"excludedFiles": [],
"patches": [],
"upstreamFileSHA256": {
"GL/glcorearb.h": "8841702a2934583c3860a19600f55e85e741cb2c1e73cd512e681e7625f10ad4",
"GL/glext.h": "d66da97d4030d0a8f2180a9338e4378f63d8343f05b56da52064f67f9493a6a3",
"GL/glxext.h": "3ded5604e6d7643632a6f50133e15d52603d1cd95399a7781d0a49d153ade8e9",
"GL/wgl.h": "e95270f0e57101478833a23d5f89c38eea91befd323894498e3c3edb199a2805",
"GL/wglext.h": "b8f828a41bf545195268aedd6677cc621a5fd87054762eb535b4304de84c2bf2",
"KHR/khrplatform.h": "7b1e01aaa7ad8f6fc34b5c7bdf79ebf5189bb09e2c4d2e79fc5d350623d11e83",
"stb/stb_image_write.h": "cbd5f0ad7a9cf4468affb36354a1d2338034f2c12473cf1a8e32053cb6914a05"
},
"vendoredFileSHA256": {
"GL/glcorearb.h": "8841702a2934583c3860a19600f55e85e741cb2c1e73cd512e681e7625f10ad4",
"GL/glext.h": "d66da97d4030d0a8f2180a9338e4378f63d8343f05b56da52064f67f9493a6a3",
"GL/glxext.h": "3ded5604e6d7643632a6f50133e15d52603d1cd95399a7781d0a49d153ade8e9",
"GL/wgl.h": "e95270f0e57101478833a23d5f89c38eea91befd323894498e3c3edb199a2805",
"GL/wglext.h": "b8f828a41bf545195268aedd6677cc621a5fd87054762eb535b4304de84c2bf2",
"KHR/khrplatform.h": "7b1e01aaa7ad8f6fc34b5c7bdf79ebf5189bb09e2c4d2e79fc5d350623d11e83",
"stb/stb_image_write.h": "cbd5f0ad7a9cf4468affb36354a1d2338034f2c12473cf1a8e32053cb6914a05"
}
}
+272
View File
@@ -0,0 +1,272 @@
# SPDX-License-Identifier: GPL-3.0-or-later
"""Authored archive fixtures exercise source selection and filesystem boundaries."""
import hashlib
import gzip
import importlib.util
import io
import json
from pathlib import Path
import tarfile
import tempfile
import unittest
import zipfile
MODULE = Path(__file__).resolve().parents[1] / "fetch.py"
SPEC = importlib.util.spec_from_file_location("fetch_dependencies", MODULE)
FETCH = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(FETCH)
def fingerprint(data):
return hashlib.sha256(data).hexdigest()
class FetchDependenciesTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="vcc-source-fetch-test-")
self.root = Path(self.temp.name)
self.metadata = self.root / "metadata"
self.metadata.mkdir()
self.cache = self.root / "cache"
self.cache.mkdir()
self.output = self.root / "output"
def tearDown(self):
self.temp.cleanup()
def archive(self, entries, name="fixture-1.tar.gz"):
path = self.cache / name
with tarfile.open(path, "w:gz") as archive:
for filename, data, mode, kind in entries:
member = tarfile.TarInfo(filename)
member.mode = mode
if kind == "symlink":
member.type = tarfile.SYMTYPE
member.linkname = "/private/escaped"
archive.addfile(member)
else:
member.size = len(data)
archive.addfile(member, io.BytesIO(data))
return path
def record(self, archive, entries=None):
hashes = entries or {"configure": fingerprint(b"#!/bin/sh\nexit 0\n"), "COPYING": fingerprint(b"Fixture notice\n")}
return {"directory": "third_party/fixture", "sourceURL": "https://example.invalid/" + archive.name,
"archiveSHA256": FETCH.sha(archive), "upstreamFileSHA256": hashes,
"vendoredFileSHA256": hashes, "excludedFiles": ["secret.rom"], "patches": []}
def ordinary_archive(self):
return self.archive([("fixture-1/configure", b"#!/bin/sh\nexit 0\n", 0o755, "file"),
("fixture-1/COPYING", b"Fixture notice\n", 0o644, "file"),
("fixture-1/secret.rom", b"excluded fixture bytes", 0o644, "file")])
def test_only_manifest_files_materialized_with_safe_modes_and_idempotent_verification(self):
archive = self.ordinary_archive()
record = self.record(archive)
first = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(first["status"], "fetched-verified")
result = self.output / "fixture"
self.assertEqual(set(p.name for p in result.iterdir()), {"configure", "COPYING"})
self.assertEqual((result / "configure").stat().st_mode & 0o7777, 0o755)
self.assertEqual((result / "COPYING").stat().st_mode & 0o7777, 0o644)
second = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(second["status"], "verified-existing")
def test_corrupt_cached_archive_is_rejected_without_output(self):
archive = self.ordinary_archive()
record = self.record(archive)
archive.write_bytes(archive.read_bytes() + b"corrupt")
with self.assertRaisesRegex(ValueError, "archive SHA-256 mismatch"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
def test_wrong_selected_bytes_do_not_leave_partial_destination(self):
archive = self.ordinary_archive()
record = self.record(archive)
record["upstreamFileSHA256"] = dict(record["upstreamFileSHA256"], COPYING="0" * 64)
with self.assertRaisesRegex(ValueError, "source file SHA-256 mismatch"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
def test_traversal_members_are_refused_even_when_not_selected(self):
for filename in ("../escaped", "/absolute", "root/../../escaped", "C:/escape", "root\\escape"):
with self.subTest(filename=filename):
archive = self.archive([(filename, b"x", 0o644, "file")])
destination = self.root / "extract"
destination.mkdir(exist_ok=True)
with self.assertRaises(ValueError):
FETCH.extract_selected(archive, destination, {"safe": fingerprint(b"x")})
self.assertEqual(list(destination.iterdir()), [])
def test_selected_links_and_duplicates_are_refused(self):
for entries in ([('root/a', b"", 0o777, "symlink")],
[('root/a', b"x", 0o644, "file"), ('root/a', b"x", 0o644, "file")]):
archive = self.archive(entries)
with self.assertRaisesRegex(ValueError, "linked|Duplicate"):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_unselected_oversized_members_are_refused(self):
archive = self.cache / "oversized.tar.gz"
member = tarfile.TarInfo("root/unselected")
member.size = FETCH.MAX_FILE_BYTES + 1
# An authored header is enough: bounds reject before reading payload.
with gzip.open(archive, "wb") as source:
source.write(member.tobuf() + bytes(1024))
with self.assertRaisesRegex(ValueError, "size limit"):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_zip_traversal_and_unix_symlinks_refused(self):
for filename, mode in [("../escape", 0o100644), ("root/a", 0o120777)]:
archive = self.cache / "fixture.zip"
with zipfile.ZipFile(archive, "w") as target:
entry = zipfile.ZipInfo(filename)
entry.create_system = 3
entry.external_attr = mode << 16
target.writestr(entry, b"x")
with self.assertRaises(ValueError):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_pinned_git_archive_prefix_and_per_file_hashes(self):
archive = self.archive([("upstream-abc/3rdparty/GL/header.h", b"header", 0o644, "file")],
"fixture-" + "a" * 40 + ".tar.gz")
record = self.record(archive, {"GL/header.h": fingerprint(b"header")})
record.pop("archiveSHA256")
record.update(sourceURL="https://github.com/example/project", sourceRevision="a" * 40,
archiveRootPrefix="3rdparty/")
self.assertEqual(FETCH.source_url(record), "https://codeload.github.com/example/project/tar.gz/" + "a" * 40)
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual((self.output / "fixture/GL/header.h").read_bytes(), b"header")
def test_import_patch_reproduces_final_hashes(self):
archive = self.archive([("fixture-1/a.txt", b"before\n", 0o644, "file")])
record = self.record(archive, {"a.txt": fingerprint(b"before\n")})
record["vendoredFileSHA256"] = {"a.txt": fingerprint(b"after\n")}
record["patches"] = ["change.patch"]
(self.metadata / "change.patch").write_text("--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-before\n+after\n")
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual((self.output / "fixture/a.txt").read_bytes(), b"after\n")
def test_existing_modified_tree_is_not_replaced(self):
archive = self.ordinary_archive()
record = self.record(archive)
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
changed = self.output / "fixture/COPYING"
changed.write_bytes(b"local edit")
with self.assertRaisesRegex(ValueError, "differs from reviewed import"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(changed.read_bytes(), b"local edit")
def test_linked_source_destination_and_archive_cache_are_refused(self):
archive = self.ordinary_archive()
record = self.record(archive)
target = self.root / "outside"
target.mkdir()
self.output.symlink_to(target, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(list(target.iterdir()), [])
linked_cache = self.root / "linked-cache"
linked_cache.symlink_to(self.cache, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.obtain_archive("fixture", record, linked_cache, offline=True)
def test_linked_ancestor_output_and_cache_never_write_through(self):
archive = self.ordinary_archive()
record = self.record(archive)
outside = self.root / "outside"
outside.mkdir()
sentinel = outside / "keep.txt"
sentinel.write_bytes(b"preserved")
linked_parent = self.root / "link"
linked_parent.symlink_to(outside, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.materialize("fixture", record, self.metadata,
linked_parent / "new-third-party", self.cache, offline=True)
self.assertEqual(sentinel.read_bytes(), b"preserved")
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.obtain_archive("fixture", record, linked_parent / "new-cache", offline=True)
self.assertEqual(sentinel.read_bytes(), b"preserved")
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
def test_only_canonical_macos_aliases_are_allowed_in_ancestry(self):
# These public spellings must remain usable by Xcode on macOS. A
# similarly named user link is still covered by the preceding test.
import os
if os.uname().sysname != "Darwin":
self.skipTest("macOS public aliases are platform-specific")
for public, canonical in [(Path("/tmp"), Path("/private/tmp")),
(Path("/var"), Path("/private/var"))]:
if public.is_symlink() and public.resolve() == canonical:
FETCH.require_unlinked_ancestors(public / "uncreated-dependency-cache", "Fixture")
def test_verify_only_missing_tree_does_not_create_output_or_fetch(self):
archive = self.ordinary_archive()
record = self.record(archive)
with self.assertRaisesRegex(ValueError, "has not been fetched"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache,
offline=True, verify_only=True)
self.assertFalse(self.output.exists())
def test_supplemental_metadata_merges_without_replacing_import_records(self):
archive = self.ordinary_archive()
record = self.record(archive)
normal = self.root / "third_party/manifests"
supplement = self.root / "tools/dependencies/manifests"
normal.mkdir(parents=True)
supplement.mkdir(parents=True)
first = normal / "fixture.json"
first.write_text(json.dumps(record))
other = dict(record, directory="third_party/legacy")
second = supplement / "legacy.json"
second.write_text(json.dumps(other))
records, locations = FETCH.load_records(self.root)
self.assertEqual(set(records), {"fixture", "legacy"})
self.assertEqual(records["fixture"], record)
self.assertEqual(locations, {"fixture": normal, "legacy": supplement})
(supplement / "fixture.json").write_text(json.dumps(other))
with self.assertRaisesRegex(ValueError, "Duplicate"):
FETCH.load_records(self.root)
self.assertEqual(json.loads(first.read_text()), record)
def test_overlapping_source_roots_and_linked_metadata_are_refused(self):
archive = self.ordinary_archive()
record = self.record(archive)
normal = self.root / "third_party/manifests"
normal.mkdir(parents=True)
(normal / "fixture.json").write_text(json.dumps(record))
other = normal / "other.json"
other.write_text(json.dumps(dict(record, directory="third_party/fixture/nested")))
with self.assertRaisesRegex(ValueError, "Overlapping"):
FETCH.load_records(self.root)
other.unlink()
other.symlink_to(normal / "fixture.json")
with self.assertRaisesRegex(ValueError, "ordinary file"):
FETCH.load_records(self.root)
def test_release_hash_or_full_immutable_revision_is_required(self):
archive = self.ordinary_archive()
record = self.record(archive)
record.pop("archiveSHA256")
with self.assertRaisesRegex(ValueError, "require a recorded"):
FETCH.source_url(record)
for revision in ("main", "a" * 12, "a" * 39, "a" * 41):
with self.subTest(revision=revision):
candidate = dict(record, sourceURL="https://github.com/example/project", sourceRevision=revision)
with self.assertRaisesRegex(ValueError, "pinned Git"):
FETCH.source_url(candidate)
def test_unapproved_import_patch_target_leaves_no_destination(self):
archive = self.ordinary_archive()
record = self.record(archive)
record["patches"] = ["escape.patch"]
(self.metadata / "escape.patch").write_text("--- a/../escape\n+++ b/../escape\n@@ -1 +1 @@\n-before\n+after\n")
with self.assertRaises(ValueError):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
if __name__ == "__main__":
unittest.main()