Publish VCC Plus OSS source distribution for 3.6.2 build 60
This commit is contained in:
commit
c652a5276d
1192 files changed
+272368
No files matched your search
@@ -0,0 +1,368 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Fetch and verify pinned source dependencies without executing their code.
|
||||
|
||||
Source archives are retained in ignored storage. Only manifest-selected regular
|
||||
files are materialized, excluded payloads are never extracted, and import patches
|
||||
must reproduce the exact reviewed file hashes. No Git repository is cloned.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
import zipfile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MAX_ARCHIVE_BYTES = 128 * 1024 * 1024
|
||||
MAX_FILE_BYTES = 64 * 1024 * 1024
|
||||
MAX_EXPANDED_BYTES = 1024 * 1024 * 1024
|
||||
MAX_MEMBERS = 100_000
|
||||
HASH = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def sha(path):
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def relative(name):
|
||||
if not isinstance(name, str) or not name or "\\" in name or "\x00" in name:
|
||||
raise ValueError("Invalid dependency path: " + repr(name))
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or ".." in path.parts or ".git" in path.parts or re.match(r"^[A-Za-z]:", name):
|
||||
raise ValueError("Unsafe dependency path: " + name)
|
||||
value = path.as_posix()
|
||||
if value == ".":
|
||||
return ""
|
||||
return value
|
||||
|
||||
|
||||
def validate_record(record):
|
||||
if 'optional' in record and not isinstance(record['optional'], bool):
|
||||
raise ValueError('Dependency optional flag must be boolean')
|
||||
directory = relative(record["directory"])
|
||||
if not directory.startswith("third_party/") or directory == "third_party/manifests":
|
||||
raise ValueError("Dependency directory must be beneath third_party")
|
||||
suffix = directory.removeprefix("third_party/")
|
||||
if suffix.startswith("manifests/"):
|
||||
raise ValueError("Dependency cannot overwrite metadata")
|
||||
for key in ("upstreamFileSHA256", "vendoredFileSHA256"):
|
||||
if not isinstance(record[key], dict) or not record[key]:
|
||||
raise ValueError("Dependency requires nonempty file hash maps")
|
||||
for name, digest in record[key].items():
|
||||
if relative(name) != name or (not isinstance(digest, str) or not HASH.fullmatch(digest)):
|
||||
raise ValueError("Invalid file fingerprint: " + name)
|
||||
if set(record["upstreamFileSHA256"]) != set(record["vendoredFileSHA256"]):
|
||||
raise ValueError("Import patches may change bytes, not the approved file inventory")
|
||||
if record.get("archiveSHA256") is not None and (not isinstance(record["archiveSHA256"], str) or not HASH.fullmatch(record["archiveSHA256"])):
|
||||
raise ValueError("Invalid archive fingerprint")
|
||||
for excluded in record.get("excludedFiles", []):
|
||||
excluded = relative(excluded).rstrip("/")
|
||||
if any(p == excluded or p.startswith(excluded + "/") for p in record["upstreamFileSHA256"]):
|
||||
raise ValueError("Selected dependency file overlaps an exclusion: " + excluded)
|
||||
if record.get("archiveRootPrefix"):
|
||||
if not relative(record["archiveRootPrefix"].rstrip("/")):
|
||||
raise ValueError("Archive root prefix cannot be empty")
|
||||
return suffix
|
||||
|
||||
|
||||
def source_url(record):
|
||||
"""Use a release URL or a full immutable GitHub revision archive."""
|
||||
url = record["sourceURL"]
|
||||
if record.get("sourceRevision"):
|
||||
revision = record["sourceRevision"]
|
||||
parts = urlsplit(url)
|
||||
fields = parts.path.strip("/").removesuffix(".git").split("/")
|
||||
if parts.scheme != "https" or parts.hostname != "github.com" or len(fields) != 2 or not re.fullmatch(r"[0-9a-f]{40}", revision):
|
||||
raise ValueError("Unsupported pinned Git source")
|
||||
return f"https://codeload.github.com/{fields[0]}/{fields[1]}/tar.gz/{revision}"
|
||||
if not record.get("archiveSHA256"):
|
||||
raise ValueError("Release archives require a recorded SHA-256")
|
||||
if urlsplit(url).scheme not in ("http", "https"):
|
||||
raise ValueError("Dependency source must use HTTP or HTTPS")
|
||||
return url
|
||||
|
||||
|
||||
class SafeRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
if urlsplit(newurl).scheme not in ("http", "https"):
|
||||
raise ValueError("Unsupported dependency redirect")
|
||||
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
||||
|
||||
|
||||
def archive_names(name, record):
|
||||
filename = Path(urlsplit(record["sourceURL"]).path).name
|
||||
if record.get("sourceRevision"):
|
||||
filename = name + "-" + record["sourceRevision"] + ".tar.gz"
|
||||
if not filename or filename in (".", ".."):
|
||||
raise ValueError("Invalid source archive filename")
|
||||
digest = record.get("archiveSHA256")
|
||||
return ([digest + "-" + filename, filename, digest + ".tar.gz"] if digest else [filename])
|
||||
|
||||
|
||||
def verify_archive(path, record):
|
||||
if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_ARCHIVE_BYTES:
|
||||
raise ValueError("Source archive must be a bounded regular file: " + str(path))
|
||||
digest = sha(path)
|
||||
if record.get("archiveSHA256") and digest != record["archiveSHA256"]:
|
||||
raise ValueError("Source archive SHA-256 mismatch: " + path.name)
|
||||
return digest
|
||||
|
||||
|
||||
|
||||
def require_unlinked_ancestors(path, description):
|
||||
"""Reject redirected destinations, retaining only macOS's public aliases.
|
||||
|
||||
Do not resolve an arbitrary user path before examining its full ancestry:
|
||||
doing so would erase precisely the linked parent this check must detect.
|
||||
"""
|
||||
aliases = {Path("/tmp"): Path("/private/tmp"), Path("/var"): Path("/private/var")}
|
||||
absolute = Path(path).absolute()
|
||||
for candidate in (absolute, *absolute.parents):
|
||||
if candidate.is_symlink() and not (
|
||||
candidate in aliases and candidate.resolve() == aliases[candidate]):
|
||||
raise ValueError(description + " cannot traverse a symbolic link")
|
||||
|
||||
|
||||
def obtain_archive(name, record, cache, offline=False, timeout=30):
|
||||
require_unlinked_ancestors(cache, "Archive cache")
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
names = archive_names(name, record)
|
||||
for filename in names:
|
||||
cached = cache / filename
|
||||
if cached.exists() or cached.is_symlink():
|
||||
verify_archive(cached, record)
|
||||
return cached
|
||||
if offline:
|
||||
raise ValueError("Offline source archive missing: " + names[0])
|
||||
url = source_url(record)
|
||||
request = Request(url, headers={"User-Agent": "VCCPlus-source-fetch/1"})
|
||||
target = cache / names[0]
|
||||
temporary = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(prefix=".download-", dir=cache, delete=False) as output:
|
||||
temporary = Path(output.name)
|
||||
with build_opener(SafeRedirect()).open(request, timeout=timeout) as response:
|
||||
if urlsplit(response.geturl()).scheme not in ("http", "https"):
|
||||
raise ValueError("Unsupported dependency response URL")
|
||||
size = 0
|
||||
while chunk := response.read(1024 * 1024):
|
||||
size += len(chunk)
|
||||
if size > MAX_ARCHIVE_BYTES:
|
||||
raise ValueError("Dependency archive exceeds download limit")
|
||||
output.write(chunk)
|
||||
verify_archive(temporary, record)
|
||||
require_unlinked_ancestors(cache, "Archive cache")
|
||||
if target.exists() or target.is_symlink():
|
||||
raise ValueError("Archive cache target changed during download")
|
||||
os.replace(temporary, target)
|
||||
return target
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def extract_selected(archive, destination, expected, archive_prefix=""):
|
||||
"""Stream only known regular members; never use extract/extractall."""
|
||||
observed = set()
|
||||
all_names = set()
|
||||
prefix = None
|
||||
expanded = 0
|
||||
count = 0
|
||||
|
||||
def consume(name, size, mode, regular, opener):
|
||||
nonlocal prefix, expanded, count
|
||||
count += 1
|
||||
if count > MAX_MEMBERS:
|
||||
raise ValueError("Source archive has too many members")
|
||||
if not 0 <= size <= MAX_FILE_BYTES:
|
||||
raise ValueError("Source archive member exceeds size limit: " + name)
|
||||
expanded += size
|
||||
if expanded > MAX_EXPANDED_BYTES:
|
||||
raise ValueError("Source archive exceeds expansion limit")
|
||||
normalized = relative(name)
|
||||
if normalized in all_names:
|
||||
raise ValueError("Duplicate source archive member: " + normalized)
|
||||
all_names.add(normalized)
|
||||
parts = normalized.split("/")
|
||||
candidates = [(normalized, "")]
|
||||
if len(parts) > 1:
|
||||
candidates.append(("/".join(parts[1:]), parts[0] + "/"))
|
||||
archive_root = archive_prefix.rstrip("/") + "/" if archive_prefix else ""
|
||||
matches = [(path[len(archive_root):], root) for path, root in candidates
|
||||
if path.startswith(archive_root) and path[len(archive_root):] in expected]
|
||||
if not matches:
|
||||
return
|
||||
if len(matches) != 1:
|
||||
raise ValueError("Ambiguous source archive member: " + normalized)
|
||||
path, root = matches[0]
|
||||
if prefix is not None and root != prefix:
|
||||
raise ValueError("Source archive has inconsistent roots")
|
||||
prefix = root
|
||||
if path in observed or not regular:
|
||||
raise ValueError("Selected member is duplicate, linked or not a regular file: " + normalized)
|
||||
with opener() as stream:
|
||||
data = stream.read(MAX_FILE_BYTES + 1)
|
||||
if len(data) != size or hashlib.sha256(data).hexdigest() != expected[path]:
|
||||
raise ValueError("Upstream source file SHA-256 mismatch: " + path)
|
||||
target = destination / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(data)
|
||||
target.chmod(0o755 if mode & 0o111 else 0o644)
|
||||
observed.add(path)
|
||||
|
||||
if zipfile.is_zipfile(archive):
|
||||
with zipfile.ZipFile(archive) as source:
|
||||
for member in source.infolist():
|
||||
mode = member.external_attr >> 16
|
||||
regular = not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG) and not (member.flag_bits & 1)
|
||||
consume(member.filename, member.file_size, mode, regular, lambda member=member: source.open(member))
|
||||
else:
|
||||
with tarfile.open(archive, mode="r|*") as source:
|
||||
for member in source:
|
||||
consume(member.name, member.size, member.mode, member.isfile(), lambda member=member: source.extractfile(member))
|
||||
if observed != set(expected):
|
||||
raise ValueError("Source archive is missing selected files: " + ", ".join(sorted(set(expected) - observed)[:12]))
|
||||
|
||||
|
||||
def verify_tree(root, expected):
|
||||
if root.is_symlink() or not root.is_dir():
|
||||
raise ValueError("Dependency source must be an ordinary directory")
|
||||
actual = {}
|
||||
for path in root.rglob("*"):
|
||||
if path.is_symlink():
|
||||
raise ValueError("Dependency contains a symbolic link: " + str(path))
|
||||
if path.is_file():
|
||||
actual[path.relative_to(root).as_posix()] = sha(path)
|
||||
elif not path.is_dir():
|
||||
raise ValueError("Dependency contains a special file")
|
||||
if actual != expected:
|
||||
changed = sorted(p for p in actual.keys() | expected.keys() if actual.get(p) != expected.get(p))
|
||||
raise ValueError("Dependency source differs from reviewed import: " + ", ".join(changed[:12]))
|
||||
|
||||
|
||||
def apply_patches(root, record, metadata):
|
||||
allowed = set(record["upstreamFileSHA256"])
|
||||
for filename in record.get("patches", []):
|
||||
if relative(filename) != filename or "/" in filename:
|
||||
raise ValueError("Import patch must be a metadata filename")
|
||||
path = metadata / filename
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("Missing import patch: " + filename)
|
||||
text = path.read_text()
|
||||
targets = []
|
||||
for line in text.splitlines():
|
||||
if line.startswith(("--- ", "+++ ")):
|
||||
name = line[4:].split("\t", 1)[0]
|
||||
if name[:2] not in ("a/", "b/") or relative(name[2:]) not in allowed:
|
||||
raise ValueError("Import patch touches an unapproved path")
|
||||
targets.append(name[2:])
|
||||
if not targets:
|
||||
raise ValueError("Import patch contains no unified file headers")
|
||||
subprocess.run(["/usr/bin/patch", "-u", "-t", "-F", "0", "-p1", "-i", str(path.resolve())],
|
||||
cwd=root, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
# A byte-perfect patch may still leave editor backups on some patch versions.
|
||||
# Do not remove them silently; verify_tree fails on any extra output.
|
||||
|
||||
|
||||
def load_records(project_root=ROOT):
|
||||
"""Read ordinary import records plus supplemental pinned-source metadata.
|
||||
|
||||
Supplemental records fill provenance gaps only; they cannot replace or
|
||||
overlap the existing reviewed dependency inventory.
|
||||
"""
|
||||
ordinary = project_root / "third_party/manifests"
|
||||
supplemental = project_root / "tools/dependencies/manifests"
|
||||
records, locations = {}, {}
|
||||
for metadata in (ordinary, supplemental):
|
||||
if metadata.is_symlink():
|
||||
raise ValueError("Dependency metadata cannot be a symbolic link")
|
||||
for path in sorted(metadata.glob("*.json")):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("Dependency manifest must be an ordinary file")
|
||||
name = path.stem
|
||||
if name in records:
|
||||
raise ValueError("Duplicate dependency manifest: " + name)
|
||||
record = json.loads(path.read_text())
|
||||
suffix = validate_record(record)
|
||||
for existing in records.values():
|
||||
other = validate_record(existing)
|
||||
if suffix == other or suffix.startswith(other + "/") or other.startswith(suffix + "/"):
|
||||
raise ValueError("Overlapping dependency source directories")
|
||||
source_url(record) # Reject unpinned/unsupported sources before any writes.
|
||||
records[name], locations[name] = record, metadata
|
||||
return records, locations
|
||||
|
||||
|
||||
def materialize(name, record, metadata, output, cache, offline=False, verify_only=False):
|
||||
suffix = validate_record(record)
|
||||
destination = output / suffix
|
||||
require_unlinked_ancestors(destination, "Dependency destination")
|
||||
if destination.exists():
|
||||
verify_tree(destination, record["vendoredFileSHA256"])
|
||||
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "verified-existing"}
|
||||
if verify_only:
|
||||
raise ValueError("Dependency source has not been fetched: " + name)
|
||||
archive = obtain_archive(name, record, cache, offline)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="." + name + "-", dir=destination.parent) as temporary:
|
||||
staged = Path(temporary) / "source"
|
||||
staged.mkdir()
|
||||
extract_selected(archive, staged, record["upstreamFileSHA256"], record.get("archiveRootPrefix", ""))
|
||||
apply_patches(staged, record, metadata)
|
||||
verify_tree(staged, record["vendoredFileSHA256"])
|
||||
require_unlinked_ancestors(destination, "Dependency destination")
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise ValueError("Dependency destination changed during preparation")
|
||||
staged.rename(destination)
|
||||
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "fetched-verified",
|
||||
"archive_sha256": sha(archive)}
|
||||
|
||||
|
||||
def default_selection(records):
|
||||
"""Optional firmware preparation is explicit, never part of ordinary fetches."""
|
||||
return [name for name, record in records.items() if not record.get('optional', False)]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--only", nargs="+", help="Manifest names, for example: --only toolshed")
|
||||
parser.add_argument("--archive-cache", type=Path, default=ROOT / ".build/dependencies/downloads")
|
||||
parser.add_argument("--output", type=Path, default=ROOT / "third_party", help="Ignored dependency source root")
|
||||
parser.add_argument("--offline", action="store_true", help="Use only already cached original source archives")
|
||||
parser.add_argument("--verify-only", action="store_true", help="Verify existing source trees without fetching")
|
||||
args = parser.parse_args()
|
||||
records, locations = load_records()
|
||||
selected = args.only or default_selection(records)
|
||||
unknown = set(selected) - records.keys()
|
||||
if unknown:
|
||||
parser.error("Unknown dependency manifest: " + ", ".join(sorted(unknown)))
|
||||
if not records:
|
||||
parser.error("No dependency manifests found")
|
||||
output = args.output.absolute()
|
||||
canonical_output = output.resolve()
|
||||
metadata_roots = {ROOT / "third_party/manifests", ROOT / "tools/dependencies/manifests"}
|
||||
if canonical_output == ROOT or canonical_output in ROOT.parents or canonical_output in metadata_roots:
|
||||
parser.error("Choose a dependency source root separate from project metadata")
|
||||
for name in dict.fromkeys(selected):
|
||||
result = materialize(name, records[name], locations[name], output, args.archive_cache, args.offline, args.verify_only)
|
||||
print(json.dumps(result, sort_keys=True), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (ValueError, OSError, tarfile.TarError, zipfile.BadZipFile, subprocess.CalledProcessError) as error:
|
||||
raise SystemExit(str(error))
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"archiveRootPrefix": "cocoroms/",
|
||||
"directory": "third_party/cocoroms",
|
||||
"excludedFiles": [
|
||||
"bas13.rom",
|
||||
"extbas11.rom",
|
||||
"disk11.rom",
|
||||
"coco3.rom"
|
||||
],
|
||||
"license": "File-specific Tandy, Microsoft and Microware notices retained in reconstructed firmware and assembly comments.",
|
||||
"optional": true,
|
||||
"patches": [],
|
||||
"project": "toolshed-cocoroms",
|
||||
"romOutputs": {
|
||||
"bas13.rom": {
|
||||
"bytes": 8192,
|
||||
"sha256": "5375b1caef9df815b781d9cd6df81ecdaddd2642b06a01adac521e40f1329751"
|
||||
},
|
||||
"coco3.rom": {
|
||||
"bytes": 32768,
|
||||
"sha256": "ad358430568250eb8330a37f85361d9ce67e5ae6d5a28c17b31802cd3953a5d2"
|
||||
},
|
||||
"disk11.rom": {
|
||||
"bytes": 8192,
|
||||
"sha256": "1c40c5a74e72edaeecc2511fee1aab347eae2c3d5fc6778cb622e3df14d63157"
|
||||
},
|
||||
"extbas11.rom": {
|
||||
"bytes": 8192,
|
||||
"sha256": "a18f270d3495d82b9f515de049f46253a623a2d9b1bb226c9ad338508344c5ee"
|
||||
}
|
||||
},
|
||||
"selection": "Assembly-only inputs, including the referenced authors-picture assembly data, for Color BASIC 1.3, Extended Color BASIC 1.1, Disk BASIC 1.1 and Super Color BASIC 2.0; no preassembled ROM or binary input.",
|
||||
"sourceRevision": "3667a41c9605b7fcb150eace29cc7f42d460be1a",
|
||||
"sourceURL": "https://github.com/nitros9project/toolshed",
|
||||
"upstreamFileSHA256": {
|
||||
"Makefile": "4a29262fc1ba0b6106887425f5b0a4edb8ab81fd343af548cb3a863bda3aea51",
|
||||
"bas.asm": "ca2e0fba27a1c960f56648ab7be6571a89a4aff6c433b9a2649924e308e8bf24",
|
||||
"coco3.asm": "01147f426baf1b48fc6d590f18ca5974bedc3dfac486020e694d9f4b890d74f2",
|
||||
"disk.asm": "735443637ebaddc358f6a12856c495cc92e08f3aaa8bb68cf9a0494d7655faae",
|
||||
"equates.asm": "099d4ae145ee5151aa1eb7e9805388d338b2613fd53e2ba666ab757734d94442",
|
||||
"extbas.asm": "75ce20734d0272f636a961b318fb898c555061194da12d9bfc26782fc4b5e42b",
|
||||
"md5sums.txt": "f0374ad4bdd32f9cd322cfa5eafe04a0b27d2e90698b536aa26c00d1ea7c8eab",
|
||||
"mugateerspic.asm": "2878b1a9fbf7cc3bde25475f5e256f25910fdb44b1b2f8e6f01883bcf13548a1"
|
||||
},
|
||||
"vendoredFileSHA256": {
|
||||
"Makefile": "4a29262fc1ba0b6106887425f5b0a4edb8ab81fd343af548cb3a863bda3aea51",
|
||||
"bas.asm": "ca2e0fba27a1c960f56648ab7be6571a89a4aff6c433b9a2649924e308e8bf24",
|
||||
"coco3.asm": "01147f426baf1b48fc6d590f18ca5974bedc3dfac486020e694d9f4b890d74f2",
|
||||
"disk.asm": "735443637ebaddc358f6a12856c495cc92e08f3aaa8bb68cf9a0494d7655faae",
|
||||
"equates.asm": "099d4ae145ee5151aa1eb7e9805388d338b2613fd53e2ba666ab757734d94442",
|
||||
"extbas.asm": "75ce20734d0272f636a961b318fb898c555061194da12d9bfc26782fc4b5e42b",
|
||||
"md5sums.txt": "f0374ad4bdd32f9cd322cfa5eafe04a0b27d2e90698b536aa26c00d1ea7c8eab",
|
||||
"mugateerspic.asm": "2878b1a9fbf7cc3bde25475f5e256f25910fdb44b1b2f8e6f01883bcf13548a1"
|
||||
},
|
||||
"version": "2.6.1-3667a41"
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"project": "legacy-graphics",
|
||||
"version": "74c571ecf055",
|
||||
"sourceURL": "https://github.com/VCCE/VCC",
|
||||
"sourceRevision": "74c571ecf055e843b48e1e91fd8ae2303e5bca69",
|
||||
"archiveRootPrefix": "3rdparty/",
|
||||
"directory": "third_party/legacy_graphics",
|
||||
"license": "Original file-specific Khronos/OpenGL/stb notices retained",
|
||||
"excludedFiles": [],
|
||||
"patches": [],
|
||||
"upstreamFileSHA256": {
|
||||
"GL/glcorearb.h": "8841702a2934583c3860a19600f55e85e741cb2c1e73cd512e681e7625f10ad4",
|
||||
"GL/glext.h": "d66da97d4030d0a8f2180a9338e4378f63d8343f05b56da52064f67f9493a6a3",
|
||||
"GL/glxext.h": "3ded5604e6d7643632a6f50133e15d52603d1cd95399a7781d0a49d153ade8e9",
|
||||
"GL/wgl.h": "e95270f0e57101478833a23d5f89c38eea91befd323894498e3c3edb199a2805",
|
||||
"GL/wglext.h": "b8f828a41bf545195268aedd6677cc621a5fd87054762eb535b4304de84c2bf2",
|
||||
"KHR/khrplatform.h": "7b1e01aaa7ad8f6fc34b5c7bdf79ebf5189bb09e2c4d2e79fc5d350623d11e83",
|
||||
"stb/stb_image_write.h": "cbd5f0ad7a9cf4468affb36354a1d2338034f2c12473cf1a8e32053cb6914a05"
|
||||
},
|
||||
"vendoredFileSHA256": {
|
||||
"GL/glcorearb.h": "8841702a2934583c3860a19600f55e85e741cb2c1e73cd512e681e7625f10ad4",
|
||||
"GL/glext.h": "d66da97d4030d0a8f2180a9338e4378f63d8343f05b56da52064f67f9493a6a3",
|
||||
"GL/glxext.h": "3ded5604e6d7643632a6f50133e15d52603d1cd95399a7781d0a49d153ade8e9",
|
||||
"GL/wgl.h": "e95270f0e57101478833a23d5f89c38eea91befd323894498e3c3edb199a2805",
|
||||
"GL/wglext.h": "b8f828a41bf545195268aedd6677cc621a5fd87054762eb535b4304de84c2bf2",
|
||||
"KHR/khrplatform.h": "7b1e01aaa7ad8f6fc34b5c7bdf79ebf5189bb09e2c4d2e79fc5d350623d11e83",
|
||||
"stb/stb_image_write.h": "cbd5f0ad7a9cf4468affb36354a1d2338034f2c12473cf1a8e32053cb6914a05"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Authored archive fixtures exercise source selection and filesystem boundaries."""
|
||||
import hashlib
|
||||
import gzip
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
|
||||
|
||||
MODULE = Path(__file__).resolve().parents[1] / "fetch.py"
|
||||
SPEC = importlib.util.spec_from_file_location("fetch_dependencies", MODULE)
|
||||
FETCH = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(FETCH)
|
||||
|
||||
|
||||
def fingerprint(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
class FetchDependenciesTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(prefix="vcc-source-fetch-test-")
|
||||
self.root = Path(self.temp.name)
|
||||
self.metadata = self.root / "metadata"
|
||||
self.metadata.mkdir()
|
||||
self.cache = self.root / "cache"
|
||||
self.cache.mkdir()
|
||||
self.output = self.root / "output"
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def archive(self, entries, name="fixture-1.tar.gz"):
|
||||
path = self.cache / name
|
||||
with tarfile.open(path, "w:gz") as archive:
|
||||
for filename, data, mode, kind in entries:
|
||||
member = tarfile.TarInfo(filename)
|
||||
member.mode = mode
|
||||
if kind == "symlink":
|
||||
member.type = tarfile.SYMTYPE
|
||||
member.linkname = "/private/escaped"
|
||||
archive.addfile(member)
|
||||
else:
|
||||
member.size = len(data)
|
||||
archive.addfile(member, io.BytesIO(data))
|
||||
return path
|
||||
|
||||
def record(self, archive, entries=None):
|
||||
hashes = entries or {"configure": fingerprint(b"#!/bin/sh\nexit 0\n"), "COPYING": fingerprint(b"Fixture notice\n")}
|
||||
return {"directory": "third_party/fixture", "sourceURL": "https://example.invalid/" + archive.name,
|
||||
"archiveSHA256": FETCH.sha(archive), "upstreamFileSHA256": hashes,
|
||||
"vendoredFileSHA256": hashes, "excludedFiles": ["secret.rom"], "patches": []}
|
||||
|
||||
def ordinary_archive(self):
|
||||
return self.archive([("fixture-1/configure", b"#!/bin/sh\nexit 0\n", 0o755, "file"),
|
||||
("fixture-1/COPYING", b"Fixture notice\n", 0o644, "file"),
|
||||
("fixture-1/secret.rom", b"excluded fixture bytes", 0o644, "file")])
|
||||
|
||||
def test_only_manifest_files_materialized_with_safe_modes_and_idempotent_verification(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
first = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual(first["status"], "fetched-verified")
|
||||
result = self.output / "fixture"
|
||||
self.assertEqual(set(p.name for p in result.iterdir()), {"configure", "COPYING"})
|
||||
self.assertEqual((result / "configure").stat().st_mode & 0o7777, 0o755)
|
||||
self.assertEqual((result / "COPYING").stat().st_mode & 0o7777, 0o644)
|
||||
second = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual(second["status"], "verified-existing")
|
||||
|
||||
def test_corrupt_cached_archive_is_rejected_without_output(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
archive.write_bytes(archive.read_bytes() + b"corrupt")
|
||||
with self.assertRaisesRegex(ValueError, "archive SHA-256 mismatch"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertFalse((self.output / "fixture").exists())
|
||||
|
||||
def test_wrong_selected_bytes_do_not_leave_partial_destination(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
record["upstreamFileSHA256"] = dict(record["upstreamFileSHA256"], COPYING="0" * 64)
|
||||
with self.assertRaisesRegex(ValueError, "source file SHA-256 mismatch"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertFalse((self.output / "fixture").exists())
|
||||
|
||||
def test_traversal_members_are_refused_even_when_not_selected(self):
|
||||
for filename in ("../escaped", "/absolute", "root/../../escaped", "C:/escape", "root\\escape"):
|
||||
with self.subTest(filename=filename):
|
||||
archive = self.archive([(filename, b"x", 0o644, "file")])
|
||||
destination = self.root / "extract"
|
||||
destination.mkdir(exist_ok=True)
|
||||
with self.assertRaises(ValueError):
|
||||
FETCH.extract_selected(archive, destination, {"safe": fingerprint(b"x")})
|
||||
self.assertEqual(list(destination.iterdir()), [])
|
||||
|
||||
def test_selected_links_and_duplicates_are_refused(self):
|
||||
for entries in ([('root/a', b"", 0o777, "symlink")],
|
||||
[('root/a', b"x", 0o644, "file"), ('root/a', b"x", 0o644, "file")]):
|
||||
archive = self.archive(entries)
|
||||
with self.assertRaisesRegex(ValueError, "linked|Duplicate"):
|
||||
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
|
||||
|
||||
def test_unselected_oversized_members_are_refused(self):
|
||||
archive = self.cache / "oversized.tar.gz"
|
||||
member = tarfile.TarInfo("root/unselected")
|
||||
member.size = FETCH.MAX_FILE_BYTES + 1
|
||||
# An authored header is enough: bounds reject before reading payload.
|
||||
with gzip.open(archive, "wb") as source:
|
||||
source.write(member.tobuf() + bytes(1024))
|
||||
with self.assertRaisesRegex(ValueError, "size limit"):
|
||||
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
|
||||
|
||||
def test_zip_traversal_and_unix_symlinks_refused(self):
|
||||
for filename, mode in [("../escape", 0o100644), ("root/a", 0o120777)]:
|
||||
archive = self.cache / "fixture.zip"
|
||||
with zipfile.ZipFile(archive, "w") as target:
|
||||
entry = zipfile.ZipInfo(filename)
|
||||
entry.create_system = 3
|
||||
entry.external_attr = mode << 16
|
||||
target.writestr(entry, b"x")
|
||||
with self.assertRaises(ValueError):
|
||||
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
|
||||
|
||||
def test_pinned_git_archive_prefix_and_per_file_hashes(self):
|
||||
archive = self.archive([("upstream-abc/3rdparty/GL/header.h", b"header", 0o644, "file")],
|
||||
"fixture-" + "a" * 40 + ".tar.gz")
|
||||
record = self.record(archive, {"GL/header.h": fingerprint(b"header")})
|
||||
record.pop("archiveSHA256")
|
||||
record.update(sourceURL="https://github.com/example/project", sourceRevision="a" * 40,
|
||||
archiveRootPrefix="3rdparty/")
|
||||
self.assertEqual(FETCH.source_url(record), "https://codeload.github.com/example/project/tar.gz/" + "a" * 40)
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual((self.output / "fixture/GL/header.h").read_bytes(), b"header")
|
||||
|
||||
def test_import_patch_reproduces_final_hashes(self):
|
||||
archive = self.archive([("fixture-1/a.txt", b"before\n", 0o644, "file")])
|
||||
record = self.record(archive, {"a.txt": fingerprint(b"before\n")})
|
||||
record["vendoredFileSHA256"] = {"a.txt": fingerprint(b"after\n")}
|
||||
record["patches"] = ["change.patch"]
|
||||
(self.metadata / "change.patch").write_text("--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-before\n+after\n")
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual((self.output / "fixture/a.txt").read_bytes(), b"after\n")
|
||||
|
||||
def test_existing_modified_tree_is_not_replaced(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
changed = self.output / "fixture/COPYING"
|
||||
changed.write_bytes(b"local edit")
|
||||
with self.assertRaisesRegex(ValueError, "differs from reviewed import"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual(changed.read_bytes(), b"local edit")
|
||||
|
||||
|
||||
def test_linked_source_destination_and_archive_cache_are_refused(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
target = self.root / "outside"
|
||||
target.mkdir()
|
||||
self.output.symlink_to(target, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual(list(target.iterdir()), [])
|
||||
linked_cache = self.root / "linked-cache"
|
||||
linked_cache.symlink_to(self.cache, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
||||
FETCH.obtain_archive("fixture", record, linked_cache, offline=True)
|
||||
|
||||
def test_linked_ancestor_output_and_cache_never_write_through(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
outside = self.root / "outside"
|
||||
outside.mkdir()
|
||||
sentinel = outside / "keep.txt"
|
||||
sentinel.write_bytes(b"preserved")
|
||||
linked_parent = self.root / "link"
|
||||
linked_parent.symlink_to(outside, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
||||
FETCH.materialize("fixture", record, self.metadata,
|
||||
linked_parent / "new-third-party", self.cache, offline=True)
|
||||
self.assertEqual(sentinel.read_bytes(), b"preserved")
|
||||
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
|
||||
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
||||
FETCH.obtain_archive("fixture", record, linked_parent / "new-cache", offline=True)
|
||||
self.assertEqual(sentinel.read_bytes(), b"preserved")
|
||||
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
|
||||
|
||||
def test_only_canonical_macos_aliases_are_allowed_in_ancestry(self):
|
||||
# These public spellings must remain usable by Xcode on macOS. A
|
||||
# similarly named user link is still covered by the preceding test.
|
||||
import os
|
||||
if os.uname().sysname != "Darwin":
|
||||
self.skipTest("macOS public aliases are platform-specific")
|
||||
for public, canonical in [(Path("/tmp"), Path("/private/tmp")),
|
||||
(Path("/var"), Path("/private/var"))]:
|
||||
if public.is_symlink() and public.resolve() == canonical:
|
||||
FETCH.require_unlinked_ancestors(public / "uncreated-dependency-cache", "Fixture")
|
||||
|
||||
def test_verify_only_missing_tree_does_not_create_output_or_fetch(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
with self.assertRaisesRegex(ValueError, "has not been fetched"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache,
|
||||
offline=True, verify_only=True)
|
||||
self.assertFalse(self.output.exists())
|
||||
|
||||
def test_supplemental_metadata_merges_without_replacing_import_records(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
normal = self.root / "third_party/manifests"
|
||||
supplement = self.root / "tools/dependencies/manifests"
|
||||
normal.mkdir(parents=True)
|
||||
supplement.mkdir(parents=True)
|
||||
first = normal / "fixture.json"
|
||||
first.write_text(json.dumps(record))
|
||||
other = dict(record, directory="third_party/legacy")
|
||||
second = supplement / "legacy.json"
|
||||
second.write_text(json.dumps(other))
|
||||
records, locations = FETCH.load_records(self.root)
|
||||
self.assertEqual(set(records), {"fixture", "legacy"})
|
||||
self.assertEqual(records["fixture"], record)
|
||||
self.assertEqual(locations, {"fixture": normal, "legacy": supplement})
|
||||
(supplement / "fixture.json").write_text(json.dumps(other))
|
||||
with self.assertRaisesRegex(ValueError, "Duplicate"):
|
||||
FETCH.load_records(self.root)
|
||||
self.assertEqual(json.loads(first.read_text()), record)
|
||||
|
||||
def test_overlapping_source_roots_and_linked_metadata_are_refused(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
normal = self.root / "third_party/manifests"
|
||||
normal.mkdir(parents=True)
|
||||
(normal / "fixture.json").write_text(json.dumps(record))
|
||||
other = normal / "other.json"
|
||||
other.write_text(json.dumps(dict(record, directory="third_party/fixture/nested")))
|
||||
with self.assertRaisesRegex(ValueError, "Overlapping"):
|
||||
FETCH.load_records(self.root)
|
||||
other.unlink()
|
||||
other.symlink_to(normal / "fixture.json")
|
||||
with self.assertRaisesRegex(ValueError, "ordinary file"):
|
||||
FETCH.load_records(self.root)
|
||||
|
||||
def test_release_hash_or_full_immutable_revision_is_required(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
record.pop("archiveSHA256")
|
||||
with self.assertRaisesRegex(ValueError, "require a recorded"):
|
||||
FETCH.source_url(record)
|
||||
for revision in ("main", "a" * 12, "a" * 39, "a" * 41):
|
||||
with self.subTest(revision=revision):
|
||||
candidate = dict(record, sourceURL="https://github.com/example/project", sourceRevision=revision)
|
||||
with self.assertRaisesRegex(ValueError, "pinned Git"):
|
||||
FETCH.source_url(candidate)
|
||||
|
||||
def test_unapproved_import_patch_target_leaves_no_destination(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
record["patches"] = ["escape.patch"]
|
||||
(self.metadata / "escape.patch").write_text("--- a/../escape\n+++ b/../escape\n@@ -1 +1 @@\n-before\n+after\n")
|
||||
with self.assertRaises(ValueError):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertFalse((self.output / "fixture").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user