Files
VCCPlusOSS/tools/dependencies/tests/test_fetch.py
T

273 lines
14 KiB
Python

# SPDX-License-Identifier: GPL-3.0-or-later
"""Authored archive fixtures exercise source selection and filesystem boundaries."""
import hashlib
import gzip
import importlib.util
import io
import json
from pathlib import Path
import tarfile
import tempfile
import unittest
import zipfile
MODULE = Path(__file__).resolve().parents[1] / "fetch.py"
SPEC = importlib.util.spec_from_file_location("fetch_dependencies", MODULE)
FETCH = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(FETCH)
def fingerprint(data):
return hashlib.sha256(data).hexdigest()
class FetchDependenciesTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="vcc-source-fetch-test-")
self.root = Path(self.temp.name)
self.metadata = self.root / "metadata"
self.metadata.mkdir()
self.cache = self.root / "cache"
self.cache.mkdir()
self.output = self.root / "output"
def tearDown(self):
self.temp.cleanup()
def archive(self, entries, name="fixture-1.tar.gz"):
path = self.cache / name
with tarfile.open(path, "w:gz") as archive:
for filename, data, mode, kind in entries:
member = tarfile.TarInfo(filename)
member.mode = mode
if kind == "symlink":
member.type = tarfile.SYMTYPE
member.linkname = "/private/escaped"
archive.addfile(member)
else:
member.size = len(data)
archive.addfile(member, io.BytesIO(data))
return path
def record(self, archive, entries=None):
hashes = entries or {"configure": fingerprint(b"#!/bin/sh\nexit 0\n"), "COPYING": fingerprint(b"Fixture notice\n")}
return {"directory": "third_party/fixture", "sourceURL": "https://example.invalid/" + archive.name,
"archiveSHA256": FETCH.sha(archive), "upstreamFileSHA256": hashes,
"vendoredFileSHA256": hashes, "excludedFiles": ["secret.rom"], "patches": []}
def ordinary_archive(self):
return self.archive([("fixture-1/configure", b"#!/bin/sh\nexit 0\n", 0o755, "file"),
("fixture-1/COPYING", b"Fixture notice\n", 0o644, "file"),
("fixture-1/secret.rom", b"excluded fixture bytes", 0o644, "file")])
def test_only_manifest_files_materialized_with_safe_modes_and_idempotent_verification(self):
archive = self.ordinary_archive()
record = self.record(archive)
first = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(first["status"], "fetched-verified")
result = self.output / "fixture"
self.assertEqual(set(p.name for p in result.iterdir()), {"configure", "COPYING"})
self.assertEqual((result / "configure").stat().st_mode & 0o7777, 0o755)
self.assertEqual((result / "COPYING").stat().st_mode & 0o7777, 0o644)
second = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(second["status"], "verified-existing")
def test_corrupt_cached_archive_is_rejected_without_output(self):
archive = self.ordinary_archive()
record = self.record(archive)
archive.write_bytes(archive.read_bytes() + b"corrupt")
with self.assertRaisesRegex(ValueError, "archive SHA-256 mismatch"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
def test_wrong_selected_bytes_do_not_leave_partial_destination(self):
archive = self.ordinary_archive()
record = self.record(archive)
record["upstreamFileSHA256"] = dict(record["upstreamFileSHA256"], COPYING="0" * 64)
with self.assertRaisesRegex(ValueError, "source file SHA-256 mismatch"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
def test_traversal_members_are_refused_even_when_not_selected(self):
for filename in ("../escaped", "/absolute", "root/../../escaped", "C:/escape", "root\\escape"):
with self.subTest(filename=filename):
archive = self.archive([(filename, b"x", 0o644, "file")])
destination = self.root / "extract"
destination.mkdir(exist_ok=True)
with self.assertRaises(ValueError):
FETCH.extract_selected(archive, destination, {"safe": fingerprint(b"x")})
self.assertEqual(list(destination.iterdir()), [])
def test_selected_links_and_duplicates_are_refused(self):
for entries in ([('root/a', b"", 0o777, "symlink")],
[('root/a', b"x", 0o644, "file"), ('root/a', b"x", 0o644, "file")]):
archive = self.archive(entries)
with self.assertRaisesRegex(ValueError, "linked|Duplicate"):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_unselected_oversized_members_are_refused(self):
archive = self.cache / "oversized.tar.gz"
member = tarfile.TarInfo("root/unselected")
member.size = FETCH.MAX_FILE_BYTES + 1
# An authored header is enough: bounds reject before reading payload.
with gzip.open(archive, "wb") as source:
source.write(member.tobuf() + bytes(1024))
with self.assertRaisesRegex(ValueError, "size limit"):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_zip_traversal_and_unix_symlinks_refused(self):
for filename, mode in [("../escape", 0o100644), ("root/a", 0o120777)]:
archive = self.cache / "fixture.zip"
with zipfile.ZipFile(archive, "w") as target:
entry = zipfile.ZipInfo(filename)
entry.create_system = 3
entry.external_attr = mode << 16
target.writestr(entry, b"x")
with self.assertRaises(ValueError):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_pinned_git_archive_prefix_and_per_file_hashes(self):
archive = self.archive([("upstream-abc/3rdparty/GL/header.h", b"header", 0o644, "file")],
"fixture-" + "a" * 40 + ".tar.gz")
record = self.record(archive, {"GL/header.h": fingerprint(b"header")})
record.pop("archiveSHA256")
record.update(sourceURL="https://github.com/example/project", sourceRevision="a" * 40,
archiveRootPrefix="3rdparty/")
self.assertEqual(FETCH.source_url(record), "https://codeload.github.com/example/project/tar.gz/" + "a" * 40)
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual((self.output / "fixture/GL/header.h").read_bytes(), b"header")
def test_import_patch_reproduces_final_hashes(self):
archive = self.archive([("fixture-1/a.txt", b"before\n", 0o644, "file")])
record = self.record(archive, {"a.txt": fingerprint(b"before\n")})
record["vendoredFileSHA256"] = {"a.txt": fingerprint(b"after\n")}
record["patches"] = ["change.patch"]
(self.metadata / "change.patch").write_text("--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-before\n+after\n")
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual((self.output / "fixture/a.txt").read_bytes(), b"after\n")
def test_existing_modified_tree_is_not_replaced(self):
archive = self.ordinary_archive()
record = self.record(archive)
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
changed = self.output / "fixture/COPYING"
changed.write_bytes(b"local edit")
with self.assertRaisesRegex(ValueError, "differs from reviewed import"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(changed.read_bytes(), b"local edit")
def test_linked_source_destination_and_archive_cache_are_refused(self):
archive = self.ordinary_archive()
record = self.record(archive)
target = self.root / "outside"
target.mkdir()
self.output.symlink_to(target, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(list(target.iterdir()), [])
linked_cache = self.root / "linked-cache"
linked_cache.symlink_to(self.cache, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.obtain_archive("fixture", record, linked_cache, offline=True)
def test_linked_ancestor_output_and_cache_never_write_through(self):
archive = self.ordinary_archive()
record = self.record(archive)
outside = self.root / "outside"
outside.mkdir()
sentinel = outside / "keep.txt"
sentinel.write_bytes(b"preserved")
linked_parent = self.root / "link"
linked_parent.symlink_to(outside, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.materialize("fixture", record, self.metadata,
linked_parent / "new-third-party", self.cache, offline=True)
self.assertEqual(sentinel.read_bytes(), b"preserved")
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.obtain_archive("fixture", record, linked_parent / "new-cache", offline=True)
self.assertEqual(sentinel.read_bytes(), b"preserved")
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
def test_only_canonical_macos_aliases_are_allowed_in_ancestry(self):
# These public spellings must remain usable by Xcode on macOS. A
# similarly named user link is still covered by the preceding test.
import os
if os.uname().sysname != "Darwin":
self.skipTest("macOS public aliases are platform-specific")
for public, canonical in [(Path("/tmp"), Path("/private/tmp")),
(Path("/var"), Path("/private/var"))]:
if public.is_symlink() and public.resolve() == canonical:
FETCH.require_unlinked_ancestors(public / "uncreated-dependency-cache", "Fixture")
def test_verify_only_missing_tree_does_not_create_output_or_fetch(self):
archive = self.ordinary_archive()
record = self.record(archive)
with self.assertRaisesRegex(ValueError, "has not been fetched"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache,
offline=True, verify_only=True)
self.assertFalse(self.output.exists())
def test_supplemental_metadata_merges_without_replacing_import_records(self):
archive = self.ordinary_archive()
record = self.record(archive)
normal = self.root / "third_party/manifests"
supplement = self.root / "tools/dependencies/manifests"
normal.mkdir(parents=True)
supplement.mkdir(parents=True)
first = normal / "fixture.json"
first.write_text(json.dumps(record))
other = dict(record, directory="third_party/legacy")
second = supplement / "legacy.json"
second.write_text(json.dumps(other))
records, locations = FETCH.load_records(self.root)
self.assertEqual(set(records), {"fixture", "legacy"})
self.assertEqual(records["fixture"], record)
self.assertEqual(locations, {"fixture": normal, "legacy": supplement})
(supplement / "fixture.json").write_text(json.dumps(other))
with self.assertRaisesRegex(ValueError, "Duplicate"):
FETCH.load_records(self.root)
self.assertEqual(json.loads(first.read_text()), record)
def test_overlapping_source_roots_and_linked_metadata_are_refused(self):
archive = self.ordinary_archive()
record = self.record(archive)
normal = self.root / "third_party/manifests"
normal.mkdir(parents=True)
(normal / "fixture.json").write_text(json.dumps(record))
other = normal / "other.json"
other.write_text(json.dumps(dict(record, directory="third_party/fixture/nested")))
with self.assertRaisesRegex(ValueError, "Overlapping"):
FETCH.load_records(self.root)
other.unlink()
other.symlink_to(normal / "fixture.json")
with self.assertRaisesRegex(ValueError, "ordinary file"):
FETCH.load_records(self.root)
def test_release_hash_or_full_immutable_revision_is_required(self):
archive = self.ordinary_archive()
record = self.record(archive)
record.pop("archiveSHA256")
with self.assertRaisesRegex(ValueError, "require a recorded"):
FETCH.source_url(record)
for revision in ("main", "a" * 12, "a" * 39, "a" * 41):
with self.subTest(revision=revision):
candidate = dict(record, sourceURL="https://github.com/example/project", sourceRevision=revision)
with self.assertRaisesRegex(ValueError, "pinned Git"):
FETCH.source_url(candidate)
def test_unapproved_import_patch_target_leaves_no_destination(self):
archive = self.ordinary_archive()
record = self.record(archive)
record["patches"] = ["escape.patch"]
(self.metadata / "escape.patch").write_text("--- a/../escape\n+++ b/../escape\n@@ -1 +1 @@\n-before\n+after\n")
with self.assertRaises(ValueError):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
if __name__ == "__main__":
unittest.main()