Publish VCC Plus OSS source distribution for 3.6.2 build 60

This commit is contained in:
Brian Holman committed 2026-11-13 00:14:03 -07:00
commit c652a5276d
1192 files changed
+272368

No files matched your search

+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Explicit app build-content selection and schema-only resource staging.
Normal is the existing maintainer/developer build. Source builds deliberately
omit prepared libraries; they never need to probe a maintainer input location.
"""
from contextlib import closing
import hashlib
import os
from pathlib import Path
import sqlite3
import tempfile
PROFILES = ('normal', 'source')
PROFILE_ENV = 'VCC_BUILD_CONTENT_PROFILE'
def content_profile(value=None, environ=None):
"""CLI values override the environment; an unset profile preserves defaults."""
selected = value if value is not None else (os.environ if environ is None else environ).get(PROFILE_ENV, 'normal')
if selected not in PROFILES:
raise ValueError('Build content profile must be normal or source')
return selected
def add_content_profile_argument(parser):
parser.add_argument('--content-profile', choices=PROFILES,
help='App content: normal (default) or source (empty libraries); overrides ' + PROFILE_ENV)
def _check_output(output, expected_name, expected_parent):
# Preserve macOS's public /tmp and /var spellings without resolving arbitrary
# links supplied inside a build directory. No other ancestor link is allowed.
aliases = {Path('/tmp'): Path('/private/tmp'), Path('/var'): Path('/private/var')}
absolute = output if output.is_absolute() else Path.cwd() / output
for parent in absolute.parents:
if parent.is_symlink() and not (parent in aliases and parent.resolve() == aliases[parent]):
raise ValueError('Build resource destination must not have a linked ancestor directory')
if output.name != expected_name or output.parent.name != expected_parent:
raise ValueError('Expected a ' + expected_parent + '/' + expected_name + ' build resource destination')
if output.is_symlink() or (output.exists() and (not output.is_file() or output.stat().st_nlink != 1)):
raise ValueError('Build resource destination must be a regular, unlinked file')
if any(Path(str(output) + suffix).exists() or Path(str(output) + suffix).is_symlink()
for suffix in ('-wal', '-shm', '-journal')):
raise ValueError('Remove SQLite transaction sidecars from this build resource before regenerating it')
def atomic_empty_database(output, schema, metadata, content_tables, *, expected_name,
expected_parent, application_id=None, user_version, fts_integrity=False):
"""Validate a new schema-only database before replacing a disposable resource.
No source database is accepted. Existing identical resource bytes retain their
timestamp. Failed schema/integrity checks leave the previous resource intact.
"""
output = Path(output)
_check_output(output, expected_name, expected_parent)
output.parent.mkdir(parents=True, exist_ok=True)
descriptor, name = tempfile.mkstemp(prefix='.empty-library-', suffix='.sqlite', dir=output.parent)
os.close(descriptor)
temporary = Path(name)
try:
with closing(sqlite3.connect(temporary)) as db:
db.execute('PRAGMA page_size=4096')
db.execute('PRAGMA foreign_keys=ON')
db.executescript(Path(schema).read_text(encoding='utf-8'))
db.executemany('INSERT INTO metadata VALUES(?,?)', metadata)
if fts_integrity:
db.execute("INSERT INTO page_terms(page_terms) VALUES('integrity-check')")
db.commit()
db.execute('VACUUM')
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
raise ValueError('Empty library integrity check failed')
if db.execute('PRAGMA foreign_key_check').fetchall():
raise ValueError('Empty library has broken relationships')
if db.execute('PRAGMA user_version').fetchone() != (user_version,):
raise ValueError('Unsupported empty library schema version')
if application_id is not None and db.execute('PRAGMA application_id').fetchone() != (application_id,):
raise ValueError('Unsupported empty library application identity')
for table in content_tables:
# Table names are authored constants, never command-line values.
if db.execute('SELECT count(*) FROM "' + table + '"').fetchone() != (0,):
raise ValueError('Library schema unexpectedly inserts content')
data = temporary.read_bytes()
result = dict(bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
os.chmod(temporary, 0o444)
_check_output(output, expected_name, expected_parent)
if output.is_file() and output.read_bytes() == data:
# chmod only a verified regular file; never follow a linked output.
os.chmod(output, 0o444)
else:
os.replace(temporary, output)
return result
finally:
temporary.unlink(missing_ok=True)
+44
View File
@@ -0,0 +1,44 @@
-- SPDX-License-Identifier: GPL-3.0-or-later
-- Review/export schema; contains no local cache paths or emulator code.
PRAGMA foreign_keys=ON;
CREATE TABLE metadata(key TEXT PRIMARY KEY,value TEXT NOT NULL);
CREATE TABLE title(id TEXT PRIMARY KEY,name TEXT NOT NULL,identity_status TEXT NOT NULL);
CREATE TABLE package(sha256 TEXT PRIMARY KEY,byte_size INTEGER NOT NULL CHECK(byte_size>=0));
CREATE TABLE release(
id TEXT PRIMARY KEY,title_id TEXT NOT NULL REFERENCES title(id),name TEXT NOT NULL,
category TEXT NOT NULL,media_type TEXT NOT NULL,content_role TEXT NOT NULL,
source_url TEXT NOT NULL UNIQUE,modified_at TEXT,modified_precision TEXT,modified_timezone TEXT,
package_sha256 TEXT NOT NULL REFERENCES package(sha256),language_status TEXT NOT NULL,
metadata_json TEXT NOT NULL CHECK(json_valid(metadata_json)));
CREATE TABLE member(
release_id TEXT NOT NULL REFERENCES release(id),path TEXT NOT NULL,sha256 TEXT NOT NULL,
byte_size INTEGER NOT NULL,kind TEXT NOT NULL,role TEXT NOT NULL,
PRIMARY KEY(release_id,path));
CREATE TABLE download_location(
package_sha256 TEXT NOT NULL REFERENCES package(sha256),url TEXT NOT NULL,priority INTEGER NOT NULL,
PRIMARY KEY(package_sha256,url));
CREATE TABLE asset(id TEXT PRIMARY KEY,name TEXT NOT NULL,kind TEXT NOT NULL,source_url TEXT NOT NULL,
sha256 TEXT,byte_size INTEGER,credit TEXT NOT NULL);
CREATE TABLE asset_delivery(
asset_id TEXT PRIMARY KEY REFERENCES asset(id),manual_url TEXT,thumbnail_url TEXT,
thumbnail_sha256 TEXT,thumbnail_byte_size INTEGER,priority INTEGER NOT NULL);
CREATE TABLE release_asset(
release_id TEXT NOT NULL REFERENCES release(id),asset_id TEXT NOT NULL REFERENCES asset(id),
status TEXT NOT NULL,reason TEXT NOT NULL,PRIMARY KEY(release_id,asset_id));
CREATE TABLE setup(
release_id TEXT PRIMARY KEY REFERENCES release(id),status TEXT NOT NULL,
configuration_json TEXT NOT NULL CHECK(json_valid(configuration_json)),
selected_members_json TEXT NOT NULL CHECK(json_valid(selected_members_json)),
command TEXT,instructions TEXT NOT NULL,reasons_json TEXT NOT NULL CHECK(json_valid(reasons_json)),
validation_json TEXT NOT NULL CHECK(json_valid(validation_json)));
CREATE TABLE source_hint(
id TEXT PRIMARY KEY,release_id TEXT NOT NULL REFERENCES release(id),page_url TEXT NOT NULL,
page_sha256 TEXT NOT NULL,source_name TEXT NOT NULL,raw_json TEXT NOT NULL CHECK(json_valid(raw_json)),
resolution TEXT NOT NULL);
CREATE INDEX release_title ON release(title_id);
CREATE INDEX release_category ON release(category);
CREATE INDEX setup_status ON setup(status);
CREATE VIEW browse AS SELECT r.*,t.name AS title,s.status,s.command,s.instructions,
s.configuration_json,s.selected_members_json,s.reasons_json,s.validation_json
FROM release r JOIN title t ON t.id=r.title_id JOIN setup s ON s.release_id=r.id;
PRAGMA user_version=1;
+282
View File
@@ -0,0 +1,282 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Stage the reviewed software catalog as a read-only app resource, never media.
Ordinary source builds get an empty schema if the private catalog is absent.
Release packaging passes --require-full and fails rather than shipping it empty.
The explicit source content profile generates an empty schema without reading
maintainer inputs; normal remains the default.
"""
import argparse
from contextlib import closing
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import tempfile
import struct
import subprocess
import zipfile
import sys
from urllib.parse import urlsplit
from concurrent.futures import ThreadPoolExecutor
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tools'))
from build_content import add_content_profile_argument, atomic_empty_database, content_profile as resolve_content_profile
SCHEMA = Path(__file__).with_name('app-schema.sql')
DEFAULT_SOURCE = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite'
DEFAULT_ARTWORK_CACHE = ROOT / 'media/software-library-full/cache'
MAX_BYTES = 256 * 1024 * 1024
PRIVATE = re.compile(r'(?:file://|/Users/|/home/|/private/|(?<![A-Za-z])[A-Za-z]:[\\/]|(?:^|[\s"\'])\.build/|software-library-full/)', re.I)
def schema_signature(db):
return {(kind, name): ' '.join(sql.split()) for kind, name, sql in db.execute(
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'") if sql}
def validate(path, require_full=False):
"""Validate a standalone finalized export without allowing writes or extensions."""
path = Path(path)
if path.is_symlink() or not path.is_file():
raise ValueError('Catalog input must be a regular, nonsymlink file')
if not 0 < path.stat().st_size <= MAX_BYTES:
raise ValueError('Catalog input has an invalid size')
for suffix in ('-wal', '-journal'):
sidecar = Path(str(path) + suffix)
if sidecar.exists() and sidecar.stat().st_size:
raise ValueError('Catalog has an active transaction sidecar; finalize it before packaging')
with closing(sqlite3.connect(path.resolve().as_uri() + '?mode=ro', uri=True)) as db:
db.execute('PRAGMA query_only=ON')
db.execute('PRAGMA trusted_schema=OFF')
if db.execute('PRAGMA user_version').fetchone()[0] != 1:
raise ValueError('Unsupported catalog schema version')
with closing(sqlite3.connect(':memory:')) as expected:
expected.executescript(SCHEMA.read_text())
if schema_signature(db) != schema_signature(expected):
raise ValueError('Catalog does not match the approved app export schema')
# Older macOS SQLite marks json_valid() unsafe with trusted_schema=OFF.
# The exact schema was just compared to our trusted, checked-in schema.
db.execute('PRAGMA trusted_schema=ON')
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
raise ValueError('Catalog integrity check failed')
if db.execute('PRAGMA foreign_key_check').fetchall():
raise ValueError('Catalog contains broken relationships')
metadata = dict(db.execute('SELECT key,value FROM metadata'))
if metadata.get('schema_version') != '1' or metadata.get('recipe_contract') != '2':
raise ValueError('Unsupported catalog metadata or loading contract')
releases = db.execute('SELECT count(*) FROM release').fetchone()[0]
if require_full and releases == 0:
raise ValueError('A populated catalog is required for release packaging')
if db.execute('SELECT count(*) FROM release LEFT JOIN setup ON release.id=setup.release_id WHERE setup.release_id IS NULL').fetchone()[0]:
raise ValueError('Catalog release is missing loading metadata')
for table, in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"):
for row in db.execute('SELECT * FROM "' + table + '"'):
for value in row:
if isinstance(value, bytes):
raise ValueError('Catalog must not contain bundled media blobs')
if isinstance(value, str) and PRIVATE.search(value):
raise ValueError('Catalog contains a private filesystem reference')
return {'titles': db.execute('SELECT count(*) FROM title').fetchone()[0],
'releases': releases, 'assets': db.execute('SELECT count(*) FROM asset').fetchone()[0]}
def empty_catalog(path):
with closing(sqlite3.connect(path)) as db:
db.executescript(SCHEMA.read_text())
db.executemany('INSERT INTO metadata VALUES(?,?)', [
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-development'),
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')])
db.commit()
def source_catalog(output):
result = atomic_empty_database(output, SCHEMA, (
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-source'),
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')),
('title', 'package', 'release', 'member', 'download_location', 'asset', 'asset_delivery',
'release_asset', 'setup', 'source_hint'), expected_name='catalog.sqlite',
expected_parent='SoftwareLibrary', user_version=1)
return dict(result, schema_version=1, recipe_contract=2, stage='empty-source', titles=0,
releases=0, assets=0, thumbnails=0, empty=True)
def package(source, output, require_full=False, *, content_profile='normal'):
if resolve_content_profile(content_profile) == 'source':
if require_full:
raise ValueError('--require-full cannot be combined with the source content profile')
return source_catalog(output)
source, output = Path(source), Path(output)
if output.is_symlink():
raise ValueError('Catalog destination must not be a symlink')
if source.resolve() == output.resolve():
raise ValueError('Catalog output must differ from the input')
output.parent.mkdir(parents=True, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix='.catalog-', suffix='.sqlite', dir=output.parent)
os.close(fd)
temporary = Path(temporary)
try:
if source.exists() or source.is_symlink():
validate(source, require_full=require_full)
shutil.copyfile(source, temporary)
elif require_full:
raise ValueError('Full catalog is missing. Prepare media/software-library-full/prepared/app-catalog.sqlite before packaging.')
else:
print('warning: Full Software Library catalog is absent; bundling an empty development catalog. See tools/catalog/README.md.')
empty_catalog(temporary)
counts = validate(temporary, require_full=require_full)
digest = hashlib.sha256(temporary.read_bytes()).hexdigest()
os.chmod(temporary, 0o444)
# Avoid touching a resource that is byte-identical across incremental builds.
if output.is_file() and hashlib.sha256(output.read_bytes()).hexdigest() == digest:
os.chmod(output, 0o444)
else:
os.replace(temporary, output)
return dict(counts, sha256=digest, empty=counts['releases'] == 0)
finally:
temporary.unlink(missing_ok=True)
def artwork_key(url, sha):
# Keep in sync with SoftwareLibraryArtworkCache.cacheKey(for:).
return hashlib.sha256(('thumbnail-v1-640\n' + url + '\n' + (sha or 'unverified')).encode()).hexdigest()
def png_thumbnail(path):
data = path.read_bytes()
if not 24 <= len(data) <= 2 * 1024 * 1024 or data[:8] != b'\x89PNG\r\n\x1a\n' or data[12:16] != b'IHDR':
return False
width, height = struct.unpack('>II', data[16:24])
return 0 < width <= 640 and 0 < height <= 640
def screenshot_member(archive):
"""Read one bounded image, never extract archive-supplied paths to disk."""
with zipfile.ZipFile(archive) as images:
members = images.infolist()
if len(members) > 256:
raise ValueError('Artwork archive has too many members')
candidates = [m for m in members if not m.is_dir()
and Path(m.filename).suffix.lower() in ('.png', '.jpg', '.jpeg', '.gif', '.webp')
and not any(part.startswith('.') or part == '__MACOSX' for part in Path(m.filename).parts)
and not (m.flag_bits & 1)
and 0 < m.file_size <= 8 * 1024 * 1024
and (m.external_attr >> 16) & 0o170000 != 0o120000]
def rank(member):
stem = Path(member.filename).stem.lower()
preferred = {'cover': 0, 'title': 1, 'intro': 2}
return (preferred.get(stem, 3), member.filename.casefold(), member.filename)
if not candidates:
return None
member = min(candidates, key=rank)
with images.open(member) as stream:
data = stream.read(8 * 1024 * 1024 + 1)
if len(data) != member.file_size or len(data) > 8 * 1024 * 1024:
raise ValueError('Artwork archive image exceeds its declared size')
return data
def package_artwork(catalog, cache, destination):
"""Generate maintainer thumbnails from previously verified local art.
The content-addressed crawl cache is optional for a developer checkout.
No network requests, original artwork or documents are copied into the app.
"""
cache, destination = Path(cache), Path(destination)
if destination.is_symlink():
raise ValueError('Artwork destination must not be a symlink')
with closing(sqlite3.connect(Path(catalog).resolve().as_uri() + '?mode=ro', uri=True)) as db:
rows = db.execute("SELECT DISTINCT a.source_url,a.sha256,a.byte_size FROM asset a JOIN release_asset ra ON ra.asset_id=a.id WHERE a.kind IN ('replacement-label','screenshot') ORDER BY a.id").fetchall()
destination.mkdir(parents=True, exist_ok=True)
def convert(row):
url, sha, size = row
parts = urlsplit(url)
suffix = Path(parts.path).suffix.lower()
if (parts.scheme != 'https' or parts.hostname != 'colorcomputerarchive.com' or not parts.path.startswith('/repo/')
or suffix not in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.zip') or not sha or not re.fullmatch('[a-f0-9]{64}', sha)):
return None
source = cache / (sha + suffix)
if not source.is_file() or source.is_symlink() or not 0 < source.stat().st_size <= 8 * 1024 * 1024:
return None
original = source.read_bytes()
if len(original) != size or hashlib.sha256(original).hexdigest() != sha:
raise ValueError('Cached artwork does not match its catalog fingerprint')
name = artwork_key(url, sha) + '.png'
target = destination / name
# A generated output is already bounded; its input identity is in its name.
if target.is_file() and not target.is_symlink() and png_thumbnail(target):
return name
# ZIP assets retain their original URL/fingerprint and attribution. Only
# a single selected image becomes a thumbnail; no archive is shipped.
image_data = screenshot_member(source) if suffix == '.zip' else None
if suffix == '.zip' and image_data is None:
return None
fd, temporary = tempfile.mkstemp(prefix='.artwork-', suffix='.png', dir=destination)
os.close(fd)
temporary = Path(temporary)
try:
with tempfile.TemporaryDirectory(prefix='vcc-artwork-') as scratch:
input_path = source
if image_data is not None:
input_path = Path(scratch) / 'image'
input_path.write_bytes(image_data)
subprocess.run(['/usr/bin/sips', '-s', 'format', 'png', '-Z', '640', str(input_path), '--out', str(temporary)],
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
if not png_thumbnail(temporary):
raise ValueError('Generated artwork thumbnail is not a bounded PNG')
os.chmod(temporary, 0o444)
os.replace(temporary, target)
finally:
temporary.unlink(missing_ok=True)
return name
with ThreadPoolExecutor(max_workers=4) as workers:
names = {name for name in workers.map(convert, rows) if name}
# Clear only our generated PNG namespace after a successful complete pass.
for path in destination.iterdir():
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and path.name not in names and not path.is_symlink():
path.unlink()
return len(names)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=Path)
add_content_profile_argument(parser)
parser.add_argument('--output', type=Path)
parser.add_argument('--check-only', action='store_true', help='Validate input without staging a resource')
parser.add_argument('--require-full', action='store_true')
args = parser.parse_args()
if not args.check_only and args.output is None:
parser.error('--output is required unless --check-only is used')
try:
profile = resolve_content_profile(args.content_profile)
if profile == 'source':
if args.require_full:
raise ValueError('--require-full cannot be combined with the source content profile')
if args.check_only:
with tempfile.TemporaryDirectory(prefix='vcc-source-catalog-check-') as temporary:
result = source_catalog(Path(temporary) / 'SoftwareLibrary/catalog.sqlite')
else:
result = source_catalog(args.output)
else:
source = args.source if args.source is not None else DEFAULT_SOURCE
result = validate(source, args.require_full) if args.check_only else package(source, args.output, args.require_full)
if not args.check_only:
artwork = args.output.parent / 'Artwork'
if artwork.is_symlink(): raise ValueError('Artwork destination must not be a symlink')
if artwork.is_dir():
for path in artwork.iterdir():
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and not path.is_symlink(): path.unlink()
result['thumbnails'] = 0
except (ValueError, OSError, sqlite3.DatabaseError, subprocess.CalledProcessError) as error:
parser.exit(1, 'error: Software Library catalog: ' + str(error) + '\n')
print('Software Library catalog: ' + json.dumps(result, sort_keys=True))
if __name__ == '__main__':
main()
+368
View File
@@ -0,0 +1,368 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Fetch and verify pinned source dependencies without executing their code.
Source archives are retained in ignored storage. Only manifest-selected regular
files are materialized, excluded payloads are never extracted, and import patches
must reproduce the exact reviewed file hashes. No Git repository is cloned.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import stat
import subprocess
import tarfile
import tempfile
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
import zipfile
ROOT = Path(__file__).resolve().parents[2]
MAX_ARCHIVE_BYTES = 128 * 1024 * 1024
MAX_FILE_BYTES = 64 * 1024 * 1024
MAX_EXPANDED_BYTES = 1024 * 1024 * 1024
MAX_MEMBERS = 100_000
HASH = re.compile(r"^[0-9a-f]{64}$")
def sha(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def relative(name):
if not isinstance(name, str) or not name or "\\" in name or "\x00" in name:
raise ValueError("Invalid dependency path: " + repr(name))
path = PurePosixPath(name)
if path.is_absolute() or ".." in path.parts or ".git" in path.parts or re.match(r"^[A-Za-z]:", name):
raise ValueError("Unsafe dependency path: " + name)
value = path.as_posix()
if value == ".":
return ""
return value
def validate_record(record):
if 'optional' in record and not isinstance(record['optional'], bool):
raise ValueError('Dependency optional flag must be boolean')
directory = relative(record["directory"])
if not directory.startswith("third_party/") or directory == "third_party/manifests":
raise ValueError("Dependency directory must be beneath third_party")
suffix = directory.removeprefix("third_party/")
if suffix.startswith("manifests/"):
raise ValueError("Dependency cannot overwrite metadata")
for key in ("upstreamFileSHA256", "vendoredFileSHA256"):
if not isinstance(record[key], dict) or not record[key]:
raise ValueError("Dependency requires nonempty file hash maps")
for name, digest in record[key].items():
if relative(name) != name or (not isinstance(digest, str) or not HASH.fullmatch(digest)):
raise ValueError("Invalid file fingerprint: " + name)
if set(record["upstreamFileSHA256"]) != set(record["vendoredFileSHA256"]):
raise ValueError("Import patches may change bytes, not the approved file inventory")
if record.get("archiveSHA256") is not None and (not isinstance(record["archiveSHA256"], str) or not HASH.fullmatch(record["archiveSHA256"])):
raise ValueError("Invalid archive fingerprint")
for excluded in record.get("excludedFiles", []):
excluded = relative(excluded).rstrip("/")
if any(p == excluded or p.startswith(excluded + "/") for p in record["upstreamFileSHA256"]):
raise ValueError("Selected dependency file overlaps an exclusion: " + excluded)
if record.get("archiveRootPrefix"):
if not relative(record["archiveRootPrefix"].rstrip("/")):
raise ValueError("Archive root prefix cannot be empty")
return suffix
def source_url(record):
"""Use a release URL or a full immutable GitHub revision archive."""
url = record["sourceURL"]
if record.get("sourceRevision"):
revision = record["sourceRevision"]
parts = urlsplit(url)
fields = parts.path.strip("/").removesuffix(".git").split("/")
if parts.scheme != "https" or parts.hostname != "github.com" or len(fields) != 2 or not re.fullmatch(r"[0-9a-f]{40}", revision):
raise ValueError("Unsupported pinned Git source")
return f"https://codeload.github.com/{fields[0]}/{fields[1]}/tar.gz/{revision}"
if not record.get("archiveSHA256"):
raise ValueError("Release archives require a recorded SHA-256")
if urlsplit(url).scheme not in ("http", "https"):
raise ValueError("Dependency source must use HTTP or HTTPS")
return url
class SafeRedirect(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
if urlsplit(newurl).scheme not in ("http", "https"):
raise ValueError("Unsupported dependency redirect")
return super().redirect_request(req, fp, code, msg, headers, newurl)
def archive_names(name, record):
filename = Path(urlsplit(record["sourceURL"]).path).name
if record.get("sourceRevision"):
filename = name + "-" + record["sourceRevision"] + ".tar.gz"
if not filename or filename in (".", ".."):
raise ValueError("Invalid source archive filename")
digest = record.get("archiveSHA256")
return ([digest + "-" + filename, filename, digest + ".tar.gz"] if digest else [filename])
def verify_archive(path, record):
if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_ARCHIVE_BYTES:
raise ValueError("Source archive must be a bounded regular file: " + str(path))
digest = sha(path)
if record.get("archiveSHA256") and digest != record["archiveSHA256"]:
raise ValueError("Source archive SHA-256 mismatch: " + path.name)
return digest
def require_unlinked_ancestors(path, description):
"""Reject redirected destinations, retaining only macOS's public aliases.
Do not resolve an arbitrary user path before examining its full ancestry:
doing so would erase precisely the linked parent this check must detect.
"""
aliases = {Path("/tmp"): Path("/private/tmp"), Path("/var"): Path("/private/var")}
absolute = Path(path).absolute()
for candidate in (absolute, *absolute.parents):
if candidate.is_symlink() and not (
candidate in aliases and candidate.resolve() == aliases[candidate]):
raise ValueError(description + " cannot traverse a symbolic link")
def obtain_archive(name, record, cache, offline=False, timeout=30):
require_unlinked_ancestors(cache, "Archive cache")
cache.mkdir(parents=True, exist_ok=True)
names = archive_names(name, record)
for filename in names:
cached = cache / filename
if cached.exists() or cached.is_symlink():
verify_archive(cached, record)
return cached
if offline:
raise ValueError("Offline source archive missing: " + names[0])
url = source_url(record)
request = Request(url, headers={"User-Agent": "VCCPlus-source-fetch/1"})
target = cache / names[0]
temporary = None
try:
with tempfile.NamedTemporaryFile(prefix=".download-", dir=cache, delete=False) as output:
temporary = Path(output.name)
with build_opener(SafeRedirect()).open(request, timeout=timeout) as response:
if urlsplit(response.geturl()).scheme not in ("http", "https"):
raise ValueError("Unsupported dependency response URL")
size = 0
while chunk := response.read(1024 * 1024):
size += len(chunk)
if size > MAX_ARCHIVE_BYTES:
raise ValueError("Dependency archive exceeds download limit")
output.write(chunk)
verify_archive(temporary, record)
require_unlinked_ancestors(cache, "Archive cache")
if target.exists() or target.is_symlink():
raise ValueError("Archive cache target changed during download")
os.replace(temporary, target)
return target
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
def extract_selected(archive, destination, expected, archive_prefix=""):
"""Stream only known regular members; never use extract/extractall."""
observed = set()
all_names = set()
prefix = None
expanded = 0
count = 0
def consume(name, size, mode, regular, opener):
nonlocal prefix, expanded, count
count += 1
if count > MAX_MEMBERS:
raise ValueError("Source archive has too many members")
if not 0 <= size <= MAX_FILE_BYTES:
raise ValueError("Source archive member exceeds size limit: " + name)
expanded += size
if expanded > MAX_EXPANDED_BYTES:
raise ValueError("Source archive exceeds expansion limit")
normalized = relative(name)
if normalized in all_names:
raise ValueError("Duplicate source archive member: " + normalized)
all_names.add(normalized)
parts = normalized.split("/")
candidates = [(normalized, "")]
if len(parts) > 1:
candidates.append(("/".join(parts[1:]), parts[0] + "/"))
archive_root = archive_prefix.rstrip("/") + "/" if archive_prefix else ""
matches = [(path[len(archive_root):], root) for path, root in candidates
if path.startswith(archive_root) and path[len(archive_root):] in expected]
if not matches:
return
if len(matches) != 1:
raise ValueError("Ambiguous source archive member: " + normalized)
path, root = matches[0]
if prefix is not None and root != prefix:
raise ValueError("Source archive has inconsistent roots")
prefix = root
if path in observed or not regular:
raise ValueError("Selected member is duplicate, linked or not a regular file: " + normalized)
with opener() as stream:
data = stream.read(MAX_FILE_BYTES + 1)
if len(data) != size or hashlib.sha256(data).hexdigest() != expected[path]:
raise ValueError("Upstream source file SHA-256 mismatch: " + path)
target = destination / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
target.chmod(0o755 if mode & 0o111 else 0o644)
observed.add(path)
if zipfile.is_zipfile(archive):
with zipfile.ZipFile(archive) as source:
for member in source.infolist():
mode = member.external_attr >> 16
regular = not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG) and not (member.flag_bits & 1)
consume(member.filename, member.file_size, mode, regular, lambda member=member: source.open(member))
else:
with tarfile.open(archive, mode="r|*") as source:
for member in source:
consume(member.name, member.size, member.mode, member.isfile(), lambda member=member: source.extractfile(member))
if observed != set(expected):
raise ValueError("Source archive is missing selected files: " + ", ".join(sorted(set(expected) - observed)[:12]))
def verify_tree(root, expected):
if root.is_symlink() or not root.is_dir():
raise ValueError("Dependency source must be an ordinary directory")
actual = {}
for path in root.rglob("*"):
if path.is_symlink():
raise ValueError("Dependency contains a symbolic link: " + str(path))
if path.is_file():
actual[path.relative_to(root).as_posix()] = sha(path)
elif not path.is_dir():
raise ValueError("Dependency contains a special file")
if actual != expected:
changed = sorted(p for p in actual.keys() | expected.keys() if actual.get(p) != expected.get(p))
raise ValueError("Dependency source differs from reviewed import: " + ", ".join(changed[:12]))
def apply_patches(root, record, metadata):
allowed = set(record["upstreamFileSHA256"])
for filename in record.get("patches", []):
if relative(filename) != filename or "/" in filename:
raise ValueError("Import patch must be a metadata filename")
path = metadata / filename
if path.is_symlink() or not path.is_file():
raise ValueError("Missing import patch: " + filename)
text = path.read_text()
targets = []
for line in text.splitlines():
if line.startswith(("--- ", "+++ ")):
name = line[4:].split("\t", 1)[0]
if name[:2] not in ("a/", "b/") or relative(name[2:]) not in allowed:
raise ValueError("Import patch touches an unapproved path")
targets.append(name[2:])
if not targets:
raise ValueError("Import patch contains no unified file headers")
subprocess.run(["/usr/bin/patch", "-u", "-t", "-F", "0", "-p1", "-i", str(path.resolve())],
cwd=root, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
# A byte-perfect patch may still leave editor backups on some patch versions.
# Do not remove them silently; verify_tree fails on any extra output.
def load_records(project_root=ROOT):
"""Read ordinary import records plus supplemental pinned-source metadata.
Supplemental records fill provenance gaps only; they cannot replace or
overlap the existing reviewed dependency inventory.
"""
ordinary = project_root / "third_party/manifests"
supplemental = project_root / "tools/dependencies/manifests"
records, locations = {}, {}
for metadata in (ordinary, supplemental):
if metadata.is_symlink():
raise ValueError("Dependency metadata cannot be a symbolic link")
for path in sorted(metadata.glob("*.json")):
if path.is_symlink() or not path.is_file():
raise ValueError("Dependency manifest must be an ordinary file")
name = path.stem
if name in records:
raise ValueError("Duplicate dependency manifest: " + name)
record = json.loads(path.read_text())
suffix = validate_record(record)
for existing in records.values():
other = validate_record(existing)
if suffix == other or suffix.startswith(other + "/") or other.startswith(suffix + "/"):
raise ValueError("Overlapping dependency source directories")
source_url(record) # Reject unpinned/unsupported sources before any writes.
records[name], locations[name] = record, metadata
return records, locations
def materialize(name, record, metadata, output, cache, offline=False, verify_only=False):
suffix = validate_record(record)
destination = output / suffix
require_unlinked_ancestors(destination, "Dependency destination")
if destination.exists():
verify_tree(destination, record["vendoredFileSHA256"])
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "verified-existing"}
if verify_only:
raise ValueError("Dependency source has not been fetched: " + name)
archive = obtain_archive(name, record, cache, offline)
destination.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="." + name + "-", dir=destination.parent) as temporary:
staged = Path(temporary) / "source"
staged.mkdir()
extract_selected(archive, staged, record["upstreamFileSHA256"], record.get("archiveRootPrefix", ""))
apply_patches(staged, record, metadata)
verify_tree(staged, record["vendoredFileSHA256"])
require_unlinked_ancestors(destination, "Dependency destination")
if destination.exists() or destination.is_symlink():
raise ValueError("Dependency destination changed during preparation")
staged.rename(destination)
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "fetched-verified",
"archive_sha256": sha(archive)}
def default_selection(records):
"""Optional firmware preparation is explicit, never part of ordinary fetches."""
return [name for name, record in records.items() if not record.get('optional', False)]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--only", nargs="+", help="Manifest names, for example: --only toolshed")
parser.add_argument("--archive-cache", type=Path, default=ROOT / ".build/dependencies/downloads")
parser.add_argument("--output", type=Path, default=ROOT / "third_party", help="Ignored dependency source root")
parser.add_argument("--offline", action="store_true", help="Use only already cached original source archives")
parser.add_argument("--verify-only", action="store_true", help="Verify existing source trees without fetching")
args = parser.parse_args()
records, locations = load_records()
selected = args.only or default_selection(records)
unknown = set(selected) - records.keys()
if unknown:
parser.error("Unknown dependency manifest: " + ", ".join(sorted(unknown)))
if not records:
parser.error("No dependency manifests found")
output = args.output.absolute()
canonical_output = output.resolve()
metadata_roots = {ROOT / "third_party/manifests", ROOT / "tools/dependencies/manifests"}
if canonical_output == ROOT or canonical_output in ROOT.parents or canonical_output in metadata_roots:
parser.error("Choose a dependency source root separate from project metadata")
for name in dict.fromkeys(selected):
result = materialize(name, records[name], locations[name], output, args.archive_cache, args.offline, args.verify_only)
print(json.dumps(result, sort_keys=True), flush=True)
if __name__ == "__main__":
try:
main()
except (ValueError, OSError, tarfile.TarError, zipfile.BadZipFile, subprocess.CalledProcessError) as error:
raise SystemExit(str(error))
@@ -0,0 +1,56 @@
{
"archiveRootPrefix": "cocoroms/",
"directory": "third_party/cocoroms",
"excludedFiles": [
"bas13.rom",
"extbas11.rom",
"disk11.rom",
"coco3.rom"
],
"license": "File-specific Tandy, Microsoft and Microware notices retained in reconstructed firmware and assembly comments.",
"optional": true,
"patches": [],
"project": "toolshed-cocoroms",
"romOutputs": {
"bas13.rom": {
"bytes": 8192,
"sha256": "5375b1caef9df815b781d9cd6df81ecdaddd2642b06a01adac521e40f1329751"
},
"coco3.rom": {
"bytes": 32768,
"sha256": "ad358430568250eb8330a37f85361d9ce67e5ae6d5a28c17b31802cd3953a5d2"
},
"disk11.rom": {
"bytes": 8192,
"sha256": "1c40c5a74e72edaeecc2511fee1aab347eae2c3d5fc6778cb622e3df14d63157"
},
"extbas11.rom": {
"bytes": 8192,
"sha256": "a18f270d3495d82b9f515de049f46253a623a2d9b1bb226c9ad338508344c5ee"
}
},
"selection": "Assembly-only inputs, including the referenced authors-picture assembly data, for Color BASIC 1.3, Extended Color BASIC 1.1, Disk BASIC 1.1 and Super Color BASIC 2.0; no preassembled ROM or binary input.",
"sourceRevision": "3667a41c9605b7fcb150eace29cc7f42d460be1a",
"sourceURL": "https://github.com/nitros9project/toolshed",
"upstreamFileSHA256": {
"Makefile": "4a29262fc1ba0b6106887425f5b0a4edb8ab81fd343af548cb3a863bda3aea51",
"bas.asm": "ca2e0fba27a1c960f56648ab7be6571a89a4aff6c433b9a2649924e308e8bf24",
"coco3.asm": "01147f426baf1b48fc6d590f18ca5974bedc3dfac486020e694d9f4b890d74f2",
"disk.asm": "735443637ebaddc358f6a12856c495cc92e08f3aaa8bb68cf9a0494d7655faae",
"equates.asm": "099d4ae145ee5151aa1eb7e9805388d338b2613fd53e2ba666ab757734d94442",
"extbas.asm": "75ce20734d0272f636a961b318fb898c555061194da12d9bfc26782fc4b5e42b",
"md5sums.txt": "f0374ad4bdd32f9cd322cfa5eafe04a0b27d2e90698b536aa26c00d1ea7c8eab",
"mugateerspic.asm": "2878b1a9fbf7cc3bde25475f5e256f25910fdb44b1b2f8e6f01883bcf13548a1"
},
"vendoredFileSHA256": {
"Makefile": "4a29262fc1ba0b6106887425f5b0a4edb8ab81fd343af548cb3a863bda3aea51",
"bas.asm": "ca2e0fba27a1c960f56648ab7be6571a89a4aff6c433b9a2649924e308e8bf24",
"coco3.asm": "01147f426baf1b48fc6d590f18ca5974bedc3dfac486020e694d9f4b890d74f2",
"disk.asm": "735443637ebaddc358f6a12856c495cc92e08f3aaa8bb68cf9a0494d7655faae",
"equates.asm": "099d4ae145ee5151aa1eb7e9805388d338b2613fd53e2ba666ab757734d94442",
"extbas.asm": "75ce20734d0272f636a961b318fb898c555061194da12d9bfc26782fc4b5e42b",
"md5sums.txt": "f0374ad4bdd32f9cd322cfa5eafe04a0b27d2e90698b536aa26c00d1ea7c8eab",
"mugateerspic.asm": "2878b1a9fbf7cc3bde25475f5e256f25910fdb44b1b2f8e6f01883bcf13548a1"
},
"version": "2.6.1-3667a41"
}
@@ -0,0 +1,29 @@
{
"project": "legacy-graphics",
"version": "74c571ecf055",
"sourceURL": "https://github.com/VCCE/VCC",
"sourceRevision": "74c571ecf055e843b48e1e91fd8ae2303e5bca69",
"archiveRootPrefix": "3rdparty/",
"directory": "third_party/legacy_graphics",
"license": "Original file-specific Khronos/OpenGL/stb notices retained",
"excludedFiles": [],
"patches": [],
"upstreamFileSHA256": {
"GL/glcorearb.h": "8841702a2934583c3860a19600f55e85e741cb2c1e73cd512e681e7625f10ad4",
"GL/glext.h": "d66da97d4030d0a8f2180a9338e4378f63d8343f05b56da52064f67f9493a6a3",
"GL/glxext.h": "3ded5604e6d7643632a6f50133e15d52603d1cd95399a7781d0a49d153ade8e9",
"GL/wgl.h": "e95270f0e57101478833a23d5f89c38eea91befd323894498e3c3edb199a2805",
"GL/wglext.h": "b8f828a41bf545195268aedd6677cc621a5fd87054762eb535b4304de84c2bf2",
"KHR/khrplatform.h": "7b1e01aaa7ad8f6fc34b5c7bdf79ebf5189bb09e2c4d2e79fc5d350623d11e83",
"stb/stb_image_write.h": "cbd5f0ad7a9cf4468affb36354a1d2338034f2c12473cf1a8e32053cb6914a05"
},
"vendoredFileSHA256": {
"GL/glcorearb.h": "8841702a2934583c3860a19600f55e85e741cb2c1e73cd512e681e7625f10ad4",
"GL/glext.h": "d66da97d4030d0a8f2180a9338e4378f63d8343f05b56da52064f67f9493a6a3",
"GL/glxext.h": "3ded5604e6d7643632a6f50133e15d52603d1cd95399a7781d0a49d153ade8e9",
"GL/wgl.h": "e95270f0e57101478833a23d5f89c38eea91befd323894498e3c3edb199a2805",
"GL/wglext.h": "b8f828a41bf545195268aedd6677cc621a5fd87054762eb535b4304de84c2bf2",
"KHR/khrplatform.h": "7b1e01aaa7ad8f6fc34b5c7bdf79ebf5189bb09e2c4d2e79fc5d350623d11e83",
"stb/stb_image_write.h": "cbd5f0ad7a9cf4468affb36354a1d2338034f2c12473cf1a8e32053cb6914a05"
}
}
+272
View File
@@ -0,0 +1,272 @@
# SPDX-License-Identifier: GPL-3.0-or-later
"""Authored archive fixtures exercise source selection and filesystem boundaries."""
import hashlib
import gzip
import importlib.util
import io
import json
from pathlib import Path
import tarfile
import tempfile
import unittest
import zipfile
MODULE = Path(__file__).resolve().parents[1] / "fetch.py"
SPEC = importlib.util.spec_from_file_location("fetch_dependencies", MODULE)
FETCH = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(FETCH)
def fingerprint(data):
return hashlib.sha256(data).hexdigest()
class FetchDependenciesTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="vcc-source-fetch-test-")
self.root = Path(self.temp.name)
self.metadata = self.root / "metadata"
self.metadata.mkdir()
self.cache = self.root / "cache"
self.cache.mkdir()
self.output = self.root / "output"
def tearDown(self):
self.temp.cleanup()
def archive(self, entries, name="fixture-1.tar.gz"):
path = self.cache / name
with tarfile.open(path, "w:gz") as archive:
for filename, data, mode, kind in entries:
member = tarfile.TarInfo(filename)
member.mode = mode
if kind == "symlink":
member.type = tarfile.SYMTYPE
member.linkname = "/private/escaped"
archive.addfile(member)
else:
member.size = len(data)
archive.addfile(member, io.BytesIO(data))
return path
def record(self, archive, entries=None):
hashes = entries or {"configure": fingerprint(b"#!/bin/sh\nexit 0\n"), "COPYING": fingerprint(b"Fixture notice\n")}
return {"directory": "third_party/fixture", "sourceURL": "https://example.invalid/" + archive.name,
"archiveSHA256": FETCH.sha(archive), "upstreamFileSHA256": hashes,
"vendoredFileSHA256": hashes, "excludedFiles": ["secret.rom"], "patches": []}
def ordinary_archive(self):
return self.archive([("fixture-1/configure", b"#!/bin/sh\nexit 0\n", 0o755, "file"),
("fixture-1/COPYING", b"Fixture notice\n", 0o644, "file"),
("fixture-1/secret.rom", b"excluded fixture bytes", 0o644, "file")])
def test_only_manifest_files_materialized_with_safe_modes_and_idempotent_verification(self):
archive = self.ordinary_archive()
record = self.record(archive)
first = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(first["status"], "fetched-verified")
result = self.output / "fixture"
self.assertEqual(set(p.name for p in result.iterdir()), {"configure", "COPYING"})
self.assertEqual((result / "configure").stat().st_mode & 0o7777, 0o755)
self.assertEqual((result / "COPYING").stat().st_mode & 0o7777, 0o644)
second = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(second["status"], "verified-existing")
def test_corrupt_cached_archive_is_rejected_without_output(self):
archive = self.ordinary_archive()
record = self.record(archive)
archive.write_bytes(archive.read_bytes() + b"corrupt")
with self.assertRaisesRegex(ValueError, "archive SHA-256 mismatch"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
def test_wrong_selected_bytes_do_not_leave_partial_destination(self):
archive = self.ordinary_archive()
record = self.record(archive)
record["upstreamFileSHA256"] = dict(record["upstreamFileSHA256"], COPYING="0" * 64)
with self.assertRaisesRegex(ValueError, "source file SHA-256 mismatch"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
def test_traversal_members_are_refused_even_when_not_selected(self):
for filename in ("../escaped", "/absolute", "root/../../escaped", "C:/escape", "root\\escape"):
with self.subTest(filename=filename):
archive = self.archive([(filename, b"x", 0o644, "file")])
destination = self.root / "extract"
destination.mkdir(exist_ok=True)
with self.assertRaises(ValueError):
FETCH.extract_selected(archive, destination, {"safe": fingerprint(b"x")})
self.assertEqual(list(destination.iterdir()), [])
def test_selected_links_and_duplicates_are_refused(self):
for entries in ([('root/a', b"", 0o777, "symlink")],
[('root/a', b"x", 0o644, "file"), ('root/a', b"x", 0o644, "file")]):
archive = self.archive(entries)
with self.assertRaisesRegex(ValueError, "linked|Duplicate"):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_unselected_oversized_members_are_refused(self):
archive = self.cache / "oversized.tar.gz"
member = tarfile.TarInfo("root/unselected")
member.size = FETCH.MAX_FILE_BYTES + 1
# An authored header is enough: bounds reject before reading payload.
with gzip.open(archive, "wb") as source:
source.write(member.tobuf() + bytes(1024))
with self.assertRaisesRegex(ValueError, "size limit"):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_zip_traversal_and_unix_symlinks_refused(self):
for filename, mode in [("../escape", 0o100644), ("root/a", 0o120777)]:
archive = self.cache / "fixture.zip"
with zipfile.ZipFile(archive, "w") as target:
entry = zipfile.ZipInfo(filename)
entry.create_system = 3
entry.external_attr = mode << 16
target.writestr(entry, b"x")
with self.assertRaises(ValueError):
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
def test_pinned_git_archive_prefix_and_per_file_hashes(self):
archive = self.archive([("upstream-abc/3rdparty/GL/header.h", b"header", 0o644, "file")],
"fixture-" + "a" * 40 + ".tar.gz")
record = self.record(archive, {"GL/header.h": fingerprint(b"header")})
record.pop("archiveSHA256")
record.update(sourceURL="https://github.com/example/project", sourceRevision="a" * 40,
archiveRootPrefix="3rdparty/")
self.assertEqual(FETCH.source_url(record), "https://codeload.github.com/example/project/tar.gz/" + "a" * 40)
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual((self.output / "fixture/GL/header.h").read_bytes(), b"header")
def test_import_patch_reproduces_final_hashes(self):
archive = self.archive([("fixture-1/a.txt", b"before\n", 0o644, "file")])
record = self.record(archive, {"a.txt": fingerprint(b"before\n")})
record["vendoredFileSHA256"] = {"a.txt": fingerprint(b"after\n")}
record["patches"] = ["change.patch"]
(self.metadata / "change.patch").write_text("--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-before\n+after\n")
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual((self.output / "fixture/a.txt").read_bytes(), b"after\n")
def test_existing_modified_tree_is_not_replaced(self):
archive = self.ordinary_archive()
record = self.record(archive)
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
changed = self.output / "fixture/COPYING"
changed.write_bytes(b"local edit")
with self.assertRaisesRegex(ValueError, "differs from reviewed import"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(changed.read_bytes(), b"local edit")
def test_linked_source_destination_and_archive_cache_are_refused(self):
archive = self.ordinary_archive()
record = self.record(archive)
target = self.root / "outside"
target.mkdir()
self.output.symlink_to(target, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertEqual(list(target.iterdir()), [])
linked_cache = self.root / "linked-cache"
linked_cache.symlink_to(self.cache, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.obtain_archive("fixture", record, linked_cache, offline=True)
def test_linked_ancestor_output_and_cache_never_write_through(self):
archive = self.ordinary_archive()
record = self.record(archive)
outside = self.root / "outside"
outside.mkdir()
sentinel = outside / "keep.txt"
sentinel.write_bytes(b"preserved")
linked_parent = self.root / "link"
linked_parent.symlink_to(outside, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.materialize("fixture", record, self.metadata,
linked_parent / "new-third-party", self.cache, offline=True)
self.assertEqual(sentinel.read_bytes(), b"preserved")
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
with self.assertRaisesRegex(ValueError, "symbolic link"):
FETCH.obtain_archive("fixture", record, linked_parent / "new-cache", offline=True)
self.assertEqual(sentinel.read_bytes(), b"preserved")
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
def test_only_canonical_macos_aliases_are_allowed_in_ancestry(self):
# These public spellings must remain usable by Xcode on macOS. A
# similarly named user link is still covered by the preceding test.
import os
if os.uname().sysname != "Darwin":
self.skipTest("macOS public aliases are platform-specific")
for public, canonical in [(Path("/tmp"), Path("/private/tmp")),
(Path("/var"), Path("/private/var"))]:
if public.is_symlink() and public.resolve() == canonical:
FETCH.require_unlinked_ancestors(public / "uncreated-dependency-cache", "Fixture")
def test_verify_only_missing_tree_does_not_create_output_or_fetch(self):
archive = self.ordinary_archive()
record = self.record(archive)
with self.assertRaisesRegex(ValueError, "has not been fetched"):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache,
offline=True, verify_only=True)
self.assertFalse(self.output.exists())
def test_supplemental_metadata_merges_without_replacing_import_records(self):
archive = self.ordinary_archive()
record = self.record(archive)
normal = self.root / "third_party/manifests"
supplement = self.root / "tools/dependencies/manifests"
normal.mkdir(parents=True)
supplement.mkdir(parents=True)
first = normal / "fixture.json"
first.write_text(json.dumps(record))
other = dict(record, directory="third_party/legacy")
second = supplement / "legacy.json"
second.write_text(json.dumps(other))
records, locations = FETCH.load_records(self.root)
self.assertEqual(set(records), {"fixture", "legacy"})
self.assertEqual(records["fixture"], record)
self.assertEqual(locations, {"fixture": normal, "legacy": supplement})
(supplement / "fixture.json").write_text(json.dumps(other))
with self.assertRaisesRegex(ValueError, "Duplicate"):
FETCH.load_records(self.root)
self.assertEqual(json.loads(first.read_text()), record)
def test_overlapping_source_roots_and_linked_metadata_are_refused(self):
archive = self.ordinary_archive()
record = self.record(archive)
normal = self.root / "third_party/manifests"
normal.mkdir(parents=True)
(normal / "fixture.json").write_text(json.dumps(record))
other = normal / "other.json"
other.write_text(json.dumps(dict(record, directory="third_party/fixture/nested")))
with self.assertRaisesRegex(ValueError, "Overlapping"):
FETCH.load_records(self.root)
other.unlink()
other.symlink_to(normal / "fixture.json")
with self.assertRaisesRegex(ValueError, "ordinary file"):
FETCH.load_records(self.root)
def test_release_hash_or_full_immutable_revision_is_required(self):
archive = self.ordinary_archive()
record = self.record(archive)
record.pop("archiveSHA256")
with self.assertRaisesRegex(ValueError, "require a recorded"):
FETCH.source_url(record)
for revision in ("main", "a" * 12, "a" * 39, "a" * 41):
with self.subTest(revision=revision):
candidate = dict(record, sourceURL="https://github.com/example/project", sourceRevision=revision)
with self.assertRaisesRegex(ValueError, "pinned Git"):
FETCH.source_url(candidate)
def test_unapproved_import_patch_target_leaves_no_destination(self):
archive = self.ordinary_archive()
record = self.record(archive)
record["patches"] = ["escape.patch"]
(self.metadata / "escape.patch").write_text("--- a/../escape\n+++ b/../escape\n@@ -1 +1 @@\n-before\n+after\n")
with self.assertRaises(ValueError):
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
self.assertFalse((self.output / "fixture").exists())
if __name__ == "__main__":
unittest.main()
+57
View File
@@ -0,0 +1,57 @@
-- SPDX-License-Identifier: GPL-3.0-or-later
PRAGMA user_version=2;
PRAGMA application_id=0x56434341;
CREATE TABLE metadata(key TEXT PRIMARY KEY,value TEXT NOT NULL) WITHOUT ROWID;
CREATE TABLE document(
id INTEGER PRIMARY KEY,
sha256 TEXT NOT NULL UNIQUE CHECK(length(sha256)=64 AND sha256 NOT GLOB '*[^a-f0-9]*'),
title TEXT NOT NULL,
category TEXT NOT NULL,
collection TEXT NOT NULL,
format TEXT NOT NULL,
source_url TEXT NOT NULL,
member_path TEXT,
delivery_url TEXT,
byte_size INTEGER NOT NULL CHECK(byte_size > 0),
page_count INTEGER NOT NULL CHECK(page_count > 0),
issue_year INTEGER CHECK(issue_year BETWEEN 1900 AND 2099),
member_offset INTEGER,
member_compressed_bytes INTEGER,
member_method INTEGER,
archive_bytes INTEGER
);
CREATE INDEX document_browse ON document(category,collection,title COLLATE NOCASE,sha256);
CREATE INDEX document_issue_year ON document(collection,issue_year,title COLLATE NOCASE,sha256)
WHERE issue_year IS NOT NULL;
CREATE INDEX document_title ON document(title COLLATE NOCASE,sha256);
CREATE TABLE page(
id INTEGER PRIMARY KEY,
document_id INTEGER NOT NULL REFERENCES document(id),
page_number INTEGER NOT NULL CHECK(page_number > 0),
page_label TEXT,
locator_kind TEXT NOT NULL CHECK(locator_kind IN ('pdf_page','worksheet','image','section')),
is_ocr INTEGER NOT NULL CHECK(is_ocr IN (0,1)),
UNIQUE(document_id,page_number)
);
-- Rendered text for complex non-PDF formats. PDF and image OCR stays in the research store.
CREATE TABLE reader_section(
page_id INTEGER PRIMARY KEY REFERENCES page(id),
text TEXT NOT NULL
);
-- Stored positions refer only to the synthetic alphabetical term bag, never to source order.
CREATE VIRTUAL TABLE page_terms USING fts5(
terms,title_terms,content='',tokenize='unicode61 remove_diacritics 2',detail=full,columnsize=1
);
CREATE TABLE reviewed_example(
original_sha256 TEXT NOT NULL REFERENCES document(sha256),
first_page INTEGER NOT NULL CHECK(first_page > 0),
last_page INTEGER NOT NULL CHECK(last_page >= first_page),
title TEXT NOT NULL,
kind TEXT NOT NULL CHECK(kind IN ('snippet','basic_program','source')),
language TEXT NOT NULL,
author TEXT NOT NULL,
provenance TEXT NOT NULL,
source TEXT NOT NULL,
source_sha256 TEXT NOT NULL,
PRIMARY KEY(original_sha256,first_page,last_page,title)
) WITHOUT ROWID;
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Stage the exported public Archive index as an Xcode resource.
The explicit source content profile stages only the checked-in empty schema.
Normal builds retain the existing prepared-index lookup and verification.
"""
import argparse
from contextlib import closing
import hashlib
import json
import os
from pathlib import Path
import sqlite3
import tempfile
import sys
NAME = 'ArchiveLibrary.sqlite'
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tools'))
from build_content import add_content_profile_argument, atomic_empty_database, content_profile as resolve_content_profile
DEFAULT = ROOT / 'media/app-inputs/archive-library' / NAME
SCHEMA = Path(__file__).with_name('archive-library.sql')
def source_archive(output):
result = atomic_empty_database(output, SCHEMA, (
('schema_version', '2'), ('application_id', str(0x56434341)), ('source_snapshot', 'empty-source'),
('documents', '0'), ('pages', '0'), ('searchable_pages', '0'), ('verified_deliveries', '0'),
('ranged_members', '0'), ('reader_sections', '0'), ('reviewed_examples', '0')),
('document', 'page', 'reader_section', 'page_terms', 'reviewed_example'),
expected_name=NAME, expected_parent='ArchiveLibrary', application_id=0x56434341,
user_version=2, fts_integrity=True)
return dict(result, schema_version=2, application_id=0x56434341, source_snapshot='empty-source',
documents=0, pages=0, searchable_pages=0, verified_deliveries=0, ranged_members=0,
reader_sections=0, reviewed_examples=0, empty=True)
def stage(source, output, required=False, *, content_profile='normal'):
if resolve_content_profile(content_profile) == 'source':
if required:
raise ValueError('--require cannot be combined with the source content profile')
if output is not None:
return source_archive(output)
with tempfile.TemporaryDirectory(prefix='vcc-source-archive-check-') as temporary:
return source_archive(Path(temporary) / 'ArchiveLibrary' / NAME)
if source is None:
source = DEFAULT
if not source.exists():
source = ROOT / 'media/knowledge-base/archive-library' / NAME
source = Path(source)
if output is not None and (output.name != NAME or output.parent.name != 'ArchiveLibrary' or output.is_symlink()):
raise ValueError('Expected an ArchiveLibrary/ArchiveLibrary.sqlite app resource destination')
if not source.exists():
if required:
raise ValueError('Prepare the Archive Library index before packaging this release.')
if output is not None:
output.unlink(missing_ok=True)
return
manifest = json.loads((source.parent / 'archive-library.json').read_bytes())
checksum = (source.parent / (NAME + '.sha256')).read_text().split()
version = manifest['schema_version']
if (checksum != [manifest['sha256'], NAME] or manifest['bytes'] != source.stat().st_size
or version not in (1, 2)):
raise ValueError('The Archive index manifest and sidecar do not match the export')
if required and version < 2:
raise ValueError('Re-export the Archive Library index with inline reader support before packaging this release.')
with closing(sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True)) as db:
if (db.execute('PRAGMA application_id').fetchone() != (0x56434341,)
or db.execute('PRAGMA user_version').fetchone() != (version,)
or db.execute('SELECT COUNT(*) FROM document').fetchone() != (manifest['documents'],)
or not manifest['documents']):
raise ValueError('Incompatible Archive Library input')
if version == 2 and (db.execute('SELECT COUNT(*) FROM reader_section').fetchone() != (manifest['reader_sections'],)
or db.execute('SELECT COUNT(*) FROM document WHERE member_offset IS NOT NULL').fetchone() != (manifest['ranged_members'],)):
raise ValueError('Archive reader coverage differs from the verified export')
if output is not None:
output.parent.mkdir(parents=True, exist_ok=True)
# Copy and verify the same stream. Replace only after both checks complete.
fd = None
temporary = None
try:
if output is not None:
fd = tempfile.NamedTemporaryFile(dir=output.parent, delete=False)
temporary = Path(fd.name)
checksum = hashlib.sha256()
with source.open('rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
checksum.update(chunk)
if fd is not None:
fd.write(chunk)
if checksum.hexdigest() != manifest['sha256']:
raise ValueError('Archive Library fingerprint differs from the verified export')
if fd is not None:
fd.close()
os.chmod(temporary, 0o444)
os.replace(temporary, output)
finally:
if fd is not None:
fd.close()
temporary.unlink(missing_ok=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=Path)
add_content_profile_argument(parser)
parser.add_argument('--output', type=Path)
parser.add_argument('--require', action='store_true')
args = parser.parse_args()
try:
result = stage(args.source, args.output, args.require, content_profile=resolve_content_profile(args.content_profile))
except (OSError, ValueError, KeyError, sqlite3.DatabaseError) as error:
parser.exit(1, 'error: Archive Library: ' + str(error) + '\n')
if result is not None:
print('Archive Library index: ' + json.dumps(result, sort_keys=True))
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Generate the source content profile lookup from production reference catalogs.
The reader callback is restricted to explicitly enumerated application inputs;
full manual chapters, examples and maintainer collections are never opened.
"""
from html import escape
import hashlib
import json
LOOKUP_ROOT = "docs/manuals/"
def _json(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2)
def _bytes(value):
return value.encode("utf-8")
def basic_anchor(entry):
if entry["id"].startswith("ERROR."):
return "err-" + entry["keyword"].lower().replace("/0", "div-zero")
return "term-" + entry["id"].lower().replace(".", "-").replace("$", "-string").replace("@", "-at")
def _page(title, body, home="../index.html"):
return _bytes(f'''<!doctype html>
<html lang="en"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>{escape(title)} · VCC Plus Reference Lookup</title>
<style>body{{font:17px/1.6 -apple-system,BlinkMacSystemFont,sans-serif;max-width:78ch;margin:2rem auto;padding:0 1rem;color:#182721;background:#fff}}a{{color:#175b43}}pre{{white-space:pre-wrap;overflow-wrap:anywhere;background:#f1f5f2;padding:1rem}}section{{border-top:1px solid #d5ded8;margin-top:2rem;padding-top:1rem}}input{{font:inherit;max-width:100%;box-sizing:border-box;width:100%;padding:.5rem}}small{{display:block}}h1,h2{{line-height:1.25}}:target{{background:#f2f7ed}}</style>
</head><body><nav><a href="{home}">Lookup library</a></nav><main>
<h1>{escape(title)}</h1>{body}</main></body></html>\n''')
def _index(book, directory, chapters):
return _bytes("(globalThis.manualBooks ??= []).push(" + _json({
"book": book, "directory": directory, "chapters": chapters,
}) + ");\n")
def build_resources(read_source):
"""Return generated lookup reader files; never read full manual content."""
catalog = json.loads(read_source("platforms/macos/Resources/BASICReference.json"))
assembly = json.loads(read_source("platforms/macos/Resources/AssemblyReference.json"))
signature = hashlib.sha256()
for suffix in ("", "Instructions", "Topics"):
name = f"AssemblyReference{suffix}.swift"
data = read_source("platforms/macos/App/" + name).replace(b"\r\n", b"\n")
signature.update(name.encode() + b"\0" + data + b"\0")
if assembly["sourceSHA256"] != signature.hexdigest():
raise ValueError("Assembly lookup snapshot differs from the selected production catalog")
resources = {}
chapters = []
for errors in (False, True):
page = "errors.html" if errors else "reference.html"
title = "BASIC errors" if errors else "BASIC command lookup"
rows = [r for r in catalog["entries"] if r["id"].startswith("ERROR.") == errors]
sections, html = [], []
for row in rows:
anchor = basic_anchor(row)
paragraphs = [row["summary"], *row.get("parameters", []), *row.get("notes", [])]
paragraphs = [p if isinstance(p, str) else _json(p) for p in paragraphs]
citation = f"Source: {row['source']}, {row['sourcePage']}."
body = "".join("<p>" + escape(p) + "</p>" for p in paragraphs)
for syntax in row.get("syntax", []):
body += "<pre><code>" + escape(syntax) + "</code></pre>"
if row.get("sample"):
body += "<pre><code>" + escape(row["sample"]) + "</code></pre>"
body += "<p>Dialect: " + escape(row["dialect"]) + ". " + escape(citation) + "</p>"
html.append(f'<section id="{escape(anchor)}"><h2>{escape(row["title"])}</h2>{body}</section>')
sections.append({"id": anchor, "title": row["title"],
"passages": [*paragraphs, *row.get("syntax", []), row.get("sample", ""), citation]})
resources[LOOKUP_ROOT + "programming-in-coco-basic/" + page] = _page(title, "\n".join(html))
chapters.append({"page": page, "label": "Lookup", "title": title, "sections": sections})
resources[LOOKUP_ROOT + "book-index.js"] = _index("BASIC Lookup", "programming-in-coco-basic", chapters)
sections, html = [], []
for i, row in enumerate(assembly["entries"]):
anchor = f"assembly-{i:04d}"
paragraphs = [str(row[key]) for key in ("category", "operation", "syntax", "flags", "detail") if row.get(key)]
if row.get("aliases"):
paragraphs.append("Aliases: " + ", ".join(row["aliases"]))
if row.get("hd6309"):
paragraphs.append("HD6309 only.")
paragraphs.extend(row.get("sources", []))
sections.append({"id": anchor, "title": row["name"], "passages": paragraphs})
html.append(f'<section id="{anchor}"><h2>{escape(row["name"])}</h2>' +
"".join("<p>" + escape(p) + "</p>" for p in paragraphs) + "</section>")
resources[LOOKUP_ROOT + "assembly-language-programming/instructions.html"] = _page("Assembly lookup", "\n".join(html))
resources[LOOKUP_ROOT + "assembly-language-programming/catalog.json"] = read_source("platforms/macos/Resources/AssemblyReference.json")
resources[LOOKUP_ROOT + "assembly-language-programming/book-index.js"] = _index(
"Assembly Lookup", "assembly-language-programming", [{"page": "instructions.html", "label": "Lookup", "title": "Assembly", "sections": sections}])
home = '''<p>This source edition includes BASIC and Assembly lookup. The complete native quick-reference browsers are also available from the Develop menu.</p>
<ul><li><a href="programming-in-coco-basic/reference.html">BASIC commands</a></li><li><a href="programming-in-coco-basic/errors.html">BASIC errors</a></li><li><a href="assembly-language-programming/instructions.html">Assembly instructions, addressing and directives</a></li></ul>
<section id="search"><h2>Search the lookup library</h2><form id="search-form"><label for="manual-query">Commands, instructions or keywords</label><input id="manual-query" type="search" autocomplete="off"></form><p id="search-status" aria-live="polite"></p><ol id="search-results"></ol></section>
<script src="book-index.js"></script><script src="assembly-language-programming/book-index.js"></script><script src="lookup-search.js"></script>'''
resources[LOOKUP_ROOT + "index.html"] = _page("VCC Plus Reference Lookup", home, "index.html")
resources[LOOKUP_ROOT + "lookup-search.js"] = _bytes('''// SPDX-License-Identifier: GPL-3.0-or-later
const field = document.getElementById("manual-query"), results = document.getElementById("search-results"), statusLine = document.getElementById("search-status");
const topics = globalThis.manualBooks.flatMap(book => book.chapters.flatMap(chapter => chapter.sections.map(section => ({...section, url: book.directory + "/" + chapter.page + "#" + section.id}))));
document.getElementById("search-form").addEventListener("submit", event => event.preventDefault());
field.addEventListener("input", () => {
const words = field.value.trim().toLowerCase().split(/\\s+/).filter(Boolean);
const matches = words.length ? topics.filter(topic => words.every(word => (topic.title + " " + topic.passages.join(" ")).toLowerCase().includes(word))) : [];
results.replaceChildren();
for (const topic of matches) { const item = document.createElement("li"), link = document.createElement("a"); link.href = topic.url; link.textContent = topic.title; item.append(link); results.append(item); }
statusLine.textContent = words.length ? matches.length + " matching topics" : "Enter a search term.";
});
''')
return resources
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Check an exported source distribution without private inputs or network access."""
import hashlib
import ast
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import sqlite3
import re
import xml.etree.ElementTree as ET
from publication_urls import unapproved_app_links
ROOT = Path(__file__).resolve().parents[2]
def artwork_provenance_issues(name, data):
"""Inspect artwork descriptions and generator strings, preserving attribution."""
if name.endswith('.svg'):
root = ET.fromstring(data)
text = '\n'.join(''.join(node.itertext()) for node in root.iter()
if node.tag.rsplit('}', 1)[-1] in {'metadata', 'desc'})
elif name.startswith('platforms/macos/Tools/') and name.endswith(('.py', '.swift')):
source = data.decode('utf-8')
if name.endswith('.py'):
text = '\n'.join(node.value for node in ast.walk(ast.parse(source))
if isinstance(node, ast.Constant) and isinstance(node.value, str))
else:
text = source
else:
return []
patterns = [r'\buser[- ]supplied\b', r'\brequested by (?:the )?user\b',
r'\b(?:supplied|provided) by Brian\b', r'\bIMG_\d+\.(?:jpg|jpeg|png)\b',
r'\b\d{12,}_[\w]+_n\.jpg\b',
r'\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\.(?:webp|jpg|png)\b',
r'\bapproved\s+(?:[\w-]+\s+){0,3}concept\b',
r'\brequested\s+(?:TANDY\s+)?branding\b',
r'\bno image-generation service\b']
return [name + ': private-reference or authoring-session metadata'
for pattern in patterns if re.search(pattern, text, re.I)]
def source_mode_matches(mode, expected):
"""Git preserves executable status; checkout write bits follow local umask."""
return bool(mode & 0o111) == bool(int(expected, 8) & 0o111)
def source_inventory():
result = set()
for target in ROOT.rglob('*'):
relative = target.relative_to(ROOT)
if '__pycache__' in relative.parts or target.name in {'.DS_Store'} or target.suffix == '.pyc':
continue
if relative.parts[0] in {'.git', '.build', 'build', 'dist'} or relative.parts[:3] == ('platforms', 'macos', 'LocalResources'):
continue
if relative.parts[0] == 'third_party' and len(relative.parts) > 1 and relative.parts[1] not in {'README.md', 'manifests'}:
continue
if target.is_symlink() or target.is_file():
result.add(relative.as_posix())
if (ROOT / '.git').exists():
result.update(subprocess.check_output(['git', '-C', str(ROOT), 'ls-files', '-z']).decode().strip('\0').split('\0'))
result.discard('')
return result
def main():
record = json.loads((ROOT / 'EXPORT.json').read_text())
expected_paths = set(record['files']) | {'EXPORT.json'}
if source_inventory() != expected_paths:
raise ValueError('Unexpected or missing distribution source: ' + str(sorted(source_inventory() ^ expected_paths)))
for relative, expected in record['files'].items():
path = ROOT / relative
if path.is_symlink() or not path.is_file():
raise ValueError('Missing or linked distribution file: ' + relative)
if hashlib.sha256(path.read_bytes()).hexdigest() != expected['sha256']:
raise ValueError('Distribution file changed: ' + relative)
if not source_mode_matches(path.stat().st_mode, expected['mode']):
raise ValueError('Distribution file mode changed: ' + relative)
# Check tracked/source files rather than generated, ignored build products.
for relative in record['files']:
path = ROOT / relative
if path.suffix.lower() in {'.rom', '.sqlite', '.sqlite3', '.db'}:
raise ValueError('Forbidden source payload: ' + relative)
if path.read_bytes().startswith(b'SQLite format 3\x00'):
raise ValueError('Database payload: ' + relative)
if unapproved_app_links(path.read_bytes().decode('utf-8', errors='ignore')):
raise ValueError('Unreviewed application repository link: ' + relative)
issues = artwork_provenance_issues(relative, path.read_bytes())
if issues:
raise ValueError('\n'.join(issues))
project = (ROOT / 'platforms/macos/VCC.xcodeproj/project.pbxproj').read_text()
if '"VCC_BUILD_CONTENT_PROFILE" = "normal";' in project or project.count('"VCC_BUILD_CONTENT_PROFILE" = "source";') != 6:
raise ValueError('Distribution must default to the shared source content profile')
if '"CODE_SIGN_STYLE" = "Automatic";' in project or '"DEVELOPMENT_TEAM" = "";' not in project:
raise ValueError('Distribution must use local ad-hoc signing without a developer team')
if project.count('"PRODUCT_NAME" = "VCC Plus OSS";') != 3 or '"PRODUCT_BUNDLE_IDENTIFIER" = "com.brianholman.vccplus.oss.mac";' not in project:
raise ValueError('OSS must have an independent app identity')
forbidden = {'platforms/macos/App/AppUpdateService.swift', 'platforms/macos/Tests/AppUpdateTests.swift'}
if forbidden.intersection(record['files']) or 'AppUpdate' in project or 'AppUpdate' in (ROOT / 'platforms/macos/App/VCCMacApp.swift').read_text():
raise ValueError('Commercial updater source, membership or hooks remain')
if 'static let bundleID = "com.brianholman.vccplus.oss.mac"' not in (ROOT / 'platforms/macos/App/StartupDefaults.swift').read_text():
raise ValueError('OSS must use an independent application-support directory')
with tempfile.TemporaryDirectory(prefix='vcc-oss-schema-') as folder:
for script, name, table in [('catalog/package.py', 'SoftwareLibrary/catalog.sqlite', 'release'),
('knowledge/package_archive.py', 'ArchiveLibrary/ArchiveLibrary.sqlite', 'document')]:
output = Path(folder) / name
subprocess.run([sys.executable, str(ROOT / 'tools' / script), '--output', str(output), '--content-profile', 'source'], check=True)
with sqlite3.connect(output.resolve().as_uri() + '?mode=ro', uri=True) as db:
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
raise ValueError('Empty dataset is damaged')
if db.execute('SELECT count(*) FROM ' + table).fetchone() != (0,):
raise ValueError('Dataset unexpectedly contains content')
print(f"PASS: {len(record['files'])} source fingerprints; no ROM/database payload; both empty datasets valid")
if __name__ == '__main__':
main()
+31
View File
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: GPL-3.0-or-later
"""Closed public-app link policy shared by both publication checkers.
An organization name is not proof of repository visibility. This inventory
records the specific external app repository reviewed for public references.
No network lookup or permission change occurs during publication checks.
"""
import re
from urllib.parse import unquote, urlsplit
PUBLIC_HOST = 'vccplus-git.brianholman.com'
APPS_NAMESPACE = '/VCCPlus' + 'Apps/'
PUBLIC_APP_REPOSITORIES = frozenset({'cococapitalquiz'})
def unapproved_app_links(text):
problems = []
for match in re.finditer(r'https?://[^\s<>"`\)\]]+', text):
value = match.group(0)
parsed = urlsplit(value)
if parsed.hostname != PUBLIC_HOST:
continue
path = unquote(parsed.path)
if not path.lower().startswith(APPS_NAMESPACE.lower()):
continue
rest = path[len(APPS_NAMESPACE):]
name = rest.split('/', 1)[0].lower()
if name not in PUBLIC_APP_REPOSITORIES or '..' in path.split('/'):
problems.append(value)
return problems
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Reconstruct pinned upstream firmware from verified assembly-only inputs.
Preparation is explicit and offline. Fetch cocoroms/LWTools separately, or supply
an existing LWASM. Builds use fresh copies and verify all generated ROM hashes.
Only the two boot ROMs, provenance manifest and notice enter an optional bundle.
"""
import argparse
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import time
ROOT = Path(__file__).resolve().parents[2]
SPEC = importlib.util.spec_from_file_location('vcc_dependency_fetch', ROOT / 'tools/dependencies/fetch.py')
DEPENDENCIES = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(DEPENDENCIES)
ROM_NAMES = ('bas13.rom', 'extbas11.rom', 'disk11.rom', 'coco3.rom')
BUNDLE_NAMES = ('coco3.rom', 'disk11.rom')
SOURCE_NAMES = {'Makefile', 'equates.asm', 'bas.asm', 'extbas.asm', 'disk.asm', 'coco3.asm', 'md5sums.txt', 'mugateerspic.asm'}
NOTICE = '''Reconstructed upstream Color Computer firmware
Source: ToolShed cocoroms, https://github.com/nitros9project/toolshed
Pinned revision: 3667a41c9605b7fcb150eace29cc7f42d460be1a
The original assembly headers identify material copied from Color BASIC
Unravelled, Extended Color BASIC Unravelled, Disk Color BASIC Unravelled and
Super Color BASIC Unravelled, adjusted for assembly. Their original source
comments are retained in the explicitly prepared source dependency.
Original firmware notices retained in the generated ROM data include:
Color BASIC: (C) 1982 TANDY; MICROSOFT
Extended Color BASIC: COPYRIGHT (C) 1982 BY TANDY;
UNDER LICENSE FROM MICROSOFT
Disk Extended Color BASIC: COPYRIGHT (C) 1982 BY TANDY;
UNDER LICENSE FROM MICROSOFT
Super Color BASIC: COPR. 1982, 1986 BY TANDY;
UNDER LICENSE FROM MICROSOFT; AND MICROWARE SYSTEMS CORP.
These are reconstructed upstream firmware, not newly authored or clean-room
replacement ROMs. The emulator's GPL notice does not relabel this firmware.
No preassembled ROM or binary input is used by this reconstruction process.
'''
def encoded(value):
return (json.dumps(value, sort_keys=True, indent=2) + '\n').encode('utf-8')
def sha(path):
return DEPENDENCIES.sha(path)
def source_record(records):
record = records['cocoroms']
if set(record['vendoredFileSHA256']) != SOURCE_NAMES or set(record['upstreamFileSHA256']) != SOURCE_NAMES:
raise ValueError('Firmware input inventory must contain exactly the eight assembly/build reference files')
if (record.get('sourceRevision') != '3667a41c9605b7fcb150eace29cc7f42d460be1a'
or record.get('archiveRootPrefix') != 'cocoroms/' or record.get('optional') is not True):
raise ValueError('Unsupported firmware source identity')
outputs = record.get('romOutputs')
if not isinstance(outputs, dict) or set(outputs) != set(ROM_NAMES):
raise ValueError('Firmware output inventory must contain the four qualified ROMs')
sizes = {'coco3.rom': 32768, 'disk11.rom': 8192, 'bas13.rom': 8192, 'extbas11.rom': 8192}
for name, spec in outputs.items():
if (not isinstance(spec, dict) or spec.get('bytes') != sizes[name]
or not isinstance(spec.get('sha256'), str) or not DEPENDENCIES.HASH.fullmatch(spec['sha256'])):
raise ValueError('Invalid generated firmware fingerprint: ' + name)
return record
def copy_verified(source, destination, fingerprints):
DEPENDENCIES.verify_tree(source, fingerprints)
destination.mkdir(parents=True, exist_ok=False)
for name in sorted(fingerprints):
original = source / name
target = destination / name
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(original, target)
target.chmod(0o755 if original.stat().st_mode & 0o111 else 0o644)
DEPENDENCIES.verify_tree(destination, fingerprints)
def validate_outputs(directory, expected):
if set(expected) != set(ROM_NAMES):
raise ValueError('Expected all four reconstructed ROM outputs')
results = {}
for name in ROM_NAMES:
path = directory / name
if path.is_symlink() or not path.is_file() or path.stat().st_nlink != 1:
raise ValueError('Generated ROM must be an ordinary unlinked file: ' + name)
result = {'bytes': path.stat().st_size, 'sha256': sha(path)}
if result != expected[name]:
raise ValueError('Generated ROM does not match the qualified fingerprint: ' + name)
results[name] = result
return results
def bundle_manifest(record, metadata, assembler, outputs):
return {
'schemaVersion': 1,
'generator': 'vccplus-rom-build',
'source': {
'project': 'ToolShed cocoroms', 'url': record['sourceURL'],
'revision': record['sourceRevision'], 'manifestSHA256': sha(metadata),
'filesSHA256': record['vendoredFileSHA256'], 'preassembledROMInputs': False,
},
'assembler': assembler,
'roms': {name: outputs[name] for name in BUNDLE_NAMES},
}
def write_bundle(destination, source, manifest):
"""Atomically create a fresh optional payload; never replace a prior bundle."""
destination = Path(destination).absolute()
DEPENDENCIES.require_unlinked_ancestors(destination, 'ROM bundle destination')
if destination.exists() or destination.is_symlink():
raise ValueError('Choose a fresh ROM bundle destination; existing payloads are never overwritten')
for name in BUNDLE_NAMES:
path = source / name
spec = manifest['roms'][name]
if path.is_symlink() or not path.is_file() or path.stat().st_nlink != 1 or path.stat().st_size != spec['bytes'] or sha(path) != spec['sha256']:
raise ValueError('ROM output changed before optional bundle staging')
destination.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix='.rom-bundle-', dir=destination.parent) as temporary:
staged = Path(temporary) / 'bundle'
staged.mkdir()
for name in BUNDLE_NAMES:
shutil.copyfile(source / name, staged / name)
if (staged / name).stat().st_size != manifest['roms'][name]['bytes'] or sha(staged / name) != manifest['roms'][name]['sha256']:
raise ValueError('ROM output changed while copying the optional bundle')
(staged / 'ROMBuild.json').write_bytes(encoded(manifest))
(staged / 'ROM_SOURCE_NOTICE.txt').write_text(NOTICE, encoding='utf-8')
if destination.exists() or destination.is_symlink():
raise ValueError('ROM bundle destination changed during preparation')
staged.rename(destination)
def build(output, dependency_root, lwasm=None, bundle_output=None, jobs=4):
output, dependency_root = Path(output).absolute(), Path(dependency_root).absolute()
DEPENDENCIES.require_unlinked_ancestors(output, 'Firmware build output')
DEPENDENCIES.require_unlinked_ancestors(dependency_root, 'Firmware dependency root')
if output.exists() or output.is_symlink():
raise ValueError('Choose a fresh build output; previous evidence is never overwritten')
if output == ROOT or output in ROOT.parents or output == dependency_root or dependency_root in output.parents:
raise ValueError('Choose build output separate from the source/dependency tree')
records, locations = DEPENDENCIES.load_records(ROOT)
record = source_record(records)
source = dependency_root / DEPENDENCIES.validate_record(record)
DEPENDENCIES.verify_tree(source, record['vendoredFileSHA256'])
if lwasm is None:
lwrecord = records['lwtools']
lwsource = dependency_root / DEPENDENCIES.validate_record(lwrecord)
DEPENDENCIES.verify_tree(lwsource, lwrecord['vendoredFileSHA256'])
else:
lwasm = Path(lwasm).resolve()
if not lwasm.is_file() or not os.access(lwasm, os.X_OK):
raise ValueError('The explicitly selected LWASM is unavailable')
if bundle_output is not None:
bundle = Path(bundle_output).absolute()
DEPENDENCIES.require_unlinked_ancestors(bundle, 'ROM bundle destination')
# An optional package must not alter verified dependency inputs. Reject
# a parent of the build too: creating the build would occupy that bundle
# destination before its final atomic installation.
canonical_bundle, canonical_dependencies, canonical_output = bundle.resolve(), dependency_root.resolve(), output.resolve()
if (canonical_bundle == canonical_dependencies or canonical_dependencies in canonical_bundle.parents
or canonical_bundle == canonical_output or canonical_bundle in canonical_output.parents):
raise ValueError('ROM bundle must be separate from dependency inputs and the build ancestor path')
if bundle.exists() or bundle.is_symlink():
raise ValueError('Choose a fresh ROM bundle destination')
output.mkdir(parents=True)
report = {'schemaVersion': 1, 'status': 'running', 'networkUsed': False,
'sourceRevision': record['sourceRevision'], 'preassembledROMInputs': False, 'commands': []}
started = time.monotonic()
try:
with (output / 'build.log').open('wb') as log:
def run(command, cwd, stdout=None):
report['commands'].append({'argv': [str(p) for p in command], 'cwd': str(cwd)})
log.write(('$ ' + ' '.join(str(p) for p in command) + '\n').encode())
log.flush()
subprocess.run(command, cwd=cwd, check=True, stdout=log if stdout is None else stdout, stderr=log)
if lwasm is None:
lwcopy = output / 'work/lwtools'
copy_verified(lwsource, lwcopy, lwrecord['vendoredFileSHA256'])
run(['/usr/bin/make', '-j' + str(jobs), 'lwasm'], lwcopy)
lwasm = lwcopy / 'lwasm/lwasm'
assembler = {'version': subprocess.check_output([str(lwasm), '--version'], stderr=subprocess.STDOUT).decode().strip(),
'sha256': sha(lwasm)}
report['assembler'] = assembler
work = output / 'work/cocoroms'
copy_verified(source, work, record['vendoredFileSHA256'])
if any(p.suffix.lower() in ('.rom', '.bin') for p in work.rglob('*')):
raise ValueError('Preassembled firmware input is forbidden')
# Execute the pinned Makefile's assembly sequence directly. A fresh
# directory forces every output and --lwasm selects the exact tool,
# independent of executable names or the user's PATH.
run([str(lwasm), '--raw', '-o', 'bas13.rom', 'equates.asm', 'bas.asm'], work)
with (work / 'cb_equates.asm').open('wb') as symbols:
run([str(lwasm), '--symbol-dump', '--no-output', 'equates.asm', 'bas.asm'], work, symbols)
run([str(lwasm), '--raw', '-o', 'extbas11.rom', 'cb_equates.asm', 'extbas.asm'], work)
with (work / 'ecb_equates.asm').open('wb') as symbols:
run([str(lwasm), '--symbol-dump', '--no-output', 'equates.asm', 'bas.asm', 'extbas.asm'], work, symbols)
run([str(lwasm), '--raw', '--pragma=noindex0tonone', '-o', 'disk11.rom', 'ecb_equates.asm', 'disk.asm'], work)
run([str(lwasm), '--raw', '-o', 'coco3.rom', 'equates.asm', 'coco3.asm'], work)
outputs = validate_outputs(work, record['romOutputs'])
for name in ROM_NAMES:
shutil.copyfile(work / name, output / name)
manifest = bundle_manifest(record, locations['cocoroms'] / 'cocoroms.json', assembler, outputs)
(output / 'ROMBuild.json').write_bytes(encoded(manifest))
(output / 'ROM_SOURCE_NOTICE.txt').write_text(NOTICE, encoding='utf-8')
if bundle_output is not None:
write_bundle(Path(bundle_output), output, manifest)
report.update(status='passed', roms=outputs)
except Exception as error:
report.update(status='failed', failure=str(error))
raise
finally:
report['elapsedSeconds'] = round(time.monotonic() - started, 3)
(output / 'BuildReport.json').write_bytes(encoded(report))
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, default=ROOT / '.build/roms')
parser.add_argument('--dependency-root', type=Path, default=ROOT / 'third_party')
parser.add_argument('--lwasm', type=Path, help='Explicit existing assembler; otherwise build verified LWTools lwasm only')
parser.add_argument('--bundle-output', type=Path, help='Fresh optional SourceROMBundle destination')
parser.add_argument('--jobs', type=int, default=4)
args = parser.parse_args()
if not 1 <= args.jobs <= 32:
parser.error('--jobs must be between 1 and 32')
result = build(args.output, args.dependency_root, args.lwasm, args.bundle_output, args.jobs)
print(json.dumps({'status': result['status'], 'roms': result['roms'], 'networkUsed': False}, indent=2))
if __name__ == '__main__':
try:
main()
except (ValueError, OSError, subprocess.CalledProcessError) as error:
raise SystemExit(str(error))
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Validate an explicitly selected source-built core ROM bundle.
Use the same bounded directory, manifest and byte checks as source-profile
staging. Validation establishes byte identity, not a redistribution license.
"""
import argparse
from pathlib import Path
import runpy
_STAGE = runpy.run_path(str(Path(__file__).with_name('stage.py')))
def validate_bundle(path):
source = _STAGE['absolute'](path)
_STAGE['directory'](source, 'Source ROM bundle')
return _STAGE['read_bundle'](source)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['validate'])
parser.add_argument('bundle', type=Path)
args = parser.parse_args()
try:
validate_bundle(args.bundle)
except (OSError, ValueError, UnicodeError) as error:
parser.error(str(error))
print('PASS: source ROM bundle validated')
if __name__ == '__main__':
main()
+319
View File
@@ -0,0 +1,319 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Stage an optional explicitly built ROM bundle for source-profile Mac builds.
Validation establishes byte identity and a safe copy boundary, not redistribution
permission or hardware compatibility. No maintainer ROM or media path is probed.
Isolated UI-test products omit all bundles without inspecting their input paths.
"""
import argparse
import ctypes
import errno
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import stat
import sys
import tempfile
ROM_SIZES = {'coco3.rom': 32768, 'disk11.rom': 8192}
BUNDLE_FILES = set(ROM_SIZES) | {'ROMBuild.json', 'ROM_SOURCE_NOTICE.txt'}
STARTER_FILES = ('Hello World.dsk', 'Hello World.cas', 'Blank Disk.dsk',
'Hello Hard Disk.dsk', 'Blank OS9 Header.bin')
HASH = re.compile(r'^[0-9a-f]{64}$')
MANIFEST_LIMIT = 256 * 1024
NOTICE_LIMIT = 64 * 1024
STARTER_LIMIT = 2 * 1024 * 1024
def absolute(path):
value = Path(path)
if '..' in value.parts:
raise ValueError('Staging paths must not contain parent traversal')
return value if value.is_absolute() else Path.cwd() / value
def check_ancestors(path, description):
"""Reject redirected inputs/outputs, except the two public macOS aliases."""
aliases = {Path('/tmp'): Path('/private/tmp'), Path('/var'): Path('/private/var')}
for parent in path.parents:
try:
info = parent.lstat()
except FileNotFoundError:
continue
if stat.S_ISLNK(info.st_mode):
if parent in aliases and parent.resolve() == aliases[parent]:
continue
raise ValueError(description + ' must not have a linked ancestor')
if not stat.S_ISDIR(info.st_mode):
raise ValueError(description + ' must have directory ancestors')
def directory(path, description, optional=False):
check_ancestors(path, description)
try:
info = path.lstat()
except FileNotFoundError:
if optional:
return False
raise ValueError(description + ' is missing')
if not stat.S_ISDIR(info.st_mode):
raise ValueError(description + ' must be an unlinked directory')
return True
def signature(info):
return (info.st_dev, info.st_ino, info.st_mode, info.st_nlink,
info.st_size, info.st_mtime_ns, info.st_ctime_ns)
def read_regular(path, maximum, description, expected_size=None):
check_ancestors(path, description)
try:
before = path.lstat()
except FileNotFoundError:
raise ValueError(description + ' is missing')
if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
raise ValueError(description + ' must be a regular file without links')
if not 0 < before.st_size <= maximum or (expected_size is not None and before.st_size != expected_size):
raise ValueError(description + ' has an invalid size')
flags = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
with os.fdopen(os.open(path, flags), 'rb') as stream:
if signature(os.fstat(stream.fileno())) != signature(before):
raise ValueError(description + ' changed before it could be read')
data = stream.read(maximum + 1)
after = os.fstat(stream.fileno())
if signature(before) != signature(after) or len(data) != before.st_size:
raise ValueError(description + ' changed while it was read')
# A concurrently replaced pathname must not be accepted as the same input.
if signature(path.lstat()) != signature(before):
raise ValueError(description + ' was replaced while it was read')
return data
def unique_object(pairs):
value = {}
for key, item in pairs:
if key in value:
raise ValueError('ROM build manifest contains a duplicate field')
value[key] = item
return value
def read_bundle(source):
if set(item.name for item in source.iterdir()) != BUNDLE_FILES:
raise ValueError('Source ROM bundle must contain exactly the manifest, notice, and two required ROMs')
manifest_bytes = read_regular(source / 'ROMBuild.json', MANIFEST_LIMIT, 'ROM build manifest')
try:
record = json.loads(manifest_bytes.decode('utf-8'), object_pairs_hook=unique_object)
except RecursionError:
raise ValueError('ROM build manifest nesting is excessive')
if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record['schemaVersion'] != 1:
raise ValueError('Unsupported ROM build manifest schema')
if record.get('generator') != 'vccplus-rom-build':
raise ValueError('Unsupported ROM bundle generator')
if not isinstance(record.get('source'), dict) or not record['source']:
raise ValueError('ROM build manifest requires source metadata')
roms = record.get('roms')
if not isinstance(roms, dict) or set(roms) != set(ROM_SIZES):
raise ValueError('ROM build manifest must identify both required ROMs only')
payload = {'ROMBuild.json': manifest_bytes}
notice = read_regular(source / 'ROM_SOURCE_NOTICE.txt', NOTICE_LIMIT, 'ROM source notice')
if not notice.decode('utf-8').strip() or b'\0' in notice:
raise ValueError('ROM source notice must be nonempty UTF-8 text')
payload['ROM_SOURCE_NOTICE.txt'] = notice
for name, size in ROM_SIZES.items():
entry = roms[name]
if (not isinstance(entry, dict) or type(entry.get('bytes')) is not int or entry['bytes'] != size
or not isinstance(entry.get('sha256'), str) or not HASH.fullmatch(entry['sha256'])):
raise ValueError('Invalid ROM manifest entry: ' + name)
data = read_regular(source / name, size, name, expected_size=size)
if hashlib.sha256(data).hexdigest() != entry['sha256']:
raise ValueError('ROM bytes do not match their build manifest: ' + name)
payload[name] = data
return payload
def check_resources(resources):
if (resources.name != 'Resources' or resources.parent.name != 'Contents'
or resources.parent.parent.suffix != '.app'):
raise ValueError('Destination must be APP.app/Contents/Resources')
directory(resources, 'App Resources destination', optional=True)
def check_kit(kit):
if not directory(kit, 'Build-owned StarterKit', optional=True):
return False
# Cleanup is confined to this build-owned directory. Reject redirected files
# before either swapping an existing kit or removing an obsolete payload.
for folder, names, files in os.walk(kit, followlinks=False):
for name in names:
directory(Path(folder) / name, 'StarterKit member directory')
for name in files:
info = (Path(folder) / name).lstat()
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
raise ValueError('StarterKit members must be regular files without links')
return True
def overlaps(first, second):
# Ancestors were checked before canonicalizing the accepted /tmp,/var aliases.
first, second = first.resolve(), second.resolve()
return first == second or first in second.parents or second in first.parents
def atomic_file(destination, data):
check_ancestors(destination, 'Build resource')
if destination.exists() or destination.is_symlink():
info = destination.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
raise ValueError('Build resource must be a regular file without links')
if info.st_size == len(data) and read_regular(destination, STARTER_LIMIT, 'Existing build resource') == data:
return
temporary = None
try:
with tempfile.NamedTemporaryFile(prefix='.source-rom-resource-', dir=destination.parent, delete=False) as stream:
temporary = Path(stream.name)
stream.write(data)
os.chmod(temporary, 0o644)
# Recheck the owned output boundary immediately before replacement.
check_ancestors(destination, 'Build resource')
if destination.exists() or destination.is_symlink():
info = destination.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
raise ValueError('Build resource changed to a linked or nonregular file')
os.replace(temporary, destination)
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
def exchange_directories(first, second):
"""Atomically exchange complete directories; never expose a partial kit."""
library = ctypes.CDLL(None, use_errno=True)
if sys.platform == 'darwin':
operation = getattr(library, 'renamex_np', None)
if operation is not None:
operation.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint)
operation.restype = ctypes.c_int
result = operation(os.fsencode(first), os.fsencode(second), 2) # RENAME_SWAP
else:
raise OSError(errno.ENOTSUP, 'Atomic directory exchange is unavailable')
elif sys.platform.startswith('linux'):
operation = getattr(library, 'renameat2', None)
if operation is not None:
operation.argtypes = (ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint)
operation.restype = ctypes.c_int
result = operation(-100, os.fsencode(first), -100, os.fsencode(second), 2) # RENAME_EXCHANGE
else:
raise OSError(errno.ENOTSUP, 'Atomic directory exchange is unavailable')
else:
raise OSError(errno.ENOTSUP, 'Atomic directory exchange is unavailable')
if result:
error = ctypes.get_errno()
raise OSError(error, os.strerror(error))
def selected_source(srcroot, explicit=None, environ=None):
environment = os.environ if environ is None else environ
return absolute(explicit if explicit is not None else
environment.get('VCC_SOURCE_ROM_DIR') or srcroot / 'LocalResources/SourceROMBundle')
def isolated_ui_testing(environment):
if environment.get('CONFIGURATION') == 'UITesting':
return True
swift_conditions = environment.get('SWIFT_ACTIVE_COMPILATION_CONDITIONS', '').split()
if 'VCC_UI_TESTING' in swift_conditions:
return True
definitions = environment.get('GCC_PREPROCESSOR_DEFINITIONS', '').split()
return any(value in ('VCC_UI_TESTING', 'VCC_UI_TESTING=1') for value in definitions)
def remove_interrupted_staging(resources):
# After an exchange, the temporary pathname contains the former StarterKit.
# A killed build must not leave that old profile's payload in the next app.
# Match only this helper's generated eight-character temporary names.
for path in resources.iterdir():
if re.fullmatch(r'\.source-rom-starter-[a-z0-9_]{8}', path.name):
check_kit(path)
shutil.rmtree(path)
elif re.fullmatch(r'\.source-rom-resource-[a-z0-9_]{8}', path.name):
info = path.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
raise ValueError('Interrupted build resource must be a regular file without links')
path.unlink()
def stage_source_roms(srcroot, resources, source=None, environ=None):
srcroot, resources = absolute(srcroot), absolute(resources)
directory(srcroot, 'Mac source directory')
check_resources(resources)
environment = os.environ if environ is None else environ
ui_testing = isolated_ui_testing(environment)
# Do not even resolve or inspect an explicit/generated ROM location for the
# isolated UI app. Only its build-owned output and authored header are used.
bundle = None if ui_testing else selected_source(srcroot, source, environment)
present = directory(bundle, 'Source ROM bundle', optional=True) if bundle is not None else False
starter = srcroot / 'Resources/StarterMedia'
directory(starter, 'Authored starter media')
if (bundle is not None and overlaps(bundle, resources)) or overlaps(starter, resources):
raise ValueError('Build resources must not overlap source inputs')
kit = resources / 'StarterKit'
check_kit(kit)
# Validate all selected inputs before creating or changing a build resource.
payload = read_bundle(bundle) if present else None
header = read_regular(starter / 'Blank OS9 Header.bin', STARTER_LIMIT, 'Authored OS-9 header', expected_size=8192)
if payload is not None:
for name in STARTER_FILES:
payload[name] = read_regular(starter / name, STARTER_LIMIT, 'Authored starter media: ' + name)
resources.mkdir(parents=True, exist_ok=True)
check_resources(resources)
remove_interrupted_staging(resources)
atomic_file(resources / 'Blank OS9 Header.bin', header)
if payload is None:
if check_kit(kit):
shutil.rmtree(kit)
return {'staged': False, 'roms': {}, 'ui_testing': ui_testing}
staging = Path(tempfile.mkdtemp(prefix='.source-rom-starter-', dir=resources))
try:
for name, data in payload.items():
(staging / name).write_bytes(data)
os.chmod(staging / name, 0o644)
os.chmod(staging, 0o755)
check_resources(resources)
if check_kit(kit):
exchange_directories(staging, kit)
# The staging pathname now contains only the previous owned kit.
else:
os.rename(staging, kit)
return {'staged': True, 'roms': {name: {'bytes': len(payload[name]), 'sha256': hashlib.sha256(payload[name]).hexdigest()}
for name in ROM_SIZES}}
finally:
if staging.exists():
shutil.rmtree(staging)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--srcroot', type=Path, required=True, help='Mac project directory (platforms/macos)')
parser.add_argument('--resources', type=Path, required=True, help='Build-owned APP.app/Contents/Resources')
parser.add_argument('--source', type=Path, help='Explicit source ROM bundle; overrides VCC_SOURCE_ROM_DIR')
args = parser.parse_args()
try:
result = stage_source_roms(args.srcroot, args.resources, args.source)
except (OSError, ValueError, UnicodeError) as error:
parser.error(str(error))
if result['staged']:
print('Included explicitly built source ROMs, original source notice, and authored starter media.')
elif result.get('ui_testing'):
print('Isolated UI-test build omits ROM bundles and retains only the authored standalone header.')
else:
print('Optional source ROM bundle is absent; building without bundled ROMs.')
if __name__ == '__main__':
main()
+182
View File
@@ -0,0 +1,182 @@
# SPDX-License-Identifier: GPL-3.0-or-later
"""Authored regressions for explicit firmware preparation and bundle boundaries."""
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
MODULE = Path(__file__).resolve().parents[1] / 'build.py'
SPEC = importlib.util.spec_from_file_location('rom_build', MODULE)
BUILD = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(BUILD)
def fingerprint(data):
return hashlib.sha256(data).hexdigest()
class ROMBuildTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(prefix='vcc-rom-builder-test-')
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
self.records, self.locations = BUILD.DEPENDENCIES.load_records()
self.record = json.loads(json.dumps(self.records['cocoroms']))
def fixture_outputs(self):
directory = self.root / 'roms'
directory.mkdir()
expected = {}
for name in BUILD.ROM_NAMES:
data = bytes(32768 if name == 'coco3.rom' else 8192)
(directory / name).write_bytes(data)
expected[name] = {'bytes': len(data), 'sha256': fingerprint(data)}
return directory, expected
def fixture_inputs(self):
dependencies = self.root / 'dependencies'
source = dependencies / 'cocoroms'
source.mkdir(parents=True)
hashes = {}
for name in BUILD.SOURCE_NAMES:
data = ('Authored source fixture: ' + name + '\n').encode()
(source / name).write_bytes(data)
hashes[name] = fingerprint(data)
self.record['upstreamFileSHA256'] = self.record['vendoredFileSHA256'] = hashes
_, self.record['romOutputs'] = self.fixture_outputs()
return dependencies
def fixture_assembler(self, fail=False):
tool = self.root / 'explicit-assembler-with-another-name'
tool.write_text('''#!/usr/bin/env python3
from pathlib import Path
import sys
if '--version' in sys.argv: print('Authored assembler fixture'); raise SystemExit(0)
''' + ("raise SystemExit(7)\n" if fail else '''if '--raw' in sys.argv:
name = sys.argv[sys.argv.index('-o') + 1]
Path(name).write_bytes(bytes(32768 if name == 'coco3.rom' else 8192))
else: print('fixture EQU 0')
'''))
tool.chmod(0o755)
return tool
def test_firmware_fetch_is_explicit_and_manifest_contains_only_source(self):
record = BUILD.source_record(self.records)
self.assertEqual(set(record['upstreamFileSHA256']), BUILD.SOURCE_NAMES)
self.assertNotIn('cocoroms', BUILD.DEPENDENCIES.default_selection(self.records))
self.assertEqual(len(BUILD.DEPENDENCIES.default_selection(self.records)), 17)
self.assertNotIn('.rom', ''.join(record['upstreamFileSHA256']))
self.assertEqual(record['archiveRootPrefix'], 'cocoroms/')
def test_unreviewed_firmware_input_is_refused(self):
self.record['vendoredFileSHA256']['supplied.rom'] = '0' * 64
with self.assertRaisesRegex(ValueError, 'exactly the eight'):
BUILD.source_record({'cocoroms': self.record})
def test_optional_flag_requires_a_boolean(self):
self.record['optional'] = 'true'
with self.assertRaisesRegex(ValueError, 'optional flag'):
BUILD.DEPENDENCIES.validate_record(self.record)
def test_malformed_output_metadata_is_rejected_explicitly(self):
for spec in [None, {'bytes': 32768, 'sha256': None}, {'bytes': 32768, 'sha256': 'not-a-hash'}]:
record = json.loads(json.dumps(self.records['cocoroms']))
record['romOutputs']['coco3.rom'] = spec
with self.assertRaisesRegex(ValueError, 'firmware fingerprint'):
BUILD.source_record({'cocoroms': record})
record = json.loads(json.dumps(self.records['cocoroms']))
record['romOutputs'] = None
with self.assertRaisesRegex(ValueError, 'output inventory'):
BUILD.source_record({'cocoroms': record})
def test_generated_hash_size_links_and_hardlinks_are_checked(self):
source, expected = self.fixture_outputs()
self.assertEqual(BUILD.validate_outputs(source, expected), expected)
target = source / 'disk11.rom'
target.write_bytes(b'changed')
with self.assertRaisesRegex(ValueError, 'qualified fingerprint'):
BUILD.validate_outputs(source, expected)
target.unlink()
target.symlink_to(source / 'bas13.rom')
with self.assertRaisesRegex(ValueError, 'unlinked'):
BUILD.validate_outputs(source, expected)
target.unlink()
os.link(source / 'bas13.rom', target)
with self.assertRaisesRegex(ValueError, 'unlinked'):
BUILD.validate_outputs(source, expected)
def test_bundle_has_exactly_two_roms_manifest_notice_and_no_private_paths(self):
source, expected = self.fixture_outputs()
manifest = BUILD.bundle_manifest(self.record, self.locations['cocoroms'] / 'cocoroms.json',
{'version': 'fixture', 'sha256': 'a' * 64}, expected)
destination = self.root / 'bundle'
BUILD.write_bundle(destination, source, manifest)
self.assertEqual({p.name for p in destination.iterdir()},
{'ROMBuild.json', 'ROM_SOURCE_NOTICE.txt', 'coco3.rom', 'disk11.rom'})
published = json.loads((destination / 'ROMBuild.json').read_text())
self.assertEqual(published['schemaVersion'], 1)
self.assertEqual(published['generator'], 'vccplus-rom-build')
self.assertEqual(set(published['roms']), set(BUILD.BUNDLE_NAMES))
self.assertNotIn(str(self.root), (destination / 'ROMBuild.json').read_text())
self.assertIn('MICROWARE', (destination / 'ROM_SOURCE_NOTICE.txt').read_text())
with self.assertRaisesRegex(ValueError, 'fresh ROM bundle'):
BUILD.write_bundle(destination, source, manifest)
def test_cold_build_uses_exact_explicit_assembler_and_no_existing_roms(self):
dependencies = self.fixture_inputs()
records = dict(self.records, cocoroms=self.record)
tool = self.fixture_assembler()
output, bundle = self.root / 'output', self.root / 'bundle'
with patch.object(BUILD.DEPENDENCIES, 'load_records', return_value=(records, self.locations)):
report = BUILD.build(output, dependencies, tool, bundle)
self.assertEqual(report['status'], 'passed')
self.assertEqual(len(report['commands']), 6)
self.assertTrue(all(command['argv'][0] == str(tool.resolve()) for command in report['commands']))
self.assertFalse(report['preassembledROMInputs'])
self.assertEqual(report['roms'], self.record['romOutputs'])
self.assertTrue((bundle / 'coco3.rom').is_file())
def test_preassembled_dependency_payload_is_refused_before_execution(self):
dependencies = self.fixture_inputs()
(dependencies / 'cocoroms/preassembled.rom').write_bytes(b'fixture')
records = dict(self.records, cocoroms=self.record)
with patch.object(BUILD.DEPENDENCIES, 'load_records', return_value=(records, self.locations)), \
patch.object(BUILD.subprocess, 'run') as execute:
with self.assertRaisesRegex(ValueError, 'differs from reviewed import'):
BUILD.build(self.root / 'output', dependencies, self.fixture_assembler())
execute.assert_not_called()
self.assertFalse((self.root / 'output').exists())
def test_failed_assembly_retains_evidence_without_creating_bundle(self):
dependencies = self.fixture_inputs()
records = dict(self.records, cocoroms=self.record)
output, bundle = self.root / 'output', self.root / 'bundle'
with patch.object(BUILD.DEPENDENCIES, 'load_records', return_value=(records, self.locations)):
with self.assertRaises(BUILD.subprocess.CalledProcessError):
BUILD.build(output, dependencies, self.fixture_assembler(fail=True), bundle)
self.assertEqual(json.loads((output / 'BuildReport.json').read_text())['status'], 'failed')
self.assertTrue((output / 'build.log').is_file())
self.assertFalse(bundle.exists())
def test_bundle_cannot_contaminate_dependencies_or_occupy_build_ancestor(self):
dependencies = self.fixture_inputs()
records = dict(self.records, cocoroms=self.record)
tool = self.fixture_assembler()
for bundle in [dependencies / 'cocoroms/package', dependencies.resolve() / 'cocoroms/canonical-package', self.root / 'new-build-parent']:
output = self.root / 'new-build-parent/output'
with patch.object(BUILD.DEPENDENCIES, 'load_records', return_value=(records, self.locations)), \
patch.object(BUILD.subprocess, 'run') as execute:
with self.assertRaisesRegex(ValueError, 'separate from dependency'):
BUILD.build(output, dependencies, tool, bundle)
execute.assert_not_called()
self.assertFalse(output.exists())
self.assertFalse(bundle.exists())
if __name__ == '__main__':
unittest.main()
+468
View File
@@ -0,0 +1,468 @@
# SPDX-License-Identifier: GPL-3.0-or-later
"""Source-ROM staging checks using authored synthetic bytes, never real ROMs."""
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import unittest
from unittest import mock
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / 'tools/roms/stage.py'
SPEC = importlib.util.spec_from_file_location('source_rom_stage', SCRIPT)
stage = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(stage)
class SourceROMStageTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory(prefix='vcc-source-rom-stage-')
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
self.srcroot = self.root / 'Source With Spaces/platforms/macos'
self.starter = self.srcroot / 'Resources/StarterMedia'
self.starter.mkdir(parents=True)
self.media = {
'Hello World.dsk': b'AUTHORED HELLO DISK',
'Hello World.cas': b'AUTHORED HELLO CASSETTE',
'Blank Disk.dsk': b'AUTHORED BLANK DISK',
'Hello Hard Disk.dsk': b'AUTHORED HARD DISK',
'Blank OS9 Header.bin': b'\x00' * 8192,
}
for name, data in self.media.items():
(self.starter / name).write_bytes(data)
self.resources = self.root / 'Build With Spaces/VCC Plus.app/Contents/Resources'
self.bundle = self.root / 'Source ROM Bundle'
self.create_bundle(self.bundle)
def create_bundle(self, path, seed=0x12):
path.mkdir(parents=True, exist_ok=True)
roms = {}
for index, (name, size) in enumerate(stage.ROM_SIZES.items()):
data = bytes([(seed + index) & 255]) * size
(path / name).write_bytes(data)
roms[name] = {'bytes': size, 'sha256': hashlib.sha256(data).hexdigest()}
record = {'schemaVersion': 1, 'generator': 'vccplus-rom-build',
'source': {'project': 'authored unit fixture', 'revision': '0' * 40},
'assembler': {'version': 'authored fixture'}, 'roms': roms}
(path / 'ROMBuild.json').write_text(json.dumps(record, indent=2) + '\n')
(path / 'ROM_SOURCE_NOTICE.txt').write_text('Authored synthetic test bytes. No hardware ROM payload.\n')
return record
def apply(self, source=None, environ=None):
return stage.stage_source_roms(self.srcroot, self.resources, source,
environ={} if environ is None else environ)
def tree(self, root):
if not root.exists():
return None
return {str(path.relative_to(root)): (path.read_bytes(), path.stat().st_mode & 0o777,
path.stat().st_mtime_ns)
for path in root.rglob('*') if path.is_file()}
def alter_manifest(self, operation):
path = self.bundle / 'ROMBuild.json'
record = json.loads(path.read_text())
operation(record)
path.write_text(json.dumps(record))
def seed_old_kit(self):
kit = self.resources / 'StarterKit'
kit.mkdir(parents=True)
(kit / 'stale-private-marker').write_bytes(b'OLDER BUILD ONLY')
(self.resources / 'Blank OS9 Header.bin').write_bytes(self.media['Blank OS9 Header.bin'])
return kit
def test_valid_bundle_copies_owned_bytes_and_preserves_all_inputs(self):
bundle_before, starter_before = self.tree(self.bundle), self.tree(self.starter)
result = self.apply(self.bundle)
self.assertTrue(result['staged'])
kit = self.resources / 'StarterKit'
self.assertEqual(set(path.name for path in kit.iterdir()), stage.BUNDLE_FILES | set(stage.STARTER_FILES))
for name in stage.BUNDLE_FILES:
self.assertEqual((kit / name).read_bytes(), (self.bundle / name).read_bytes())
self.assertNotEqual((kit / name).stat().st_ino, (self.bundle / name).stat().st_ino)
for name, data in self.media.items():
self.assertEqual((kit / name).read_bytes(), data)
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
self.assertEqual(self.tree(self.bundle), bundle_before)
self.assertEqual(self.tree(self.starter), starter_before)
self.assertEqual(result['roms']['coco3.rom']['bytes'], 32768)
self.assertEqual(result['roms']['disk11.rom']['bytes'], 8192)
def test_missing_default_removes_only_owned_stale_kit(self):
kit = self.seed_old_kit()
untouched = self.resources / 'unrelated-resource.txt'
untouched.write_bytes(b'PRESERVE')
self.assertFalse(self.apply()['staged'])
self.assertFalse(kit.exists())
self.assertEqual(untouched.read_bytes(), b'PRESERVE')
self.assertTrue((self.resources / 'Blank OS9 Header.bin').is_file())
def test_explicit_and_environment_selection_never_fall_back(self):
default = self.srcroot / 'LocalResources/SourceROMBundle'
self.create_bundle(default, seed=0x44)
chosen = stage.selected_source(self.srcroot, self.bundle, {'VCC_SOURCE_ROM_DIR': str(default)})
self.assertEqual(chosen, self.bundle)
self.assertEqual(stage.selected_source(self.srcroot, environ={'VCC_SOURCE_ROM_DIR': str(self.bundle)}), self.bundle)
self.assertEqual(stage.selected_source(self.srcroot, environ={'VCC_SOURCE_ROM_DIR': ''}), default)
self.assertTrue(self.apply(environ={'VCC_SOURCE_ROM_DIR': str(self.bundle)})['staged'])
self.assertEqual((self.resources / 'StarterKit/coco3.rom').read_bytes(), (self.bundle / 'coco3.rom').read_bytes())
self.assertFalse(self.apply(self.root / 'Absent Explicit Bundle')['staged'])
self.assertFalse((self.resources / 'StarterKit').exists())
def test_default_bundle_is_used_only_from_source_specific_directory(self):
destination = self.srcroot / 'LocalResources/SourceROMBundle'
shutil.copytree(self.bundle, destination)
self.assertTrue(self.apply()['staged'])
shutil.rmtree(destination)
private = self.srcroot / 'LocalResources/ROMs'
private.mkdir()
(private / 'coco3.rom').write_bytes(b'PRIVATE PATH MUST NOT BE READ')
media = self.srcroot / 'LocalResources/Media'
media.mkdir()
(media / 'NitrOS-9 L2 V3.3.0.vhd').write_bytes(b'NO PRIVATE MEDIA DISCOVERY')
self.assertFalse(self.apply()['staged'])
self.assertFalse((self.resources / 'StarterKit').exists())
self.assertEqual((private / 'coco3.rom').read_bytes(), b'PRIVATE PATH MUST NOT BE READ')
def test_isolated_ui_product_never_selects_or_reads_prepared_rom_bundles(self):
default = self.srcroot / 'LocalResources/SourceROMBundle'
shutil.copytree(self.bundle, default)
original, default_original = self.tree(self.bundle), self.tree(default)
configurations = (
{'CONFIGURATION': 'UITesting'},
{'CONFIGURATION': 'Debug', 'SWIFT_ACTIVE_COMPILATION_CONDITIONS': 'DEBUG VCC_UI_TESTING'},
{'CONFIGURATION': 'Release', 'GCC_PREPROCESSOR_DEFINITIONS': 'VCC_UI_TESTING=1'},
)
for environment in configurations:
with self.subTest(environment=environment):
self.seed_old_kit()
environment['VCC_SOURCE_ROM_DIR'] = str(self.bundle)
with mock.patch.object(stage, 'selected_source', side_effect=AssertionError('UI build selected a ROM input')):
with mock.patch.object(stage, 'read_bundle', side_effect=AssertionError('UI build read a ROM input')):
result = self.apply(self.bundle, environment)
self.assertFalse(result['staged'])
self.assertTrue(result['ui_testing'])
self.assertFalse((self.resources / 'StarterKit').exists())
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
self.assertEqual(self.tree(self.bundle), original)
self.assertEqual(self.tree(default), default_original)
def test_isolated_ui_product_does_not_probe_unsafe_explicit_or_environment_paths(self):
alias = self.root / 'Unsafe ROM Alias'
alias.symlink_to(self.bundle, target_is_directory=True)
original = self.tree(self.bundle)
self.seed_old_kit()
# Parent traversal and linked input locations would both be refused for
# an ordinary source app, but neither location is probed by a UI build.
result = self.apply(alias / '..' / 'missing', {'CONFIGURATION': 'UITesting',
'VCC_SOURCE_ROM_DIR': str(alias)})
self.assertFalse(result['staged'])
self.assertFalse((self.resources / 'StarterKit').exists())
self.assertEqual(self.tree(self.bundle), original)
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
def test_atomic_replacement_removes_all_previous_profile_payload(self):
old = self.seed_old_kit()
nested = old / 'old-nested'
nested.mkdir()
(nested / 'payload').write_bytes(b'OLD')
self.assertTrue(self.apply(self.bundle)['staged'])
self.assertFalse((old / 'stale-private-marker').exists())
self.assertFalse(nested.exists())
self.assertEqual(set(path.name for path in old.iterdir()), stage.BUNDLE_FILES | set(stage.STARTER_FILES))
self.assertFalse(list(self.resources.glob('.source-rom-starter-*')))
def test_failed_atomic_exchange_preserves_previous_kit(self):
old = self.seed_old_kit()
before = self.tree(old)
with mock.patch.object(stage, 'exchange_directories', side_effect=OSError('authored exchange refusal')):
with self.assertRaises(OSError):
self.apply(self.bundle)
self.assertEqual(self.tree(old), before)
self.assertFalse(list(self.resources.glob('.source-rom-starter-*')))
def test_next_build_removes_only_interrupted_owned_staging(self):
self.resources.mkdir(parents=True)
stale = self.resources / '.source-rom-starter-abcdefgh'
stale.mkdir()
(stale / 'prior-private-rom-marker').write_bytes(b'PRIOR PROFILE')
(self.resources / '.source-rom-resource-12345678').write_bytes(b'PARTIAL HEADER')
unrelated = self.resources / '.source-rom-starter-not-a-generated-name'
unrelated.mkdir()
(unrelated / 'preserve').write_bytes(b'KEEP')
self.assertFalse(self.apply()['staged'])
self.assertFalse(stale.exists())
self.assertFalse((self.resources / '.source-rom-resource-12345678').exists())
self.assertEqual((unrelated / 'preserve').read_bytes(), b'KEEP')
# Even a helper-shaped name cannot redirect cleanup through a link.
stale.symlink_to(unrelated, target_is_directory=True)
with self.assertRaises(ValueError):
self.apply()
self.assertEqual((unrelated / 'preserve').read_bytes(), b'KEEP')
def test_invalid_manifest_fields_refuse_before_changing_build_output(self):
cases = (
lambda r: r.update(schemaVersion=2),
lambda r: r.update(schemaVersion=True),
lambda r: r.update(generator='different-generator'),
lambda r: r.update(source={}),
lambda r: r.update(roms=[]),
lambda r: r['roms'].pop('disk11.rom'),
lambda r: r['roms'].update({'extra.rom': {'bytes': 1, 'sha256': '0' * 64}}),
lambda r: r['roms']['coco3.rom'].update(bytes=32767),
lambda r: r['roms']['coco3.rom'].update(bytes=True),
lambda r: r['roms']['disk11.rom'].update(sha256='A' * 64),
lambda r: r['roms']['disk11.rom'].update(sha256='0' * 64),
)
old = self.seed_old_kit()
before = self.tree(self.resources)
for operation in cases:
with self.subTest(operation=operation):
self.create_bundle(self.bundle)
self.alter_manifest(operation)
with self.assertRaises(ValueError):
self.apply(self.bundle)
self.assertEqual(self.tree(self.resources), before)
self.assertTrue((old / 'stale-private-marker').is_file())
def test_invalid_json_duplicate_fields_and_nonobject_manifest_are_refused(self):
for data in (b'{', b'[]', b'{"schemaVersion":1,"schemaVersion":1}', b'\xff', b' ' * (stage.MANIFEST_LIMIT + 1)):
with self.subTest(data=data[:25]):
(self.bundle / 'ROMBuild.json').write_bytes(data)
with self.assertRaises((ValueError, UnicodeError)):
self.apply(self.bundle)
self.assertFalse(self.resources.exists())
def test_missing_extra_or_nonregular_bundle_members_are_refused(self):
for name in stage.BUNDLE_FILES:
with self.subTest(missing=name):
self.create_bundle(self.bundle)
(self.bundle / name).unlink()
with self.assertRaises(ValueError):
self.apply(self.bundle)
self.create_bundle(self.bundle)
(self.bundle / '.unexpected').write_bytes(b'NO EXTRA FILES')
with self.assertRaises(ValueError):
self.apply(self.bundle)
(self.bundle / '.unexpected').unlink()
(self.bundle / 'disk11.rom').unlink()
(self.bundle / 'disk11.rom').mkdir()
with self.assertRaises(ValueError):
self.apply(self.bundle)
self.assertFalse(self.resources.exists())
def test_rom_payload_sizes_and_manifest_identity_are_enforced(self):
for name, size in stage.ROM_SIZES.items():
for length in (0, size - 1, size + 1):
with self.subTest(name=name, length=length):
self.create_bundle(self.bundle)
(self.bundle / name).write_bytes(bytes(length))
with self.assertRaises(ValueError):
self.apply(self.bundle)
self.create_bundle(self.bundle)
(self.bundle / name).write_bytes(bytes([0x77]) * size)
with self.assertRaisesRegex(ValueError, 'do not match'):
self.apply(self.bundle)
self.assertFalse(self.resources.exists())
def test_notice_is_required_bounded_nonempty_utf8(self):
for notice in (b'', b' \n\t', b'\xff', b'notice\0', b'x' * (stage.NOTICE_LIMIT + 1)):
with self.subTest(notice=notice[:25]):
(self.bundle / 'ROM_SOURCE_NOTICE.txt').write_bytes(notice)
with self.assertRaises((ValueError, UnicodeError)):
self.apply(self.bundle)
self.assertFalse(self.resources.exists())
def test_source_bundle_leaf_and_ancestor_symlinks_are_refused(self):
alias = self.root / 'Linked Bundle'
alias.symlink_to(self.bundle, target_is_directory=True)
with self.assertRaises(ValueError):
self.apply(alias)
parent = self.root / 'Linked Parent'
parent.symlink_to(self.root, target_is_directory=True)
with self.assertRaisesRegex(ValueError, 'linked ancestor'):
self.apply(parent / self.bundle.name)
dangling = self.root / 'Dangling Bundle'
dangling.symlink_to(self.root / 'missing')
with self.assertRaises(ValueError):
self.apply(dangling)
self.assertFalse(self.resources.exists())
def test_source_members_must_not_be_symbolic_or_hard_links(self):
for name in stage.BUNDLE_FILES:
for kind in ('symbolic', 'hard'):
with self.subTest(name=name, kind=kind):
self.create_bundle(self.bundle)
original = self.bundle / name
victim = self.root / 'Input Victim'
victim.write_bytes(original.read_bytes())
before = victim.read_bytes()
original.unlink()
if kind == 'symbolic':
original.symlink_to(victim)
else:
os.link(victim, original)
with self.assertRaises(ValueError):
self.apply(self.bundle)
self.assertEqual(victim.read_bytes(), before)
original.unlink()
victim.unlink()
self.assertFalse(self.resources.exists())
def test_output_shape_and_linked_ancestors_cannot_redirect_writes(self):
with self.assertRaisesRegex(ValueError, 'APP.app'):
stage.stage_source_roms(self.srcroot, self.root / 'Not An App/Resources', self.bundle, {})
victim = self.root / 'Untouched Output Victim'
victim.mkdir()
build = self.root / 'Linked Build'
build.symlink_to(victim, target_is_directory=True)
with self.assertRaisesRegex(ValueError, 'linked ancestor'):
stage.stage_source_roms(self.srcroot, build / 'VCC.app/Contents/Resources', self.bundle, {})
self.assertEqual(list(victim.iterdir()), [])
def test_existing_kit_links_refuse_cleanup_or_replacement(self):
kit = self.seed_old_kit()
victim = self.root / 'Kit Victim'
victim.write_bytes(b'UNCHANGED')
for kind in ('symbolic', 'hard', 'directory'):
with self.subTest(kind=kind):
link = kit / 'unsafe-member'
if kind == 'symbolic':
link.symlink_to(victim)
elif kind == 'hard':
os.link(victim, link)
else:
link.symlink_to(self.root, target_is_directory=True)
for source in (None, self.bundle):
with self.assertRaises(ValueError):
self.apply(source)
self.assertEqual(victim.read_bytes(), b'UNCHANGED')
self.assertTrue((kit / 'stale-private-marker').is_file())
link.unlink()
shutil.rmtree(kit)
kit.symlink_to(self.bundle, target_is_directory=True)
with self.assertRaises(ValueError):
self.apply()
self.assertEqual(set(path.name for path in self.bundle.iterdir()), stage.BUNDLE_FILES)
def test_linked_standalone_header_is_never_replaced(self):
self.resources.mkdir(parents=True)
victim = self.root / 'Header Victim'
victim.write_bytes(b'UNCHANGED')
for kind in ('symbolic', 'hard'):
with self.subTest(kind=kind):
destination = self.resources / 'Blank OS9 Header.bin'
if kind == 'symbolic':
destination.symlink_to(victim)
else:
os.link(victim, destination)
with self.assertRaises(ValueError):
self.apply(self.bundle)
self.assertEqual(victim.read_bytes(), b'UNCHANGED')
self.assertFalse((self.resources / 'StarterKit').exists())
destination.unlink()
def test_authored_input_links_and_bad_header_refuse_before_mutation(self):
header = self.starter / 'Blank OS9 Header.bin'
header.write_bytes(b'SHORT')
with self.assertRaises(ValueError):
self.apply(self.bundle)
header.write_bytes(self.media[header.name])
target = self.starter / 'Hello World.cas'
victim = self.root / 'Authored Input Victim'
victim.write_bytes(target.read_bytes())
target.unlink()
target.symlink_to(victim)
with self.assertRaises(ValueError):
self.apply(self.bundle)
self.assertFalse(self.resources.exists())
def test_inputs_and_build_output_must_not_overlap(self):
self.resources.mkdir(parents=True)
embedded = self.resources / 'StarterKit'
shutil.copytree(self.bundle, embedded)
before = self.tree(embedded)
with self.assertRaisesRegex(ValueError, 'overlap'):
self.apply(embedded)
self.assertEqual(self.tree(embedded), before)
with self.assertRaisesRegex(ValueError, 'parent traversal'):
self.apply(self.bundle / '..' / self.bundle.name)
def run_build_phase(self, *, profile, configuration='Release', source=None):
helper = self.srcroot.parents[1] / 'tools/roms/stage.py'
helper.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(SCRIPT, helper)
environment = os.environ.copy()
environment.update(SRCROOT=str(self.srcroot), TARGET_BUILD_DIR=str(self.resources.parents[2]),
UNLOCALIZED_RESOURCES_FOLDER_PATH='VCC Plus.app/Contents/Resources',
VCC_BUILD_CONTENT_PROFILE=profile, CONFIGURATION=configuration)
environment.pop('VCC_SOURCE_ROM_DIR', None)
if source is not None:
environment['VCC_SOURCE_ROM_DIR'] = str(source)
return subprocess.run(['/bin/sh', str(ROOT / 'platforms/macos/Packaging/copy-development-starter.sh')],
env=environment, capture_output=True, text=True)
def test_source_build_phase_accepts_bundle_in_debug_and_release_without_private_discovery(self):
private = self.srcroot / 'LocalResources/ROMs'
private.mkdir(parents=True)
(private / 'coco3.rom').write_bytes(b'INVALID NORMAL PROFILE INPUT')
for configuration in ('Debug', 'Release'):
with self.subTest(configuration=configuration):
result = self.run_build_phase(profile='source', configuration=configuration, source=self.bundle)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual((self.resources / 'StarterKit/coco3.rom').read_bytes(), (self.bundle / 'coco3.rom').read_bytes())
self.assertEqual((private / 'coco3.rom').read_bytes(), b'INVALID NORMAL PROFILE INPUT')
def test_source_ui_testing_build_phase_omits_prepared_bundle_and_stale_kit(self):
default = self.srcroot / 'LocalResources/SourceROMBundle'
shutil.copytree(self.bundle, default)
original, default_original = self.tree(self.bundle), self.tree(default)
self.seed_old_kit()
result = self.run_build_phase(profile='source', configuration='UITesting', source=self.bundle)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('Isolated UI-test build', result.stdout)
self.assertFalse((self.resources / 'StarterKit').exists())
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
self.assertEqual(self.tree(self.bundle), original)
self.assertEqual(self.tree(default), default_original)
def test_source_build_phase_rejects_malformed_present_bundle(self):
(self.bundle / 'coco3.rom').write_bytes(b'INVALID')
result = self.run_build_phase(profile='source', source=self.bundle)
self.assertNotEqual(result.returncode, 0)
self.assertIn('invalid size', result.stderr)
self.assertFalse((self.resources / 'StarterKit').exists())
def test_normal_debug_phase_preserves_existing_private_workflow(self):
private = self.srcroot / 'LocalResources/ROMs'
private.mkdir(parents=True)
expected = {'coco3.rom': bytes([0x33]) * 32768, 'disk11.rom': bytes([0x44]) * 8192,
'orch90.rom': bytes([0x55]) * 8192}
for name, data in expected.items():
(private / name).write_bytes(data)
result = self.run_build_phase(profile='normal', configuration='Debug', source=self.bundle)
self.assertEqual(result.returncode, 0, result.stderr)
kit = self.resources / 'StarterKit'
for name, data in expected.items():
self.assertEqual((kit / name).read_bytes(), data)
self.assertFalse((kit / 'ROMBuild.json').exists())
for name, data in self.media.items():
self.assertEqual((kit / name).read_bytes(), data)
def test_normal_release_phase_still_excludes_optional_development_roms(self):
self.seed_old_kit()
result = self.run_build_phase(profile='normal', configuration='Release', source=self.bundle)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertFalse((self.resources / 'StarterKit').exists())
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
if __name__ == '__main__':
unittest.main()
+248
View File
@@ -0,0 +1,248 @@
# SPDX-License-Identifier: GPL-3.0-or-later
"""Offline content-profile checks using authored disposable inputs only."""
from contextlib import closing
import hashlib
import json
import os
from pathlib import Path
import shutil
import sqlite3
import subprocess
import sys
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
RESOURCES = (
('tools/catalog/package.py', 'tools/catalog/app-schema.sql', 'SoftwareLibrary/catalog.sqlite'),
('tools/knowledge/package_archive.py', 'tools/knowledge/archive-library.sql', 'ArchiveLibrary/ArchiveLibrary.sqlite'),
)
class BuildContentTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory(prefix='vcc-content-profile-')
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
self.source = self.root / 'Source With Spaces'
self.build = self.root / 'Build With Spaces'
for relative in ['tools/build_content.py', *(p for r in RESOURCES for p in r[:2])]:
destination = self.source / relative
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(ROOT / relative, destination)
self.tempting = []
for relative in ('media/software-library-full/prepared/app-catalog.sqlite',
'media/app-inputs/archive-library/ArchiveLibrary.sqlite',
'media/knowledge-base/archive-library/ArchiveLibrary.sqlite'):
target = self.source / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(b'PRIVATE-DATA-TEST-MARKER\x00')
self.tempting.append(target)
def run_script(self, resource, *extra, profile='source', env_profile=None, output=True, check=True):
command = [sys.executable, str(self.source / resource[0])]
if profile is not None:
command += ['--content-profile', profile]
if output:
command += ['--output', str(self.build / resource[2])]
environment = os.environ.copy()
environment.pop('VCC_BUILD_CONTENT_PROFILE', None)
if env_profile is not None:
environment['VCC_BUILD_CONTENT_PROFILE'] = env_profile
completed = subprocess.run([*command, *extra], cwd=self.source,
env=environment, capture_output=True, text=True)
if check:
self.assertEqual(completed.returncode, 0, completed.stderr)
return completed
def metadata(self, resource, **kwargs):
return json.loads(self.run_script(resource, **kwargs).stdout.split(': ', 1)[1])
def test_software_schema_and_reader_contract_have_no_content(self):
resource = RESOURCES[0]
info = self.metadata(resource)
self.assertTrue(info['empty'])
self.assertEqual((info['titles'], info['releases'], info['assets'], info['thumbnails']), (0, 0, 0, 0))
path = self.build / resource[2]
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
self.assertEqual(db.execute('PRAGMA user_version').fetchone(), (1,))
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['stage'], 'empty-source')
for table in ('title', 'package', 'release', 'member', 'download_location',
'asset', 'asset_delivery', 'release_asset', 'setup', 'source_hint'):
self.assertEqual(db.execute('SELECT count(*) FROM ' + table).fetchone(), (0,))
self.assertEqual(db.execute('SELECT * FROM browse').fetchall(), [])
self.assertEqual(db.execute('SELECT ra.release_id,a.*,d.manual_url FROM release_asset ra '
'JOIN asset a ON a.id=ra.asset_id LEFT JOIN asset_delivery d '
'ON d.asset_id=a.id').fetchall(), [])
self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(), [('ok',)])
def test_archive_schema_supports_empty_browse_search_and_reader(self):
resource = RESOURCES[1]
info = self.metadata(resource)
self.assertTrue(info['empty'])
path = self.build / resource[2]
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
db.execute('PRAGMA query_only=ON')
db.execute('PRAGMA trusted_schema=OFF')
self.assertEqual(db.execute('PRAGMA application_id').fetchone(), (0x56434341,))
self.assertEqual(db.execute('PRAGMA user_version').fetchone(), (2,))
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['source_snapshot'], 'empty-source')
for table in ('document', 'page', 'reader_section', 'page_terms', 'reviewed_example'):
self.assertEqual(db.execute('SELECT count(*) FROM ' + table).fetchone(), (0,))
self.assertEqual(db.execute('SELECT d.* FROM document d ORDER BY d.title COLLATE NOCASE,d.sha256 LIMIT ? OFFSET ?', (50, 0)).fetchall(), [])
self.assertEqual(db.execute('SELECT d.*,p.page_number FROM page_terms JOIN page p '
'ON p.id=page_terms.rowid JOIN document d ON d.id=p.document_id '
'WHERE page_terms MATCH ? ORDER BY bm25(page_terms,1.0,4.0),p.id '
'LIMIT ? OFFSET ?', ('"basic"', 50, 0)).fetchall(), [])
self.assertEqual(db.execute('SELECT p.page_label,s.text FROM document d JOIN page p '
'ON p.document_id=d.id JOIN reader_section s ON s.page_id=p.id '
'WHERE d.sha256=? AND p.page_number=?', ('0'*64, 1)).fetchall(), [])
self.assertEqual(db.execute('PRAGMA foreign_key_check').fetchall(), [])
def test_repeat_build_keeps_identical_bytes_and_timestamp(self):
for resource in RESOURCES:
with self.subTest(resource=resource[0]):
first = self.metadata(resource)
path = self.build / resource[2]
original, stamp = path.read_bytes(), path.stat().st_mtime_ns
second = self.metadata(resource)
self.assertEqual(first, second)
self.assertEqual(path.read_bytes(), original)
self.assertEqual(path.stat().st_mtime_ns, stamp)
self.assertEqual(path.stat().st_mode & 0o777, 0o444)
self.assertEqual(first['sha256'], hashlib.sha256(original).hexdigest())
self.assertNotIn(b'PRIVATE-DATA-TEST-MARKER', original)
def test_source_profile_ignores_explicit_and_default_maintainer_inputs(self):
for resource in RESOURCES:
with self.subTest(resource=resource[0]):
self.run_script(resource, '--source', str(self.tempting[0]))
for source in self.tempting:
self.assertEqual(source.read_bytes(), b'PRIVATE-DATA-TEST-MARKER\x00')
self.assertNotIn(b'PRIVATE-DATA-TEST-MARKER', (self.build / resource[2]).read_bytes())
def test_old_resource_is_replaced_but_invalid_schema_preserves_it(self):
for resource in RESOURCES:
with self.subTest(resource=resource[0]):
path = self.build / resource[2]
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b'OLD-BUILD-PRIVATE-CONTENT')
self.metadata(resource)
original = path.read_bytes()
self.assertNotIn(b'OLD-BUILD-PRIVATE-CONTENT', original)
with (self.source / resource[1]).open('a') as stream:
stream.write('\nINVALID SQL INTENTIONALLY AUTHORED FOR TEST;\n')
self.assertNotEqual(self.run_script(resource, check=False).returncode, 0)
self.assertEqual(path.read_bytes(), original)
self.assertFalse(list(path.parent.glob('.empty-library-*.sqlite')))
def test_linked_outputs_never_change_the_target(self):
for resource in RESOURCES:
for kind in ('symlink', 'hardlink', 'dangling'):
with self.subTest(resource=resource[0], kind=kind):
path = self.build / resource[2]
path.parent.mkdir(parents=True, exist_ok=True)
target = self.root / 'link-victim'
if kind != 'dangling':
target.write_bytes(b'unchanged')
if kind == 'hardlink':
os.link(target, path)
else:
path.symlink_to(target)
self.assertNotEqual(self.run_script(resource, check=False).returncode, 0)
if kind != 'dangling':
self.assertEqual(target.read_bytes(), b'unchanged')
target.unlink()
path.unlink()
def test_linked_output_ancestor_cannot_redirect_a_build_resource(self):
target = self.root / 'untouched-directory'
target.mkdir()
self.build.symlink_to(target, target_is_directory=True)
for resource in RESOURCES:
with self.subTest(resource=resource[0]):
rejected = self.run_script(resource, check=False)
self.assertNotEqual(rejected.returncode, 0)
self.assertIn('linked ancestor', rejected.stderr)
self.assertEqual(list(target.iterdir()), [])
def test_standard_temporary_path_alias_remains_usable(self):
if not Path('/tmp').is_symlink() or Path('/tmp').resolve() != Path('/private/tmp'):
self.skipTest('macOS temporary-directory alias is absent')
with tempfile.TemporaryDirectory(dir='/tmp', prefix='vcc-content-alias-') as temporary:
self.build = Path(temporary)
for resource in RESOURCES:
self.assertTrue(self.metadata(resource)['empty'])
def test_transaction_sidecars_are_rejected_including_empty_and_dangling(self):
for resource in RESOURCES:
self.metadata(resource)
path = self.build / resource[2]
original = path.read_bytes()
for suffix in ('-wal', '-shm', '-journal'):
for kind in ('empty', 'data', 'dangling'):
with self.subTest(resource=resource[0], suffix=suffix, kind=kind):
sidecar = Path(str(path) + suffix)
if kind == 'dangling':
sidecar.symlink_to(self.root / 'missing-sidecar-target')
else:
sidecar.write_bytes(b'' if kind == 'empty' else b'old transaction')
rejected = self.run_script(resource, check=False)
self.assertNotEqual(rejected.returncode, 0)
self.assertIn('sidecars', rejected.stderr)
self.assertEqual(path.read_bytes(), original)
sidecar.unlink()
def test_source_profile_rejects_full_release_requirement(self):
for resource, flag in zip(RESOURCES, ('--require-full', '--require')):
self.assertNotEqual(self.run_script(resource, flag, check=False).returncode, 0)
self.assertFalse((self.build / resource[2]).exists())
def test_source_check_only_builds_schema_without_input_or_output(self):
self.run_script(RESOURCES[0], '--check-only', output=False)
self.run_script(RESOURCES[1], output=False)
self.assertFalse(self.build.exists())
def test_environment_selects_source_and_explicit_cli_overrides_it(self):
for resource in RESOURCES:
with self.subTest(resource=resource[0]):
self.assertTrue(self.metadata(resource, profile=None, env_profile='source')['empty'])
# The normal path must read and reject the authored corrupt maintainer input.
rejected = self.run_script(resource, profile='normal', env_profile='source', check=False)
self.assertNotEqual(rejected.returncode, 0)
self.assertNotEqual(self.run_script(resource, profile=None, env_profile='unknown', check=False).returncode, 0)
self.assertEqual(self.run_script(resource, profile='source', env_profile='unknown').returncode, 0)
def test_normal_archive_retains_verified_populated_input_exactly(self):
source = self.tempting[1]
source.unlink()
with closing(sqlite3.connect(source)) as db:
db.executescript((self.source / RESOURCES[1][1]).read_text())
db.execute("INSERT INTO document(id,sha256,title,category,collection,format,source_url,byte_size,page_count) "
"VALUES(1,?,'Authored fixture','Books','Books','pdf','https://example.test/fixture.pdf',20,1)", ('a'*64,))
db.commit()
original = source.read_bytes()
digest = hashlib.sha256(original).hexdigest()
(source.parent / 'archive-library.json').write_text(json.dumps(dict(schema_version=2,
sha256=digest, bytes=len(original), documents=1, reader_sections=0, ranged_members=0)))
(source.parent / 'ArchiveLibrary.sqlite.sha256').write_text(digest + ' ArchiveLibrary.sqlite\n')
self.run_script(RESOURCES[1], '--require', profile=None)
self.assertEqual((self.build / RESOURCES[1][2]).read_bytes(), original)
self.assertEqual(source.read_bytes(), original)
def test_default_normal_behavior_still_uses_existing_fallbacks(self):
for path in self.tempting:
path.unlink()
self.assertIn('absent', self.run_script(RESOURCES[0], profile=None).stdout)
path = self.build / RESOURCES[0][2]
with closing(sqlite3.connect(path)) as db:
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['stage'], 'empty-development')
archive = self.build / RESOURCES[1][2]
archive.parent.mkdir(parents=True, exist_ok=True)
archive.write_bytes(b'old index')
self.run_script(RESOURCES[1], profile=None)
self.assertFalse(archive.exists())
if __name__ == '__main__':
unittest.main()
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Build the reviewed CoCo Letters source and a DOS-bootable DSK without downloading."""
import argparse
import json
import os
from pathlib import Path
import shutil
import subprocess
ROOT=Path(__file__).resolve().parents[2]
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--toolchain', type=Path, required=True)
parser.add_argument('--output', type=Path, default=ROOT/'.build/coco-letters')
args=parser.parse_args()
prefix=args.toolchain.resolve();out=args.output.resolve()
features=json.loads((prefix/'VCC-toolchain.json').read_text())
if not features['standaloneDisk']:
raise SystemExit('CoCo Letters requires the optional private standalone disk source; see docs/development/TOOLCHAIN.md')
if out.exists(): raise SystemExit('Choose a fresh --output directory')
out.mkdir(parents=True)
env=dict(os.environ, PATH=str(prefix/'bin')+os.pathsep+os.environ['PATH'])
env.setdefault('DEVELOPER_DIR','/Applications/Xcode.app/Contents/Developer')
for name in ('decbfile','bcontrol','bsound'):
shutil.copytree(ROOT/'third_party/cmoc-libraries'/name,out/name)
subprocess.run(['make','-j4',f'lib{name}.a'],cwd=out/name,env=env,check=True)
game=out/'game';shutil.copytree(ROOT/'third_party/examples/coco-letters',game)
# Upstream generates two outputs in one rule; serial make avoids duplicate writes.
subprocess.run(['make','DECBFILE_LIB_DIR=../decbfile','BCONTROL_LIB_DIR=../bcontrol','BSOUND_LIB_DIR=../bsound'],cwd=game,env=env,check=True)
disk=game/'coco-letters-0.1.0.dsk'
if disk.stat().st_size!=35*18*256: raise SystemExit('Unexpected disk geometry')
print(f'Built disk: {disk}\nMount in FD0, select RGB, then type DOS. Requires CoCo 3, 128 KB or more.')
if __name__=='__main__': main()
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Build pinned Perl for Make recipes; no host Perl is required after packaging."""
import argparse
import os
from pathlib import Path
import shutil
import subprocess
ROOT = Path(__file__).resolve().parents[2]
VERSION = '5.44.0'
def build(output, architectures, jobs):
source = ROOT / 'third_party/perl'
env = dict(os.environ)
env.setdefault('DEVELOPER_DIR', '/Applications/Xcode.app/Contents/Developer')
prefixes = {}
for arch in architectures:
work = output / arch / 'work'
prefix = output / arch / 'install'
shutil.copytree(source, work)
flags = f'-arch {arch} -mmacosx-version-min=14.0'
commands = [
['sh', 'Configure', '-des', f'-Dprefix={prefix}', '-Dcc=clang',
f'-Dccflags={flags}', f'-Dldflags={flags}',
f'-Dlddlflags=-bundle -undefined dynamic_lookup {flags}',
'-Duserelocatableinc', '-Uuseshrplib', '-Uuseithreads',
'-Dman1dir=none', '-Dman3dir=none'],
['/usr/bin/make', f'-j{jobs}'], ['/usr/bin/make', 'install']]
for command in commands:
print('+', ' '.join(map(str, command)), flush=True)
subprocess.run(command, cwd=work, env=env, check=True)
prefixes[arch] = prefix
return prefixes
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--architectures', nargs='+', choices=['arm64', 'x86_64'], default=['arm64', 'x86_64'])
parser.add_argument('--jobs', type=int, default=4)
args = parser.parse_args()
output = args.output.resolve()
if output.exists(): parser.error('Choose a fresh output directory')
output.mkdir(parents=True)
build(output, list(dict.fromkeys(args.architectures)), max(1, args.jobs))
if __name__ == '__main__': main()
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Build a relocatable macOS CMOC runtime; host SDK tools are build-time only."""
import argparse
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[2]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, default=ROOT / '.build/cmoc-runtime')
parser.add_argument('--architectures', nargs='+', choices=['arm64', 'x86_64'], default=['arm64', 'x86_64'])
parser.add_argument('--jobs', type=int, default=4)
parser.add_argument('--standalone-disk-source', type=Path, help='Optional private DECB-derived disk driver; never downloaded automatically')
args = parser.parse_args()
output = args.output.resolve()
if output.exists(): parser.error('Choose a fresh output directory')
output.mkdir(parents=True)
architectures = list(dict.fromkeys(args.architectures))
base = dict(os.environ)
base.setdefault('DEVELOPER_DIR', '/Applications/Xcode.app/Contents/Developer')
def run(*command, cwd=ROOT, env=base):
print('+', ' '.join(map(str, command)), flush=True)
subprocess.run(list(map(str, command)), cwd=cwd, env=env, check=True)
run(sys.executable, ROOT / 'tools/toolchain/verify-sources.py')
roots = []
for architecture in architectures:
arch_root = output / architecture
run(sys.executable, ROOT / 'tools/toolchain/build.py', '--output', arch_root,
'--host-arch', architecture, '--jobs', args.jobs)
prefix = arch_root / 'install'
env = dict(base, CC='clang', CXX='clang++',
CFLAGS=f'-O2 -arch {architecture} -mmacosx-version-min=14.0 -Wno-error=implicit-function-declaration',
CXXFLAGS=f'-O2 -arch {architecture} -mmacosx-version-min=14.0',
LDFLAGS=f'-arch {architecture} -mmacosx-version-min=14.0')
for name in ('mcpp', 'make'):
source = arch_root / 'work' / name
shutil.copytree(ROOT / 'third_party' / name, source)
for file in source.rglob('*'):
if file.is_file(): os.utime(file, (946684800, 946684800))
options = [f'--host={architecture}-apple-darwin', '--without-guile', '--disable-dependency-tracking'] if name == 'make' else []
run('./configure', f'--prefix={prefix}', *options, cwd=source, env=env)
run('/usr/bin/make', f'-j{args.jobs}', cwd=source, env=env)
run('/usr/bin/make', 'install', cwd=source, env=env)
run('xcrun', 'clang', '-O2', '-arch', architecture, '-mmacosx-version-min=14.0',
ROOT / 'tools/toolchain/cpp-mcpp.c', '-o', prefix / 'bin/cpp')
roots.append(prefix)
bundle = output / 'Toolchain'
for name in ('bin', 'libexec', 'share', 'LICENSES'): (bundle / name).mkdir(parents=True)
shutil.copytree(roots[0] / 'share/cmoc', bundle / 'share/cmoc')
shutil.copyfile(ROOT / 'tools/toolchain/cmoc-runtime.sh', bundle / 'bin/cmoc')
(bundle / 'bin/cmoc').chmod(0o755)
for tool in ('cmoc', 'cpp', 'mcpp', 'make', 'lwasm', 'lwlink', 'lwar', 'writecocofile'):
target = bundle / ('libexec' if tool == 'cmoc' else 'bin') / tool
inputs = [prefix / 'bin' / tool for prefix in roots]
if len(inputs) > 1: run('xcrun', 'lipo', '-create', *inputs, '-output', target)
else: shutil.copyfile(inputs[0], target)
target.chmod(0o755)
notices = {'CMOC-COPYING.txt': 'cmoc/COPYING', 'CMOC-README.txt': 'cmoc/README',
'LWTools-COPYING.txt': 'lwtools/COPYING', 'LWTools-GPL3.txt': 'lwtools/GPL3',
'MCPP-LICENSE.txt': 'mcpp/LICENSE', 'MAKE-COPYING.txt': 'make/COPYING'}
for target, source in notices.items(): shutil.copyfile(ROOT / 'third_party' / source, bundle / 'LICENSES' / target)
shutil.copyfile(ROOT / 'tools/toolchain/patches/README.md', bundle / 'LICENSES/CMOC-VCC-EXTENSIONS.txt')
record = {'schemaVersion': 1, 'cmoc': '0.1.100', 'lwtools': '4.25', 'mcpp': '2.7.2',
'make': '4.4.1', 'vccDebugMetadata': 1, 'vccAddZeroFlagsFix': 1, 'vccIndexedByteSubtractionFix': 1, 'vccIndexedArgumentLoadFix': 1, 'vccWriteCoCoFileBeforeDir': 1, 'vccLWASMProjectInputs': 1, 'architectures': architectures, 'privateRuntimeInputs': False,
'minimumMacOS': '14.0', 'appleDeveloperToolsRequiredAtRuntime': False}
(bundle / 'manifest.json').write_text(json.dumps(record, indent=2) + '\n')
perl_output = output / 'perl'
run(sys.executable, ROOT / 'tools/toolchain/build-perl.py', '--output', perl_output,
'--architectures', *architectures, '--jobs', args.jobs)
perl_prefixes = [argument for arch in architectures
for argument in ('--prefix', f'{arch}={perl_output / arch / "install"}')]
run(sys.executable, ROOT / 'tools/toolchain/stage-perl.py', bundle, *perl_prefixes)
run(sys.executable, ROOT / 'tools/toolchain/stage-libraries.py', bundle, '--jobs', args.jobs)
run(sys.executable, ROOT / 'tools/toolchain/stage-decbfile.py', bundle, '--jobs', args.jobs)
if args.standalone_disk_source:
run(sys.executable, ROOT / 'tools/toolchain/stage-standalone-disk.py', bundle, '--source', args.standalone_disk_source)
run(sys.executable, ROOT / 'tools/toolchain/verify-runtime.py', bundle)
print(f'Runtime ready: {bundle}\nStage at platforms/macos/LocalResources/ToolchainBundle for app builds.')
if __name__ == '__main__': main()
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Build the pinned host toolchain in ignored storage; never download or install globally."""
import argparse
import hashlib
import json
import sys
import os
from pathlib import Path
import shutil
import subprocess
ROOT = Path(__file__).resolve().parents[2]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, default=ROOT / '.build/toolchain')
parser.add_argument('--mc6839-rom', type=Path, help='Optional private Float09.bin')
parser.add_argument('--standalone-disk-source', type=Path, help='Optional private upstream dskcon-standalone.c')
parser.add_argument('--jobs', type=int, default=4)
parser.add_argument('--host-arch', choices=['native', 'arm64', 'x86_64'], default='native')
args = parser.parse_args()
out = args.output.resolve()
if out == ROOT or ROOT / 'third_party' in out.parents or out == ROOT / 'third_party':
parser.error('Choose a separate build output directory')
subprocess.run([sys.executable, str(ROOT / 'tools/toolchain/verify-sources.py')], check=True)
prefix = out / 'install'
work = out / 'work'
work.mkdir(parents=True, exist_ok=True)
env = dict(os.environ)
env['PATH'] = str(prefix / 'bin') + os.pathsep + env['PATH']
env.setdefault('DEVELOPER_DIR', '/Applications/Xcode.app/Contents/Developer')
if args.host_arch != 'native':
if sys.platform != 'darwin':
parser.error('--host-arch is currently a macOS build option')
flags = f'-O2 -arch {args.host_arch} -mmacosx-version-min=14.0'
env.update(CC='clang', CXX='clang++', CFLAGS=flags, CXXFLAGS=flags, LDFLAGS=f'-arch {args.host_arch} -mmacosx-version-min=14.0')
def run(*command, cwd):
print('+', ' '.join(map(str, command)), flush=True)
subprocess.run(list(map(str, command)), cwd=cwd, env=env, check=True)
# Copies protect pristine vendored snapshots from generated makefiles/objects.
for name in ('lwtools', 'cmoc'):
target = work / name
if target.exists():
raise SystemExit(f'{target} already exists; choose a fresh --output for a reproducible build')
shutil.copytree(ROOT / 'third_party' / name, target)
# Git does not retain mtimes. Equal times keep shipped generated files
# from unnecessarily invoking version-specific Autotools generators.
for item in target.rglob('*'):
if item.is_file():
os.utime(item, (946684800, 946684800))
# Retain pristine upstream snapshots; apply the reviewed patches only to the build copies.
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/cmoc-debug-metadata.patch', cwd=work / 'cmoc')
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/cmoc-add-zero-flags.patch', cwd=work / 'cmoc')
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/cmoc-indexed-byte-subtraction.patch', cwd=work / 'cmoc')
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/cmoc-indexed-argument-load.patch', cwd=work / 'cmoc')
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/writecocofile-before-dir.patch', cwd=work / 'cmoc')
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/lwtools-project-inputs.patch', cwd=work / 'lwtools')
run('make', f'-j{args.jobs}', f'PREFIX={prefix}', cwd=work / 'lwtools')
run('make', f'PREFIX={prefix}', 'install', cwd=work / 'lwtools')
# Optional private inputs are checked against the reviewed upstream release.
record = json.loads((ROOT / 'third_party/manifests/cmoc.json').read_text())
for supplied, relative in ((args.mc6839_rom, 'src/float/Float09.bin'),
(args.standalone_disk_source, 'src/stdlib/dskcon-standalone.c')):
if supplied:
if hashlib.sha256(supplied.read_bytes()).hexdigest() != record['excludedFileSHA256'][relative]:
raise SystemExit(f'Private input does not match reviewed CMOC release: {supplied}')
shutil.copyfile(supplied, work / 'cmoc' / relative)
for extension in ('am', 'in'):
if args.mc6839_rom:
path = work / 'cmoc/src/float' / ('Makefile.' + extension)
text = path.read_text().replace('cmoclib_DATA = ', 'cmoclib_DATA = float-ctor.mc6839_o float-ctor.mc6839_os9_o libcmoc-float-mc6839.a libcmoc-float-mc6839_os9.a ')
path.write_text(text); os.utime(path, (946684800, 946684800))
if args.standalone_disk_source:
path = work / 'cmoc/src/stdlib' / ('Makefile.' + extension)
text = path.read_text()
# Restore the omitted implementation beside the other C runtime sources.
assert 'C_COCO_BASIC =' in text
text = text.replace('C_COCO_BASIC =', 'C_COCO_BASIC = dskcon-standalone.c')
path.write_text(text); os.utime(path, (946684800, 946684800))
if sys.platform == 'darwin':
wrapper = prefix / 'bin/cpp'
shutil.copyfile(ROOT / 'tools/toolchain/cpp-macos.sh', wrapper)
wrapper.chmod(0o755)
run('./configure', f'--prefix={prefix}', '--with-writecocofile', cwd=work / 'cmoc')
run('make', f'-j{args.jobs}', cwd=work / 'cmoc')
run('make', 'install', cwd=work / 'cmoc')
(prefix / 'VCC-toolchain.json').write_text(json.dumps({'mc6839': bool(args.mc6839_rom), 'standaloneDisk': bool(args.standalone_disk_source)}, indent=2) + '\n')
for tool in ('cmoc', 'lwasm', 'lwlink', 'lwar'):
run(prefix / 'bin' / tool, '--version', cwd=out)
print(f'Toolchain ready in {prefix}; no app or global installation changed.')
if __name__ == '__main__':
main()
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Compile the pinned libraries and a CoCo program; optionally execute it in VCC."""
import argparse
import os
import json
from pathlib import Path
import shutil
import subprocess
ROOT = Path(__file__).resolve().parents[2]
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--toolchain', type=Path, default=ROOT / '.build/toolchain/install')
p.add_argument('--output', type=Path, default=ROOT / '.build/toolchain-check')
p.add_argument('--rom', type=Path, help='Optional private 32 KiB CoCo 3 ROM; read only')
p.add_argument('--disk-rom', type=Path, help='Private Disk BASIC ROM for the optional game check')
p.add_argument('--game-disk', type=Path, help='Source-built CoCo Letters DSK; read only')
args = p.parse_args()
if (args.disk_rom or args.game_disk) and not (args.rom and args.disk_rom and args.game_disk):
p.error('The game check needs --rom, --disk-rom and --game-disk together')
out=args.output.resolve();out.mkdir(parents=True,exist_ok=True)
prefix=args.toolchain.resolve()
env=dict(os.environ, PATH=str(prefix/'bin')+os.pathsep+os.environ['PATH'])
env.setdefault('DEVELOPER_DIR','/Applications/Xcode.app/Contents/Developer')
def run(*command, cwd=out, **kw):
print('+', ' '.join(map(str,command)),flush=True)
return subprocess.run(list(map(str,command)),cwd=cwd,env=env,check=True,**kw)
run('python3',ROOT/'tools/toolchain/verify-sources.py')
for source in sorted((ROOT/'third_party/cmoc-libraries').iterdir()):
if not source.is_dir(): continue
dest=out/source.name
if dest.exists(): raise SystemExit(f'{dest} already exists; use a fresh --output')
shutil.copytree(source,dest)
run('make','-j4',f'lib{source.name}.a',cwd=dest)
assert (dest/f'lib{source.name}.a').stat().st_size > 0
run(prefix/'bin/cmoc','--coco','--intermediate','-o',out/'HELLO.BIN',ROOT/'tests/toolchain/hello.c')
data=(out/'HELLO.BIN').read_bytes()
offset=0;blocks=0
while offset<len(data) and data[offset]==0:
size=int.from_bytes(data[offset+1:offset+3],'big');offset+=5+size;blocks+=1
assert blocks and offset+5==len(data) and data[offset:offset+3]==b'\xff\0\0', 'Invalid DECB executable'
# A small known assembly result verifies the checked-in assembler independently.
(out/'known.asm').write_text(' org $4000\nstart lda #$42\n sta $0400\n rts\n end start\n')
run(prefix/'bin/lwasm','--format=raw','--output=known.bin','--list=known.lst','known.asm')
assert (out/'known.bin').read_bytes()==bytes.fromhex('8642b7040039')
features=json.loads((prefix/'VCC-toolchain.json').read_text())
if features['mc6839']:
run(prefix/'bin/cmoc','--coco','--mc6839','-o',out/'FLOAT.BIN',ROOT/'tests/toolchain/float.c')
if args.rom:
build=out/'xcode'
run('xcodebuild','-project',ROOT/'platforms/macos/VCC.xcodeproj','-scheme','VCCCoreChecks','-configuration','Release','-derivedDataPath',build,'build')
run('xcrun','clang++','-std=c++17','-O2','-mmacosx-version-min=14.0',ROOT/'tests/toolchain/run-compiled.cpp',build/'Build/Products/Release/libVCCCore.a','-o',out/'run-compiled')
run(out/'run-compiled',args.rom.resolve(),out/'HELLO.BIN')
for optimization in ('-O0', '-O2'):
binary = out / f'logical-negation{optimization}.bin'
run(prefix/'bin/cmoc', '--coco', '--org=2800', optimization, '-Werror', '-o', binary,
ROOT/'tests/toolchain/logical-negation.c')
run(out/'run-compiled', args.rom.resolve(), binary, 'NOT 010 1000 END')
run('python3', ROOT/'tests/toolchain/indexed-subtraction/check.py', prefix,
build/'Build/Products/Release/libVCCCore.a', args.rom.resolve())
run('python3', ROOT/'tests/toolchain/indexed-arguments/check.py', prefix,
build/'Build/Products/Release/libVCCCore.a', args.rom.resolve())
if features['mc6839']:
run(out/'run-compiled',args.rom.resolve(),out/'FLOAT.BIN','MC6839 MATH OK')
if args.game_disk:
run('xcrun','clang++','-std=c++17','-O2','-mmacosx-version-min=14.0',ROOT/'tests/toolchain/run-coco-letters.cpp',build/'Build/Products/Release/libVCCCore.a','-o',out/'run-coco-letters')
run(out/'run-coco-letters',args.rom.resolve(),args.disk_rom.resolve(),args.game_disk.resolve(),out/'captures')
print('PASS: nine library archives, CMOC compile/link, DECB format, assembler bytes; optional MC6839 arithmetic')
if __name__=='__main__': main()
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# SPDX-License-Identifier: GPL-3.0-or-later
# The pinned CMOC release interpolates runtime-library paths into shell commands.
# A private, space-free alias makes a relocated 'VCC Plus.app' safe while
# dependency-file escaping is handled by the maintained compiler patch. Source/output names are supplied relative to
# the project's build directory by the prototype.
set -eu
runtime=$(CDPATH= cd -- "$(/usr/bin/dirname -- "$0")/.." && pwd -P)
alias_dir=$(/usr/bin/mktemp -d /tmp/vcc-cmoc.XXXXXXXX)
trap '/bin/rm -f "$alias_dir/cmoc"; /bin/rmdir "$alias_dir"' EXIT
/bin/ln -s "$runtime/share/cmoc" "$alias_dir/cmoc"
export CMOCPKGDATADIR="$alias_dir/cmoc"
export CMOCFLOATLIBDIR="$alias_dir/cmoc/lib"
export PATH="$runtime/bin:${PATH:-/usr/bin:/bin}"
if [ "${VCC_CMOC_COMPAT:-0}" = 1 ]; then
set -- "-I$alias_dir/cmoc/include/vcc/compat" "$@"
fi
"$runtime/libexec/cmoc" "$@"
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
# SPDX-License-Identifier: GPL-3.0-or-later
# CMOC expects modern preprocessing. Apple's legacy cpp shim forces traditional
# mode, which misreads valid macros in hirestxt. Use Clang directly.
exec xcrun clang -E -Wno-builtin-macro-redefined "$@"
+174
View File
@@ -0,0 +1,174 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// Adapt CMOC's GCC-style preprocessor invocation to bundled mcpp. Ordinary C
// keeps mcpp's standard token separation. Only explicit asm { ... } bodies use
// mcpp's documented assembler expansion, which preserves #MACRO and -MACRO,x.
#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <mach-o/dyld.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/wait.h>
#include <unistd.h>
typedef struct { size_t start, end; } Block;
typedef struct { char *text; size_t size; Block *blocks; size_t count; } Output;
static int preprocess(const char *executable, char **args, Output *out) {
FILE *capture = tmpfile();
if (!capture) { perror("preprocessor capture"); return 1; }
pid_t child = fork();
if (child < 0) { perror("preprocessor fork"); fclose(capture); return 1; }
if (!child) {
if (dup2(fileno(capture), STDOUT_FILENO) < 0) _exit(126);
execv(executable, args); perror("bundled mcpp"); _exit(127);
}
int status;
while (waitpid(child, &status, 0) < 0) {
if (errno != EINTR) { fclose(capture); return 1; }
}
off_t length = ftello(capture);
// Guard malformed tools/unbounded inputs; no truncated source is compiled.
if (length < 0 || length > 64 * 1024 * 1024 || fseeko(capture, 0, SEEK_SET)) {
fputs("Preprocessor output is too large or unreadable\n", stderr);
fclose(capture); return 1;
}
out->size = (size_t)length;
out->text = malloc(out->size + 1);
if (!out->text || fread(out->text, 1, out->size, capture) != out->size) {
fclose(capture); return 1;
}
out->text[out->size] = 0;
fclose(capture);
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
}
static int identifier(char c) { return isalnum((unsigned char)c) || c == '_'; }
static size_t quote_end(const char *s, size_t i, size_t length) {
char quote = s[i++];
while (i < length) {
if (s[i] == '\\' && i + 1 < length) i += 2;
else if (s[i++] == quote) break;
}
return i;
}
static int find_blocks(Output *out) {
const char *s = out->text;
size_t i = 0;
while (i < out->size) {
// Line markers contain filenames and must never count as source tokens.
if ((i == 0 || s[i - 1] == '\n') && s[i] == '#') {
while (i < out->size && s[i] != '\n') ++i;
} else if (s[i] == '"' || s[i] == '\'') i = quote_end(s, i, out->size);
else if (s[i] == '/' && i + 1 < out->size && s[i + 1] == '/') {
while (i < out->size && s[i] != '\n') ++i;
} else if (s[i] == '/' && i + 1 < out->size && s[i + 1] == '*') {
i += 2;
while (i + 1 < out->size && !(s[i] == '*' && s[i + 1] == '/')) ++i;
i = i + 2 < out->size ? i + 2 : out->size;
} else if (i + 3 <= out->size && !memcmp(s + i, "asm", 3)
&& (i == 0 || !identifier(s[i - 1]))
&& (i + 3 == out->size || !identifier(s[i + 3]))) {
i += 3;
while (i < out->size && isspace((unsigned char)s[i])) ++i;
if (i == out->size || s[i] != '{') continue;
size_t start = ++i;
unsigned depth = 1;
while (i < out->size && depth) {
// Match CMOC's balanced-brace inline-assembly token, retaining
// line markers and assembler comments byte for byte.
if (s[i] == '{') ++depth;
else if (s[i] == '}') --depth;
++i;
}
if (depth) return 0;
Block *grown = realloc(out->blocks, (out->count + 1) * sizeof(Block));
if (!grown) return 0;
out->blocks = grown;
out->blocks[out->count++] = (Block){ start, i - 1 };
} else ++i;
}
return 1;
}
static void emit(const char *text, size_t length) {
size_t start = 0;
while (start < length) {
size_t end = start;
while (end < length && text[end] != '\n') ++end;
if (end < length) ++end;
// mcpp's cur_file() prints raw path bytes inside quotes. Escape them
// for CMOC's C-style marker parser; paths may themselves contain quotes.
// CMOC accepts GCC markers (# NUMBER), not mcpp's #line NUMBER.
size_t number = start;
if (end - start >= 6 && !memcmp(text + start, "#line ", 6)) number += 6;
else if (end - start >= 2 && text[start] == '#' && text[start + 1] == ' ') number += 2;
size_t p = number;
while (p < end && isdigit((unsigned char)text[p])) ++p;
while (p < end && text[p] == ' ') ++p;
size_t last = end;
while (last > p && text[last - 1] != '"') --last;
if (number != start && p < end && text[p] == '"' && last > p + 1) {
fputs("# ", stdout);
fwrite(text + number, 1, p + 1 - number, stdout);
for (size_t i = p + 1; i < last - 1; ++i) {
if (text[i] == '"' || text[i] == '\\') fputc('\\', stdout);
fputc(text[i], stdout);
}
fwrite(text + last - 1, 1, end - last + 1, stdout);
} else fwrite(text + start, 1, end - start, stdout);
start = end;
}
}
int main(int argc, char **argv) {
char executable[PATH_MAX], resolved[PATH_MAX];
uint32_t size = sizeof executable;
if (_NSGetExecutablePath(executable, &size) || !realpath(executable, resolved)) {
perror("Cannot locate bundled preprocessor"); return 1;
}
char *last = strrchr(resolved, '/');
if (!last || (size_t)(last - resolved) + sizeof "/mcpp" > sizeof resolved) return 1;
strcpy(last + 1, "mcpp");
char **args = calloc((size_t)argc + 9, sizeof *args);
if (!args) return 1;
int count = 0;
args[count++] = resolved;
args[count++] = "-e"; args[count++] = "utf8";
args[count++] = "-W0";
for (int i = 1; i < argc; ++i) {
if (!strcmp(argv[i], "-xc++")) args[count++] = "-+";
else if (!strcmp(argv[i], "-nostdinc")) args[count++] = "-I-";
else if (!strcmp(argv[i], "-undef")) args[count++] = "-N";
else if (!strcmp(argv[i], "-Wno-builtin-macro-redefined")) continue;
else args[count++] = argv[i];
}
Output standard = {0}, assembly = {0};
int result = preprocess(resolved, args, &standard);
if (!result && !find_blocks(&standard)) {
fputs("Cannot identify CMOC inline assembly blocks\n", stderr); result = 1;
}
if (!result && standard.count) {
args[count++] = "-a";
result = preprocess(resolved, args, &assembly);
if (!result && (!find_blocks(&assembly) || assembly.count != standard.count)) {
fputs("CMOC inline assembly differs between preprocessing modes\n", stderr); result = 1;
}
if (!result) {
size_t offset = 0;
for (size_t i = 0; i < standard.count; ++i) {
Block a = standard.blocks[i], b = assembly.blocks[i];
emit(standard.text + offset, a.start - offset);
emit(assembly.text + b.start, b.end - b.start);
offset = a.end;
}
emit(standard.text + offset, standard.size - offset);
}
} else if (standard.text) emit(standard.text, standard.size);
free(standard.text); free(standard.blocks);
free(assembly.text); free(assembly.blocks); free(args);
if (ferror(stdout) || fflush(stdout)) return 1;
return result;
}
+104
View File
@@ -0,0 +1,104 @@
# Local CMOC compiler patches
`cmoc-debug-metadata.patch` is an original VCC Plus extension to CMOC 0.1.100,
modified November 3, 2026 and licensed GPL-3.0-or-later. It is applied to the ignored build copy by
`tools/toolchain/build.py`; the pinned upstream snapshot and its checksums stay
unchanged.
With `-fno-omit-frame-pointer`, the compiler writes `* VCCDBG1 {…}` JSON comments
into retained assembly/listings. Metadata comes from resolved compiler types and
declarations, not a C-source text parser. It contains:
- function entry, frame-ready body and epilogue boundaries;
- lexical block ranges, including `for`-statement scopes;
- signedness, storage size, array dimensions, member offsets and pointer types;
- explicit `floatingFormat` tags: `coco-basic-40` for Color BASIC/native 5-byte floats and `ieee754-binary32-be` for MC6839 single precision;
- linker symbols for globals/static locals and U-relative offsets for parameters
and ordinary locals.
The boundaries are zero-byte assembler labels. They can inhibit some assembly
optimizations in debug builds; default/release compiler invocations emit neither
the comments nor these labels. No instructions or data bytes are introduced by
the metadata. The compiler runtime manifest records `vccDebugMetadata: 1`.
The app accepts metadata only from objects/listings/linker maps verified against
the same frozen sources and final executable. Inspection additionally checks
live mapped code and uses the debugger's atomic, non-I/O memory snapshot. Frame
locals require `-O0` as well as the preserved frame pointer and are unavailable in assembly-only functions, alternative calling
conventions, prologues/epilogues, or builds without a preserved U frame. Unsupported
floating-point formats and recursively truncated types remain explicitly
unavailable. Legacy metadata without `floatingFormat` is accepted but not decoded by guessing from storage size. Known format tags must agree with storage size; a genuine double-precision type is not tagged as supported. CMOC 0.1.100 currently lowers source-level `double` to `float`, so that alias carries the selected single-precision format. Expressions do not execute guest code or modify memory.
To regenerate the patch, edit a separate copy of the upstream source and produce
a unified diff with `a/src/…` and `b/src/…` paths. Verify that applying it to a fresh
copy succeeds and that ordinary optimized builds retain their original bytes.
## Indexed byte arithmetic and comparisons
`cmoc-indexed-byte-subtraction.patch` corrects three peephole rules in the build copy of CMOC 0.1.100 (modified November 7, 2026, GPL-3.0-or-later). On the 6809, B is the low byte of D. A byte load or negation can change the effective address of a later B- or D-indexed operand, including an indirect one. The subtraction and comparison rules now keep the original instruction order in those cases. The rules that remove a stack push also guard operands whose address would change with the stack pointer or an index side effect. Ordinary immediate substitutions remain available.
The [guest regression](../../../tests/toolchain/indexed-subtraction/qualification.c) exercises `184 - heights[i]` with runtime indices, variable bases, byte wraparound, both operand orders and equality/inequality against an array element. The [runner](../../../tests/toolchain/indexed-subtraction/check.py) checks the output against host-calculated expectations on MC6809 and HD6309 at `-O0` and `-O2`. It also runs as part of `tools/toolchain/check.py --rom`. Packaged runtimes identify the change with `vccIndexedByteSubtractionFix: 1` in their manifest.
## Indexed byte loads in function arguments
`cmoc-indexed-argument-load.patch` (November 11, 2026, GPL-3.0-or-later)
corrects `removeUselessLdaOrLdb` in the CMOC 0.1.100 build copy. The rule used
to remove an index reload from `LDB index; LDX #array; LDB D,X`, although B is
the low byte of D and is needed to select the loaded byte. Consecutive indexed
function arguments could therefore use the preceding argument's value as an
array subscript. The rule now uses the existing instruction-effects analysis
to preserve B dependencies in both intervening and final loads, including
indirect B/D-indexed operands. It also checks the actual operand for
predecrement/postincrement side effects. Independent overwritten loads remain
eligible for removal.
The [target regression](../../../tests/toolchain/indexed-arguments/qualification.c)
passes adjacent elements from two arrays at runtime indices 0 and 2, using
byte indices, word indices and pointer bases, with distinct surrounding
sentinels. Its [runner](../../../tests/toolchain/indexed-arguments/check.py)
executes `--coco --no-relocate --switch=jump` binaries at `-O0` and `-O2` on
MC6809 and HD6309; it is included in `tools/toolchain/check.py --rom`.
The [rule check](../../../tests/toolchain/indexed-arguments/check-rule.py)
links the actual built compiler objects and checks dependency/side-effect guards
as well as safe removals. See the [qualification record](../../../docs/development/CMOC_INDEXED_ARGUMENT_LOAD.md).
Packaged runtimes identify the change with `vccIndexedArgumentLoadFix: 1`.
## Logical negation of integer results
`cmoc-add-zero-flags.patch` (November 7, 2026, GPL-3.0-or-later) corrects CMOC 0.1.100's general peephole optimizer. CMOC emits `ADDD #0` to set the flags for a word-sized comparison with zero. A function can return 1 in D with Z still set from clearing A. Removing the add before the `tfrZtoB` helper made `!getReady()` return 1 in that case. The patch removes the general rule's unproven add-zero deletions and byte-test substitutions. The separate optimizations for recognized load/compare/branch sequences still run.
The [guest regression](../../../tests/toolchain/logical-negation.c) prints actual results for zero, widened unsigned bytes and 16-bit values. Run `tools/toolchain/check.py --rom` with a local CoCo 3 ROM to check those results at `-O0` and `-O2` on MC6809 and HD6309. Rebuilt app toolchains record `vccAddZeroFlagsFix: 1` in the manifest.
## Explicit before-directory disk allocation
`writecocofile-before-dir.patch` (November 4, 2026, GPL-3.0-or-later) adds
`--allocate-before-dir` as an explicit choice of the existing forward allocator.
It starts at granule zero, skips occupied granules, and continues above the
directory track if necessary. It does not imply a strict below-directory limit.
The last allocation option wins, as with the existing choices.
`build.py` applies it only to the build copy. `build-runtime.py` records
`vccWriteCoCoFileBeforeDir: 1`. The original CoCo Letters Makefile uses this option;
its former recipe-removal patch is retired and the vendored game matches upstream.
Runtime verification checks allocation order, byte-identical default behavior,
file readback, and spillover beyond the lower half of the disk.
## LWASM project input boundary
The native Assembly workspace passes `--project-inputs` when invoking LWASM.
`lwtools-project-inputs.patch` (November 5, 2026, GPL-3.0-or-later) constrains
source and binary input reads to regular files inside the physical working
directory. It checks each open, including binary data reopened during emission.
The companion `--project-map=FILE` records physical source paths, physical
one-based lines, sections, offsets and original instruction bytes. Macro
expansions, data directives, synthesized 6800/convenience operations and
OS-9 calls cannot become source breakpoints. The Swift client verifies final
DECB bytes after relocation. A project map requested with `--6809` also
rejects source pragmas that change that target.
The app supplies an isolated, symlink-free copy of the source snapshot. Other
invocations retain ordinary LWASM path behavior. Custom Make recipes execute
as user-supplied build commands and have a separate trust boundary.
Rebuild and stage the native runtime with `build-runtime.py` before packaging
the workspace. The runtime manifest records `vccLWASMProjectInputs: 1`;
older assemblers reject the required option.
@@ -0,0 +1,57 @@
--- a/src/ASMText2.cpp
+++ b/src/ASMText2.cpp
@@ -1517,8 +1517,9 @@
}
-// Remove AND, OR, MUL or ADD operations if we can show that
-// they will not change performance of the program.
+// ADD #0 can be an explicit condition-code test. Leave it to rules that
+// prove which flags the surrounding instructions produce and consume.
+// In particular, a called runtime helper may read the flags on entry.
bool
ASMText::removeAndOrMulAddSub(size_t index)
{
@@ -1588,42 +1589,6 @@
commentOut(index, "optim: removeAndOrMulAddSub case 5");
madeChanges = true;
}
- } else if ((instr == "ADDA") || (instr == "ADDB") || (instr == "ADDD")) {
- if ((instr[3] == 'A' && simulator.regs.accum.a.known && simulator.regs.accum.a.val == 0) &&
- (instr[3] == 'B' && simulator.regs.accum.b.known && simulator.regs.accum.b.val == 0) &&
- (instr[3] == 'D' && simulator.regs.accum.dknown() && simulator.regs.accum.dval() == 0)) {
- // We know that A, B or D are zero, so change to a LD
- const Element &e1 = elements[index + 1];
- if (e1.type != INSTR || !isBasicBlockEndingInstruction(e1)) {
- char newInstr[8];
- snprintf(newInstr, sizeof(newInstr), "LD%s", instr.c_str() + 3);
- replaceWithInstr(index, newInstr, oper, "optim: removeAndOrMulAddSub case 6");
- madeChanges = true;
- }
- } else if (extractConstantLiteral(e.fields[1], val)) {
- // We know that the operand is a constant 0 and the side effects are not valued,
- // so we can comment it out
- const Element &e1 = elements[index + 1];
- if (val == 0 && (e1.type == INSTR && !isBasicBlockEndingInstruction(e1))) {
- InsEffects effects(e1);
- if (!(effects.read & CC)) {
- commentOut(index, "optim: removeAndOrMulAddSub case 7");
- madeChanges = true;
- }
- } else if (val == 0 && e1.type == INSTR && isBasicBlockEndingInstruction(e1) &&
- (((instr[3] == 'D') && (simulator.regs.accum.a.known && simulator.regs.accum.a.val == 0)) ||
- (instr[3] == 'B'))) {
- // ADDD #0 and A is definitely zero or ADDB #0 so replace with TSTB
- replaceWithInstr(index, "TSTB", "", "optim: removeAndOrMulAddSub case 8");
- madeChanges = true;
- } else if (val == 0 && e1.type == INSTR && isBasicBlockEndingInstruction(e1) &&
- (((instr[3] == 'D') && (simulator.regs.accum.b.known && simulator.regs.accum.b.val == 0)) ||
- (instr[3] == 'A'))) {
- // ADDD #0 and B is definitely zero or ADDA #0 so replace with TSTA
- replaceWithInstr(index, "TSTA", "", "optim: removeAndOrMulAddSub case 9");
- madeChanges = true;
- }
- }
} else if ((instr == "LEAX") && (oper == "D,X")) {
if (simulator.regs.accum.dknown() && simulator.regs.accum.dval() == 0) {
commentOut(index, "optim: removeAndOrMulAddSub case 10");
@@ -0,0 +1,230 @@
--- a/src/Declaration.cpp
+++ b/src/Declaration.cpp
@@ -15,6 +15,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+#include "VCCDebug.h"
#include "Declaration.h"
#include "TranslationUnit.h"
@@ -334,6 +335,8 @@
bool canBeStaticInit = (initializationExpr && initializationExpr->isStaticallyInitializable(*getTypeDesc()));
if (canBeStaticInit != withStaticInitializer)
return true; // nothing to do
+
+ if (isGlobal()) VCCDebug::variable(out, *this, "", "");
if (withStaticInitializer)
{
--- a/src/ForStmt.cpp
+++ b/src/ForStmt.cpp
@@ -15,6 +15,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+#include "VCCDebug.h"
#include "ForStmt.h"
#include "TranslationUnit.h"
@@ -115,6 +116,8 @@
//cerr << "pushing ForStmt scope:\n";
pushScopeIfExists();
+ const string debugBegin = getScope() && VCCDebug::enabled() ? VCCDebug::label() : "", debugEnd = getScope() && VCCDebug::enabled() ? VCCDebug::label() : "";
+ VCCDebug::scope(out, getScope(), debugBegin, debugEnd);
TranslationUnit &tu = TranslationUnit::instance();
string bodyLabel = TranslationUnit::instance().generateLabel('L');
@@ -128,6 +131,7 @@
tu.popBreakableLabels();
//cerr << "popping ForStmt scope:\n";
+ if (getScope() && VCCDebug::enabled()) out.emitLabel(debugEnd);
popScopeIfExists();
return cs;
}
--- a/src/FunctionDef.cpp
+++ b/src/FunctionDef.cpp
@@ -15,6 +15,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+#include "VCCDebug.h"
#include "FunctionDef.h"
#include "Scope.h"
@@ -853,11 +854,23 @@
functor.emitComments(out);
}
+ // VCC Plus: exact compiler-resolved body/scope boundaries, before epilogue.
+ const string debugBody = VCCDebug::enabled() ? VCCDebug::label() : "", debugEnd = VCCDebug::enabled() ? VCCDebug::label() : "";
+ if (VCCDebug::enabled()) out.emitComment("VCCDBG1 {\"kind\":\"function\",\"name\":" + VCCDebug::quote(functionId)
+ + ",\"location\":" + VCCDebug::quote(getLineNo())
+ + ",\"symbol\":" + VCCDebug::quote(functionLabel)
+ + ",\"begin\":" + VCCDebug::quote(debugBody) + ",\"end\":" + VCCDebug::quote(debugEnd)
+ + ",\"frame\":" + (needStackFrame && !isAssemblyOnly() && callConvention == DEFAULT_CMOC_CALL_CONV
+ && TranslationUnit::instance().getFramePointerOption() == FramePointerOption::KEEP_ALL
+ && TranslationUnit::instance().hasUnoptimizedDebugStorage() ? "true" : "false") + "}");
+
// Generate code for the body:
TranslationUnit::instance().setCurrentFunctionEndLabel(endLabel);
TranslationUnit::instance().pushScope(const_cast<Scope *>(getScope())); // const_cast should be removed...
+ VCCDebug::scope(out, getScope(), debugBody, debugEnd);
bool success = bodyStmts->emitCode(out, false);
+ if (VCCDebug::enabled()) out.emitLabel(debugEnd);
out.emitLabel(endLabel, "end of " + functionId + "()");
if (!asmOnly && callConvention == GCC6809_CALL_CONV && getTypeSize() == 2)
--- a/src/TranslationUnit.cpp
+++ b/src/TranslationUnit.cpp
@@ -2573,3 +2573,6 @@
irFile << "}\n"; // main JSON object
return true;
}
+
+// VCC Plus: optimized stores/register reuse cannot claim stable C locals.
+bool TranslationUnit::hasUnoptimizedDebugStorage() const { return params.optimizationLevel == 0; }
--- a/src/TranslationUnit.h
+++ b/src/TranslationUnit.h
@@ -194,6 +194,7 @@
bool isRelocatabilitySupported() const;
FramePointerOption getFramePointerOption() const;
+ bool hasUnoptimizedDebugStorage() const;
// Called during parsing, right after the parser recognized the #pragma directive.
// May change some fields of this TranslationUnit object.
--- a/src/TreeSequence.cpp
+++ b/src/TreeSequence.cpp
@@ -15,6 +15,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+#include "VCCDebug.h"
#include "TreeSequence.h"
#include "TranslationUnit.h"
@@ -141,6 +142,8 @@
const bool isCommaExpr = !!dynamic_cast<const CommaExpr *>(this);
pushScopeIfExists();
+ const string debugBegin = getScope() && VCCDebug::enabled() ? VCCDebug::label() : "", debugEnd = getScope() && VCCDebug::enabled() ? VCCDebug::label() : "";
+ VCCDebug::scope(out, getScope(), debugBegin, debugEnd);
bool success = true;
for (vector<Tree *>::const_iterator it = sequence.begin(); it != sequence.end(); it++)
@@ -178,6 +181,7 @@
}
}
+ if (getScope() && VCCDebug::enabled()) out.emitLabel(debugEnd);
popScopeIfExists();
return success;
--- /dev/null
+++ b/src/VCCDebug.h
@@ -0,0 +1,101 @@
+// VCC Plus compiler debug extension. SPDX-License-Identifier: GPL-3.0-or-later
+// Metadata comments and labels occupy no guest memory. Never infer C storage
+// or type layout from source text: use the compiler's resolved declarations.
+#ifndef VCC_DEBUG_H
+#define VCC_DEBUG_H
+#include "ASMText.h"
+#include "Declaration.h"
+#include "Scope.h"
+#include "TranslationUnit.h"
+#include "ClassDef.h"
+#include <sstream>
+#include <iomanip>
+namespace VCCDebug {
+inline bool enabled() { return TranslationUnit::instance().getFramePointerOption() == FramePointerOption::KEEP_ALL; }
+inline std::string arrayType(const TypeDesc *, const std::vector<uint16_t> &, unsigned);
+inline std::string quote(const std::string &s) {
+ std::ostringstream o; o << '"';
+ for (unsigned char c : s) {
+ if (c == '"' || c == '\\') o << '\\' << c;
+ else if (c < 32) o << "\\u00" << std::hex << std::setw(2) << std::setfill('0') << unsigned(c) << std::dec;
+ else o << c;
+ }
+ o << '"'; return o.str();
+}
+inline std::string type(const TypeDesc *td, unsigned depth = 0) {
+ std::ostringstream o;
+ const int size = td->type == FUNCTION_TYPE || td->type == VOID_TYPE || (td->type == ARRAY_TYPE && td->numArrayElements == TypeDesc::NO_DIMENSION) ? 0 : TranslationUnit::instance().getTypeSize(*td);
+ o << "{\"name\":" << quote(td->toString()) << ",\"size\":" << (size < 0 ? 0 : size);
+ if (depth > 5) o << ",\"kind\":\"unsupported\"";
+ else if (td->type == BYTE_TYPE || td->type == WORD_TYPE || td->isLong())
+ o << ",\"kind\":\"integer\",\"signed\":" << (td->isSigned ? "true" : "false");
+ else if (td->isReal()) {
+ auto &tu = TranslationUnit::instance();
+ const char *format = "unsupported";
+ if (!td->isDouble() && tu.isFloatingPointSupported()) {
+ if (tu.getFloatingPointLibrary() == FloatingPointLibrary::MC6839_LIB && size == 4)
+ format = "ieee754-binary32-be";
+ else if (size == 5 && (tu.getFloatingPointLibrary() == FloatingPointLibrary::NATIVE_LIB
+ || tu.getTargetPlatform() == COCO_BASIC || tu.getTargetPlatform() == DRAGON))
+ format = "coco-basic-40";
+ }
+ o << ",\"kind\":\"floating\",\"floatingFormat\":" << quote(format);
+ }
+ else if (td->type == POINTER_TYPE)
+ o << ",\"kind\":\"pointer\",\"element\":" << type(td->pointedTypeDesc, depth + 1);
+ else if (td->type == ARRAY_TYPE && td->numArrayElements != TypeDesc::NO_DIMENSION)
+ o << ",\"kind\":\"array\",\"count\":" << td->numArrayElements << ",\"element\":" << type(td->pointedTypeDesc, depth + 1);
+ else if (td->type == CLASS_TYPE) {
+ const ClassDef *cl = TranslationUnit::instance().getClassDef(td->className);
+ o << ",\"kind\":\"aggregate\",\"members\":[";
+ if (cl) for (size_t i=0;i<cl->getNumDataMembers();++i) {
+ auto m = cl->getDataMember(i);
+ if (i) o << ',';
+ auto dims = m->getArrayDimensions(); m->getTypeDesc()->appendDimensions(dims);
+ o << "{\"name\":" << quote(m->getName()) << ",\"offset\":" << cl->getDataMemberOffset(i, NULL)
+ << ",\"type\":" << (dims.empty() ? type(m->getTypeDesc(), depth+1) : arrayType(m->getTypeDesc(), dims, depth+1)) << '}';
+ }
+ o << ']';
+ } else o << ",\"kind\":\"unsupported\"";
+ o << '}'; return o.str();
+}
+inline std::string arrayType(const TypeDesc *original, const std::vector<uint16_t> &dims, unsigned depth) {
+ const TypeDesc *td=original;
+ while (td->type == ARRAY_TYPE) td=td->pointedTypeDesc;
+ std::string result=type(td, depth+1);
+ size_t size=TranslationUnit::instance().getTypeSize(*td);
+ for (auto it=dims.rbegin();it!=dims.rend();++it) {
+ size *= *it;
+ std::ostringstream o; o << "{\"name\":" << quote(original->toString())
+ << ",\"kind\":\"array\",\"size\":" << size << ",\"count\":" << *it << ",\"element\":" << result << '}';
+ result=o.str();
+ }
+ return result;
+}
+inline std::string declarationType(const Declaration &decl) {
+ return decl.getTypeDesc()->type == ARRAY_TYPE ? arrayType(decl.getTypeDesc(), decl.arrayDimensions, 0) : type(decl.getTypeDesc());
+}
+inline std::string label() { return ".vcc.debug." + TranslationUnit::instance().generateLabel('L'); }
+inline void variable(ASMText &out, const Declaration &d, const std::string &begin, const std::string &end) {
+ if (!enabled() || d.isExtern || d.getVariableId().empty() || d.getVariableId()[0] == '$') return;
+ uint16_t size=0; if (!d.getVariableSizeInBytes(size) || !size) return;
+ std::ostringstream o; o << "VCCDBG1 {\"kind\":\"variable\",\"name\":" << quote(d.getVariableId())
+ << ",\"location\":" << quote(d.getLineNo()) << ",\"type\":" << declarationType(d)
+ << ",\"begin\":" << quote(begin) << ",\"end\":" << quote(end)
+ << ",\"static\":" << (d.isStatic() ? "true" : "false");
+ if (d.isGlobal() || d.isLocalStatic()) o << ",\"symbol\":" << quote(d.getLabel());
+ else o << ",\"offset\":" << d.getFrameDisplacement();
+ o << '}'; out.emitComment(o.str());
+}
+inline void scope(ASMText &out, const Scope *s, const std::string &begin, const std::string &end) {
+ if (!enabled() || !s) return;
+ class Visitor : public Scope::DeclarationFunctor {
+ ASMText &out; const std::string &begin, &end;
+ public:
+ Visitor(ASMText &o, const std::string &b, const std::string &e):out(o),begin(b),end(e){}
+ bool operator()(Declaration &d) override { variable(out,d,begin,end); return true; }
+ } visit(out,begin,end);
+ out.emitLabel(begin); s->iterateDeclarations(visit,false);
+}
+}
+#endif
@@ -0,0 +1,45 @@
--- a/src/ASMText2.cpp
+++ b/src/ASMText2.cpp
@@ -4952,29 +4952,31 @@
// Look for this:
-// LDB {foo} ; {bar} involves not pre/post-increment, e.g., LDB ,S+
-// LDX/LEAX {bar} ; {bar} does not involve B, e.g., LDX B,U
-// LDB {baz}
-// Remove the first LDB.
+// LDB {foo} ; no pre/post-increment side effects
+// LDX/LEAX {bar} ; must not read B, including D-indexed operands
+// LDB {baz} ; must not read the previous B in its effective address
+// Remove the first LDB only when its value and side effects are unused.
//
bool
ASMText::removeUselessLdaOrLdb(size_t index)
{
if (! isInstrAnyArg(index, "LDB"))
return false;
- const string &arg = elements[index].fields[0];
- if (startsWith(arg, ",-") || endsWith(arg, "+")) // if pre/post increment
- return false;
+ if (isInstrWithPreDecrOrPostIncr(index))
+ return false; // Removing the load must not remove an index/stack update.
size_t otherLoad = findNextInstrBeforeLabel(index + 1);
if (! isInstrAnyArg(otherLoad, "LDX") && ! isInstrAnyArg(otherLoad, "LEAX"))
return false;
- const string &otherLoadArg = elements[otherLoad].fields[1];
- if (startsWith(otherLoadArg, "B,"))
- return false; // we have "B,reg" as the arg to LDX/LEAX, so previous LDB is useful
+ // B is also the low byte of D. Use the common instruction-effects
+ // analysis for both direct and indirect accumulator-indexed operands.
+ if (InsEffects(elements[otherLoad]).read & B)
+ return false;
size_t secondLdAcc = findNextInstrBeforeLabel(otherLoad + 1);
if (! isInstrAnyArg(secondLdAcc, "LDB"))
return false;
-
+ if (InsEffects(elements[secondLdAcc]).read & B)
+ return false; // The overwritten value may still select the byte to load.
+
// Apply the optimization.
commentOut(index, "optim: removeUselessLdaOrLdb (removed LDB b/c later LDB)");
return true;
@@ -0,0 +1,68 @@
--- a/src/ASMText2.cpp
+++ b/src/ASMText2.cpp
@@ -4159,16 +4159,27 @@
}
+// B is the low half of D. Both direct and indirect accumulator-indexed
+// addresses can therefore depend on the value of B before a rewrite.
+static bool
+usesBInAddress(const string &operand)
+{
+ return startsWith(operand, "B,") || startsWith(operand, "D,")
+ || startsWith(operand, "[B,") || startsWith(operand, "[D,");
+}
+
+
// Look for: LDB foo; PSHS B; LDB bar; CMPB ,S+.
// Change to: LDB bar; CMPB foo.
-// LDB args must not involve S.
+// Swapping the two reads must preserve their offsets and index registers.
//
bool
ASMText::removePushBFromLdbPushLdbCmp(size_t index)
{
if (!isInstrAnyArg(index, "LDB"))
return false;
- if (elements[index].fields[1].find(",S") != string::npos)
+ if (usesBInAddress(elements[index].fields[1]) || isInstrWithPreDecrOrPostIncr(index)
+ || elements[index].fields[1].find(",S") != string::npos)
return false;
size_t pushIndex = findNextInstrBeforeLabel(index + 1);
@@ -4178,7 +4189,8 @@
size_t secondLoadIndex = findNextInstrBeforeLabel(pushIndex + 1);
if (!isInstrAnyArg(secondLoadIndex, "LDB"))
return false;
- if (elements[secondLoadIndex].fields[1].find(",S") != string::npos)
+ if (usesBInAddress(elements[secondLoadIndex].fields[1]) || isInstrWithPreDecrOrPostIncr(secondLoadIndex)
+ || elements[secondLoadIndex].fields[1].find(",S") != string::npos)
return false;
size_t cmpIndex = findNextInstrBeforeLabel(secondLoadIndex + 1);
@@ -4440,6 +4452,13 @@
if (!isInstr(subIndex, "SUBB", ",S+"))
return false;
+ // NEGB changes B before the load's address is used, and removing the
+ // push changes S. Keep the original sequence when either matters.
+ const string &operand = elements[ldbIndex].fields[1];
+ if (usesBInAddress(operand) || operand.find(",S") != string::npos
+ || isInstrWithPreDecrOrPostIncr(ldbIndex))
+ return false;
+
// Apply the optimization.
commentOut(index, "optim: removePushBBeforeSubB"); // remove PSHS
elements[subIndex].fields[0] = "ADDB";
@@ -4833,6 +4852,11 @@
if (isInstrWithPreDecrOrPostIncr(addb))
return false;
+ // Swapping the operands also changes which value of B is available
+ // when each effective address is evaluated.
+ if (usesBInAddress(elements[index].fields[1]) || usesBInAddress(elements[addb].fields[1]))
+ return false;
+
// Apply the optimization.
const string ldbArg = elements[index].fields[1];
elements[index].fields[1] = elements[addb].fields[1];
@@ -0,0 +1,362 @@
--- a/lwasm/input.c
+++ b/lwasm/input.c
@@ -26,6 +26,9 @@
*/
#include <errno.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <sys/stat.h>
#include <stdio.h>
#include <string.h>
@@ -65,8 +68,40 @@
void *data;
int data2;
char *filespec;
+ const char *source_path;
+ int source_line;
struct input_stack_node *stack;
};
+
+/* VCC Plus native assembly builds run in a private, immutable snapshot.
+ * Resolve the target before opening it, including the second binary-include
+ * read during emission. Ordinary LWASM invocations retain upstream behavior. */
+static char *project_root;
+
+void input_restrict_to_project(void)
+{
+ project_root = realpath(".", NULL);
+ if (!project_root)
+ lw_error("Cannot resolve assembly project folder.\n");
+}
+
+FILE *input_fopen(const char *path)
+{
+ if (project_root)
+ {
+ char resolved[PATH_MAX];
+ struct stat info;
+ size_t length = strlen(project_root);
+ if (!realpath(path, resolved))
+ return NULL;
+ if (strncmp(project_root, resolved, length) || resolved[length] != '/')
+ lw_error("Assembly input leaves the project snapshot: %s\n", path);
+ if (stat(resolved, &info) != 0 || !S_ISREG(info.st_mode))
+ lw_error("Assembly input is not a regular file: %s\n", path);
+ return fopen(resolved, "rb");
+ }
+ return fopen(path, "rb");
+}
static char *make_filename(char *p, char *f)
{
@@ -110,7 +145,7 @@
}
/* this adds real filenames that were opened to a list */
-void input_add_to_resource_list(asmstate_t *as, const char *s)
+static const char *input_add_to_resource_list(asmstate_t *as, const char *s)
{
struct ifl *ifl;
@@ -127,6 +162,7 @@
ifl_head = ifl;
ifl -> fn = lw_strdup(s);
}
+ return ifl -> fn;
}
void input_init(asmstate_t *as)
@@ -155,6 +191,12 @@
char *dn, *dp;
// int o;
+ if (as->project_map_file && project_root)
+ {
+ char resolved[PATH_MAX];
+ if (realpath(fn, resolved))
+ IS->source_path = input_add_to_resource_list(as, resolved + strlen(project_root) + 1);
+ }
dn = lw_strdup(fn);
lw_stack_push(as -> includelist, dn);
@@ -186,6 +228,8 @@
struct input_stack *t;
t = lw_alloc(sizeof(struct input_stack));
+ t -> source_path = NULL;
+ t -> source_line = 0;
t -> filespec = lw_strdup(s);
t -> type = input_type_string;
@@ -204,6 +248,8 @@
char *p, *p2;
t = lw_alloc(sizeof(struct input_stack));
+ t -> source_path = NULL;
+ t -> source_line = 0;
t -> filespec = lw_strdup(s);
for (s2 = s; *s2 && (*s2 != ':'); s2++)
@@ -238,7 +284,7 @@
if (input_isabsolute(s))
{
/* absolute path */
- IS -> data = fopen(s, "rb");
+ IS -> data = input_fopen(s);
debug_message(as, 1, "Opening (abs) %s", s);
if (!IS -> data && !IGNOREERROR)
{
@@ -257,7 +303,7 @@
p = lw_stack_top(as -> file_dir);
p2 = make_filename(p, s);
debug_message(as, 1, "Open: (cd) %s\n", p2);
- IS -> data = fopen(p2, "rb");
+ IS -> data = input_fopen(p2);
if (IS -> data)
{
input_pushpath(as, p2);
@@ -274,7 +320,7 @@
{
p2 = make_filename(p, s);
debug_message(as, 1, "Open (sp): %s\n", p2);
- IS -> data = fopen(p2, "rb");
+ IS -> data = input_fopen(p2);
if (IS -> data)
{
input_pushpath(as, p2);
@@ -300,7 +346,7 @@
if (s[0] == '-' && s[1] == '\0')
IS -> data = stdin;
else
- IS -> data = fopen(s, "rb");
+ IS -> data = input_fopen(s);
if (!IS -> data)
{
@@ -328,7 +374,7 @@
debug_message(as, 2, "Open file (st abs) %s", s);
if (as -> flags & FLAG_DEPEND)
printf("%s\n", s);
- fp = fopen(s, "rb");
+ fp = input_fopen(s);
if (!fp)
{
return NULL;
@@ -343,7 +389,7 @@
p = lw_stack_top(as -> file_dir);
p2 = make_filename(p ? p : "", s);
debug_message(as, 2, "Open file (st cd) %s", p2);
- fp = fopen(p2, "rb");
+ fp = input_fopen(p2);
if (fp)
{
if (as -> flags & FLAG_DEPEND)
@@ -362,7 +408,7 @@
{
p2 = make_filename(p, s);
debug_message(as, 2, "Open file (st ip) %s", p2);
- fp = fopen(p2, "rb");
+ fp = input_fopen(p2);
if (fp)
{
if (as -> flags & FLAG_DEPEND)
@@ -471,6 +517,7 @@
if (eol)
{
s = lw_strdup(linebuff);
+ IS->source_line++;
return s;
}
}
@@ -537,6 +584,15 @@
default:
lw_error("Problem reading from unknown input type\n");
return NULL;
+ }
+}
+
+void input_project_location(asmstate_t *as, line_t *line)
+{
+ if (IS && IS->type != input_type_string)
+ {
+ line->project_source = IS->source_path;
+ line->project_line = IS->source_line;
}
}
--- a/lwasm/input.h
+++ b/lwasm/input.h
@@ -32,6 +32,9 @@
void input_stack_push(asmstate_t *as, input_stack_entry *se);
input_stack_entry *input_stack_pop(asmstate_t *as, int magic, int (*fn)(input_stack_entry *e, void *data), void *data);
+void input_restrict_to_project(void);
+FILE *input_fopen(const char *path);
+void input_project_location(asmstate_t *as, line_t *line);
void input_init(asmstate_t *as);
void input_openstring(asmstate_t *as, char *s, char *str);
void input_open(asmstate_t *as, char *s);
--- a/lwasm/list.c
+++ b/lwasm/list.c
@@ -24,6 +24,7 @@
#include <string.h>
#include <lw_alloc.h>
+#include <lw_error.h>
#include <lw_string.h>
#include "lwasm.h"
@@ -489,3 +490,43 @@
if (of && of != stdout)
fclose(of);
}
+
+/* VCC Plus project source map. Keep the normal human-readable listing intact.
+ * Physical file lines are captured at input; macro strings have no file identity.
+ * The client still validates original object bytes, link relocations and the BIN. */
+void do_project_map(asmstate_t *as)
+{
+ FILE *out = fopen(as->project_map_file, "w");
+ line_t *line;
+ const int synthetic = lwasm_insn_is6800 | lwasm_insn_is6809conv |
+ lwasm_insn_is6309conv | lwasm_insn_isemuext;
+ if (!out) lw_error("Cannot write project instruction map.\n");
+ for (line = as->line_head; line; line = line->next)
+ {
+ const char *section = line->csect ? line->csect->name : "";
+ lw_expr_t address;
+ int i;
+ if (!line->project_source || !line->emitted_opcode || line->insn < 0 ||
+ (instab[line->insn].flags & synthetic) || !strcmp(instab[line->insn].opcode, "os9") ||
+ line->outputl < 1 || line->outputl > 8 || line->outputl != line->len ||
+ strpbrk(line->project_source, "\t\r\n") || strpbrk(section, "\t\r\n"))
+ continue;
+ address = lw_expr_copy(line->addr);
+ as->exportcheck = 1;
+ as->csect = line->csect;
+ lwasm_reduce_expr(as, address);
+ as->exportcheck = 0;
+ if (lw_expr_istype(address, lw_expr_type_int))
+ {
+ int value = lw_expr_intval(address);
+ if (value >= 0 && value <= 65535)
+ {
+ fprintf(out, "%s\t%d\t%s\t%04X\t", line->project_source, line->project_line, section, value);
+ for (i = 0; i < line->outputl; ++i) fprintf(out, "%02X", line->output[i]);
+ fputc('\n', out);
+ }
+ }
+ lw_expr_destroy(address);
+ }
+ if (fclose(out)) lw_error("Cannot finish writing project instruction map.\n");
+}
--- a/lwasm/lwasm.c
+++ b/lwasm/lwasm.c
@@ -504,6 +504,7 @@
void lwasm_emitop(line_t *cl, int opc)
{
+ cl->emitted_opcode = 1;
if (cl->cycle_base == 0)
lwasm_cycle_update_count(cl, opc); /* only call first time, never on postbyte */
--- a/lwasm/lwasm.h
+++ b/lwasm/lwasm.h
@@ -278,6 +278,9 @@
struct line_s
{
+ const char *project_source;
+ int project_line;
+ int emitted_opcode;
lw_expr_t addr; // assembly address of the line
lw_expr_t daddr; // data address of the line (os9 only)
lw_expr_t phase; // the "phase" setting (assumed address for assembly)
@@ -440,6 +443,8 @@
sectiontab_t *sections; // section table
exportlist_t *exportlist; // list of exported symbols
importlist_t *importlist; // list of imported symbols
+ char *project_map_file;
+ int project_6809_only;
char *list_file; // name of file to list to
char *symbol_dump_file; // name of file to dump symbol table to
int tabwidth; // tab width in list file
--- a/lwasm/main.c
+++ b/lwasm/main.c
@@ -71,6 +71,8 @@
{ "6800compat", 0x200, 0, 0, "Enable 6800 compatibility instructions, equivalent to --pragma=6800compat" },
{ "no-output", 0x105, 0, 0, "Inhibit creation of output file" },
{ "no-warn", 0x109, "FLAG", 0, "Suppress warnings of the specified type" },
+ { "project-map", 0x10d, "FILE", 0, "Write physical source instruction locations for a project snapshot" },
+ { "project-inputs", 0x10c, 0, 0, "Restrict all assembly inputs to the current project folder" },
{ "error-format", 0x10b, "STYLE", 0, "Error output style: lwasm (default), gcc, vs" },
{ 0 }
};
@@ -82,6 +84,12 @@
switch (key)
{
+ case 0x10d:
+ as->project_map_file = lw_strdup(arg);
+ break;
+ case 0x10c:
+ input_restrict_to_project();
+ break;
case 'I':
lw_stringlist_addstring(as -> include_list, arg);
break;
@@ -333,6 +341,7 @@
void do_output(asmstate_t *as);
void do_symdump(asmstate_t *as);
void do_list(asmstate_t *as);
+void do_project_map(asmstate_t *as);
void do_map(asmstate_t *as);
lw_expr_t lwasm_evaluate_special(int t, void *ptr, void *priv);
lw_expr_t lwasm_evaluate_var(char *var, void *priv);
@@ -391,6 +400,7 @@
asmstate.output_file = lw_strdup("a.out");
}
+ asmstate.project_6809_only = asmstate.project_map_file && (asmstate.pragmas & PRAGMA_6809);
input_init(&asmstate);
for (passnum = 0; passlist[passnum].fn; passnum++)
@@ -450,6 +460,7 @@
}
do_symdump(&asmstate);
do_list(&asmstate);
+ if (asmstate.project_map_file) do_project_map(&asmstate);
do_map(&asmstate);
if (asmstate.testmode_errorcount > 0) exit(1);
--- a/lwasm/pass1.c
+++ b/lwasm/pass1.c
@@ -115,12 +115,15 @@
memset(cl, 0, sizeof(line_t));
cl -> outputl = -1;
cl -> linespec = lw_strdup(input_curspec(as));
+ if (as->project_map_file) input_project_location(as, cl);
cl -> prev = as -> line_tail;
cl -> insn = -1;
cl -> as = as;
cl -> inmod = as -> inmod;
cl -> csect = as -> csect;
cl -> pragmas = as -> pragmas;
+ if (as->project_6809_only && !(cl->pragmas & PRAGMA_6809))
+ lwasm_register_error2(as, cl, E_6309_INVALID, "Source changes the selected 6809 project target.");
cl -> context = as -> context;
cl -> hasoperand = 1; // default: assume operand present; cleared by inherent insns
cl -> ltext = lw_strdup(line);
--- a/lwasm/pseudo.c
+++ b/lwasm/pseudo.c
@@ -1662,7 +1662,7 @@
FILE *fp;
int c, i;
- fp = fopen(l -> lstr, "rb");
+ fp = input_fopen(l -> lstr);
if (!fp)
{
lwasm_register_error2(as, l, E_FILE_OPEN, "%s", "(emit)!");
@@ -0,0 +1,32 @@
--- a/src/writecocofile.cpp
+++ b/src/writecocofile.cpp
@@ -1,3 +1,8 @@
+/* VCC Plus modification, November 4, 2026: support an explicit
+ --allocate-before-dir option using the existing forward allocator.
+ This modification is licensed GPL-3.0-or-later.
+*/
+
/* writecocofile.cpp - A tool to write native files to a CoCo DECB disk image.
Copyright (C) 2003-2026 Pierre Sarrazin <http://sarrazip.com/>
@@ -874,6 +879,8 @@
"--kill|-k Kill the designated file.\n"
"--read|-r Read instead of writing. Refuses to overwrite existing native file.\n"
"--stdout|-s Like --read, but print file contents on standard out.\n"
+ "--allocate-before-dir Allocate from granule zero (the default), preferring space\n"
+ " before the directory, then continuing above it if needed.\n"
"--allocate-after-dir When adding a file, allocate granules after the directory track,\n"
" then continue allocating below that track if needed.\n"
"--allocate-from-end When adding a file, allocate granules from the end.\n"
@@ -1376,6 +1383,11 @@
granuleAllocator.reset(new BackwardGranuleAllocator());
continue;
}
+ if (curopt == "--allocate-before-dir")
+ {
+ granuleAllocator.reset(new ForwardGranuleAllocator());
+ continue;
+ }
if (curopt == "--allocate-after-dir")
{
granuleAllocator.reset(new PostDirTrackGranuleAllocator());
+30
View File
@@ -0,0 +1,30 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// Select the matching relocatable Perl installation. Architecture-specific
// Config.pm and XS modules remain paired with the interpreter that built them.
#include <mach-o/dyld.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
int main(int argc, char **argv) {
(void)argc;
char path[PATH_MAX], resolved[PATH_MAX]; uint32_t size = sizeof path;
if (_NSGetExecutablePath(path, &size) || !realpath(path, resolved)) {
perror("Cannot locate bundled Perl"); return 1;
}
char *last = strrchr(resolved, '/'); if (!last) return 1; *last = 0;
last = strrchr(resolved, '/'); if (!last) return 1; *last = 0;
#if defined(__aarch64__)
const char *relative = "/libexec/perl/arm64/bin/perl";
#elif defined(__x86_64__)
const char *relative = "/libexec/perl/x86_64/bin/perl";
#else
#error Unsupported Perl host architecture
#endif
if (strlen(resolved) + strlen(relative) >= sizeof path) return 1;
snprintf(path, sizeof path, "%s%s", resolved, relative);
argv[0] = path;
execv(path, argv);
perror("Cannot execute bundled Perl"); return 1;
}
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Prepare the real CoCo Letters Make project for local VCC Plus qualification.
This is a test fixture, not an application resource. The explicitly supplied
private standalone disk source and generated game stay in the chosen local
output folder and are never copied into the public toolchain or Git snapshot.
"""
import argparse
import hashlib
import json
from pathlib import Path
import shutil
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[2]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--standalone-disk-source', type=Path, required=True)
args = parser.parse_args()
output = args.output.resolve()
if output.exists(): parser.error('Choose a fresh project directory')
if output == ROOT or ROOT / 'third_party' == output or ROOT / 'third_party' in output.parents:
parser.error('Choose local test storage outside the source snapshots')
record = json.loads((ROOT / 'third_party/manifests/cmoc.json').read_text())
expected = record['excludedFileSHA256']['src/stdlib/dskcon-standalone.c']
if hashlib.sha256(args.standalone_disk_source.read_bytes()).hexdigest() != expected:
parser.error('Private disk source does not match the reviewed CMOC release')
subprocess.run([sys.executable, ROOT / 'tools/toolchain/verify-sources.py'], check=True)
shutil.copytree(ROOT / 'third_party/examples/coco-letters', output)
# A grouped target makes the upstream two-output dictionary recipe safe when
# GNU Make runs in parallel. The vendored source itself remains unchanged.
upstream = (output / 'Makefile').read_text()
original = '$(DATA_FILENAME) $(WORD_LENGTH_FILENAME): $(BUILD_DATA)'
if upstream.count(original) != 1: raise SystemExit('Review updated dictionary generation rule')
upstream = upstream.replace(original, '$(DATA_FILENAME) $(WORD_LENGTH_FILENAME) &: $(BUILD_DATA)')
(output / 'Game.mk').write_text(upstream)
for name in ('decbfile', 'bcontrol', 'bsound'):
shutil.copytree(ROOT / 'third_party/cmoc-libraries' / name, output / 'libraries' / name)
(output / 'private').mkdir()
shutil.copyfile(args.standalone_disk_source, output / 'private/dskcon-standalone.c')
(output / 'Makefile').write_text('''# VCC Plus local qualification wrapper. GPL-3.0-or-later.
# Game.mk is the upstream game Makefile, with its dictionary recipe grouped.
override DECBFILE_LIB_DIR = libraries/decbfile
override BCONTROL_LIB_DIR = libraries/bcontrol
override BSOUND_LIB_DIR = libraries/bsound
include Game.mk
.DEFAULT_GOAL := all
INTERMEDIATE = --intermediate
LIBS += private/dskcon-standalone.o
$(BIN): private/dskcon-standalone.o
private/dskcon-standalone.o: private/dskcon-standalone.c
\t$(CMOC) --coco --intermediate -O2 -c -o $@ $<
libraries/decbfile/libdecbfile.a: $(wildcard libraries/decbfile/*.c libraries/decbfile/*.h) libraries/decbfile/Makefile
\t$(MAKE) -C libraries/decbfile libdecbfile.a
libraries/bcontrol/libbcontrol.a: $(wildcard libraries/bcontrol/*.c libraries/bcontrol/*.h) libraries/bcontrol/Makefile
\t$(MAKE) -C libraries/bcontrol libbcontrol.a
libraries/bsound/libbsound.a: $(wildcard libraries/bsound/*.c libraries/bsound/*.h) libraries/bsound/Makefile
\t$(MAKE) -C libraries/bsound libbsound.a
.PHONY: clean-libraries
clean: clean-libraries
clean-libraries:
\t$(MAKE) -C libraries/decbfile clean
\t$(MAKE) -C libraries/bcontrol clean
\t$(MAKE) -C libraries/bsound clean
\trm -f private/*.o private/*.s private/*.lst $(DSK)
''')
(output / 'vccplus-project.json').write_text(json.dumps({
'entrySource': 'main.c', 'makefile': 'Makefile', 'buildTarget': 'all',
'cleanTarget': 'clean', 'diskTarget': 'all', 'binary': 'letters.bin',
'diskImage': 'coco-letters-0.1.0.dsk', 'launchMode': 'disk',
'launchCommands': ['DOS'], 'jobs': 4,
'configurations': [{'name': 'Debug', 'makeArguments': ['INTERMEDIATE=--intermediate']},
{'name': 'Release', 'makeArguments': []}]
}, indent=2) + '\n')
(output / '.gitignore').write_text('private/\n*.bin\n*.dsk\n*.dat\n*.o\n*.s\n*.lst\n*.map\n*.link\n*.d\nword-length-index.c\n')
(output / 'VCC-QUALIFICATION.txt').write_text(
'Local qualification fixture. Includes separately supplied private disk source.\n'
'Do not distribute this folder as part of VCC Plus or its public toolchain.\n'
'Open the folder in C Development, build the all target, then mount/boot its disk.\n'
'Requires CoCo 3, at least 128 KB, floppy controller with Disk BASIC, and RGB video.\n'
'The original optimization flags are retained to fit its memory map.\n'
'Source breakpoints are available only where exact executable mappings are verified.\n')
print('Prepared local Make project:', output)
if __name__ == '__main__': main()
+32
View File
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: GPL-3.0-or-later
"""Validate the explicitly staged private disk-driver extension."""
import hashlib
STANDALONE_FILES = {
'share/cmoc/lib/libdskcon-standalone.a',
'share/cmoc/include/dskcon-standalone.h',
'LICENSES/DSKCON-STANDALONE-NOTICE.txt',
}
def validate_private_inputs(runtime, record):
disk = record.get('standaloneDisk')
if disk is None:
if record.get('privateRuntimeInputs') is not False:
raise ValueError('Private runtime input declaration is missing')
if any((runtime / path).exists() for path in STANDALONE_FILES - {'share/cmoc/include/dskcon-standalone.h'}):
raise ValueError('Undeclared standalone disk payload')
return
if record.get('privateRuntimeInputs') is not True or disk.get('profile') != 'decb-derived-standalone':
raise ValueError('Unsupported private runtime profile')
hashes = disk.get('filesSHA256', {})
if set(hashes) != STANDALONE_FILES:
raise ValueError('Incomplete standalone disk payload declaration')
if disk.get('license') != 'public-domain-except-DECB-derived-portions':
raise ValueError('Missing standalone disk license distinction')
if not isinstance(disk.get('sourceSHA256'), str) or len(disk['sourceSHA256']) != 64:
raise ValueError('Missing standalone disk source identity')
for relative, expected in hashes.items():
path = runtime / relative
if path.is_symlink() or not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected:
raise ValueError('Standalone disk payload does not match its manifest: ' + relative)
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Prebuild the public-domain, ROM-backed DECB file API into an existing runtime.
The same bundled compiler/preprocessor builds libraries and end-user programs;
Apple developer tools are not required for this staging operation.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tempfile
from runtime_inputs import validate_private_inputs
ROOT = Path(__file__).resolve().parents[2]
SOURCE = ROOT / 'third_party/cmoc-libraries/decbfile'
VERSION = '0.1.13'
EXCLUDED = {
'drivewire.c', 'dwReadBeckerPort.c', 'dwWriteBeckerPort.c',
'dwReadBitBangerSerialPort.c', 'dwWriteBitBangerSerialPort.c',
'FileChoiceDialog.c', 'runDOSLoader.c', 'binfile.c', 'writeMemoryToFile.c',
'readFileToMemory.c', 'readFileToDynamicMemory.c', 'decbutil.c', 'crc16.c',
}
def sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('runtime', type=Path)
parser.add_argument('--jobs', type=int, default=4)
args = parser.parse_args()
runtime = args.runtime.resolve()
manifest_path = runtime / 'manifest.json'
manifest = json.loads(manifest_path.read_text())
validate_private_inputs(runtime, manifest)
source_manifest = ROOT / 'third_party/manifests/decbfile.json'
source_record = json.loads(source_manifest.read_text())
if source_record['version'] != VERSION:
parser.error('Review the ROM-backed library selection before changing upstream version')
subprocess.run([sys.executable, ROOT / 'tools/toolchain/verify-sources.py'], check=True)
source_list = re.search(r'^LIBSRC = \\\n(.*?)\n\nLIBHEADERS', (SOURCE / 'Makefile').read_text(), re.M | re.S)
if not source_list:
parser.error('Unexpected upstream source list')
sources = source_list.group(1).replace('\\', '').split()
if not EXCLUDED.issubset(sources):
parser.error('Unexpected upstream optional components')
selected = [name for name in sources if name not in EXCLUDED]
env = dict(os.environ, PATH=str(runtime / 'bin') + ':/usr/bin:/bin')
env['DEVELOPER_DIR'] = '/VCC-no-developer-tools'
env['SDKROOT'] = '/VCC-no-sdk'
# A simple temporary path also avoids upstream CMOC's shell-quoting limits.
with tempfile.TemporaryDirectory(prefix='vcc-decbfile-', dir='/tmp') as folder:
work = Path(folder)
for name in selected + ['decbfile.h', 'decbfile_private.h']:
shutil.copyfile(SOURCE / name, work / name)
def compile_source(name):
subprocess.run([str(runtime / 'bin/cmoc'), '--coco', '-O2',
'-DNDEBUG', '-Wfor-condition-sizes', '-Werror', '-I.', '-c',
'-o', name[:-2] + '.o', name], cwd=work, env=env, check=True)
with ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool:
list(pool.map(compile_source, selected))
archive = work / 'libdecbfile.a'
subprocess.run([str(runtime / 'bin/lwar'), '--create', str(archive),
*[name[:-2] + '.o' for name in selected]], cwd=work, env=env, check=True)
archive_listing = subprocess.check_output([str(runtime / 'bin/lwar'), '--list', str(archive)], text=True)
for name in EXCLUDED:
if name[:-2] + '.o' in archive_listing:
raise SystemExit('Unexpected excluded archive member: ' + name)
target_archive = runtime / 'share/cmoc/lib/libdecbfile.a'
target_header = runtime / 'share/cmoc/include/decbfile.h'
shutil.copyfile(archive, target_archive)
shutil.copyfile(SOURCE / 'decbfile.h', target_header)
# Retain the original declaration verbatim instead of assigning a GPL label
# to a component whose author explicitly released it into the public domain.
header_notice = (SOURCE / 'decbfile.h').read_text().split('*/', 1)[0] + '*/\n'
notice = ('DECB File Library ' + VERSION + ' — ROM-backed core\n\n' + header_notice +
'\nUses the Disk BASIC ROM already mapped in the emulated machine.\n'
'No Disk BASIC ROM bytes or standalone Disk BASIC-derived driver are included in this archive.\n'
'Optional standalone, DriveWire, UI and binary-loader extensions are not included.\n\n'
'Included upstream source files:\n' + ''.join(' ' + name + '\n' for name in selected))
(runtime / 'LICENSES/DECBFILE-NOTICE.txt').write_text(notice)
manifest['decbfile'] = {'version': VERSION, 'profile': 'rom-backed-core',
'license': 'public-domain', 'requiresDiskBASICROM': True,
'standaloneDriverIncluded': False, 'sourceFiles': selected,
'sourceManifestSHA256': sha(source_manifest),
'archiveSHA256': sha(target_archive), 'headerSHA256': sha(target_header)}
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
print(f'PASS: staged {len(selected)} public-domain DECB core objects; link with -ldecbfile')
if __name__ == '__main__': main()
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Compile the original VCC Plus guest libraries with the installed CMOC."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[2]
SDK = ROOT / "sdk/cmoc"
LIBRARIES = {"ssc": "ssc", "orch90": "orch90", "gime": "gime", "input": "input", "bank": "bank", "sdc": "sdc", "vcccompat": "compat"}
def stage(runtime):
manifest_file = runtime / "manifest.json"
manifest = json.loads(manifest_file.read_text())
env = dict(os.environ, PATH=str(runtime / "bin") + ":/usr/bin:/bin", LC_ALL="C",
VCC_CMOC_COMPAT="0")
with tempfile.TemporaryDirectory(dir="/tmp", prefix="vcc-sdk-") as tmp:
work = Path(tmp)
include = work / "include"
shutil.copytree(SDK / "include", include)
shutil.copytree(SDK / "compat/include", include / "vcc/compat")
# Build against the pinned public header even when the ROM library has
# not yet been staged in a fresh runtime. Only SDK headers ship here.
shutil.copyfile(ROOT / "third_party/cmoc-libraries/decbfile/decbfile.h", include / "decbfile.h")
payload = {}
for name, source in LIBRARIES.items():
folder = work / source
folder.mkdir()
objects = []
flags = ["-I" + str(include)]
if name == "vcccompat":
flags.insert(0, "-I" + str(include / "vcc/compat"))
for header in (SDK / source).glob("*.h"):
shutil.copyfile(header, folder / header.name)
for unit in sorted((SDK / source).glob("*.c")):
shutil.copyfile(unit, folder / unit.name)
obj = unit.stem + ".o"
subprocess.run([runtime / "bin/cmoc", "--coco", "-O2", "-Werror",
*flags, "-c", "-o", obj, unit.name],
cwd=folder, env=env, check=True)
objects.append(obj)
if not objects:
raise ValueError("No compilation units for " + name)
archive = folder / ("lib" + name + ".a")
subprocess.run([runtime / "bin/lwar", "--create", archive.name, *objects],
cwd=folder, env=env, check=True)
if name == "vcccompat":
headers = [*sorted((include / "vcc/compat").glob("*.h")),
include / "vcc/printer.h", include / "vcc/cassette.h", include / "vcc/decimal.h"]
else:
headers = [include / ("vcc/" + name + ".h")]
files = {"share/cmoc/lib/" + archive.name: archive, "LICENSES/VCC-Guest-SDK.txt": ROOT / "COPYING"}
files.update({"share/cmoc/include/" + h.relative_to(include).as_posix(): h for h in headers})
version = {"vcccompat": "0.14.0", "ssc": "0.2.0", "orch90": "0.2.0"}.get(name, "0.1.0")
manifest.setdefault("libraries", {})[name] = dict(version=version, profile="coco",
license="GPL-3.0-or-later", filesSHA256={k: hashlib.sha256(p.read_bytes()).hexdigest() for k, p in files.items()})
if name == "vcccompat":
manifest["libraries"][name].update(diskReadStreams=True, diskWriteStreams=True, diskStreamPositioning=True,
boundedHeap=True, integerConversions=True, reentrantTokenizer=True,
diskUpdateStreams=True, createAppendUpdateStreams=True, diskFormattedOutput=True,
diskStreamClearError=True, printerFormattedOutput=True,
cassetteTextFiles=True, diskFileManagement=True, fixedDecimal=True)
if name == "ssc":
manifest["libraries"][name]["storedSpeechAndEffects"] = True
if name == "orch90":
manifest["libraries"][name]["backgroundPlayback"] = True
payload.update(files)
for path, original in payload.items():
target = runtime / path
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(original, target)
shutil.copyfile(ROOT / "tools/toolchain/cmoc-runtime.sh", runtime / "bin/cmoc")
temporary = manifest_file.with_suffix(".json.new")
temporary.write_text(json.dumps(manifest, indent=2) + "\n")
temporary.replace(manifest_file)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("runtime", type=Path)
stage(parser.parse_args().runtime.resolve())
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Build and package reviewed CMOC libraries with the standalone compiler chain."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
from runtime_inputs import validate_private_inputs
ROOT = Path(__file__).resolve().parents[2]
LIBRARIES = ('astar', 'bcontrol', 'bgraph', 'bsound', 'cardgame', 'ebitor', 'hirestxt', 'wydealloc')
def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('runtime', type=Path)
parser.add_argument('--jobs', type=int, default=4)
args = parser.parse_args()
runtime = args.runtime.resolve()
manifest_path = runtime / 'manifest.json'
manifest = json.loads(manifest_path.read_text())
validate_private_inputs(runtime, manifest)
subprocess.run([sys.executable, ROOT / 'tools/toolchain/verify-sources.py'], check=True)
env = dict(os.environ, PATH=str(runtime / 'bin') + ':/usr/bin:/bin', LC_ALL='C',
DEVELOPER_DIR='/VCC-no-developer-tools', SDKROOT='/VCC-no-sdk')
records = {}
with tempfile.TemporaryDirectory(prefix='vcc-cmoc-libraries-', dir='/tmp') as folder:
for name in LIBRARIES:
source = ROOT / 'third_party/cmoc-libraries' / name
work = Path(folder) / name
shutil.copytree(source, work)
subprocess.run([str(runtime / 'bin/make'), f'-j{max(1, args.jobs)}', f'lib{name}.a'],
cwd=work, env=env, check=True)
output = subprocess.check_output([str(runtime / 'bin/make'), '--no-print-directory',
'--eval', 'vcc-list-headers:;@echo VCC_HEADERS $(LIBHEADERS)', 'vcc-list-headers'],
cwd=work, env=env, text=True)
matches = [line for line in output.splitlines() if line.startswith('VCC_HEADERS ')]
if len(matches) != 1: raise SystemExit('Cannot determine library headers: ' + name)
headers = matches[0].split()[1:]
archive = runtime / 'share/cmoc/lib' / f'lib{name}.a'
shutil.copyfile(work / f'lib{name}.a', archive)
files = {archive.relative_to(runtime).as_posix(): sha(archive)}
for header in headers:
if Path(header).name != header or not header.endswith('.h'):
raise SystemExit('Unexpected library header: ' + header)
target = runtime / 'share/cmoc/include' / header
if target.exists() and target.read_bytes() != (source / header).read_bytes():
raise SystemExit('Conflicting library header: ' + header)
shutil.copyfile(source / header, target)
files[target.relative_to(runtime).as_posix()] = sha(target)
record = json.loads((ROOT / 'third_party/manifests' / (name + '.json')).read_text())
notice = runtime / 'LICENSES' / (name.upper() + '-NOTICE.txt')
text = f"{name} {record['version']}\n\n{record['license']}\n\n"
# Keep declarations/acknowledgments from every compiled source and
# public header. Full corresponding source remains in third_party.
for file in sorted(source.glob('*')):
if not file.is_file(): continue
if file.name in ('COPYING', 'README.md'):
text += '\n--- ' + file.name + ' ---\n' + file.read_text()
elif file.suffix in ('.c', '.h'):
content = file.read_text()
if content.startswith('/*'):
notice_end = content.find('*/')
excerpt = content[:notice_end + 2] if notice_end >= 0 else ''
else:
lines = []
for line in content.splitlines():
if line.startswith('//') or not line.strip(): lines.append(line)
else: break
excerpt = '\n'.join(lines).strip()
if excerpt: text += '\n--- ' + file.name + ' ---\n' + excerpt + '\n'
notice.write_text(text)
records[name] = {'version': record['version'], 'license': record['license'],
'profile': 'coco', 'filesSHA256': files}
manifest['libraries'] = records
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
subprocess.run([sys.executable, ROOT / 'tools/toolchain/stage-guest-sdk.py', runtime], check=True)
print('PASS: guest SDK and eight optional library archives and headers built with bundled tools')
if __name__ == '__main__': main()
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Package locally built Perl, preserving separate architecture-dependent cores."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[2]
VERSION = '5.44.0'
MACHO_MAGIC = {b'\xcf\xfa\xed\xfe', b'\xfe\xed\xfa\xcf', b'\xca\xfe\xba\xbe', b'\xbe\xba\xfe\xca'}
def native(path):
with path.open('rb') as stream: return stream.read(4) in MACHO_MAGIC
def stage(runtime, prefixes):
manifest_path = runtime / 'manifest.json'
manifest = json.loads(manifest_path.read_text())
if set(prefixes) != set(manifest['architectures']):
raise SystemExit('Perl builds must cover every declared runtime architecture')
helpers = {p: arches for p, arches in manifest.get('nativeHelpers', {}).items()
if not p.startswith('libexec/perl/') and p != 'bin/perl'}
target = runtime / 'libexec/perl'
if target.exists(): shutil.rmtree(target)
for arch, prefix in prefixes.items():
actual = subprocess.check_output([str(prefix / 'bin/perl'), '-e', 'print $^V'], text=True)
if actual != 'v' + VERSION: raise SystemExit('Unexpected Perl version: ' + actual)
installation = target / arch
(installation / 'bin').mkdir(parents=True)
shutil.copyfile(prefix / 'bin/perl', installation / 'bin/perl')
(installation / 'bin/perl').chmod(0o755)
# Core modules and their data/XS binaries, without SDK headers, static
# development archives, manuals, command-line maintenance tools or CPAN
# download caches. Original module notices remain in their source text.
def ignored(folder, names):
return [name for name in names if name == 'CORE' or name == 'pods'
or name.endswith(('.pod', '.a', '.h')) or name == '.packlist']
shutil.copytree(prefix / 'lib', installation / 'lib', ignore=ignored)
for path in installation.rglob('*'):
if path.is_symlink(): raise SystemExit('Unexpected symlink in Perl runtime: ' + str(path))
if path.is_file() and native(path):
slices = subprocess.check_output(['/usr/bin/lipo', '-archs', str(path)], text=True).split()
if arch not in slices: raise SystemExit('Incorrect Perl architecture: ' + str(path))
helpers[path.relative_to(runtime).as_posix()] = [arch]
env = dict(os.environ)
env.setdefault('DEVELOPER_DIR', '/Applications/Xcode.app/Contents/Developer')
with tempfile.TemporaryDirectory(prefix='vcc-perl-launcher-') as folder:
slices = []
for arch in prefixes:
binary = Path(folder) / arch
subprocess.run(['xcrun', 'clang', '-O2', '-arch', arch, '-mmacosx-version-min=14.0',
str(ROOT / 'tools/toolchain/perl-launcher.c'), '-o', str(binary)], env=env, check=True)
slices.append(binary)
launcher = runtime / 'bin/perl'
if len(slices) == 1: shutil.copyfile(slices[0], launcher)
else: subprocess.run(['/usr/bin/lipo', '-create', *map(str, slices), '-output', str(launcher)], check=True)
launcher.chmod(0o755)
helpers['bin/perl'] = list(prefixes)
# Upstream script's /usr/bin/perl shebang would bypass our bundled runtime.
# Preserve the script unchanged behind a relocatable wrapper instead.
shutil.copyfile(ROOT / 'third_party/cmoc/src/install-coco-boot-loader',
runtime / 'libexec/install-coco-boot-loader.pl')
wrapper = runtime / 'bin/install-coco-boot-loader'
wrapper.write_text('#!/bin/sh\n# SPDX-License-Identifier: GPL-3.0-or-later\n'
'set -eu\nruntime=$(CDPATH= cd -- "$(/usr/bin/dirname -- "$0")/.." && pwd -P)\n'
'exec "$runtime/bin/perl" "$runtime/libexec/install-coco-boot-loader.pl" "$@"\n')
wrapper.chmod(0o755)
notices = runtime / 'LICENSES'
for source, name in [('README', 'PERL-README.txt'), ('Copying', 'PERL-COPYING.txt'),
('Artistic', 'PERL-ARTISTIC.txt'),
('cpan/Compress-Raw-Bzip2/bzip2-src/LICENSE', 'PERL-BZIP2-LICENSE.txt')]:
shutil.copyfile(ROOT / 'third_party/perl' / source, notices / name)
manifest['nativeHelpers'] = dict(sorted(helpers.items()))
manifest['perl'] = {'version': VERSION, 'license': 'GPL-1.0-or-later OR Artistic-1.0-Perl',
'profile': 'interpreter-and-core-modules', 'architectures': list(prefixes),
'sourceManifestSHA256': hashlib.sha256((ROOT / 'third_party/manifests/perl.json').read_bytes()).hexdigest()}
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
for arch in prefixes:
subprocess.run(['/usr/bin/arch', '-' + arch, str(runtime / 'bin/perl'), '-MGetopt::Long',
'-MFile::Spec', '-MDigest::SHA', '-e',
'use strict; use warnings; print "Bundled Perl $^V OK\\n";'], check=True)
print('PASS: staged independent Perl/core-module runtime for ' + ', '.join(prefixes))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('runtime', type=Path)
parser.add_argument('--prefix', action='append', required=True, help='ARCH=installed-prefix (repeat per architecture)')
args = parser.parse_args()
prefixes = {}
for item in args.prefix:
arch, separator, value = item.partition('=')
if not separator or arch not in ('arm64', 'x86_64') or arch in prefixes:
parser.error('Expected one ARCH=prefix per architecture')
prefixes[arch] = Path(value).resolve()
stage(args.runtime.resolve(), prefixes)
if __name__ == '__main__': main()
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Build an explicitly supplied, private DECB-derived driver as an optional library.
The source remains outside Git. The ROM-backed decbfile archive is unchanged.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
from runtime_inputs import validate_private_inputs
ROOT = Path(__file__).resolve().parents[2]
def sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('runtime', type=Path)
parser.add_argument('--source', type=Path, required=True)
args = parser.parse_args()
runtime = args.runtime.resolve()
manifest = runtime / 'manifest.json'
record = json.loads(manifest.read_text())
validate_private_inputs(runtime, record)
cmoc = json.loads((ROOT / 'third_party/manifests/cmoc.json').read_text())
expected = cmoc['excludedFileSHA256']['src/stdlib/dskcon-standalone.c']
if sha(args.source) != expected:
parser.error('Private driver must match the reviewed CMOC source release')
header = ROOT / 'third_party/cmoc/src/stdlib/dskcon-standalone.h'
if sha(header) != cmoc['upstreamFileSHA256']['src/stdlib/dskcon-standalone.h']:
parser.error('Standalone header differs from the reviewed source')
env = dict(os.environ, PATH=str(runtime / 'bin') + ':/usr/bin:/bin',
DEVELOPER_DIR='/VCC-no-developer-tools', SDKROOT='/VCC-no-sdk')
with tempfile.TemporaryDirectory(prefix='vcc-dskcon-', dir='/tmp') as folder:
work = Path(folder)
shutil.copyfile(args.source, work / 'dskcon-standalone.c')
shutil.copyfile(header, work / header.name)
subprocess.run([str(runtime / 'bin/cmoc'), '--coco', '-O2', '-I.', '-c',
'-o', 'dskcon-standalone.o', 'dskcon-standalone.c'], cwd=work, env=env, check=True)
subprocess.run([str(runtime / 'bin/lwar'), '--create', 'libdskcon-standalone.a',
'dskcon-standalone.o'], cwd=work, env=env, check=True)
shutil.copyfile(work / 'libdskcon-standalone.a', runtime / 'share/cmoc/lib/libdskcon-standalone.a')
shutil.copyfile(header, runtime / 'share/cmoc/include/dskcon-standalone.h')
upstream_notice = args.source.read_text().split('*/', 1)[0] + '*/\n'
notice = ('Standalone CoCo floppy disk driver — optional linked library\n\n' + upstream_notice +
'\nThis library contains portions taken from Disk Extended Color BASIC (DECB).\n'
'Those portions are separate third-party code, not covered by Pierre Sarrazin\'s\n'
'public-domain dedication or VCC Plus\'s GPL license. Original rights remain\n'
'with their respective owners; this notice grants no additional rights.\n\n'
'The ROM-backed decbfile core remains a separate public-domain library.\n'
'Use -ldskcon-standalone to explicitly link this driver into a CoCo program.\n'
'Its code runs from RAM and accesses the emulated floppy controller.\n'
'It does not require Disk BASIC to remain mapped during disk operations.\n\n'
'Rights-holder inquiries or removal requests: [email protected]\n')
(runtime / 'LICENSES/DSKCON-STANDALONE-NOTICE.txt').write_text(notice)
paths = ['share/cmoc/lib/libdskcon-standalone.a', 'share/cmoc/include/dskcon-standalone.h',
'LICENSES/DSKCON-STANDALONE-NOTICE.txt']
record['privateRuntimeInputs'] = True
record['standaloneDisk'] = {
'profile': 'decb-derived-standalone', 'cmocVersion': cmoc['version'],
'license': 'public-domain-except-DECB-derived-portions', 'sourceSHA256': expected,
'filesSHA256': {p: sha(runtime / p) for p in paths},
}
validate_private_inputs(runtime, record)
manifest.write_text(json.dumps(record, indent=2) + '\n')
print('Staged optional standalone disk API; explicitly link with -ldskcon-standalone.')
if __name__ == '__main__':
main()
+295
View File
@@ -0,0 +1,295 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Audit a staged runtime and compile through Make without Apple developer tools."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
from runtime_inputs import validate_private_inputs
TOOLS = ('libexec/cmoc', 'bin/cpp', 'bin/mcpp', 'bin/make', 'bin/lwasm', 'bin/lwlink', 'bin/lwar', 'bin/writecocofile')
def check_assembly(runtime, folder):
"""Both native slices must resolve source identity and reject escaping inputs."""
folder.mkdir()
assembler = str(runtime / 'bin/lwasm')
(folder / 'part.inc').write_text(' lda #$8f\n fcb $86,$8f\n')
source = folder / 'main.asm'
source.write_text(' org $3f00\n include "part.inc"\n end $3f00\n')
command = [assembler, '--project-inputs', '--project-map=lines.tsv', '--6809', '--decb', '-o', 'first.bin', 'main.asm']
subprocess.run(command, cwd=folder, check=True)
if (folder / 'lines.tsv').read_text() != 'part.inc\t1\t\t3F00\t868F\n':
raise SystemExit('LWASM physical source map is missing or includes data')
if (folder / 'first.bin').read_bytes() != bytes.fromhex('00 0004 3f00 868f 868f ff 0000 3f00'):
raise SystemExit('LWASM project image differs')
outside = folder.with_suffix('.inc')
outside.write_text(' nop\n')
(folder / 'linked.inc').symlink_to(outside)
for directive, path in [('include', str(outside)), ('include', 'linked.inc'), ('includebin', '../' + outside.name)]:
source.write_text(f" org $3f00\n {directive} '{path}'\n end $3f00\n")
failure = subprocess.run(command, cwd=folder, capture_output=True, text=True)
if failure.returncode == 0 or 'leaves the project snapshot' not in failure.stderr:
raise SystemExit('Unexpected LWASM result for escaping ' + directive + ': ' + failure.stderr)
def check_disk_allocation(runtime, folder):
"""The explicit before-dir option must match the existing forward allocator."""
folder.mkdir()
tool = str(runtime / 'bin/writecocofile')
payload = folder / 'DATA.DAT'
payload.write_bytes(bytes(range(256)) * (35 * 9)) # 35 granules: spills above track 17.
images = []
for name, options, first in [('default', [], 0), ('before', ['--allocate-before-dir'], 0),
('after', ['--allocate-after-dir'], 34),
('last-option', ['--allocate-after-dir', '--allocate-before-dir'], 0)]:
image = folder / (name + '.dsk')
image.write_bytes(b'\xff' * (35 * 18 * 256))
subprocess.run([tool, *options, str(image), str(payload)], check=True, stdout=subprocess.DEVNULL)
content = image.read_bytes()
entry = 17 * 18 * 256 + 2 * 256
if content[entry + 13] != first:
raise SystemExit('Incorrect disk allocation start: ' + name)
restored = subprocess.check_output([tool, '--stdout', str(image), 'DATA.DAT'])
if restored != payload.read_bytes():
raise SystemExit('Disk allocation readback mismatch: ' + name)
images.append(content)
if images[0] != images[1] or images[0] != images[3]:
raise SystemExit('Before-directory option changed default allocation semantics')
def check_guest_example(runtime, env, folder, project_name, payloads):
"""Build an editable example and compare its guest disk files with the build outputs."""
source = Path(__file__).resolve().parents[2] / "examples/cmoc" / project_name
shutil.copytree(source, folder, ignore=shutil.ignore_patterns("obj", "generated", "*.bin", "*.BIN", "*.dsk", "*.DSK", "*.map", "*.link"))
subprocess.run([runtime / "bin/make", "-B", "disk"], cwd=folder, env=env, check=True)
for disk_name, output, guest_name in payloads:
expected = (folder / output).read_bytes()
extracted = subprocess.check_output([runtime / "bin/writecocofile", "--stdout", disk_name, guest_name],
cwd=folder, env=env)
if not expected or extracted != expected:
raise SystemExit(f"{project_name}: {guest_name} differs from {output}")
subprocess.run([runtime / "bin/make", "-q", "build"], cwd=folder, env=env, check=True)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('runtime', type=Path)
args = parser.parse_args()
runtime = args.runtime.resolve()
record = json.loads((runtime / 'manifest.json').read_text())
validate_private_inputs(runtime, record)
disk = record.get('decbfile')
if disk:
if disk.get('profile') != 'rom-backed-core' or disk.get('standaloneDriverIncluded') is not False:
raise SystemExit('Unsupported disk library profile')
for relative, key in [('share/cmoc/lib/libdecbfile.a', 'archiveSHA256'), ('share/cmoc/include/decbfile.h', 'headerSHA256')]:
if hashlib.sha256((runtime / relative).read_bytes()).hexdigest() != disk[key]:
raise SystemExit('Disk library manifest hash mismatch: ' + relative)
helpers = {relative: record['architectures'] for relative in TOOLS}
helpers.update(record.get('nativeHelpers', {}))
for library in record.get('libraries', {}).values():
for relative, expected in library['filesSHA256'].items():
if hashlib.sha256((runtime / relative).read_bytes()).hexdigest() != expected:
raise SystemExit('Library manifest hash mismatch: ' + relative)
for relative, architectures in helpers.items():
binary = runtime / relative
info = subprocess.check_output(['/usr/bin/lipo', '-archs', str(binary)], text=True).split()
if not set(architectures).issubset(info): raise SystemExit(f'Missing architecture: {relative}')
links = subprocess.check_output(['/usr/bin/otool', '-L', str(binary)], text=True)
for line in links.splitlines():
if line.startswith('\t') and not line.strip().startswith(('/usr/lib/', '/System/Library/')):
raise SystemExit(f'Non-system runtime dependency: {relative}: {line}')
with tempfile.TemporaryDirectory(prefix='VCC runtime #$ quote" relocation ') as folder:
moved = Path(folder) / 'VCC Plus.app/Contents/Resources/Toolchain'
shutil.copytree(runtime, moved)
project = Path(folder) / 'C project with spaces'; project.mkdir()
(project / 'main.c').write_text('#include <cmoc.h>\n#define JOIN(a,b) a##b\n#define STRING(x) #x\nint main(void) { unsigned JOIN(co,unt)=7; printf(STRING(HELLO) " %u\\n", count); return 0; }\n')
if disk:
(project / 'disk.c').write_text('''#include <decbfile.h>
DECBDrive drives[1]; DECBFile file; byte written[256], readback[256];
int main(void) {
initCoCoSupport();
if (decb_init(drives, 1) || decb_registerDrive(0)) return 1;
memset(written, 42, 256);
if (decb_createSectorFile(&file, 0, "ROUND.DAT", DECB_TYPE_BASIC_DATA, DECB_FORMAT_BINARY)) return 2;
if (decb_writeSector(&file, written, 0) || decb_setNumBytesUsedInLastSector(&file, 256) || decb_closeSectorFile(&file)) return 3;
if (decb_openSectorFile(&file, 0, "ROUND.DAT") || decb_readSector(&file, readback, 0) || decb_closeSectorFile(&file)) return 4;
if (decb_rename(0, "ROUND.DAT", "RESULT.DAT") || decb_kill(0, "RESULT.DAT") || decb_shutdown()) return 5;
return memcmp(written, readback, 256) != 0;
}
''')
def compile_disk(toolchain, tool_env):
if disk:
subprocess.run([str(toolchain / 'bin/cmoc'), '--coco', '--intermediate', '-O0',
'-o', 'disk.bin', 'disk.c', '-ldecbfile'], cwd=project, env=tool_env, check=True)
image = (project / 'disk.bin').read_bytes()
if len(image) < 10 or image[0] != 0 or image[-5:-2] != b'\xff\0\0':
raise SystemExit('Invalid linked disk example binary')
if record.get('standaloneDisk'):
(project / 'standalone.c').write_text('#include <dskcon-standalone.h>\nint main(void) { unsigned long saved = dskcon_init(dskcon_nmiService); DCOPC=2; DCDRV=0; DCTRK=17; DCSEC=3; DCBPT=(void *)0x500; dskcon_processSector(); dskcon_irqService(); dskcon_shutdown(saved); return DCSTA; }\n')
subprocess.run([str(toolchain / 'bin/cmoc'), '--coco', '-o', 'standalone.bin',
'standalone.c', '-ldskcon-standalone'], cwd=project, env=tool_env, check=True)
(project / 'Makefile').write_text('all: main.bin\nmain.bin: main.c\n\tcmoc --coco --intermediate -O0 -o main.bin main.c\n')
env = dict(os.environ, PATH=str(moved / 'bin') + ':/usr/bin:/bin',
DEVELOPER_DIR='/VCC-validation-no-developer-tools', SDKROOT='/VCC-validation-no-sdk', LC_ALL='C')
for key in ('PERL5LIB', 'PERLLIB', 'PERL5OPT'): env.pop(key, None)
def check_perl(toolchain, tool_env):
if not record.get('perl'): return
result = subprocess.check_output([str(toolchain / 'bin/perl'), '-MGetopt::Long',
'-MDigest::SHA', '-e', 'use strict; use warnings; print $^X, \"\\n\", Digest::SHA::sha256_hex(\"VCC\");'],
cwd=project, env=tool_env, text=True)
if str(toolchain) not in result or hashlib.sha256(b'VCC').hexdigest() not in result:
raise SystemExit('Perl did not use the relocated bundled interpreter and modules')
subprocess.run([str(toolchain / 'bin/install-coco-boot-loader'), '--help'],
cwd=project, env=tool_env, check=True, stdout=subprocess.DEVNULL)
def check_dependencies(toolchain, tool_env, name):
dependency_project = Path(folder) / name
dependency_project.mkdir()
header = dependency_project / 'headers #$ spaces' / 'value.h'
header.parent.mkdir()
header.write_text('#define VALUE 42\n')
(dependency_project / 'main.c').write_text('#include <cmoc.h>\n#include "headers #$ spaces/value.h"\nint main(void) { return VALUE; }\n')
# Include before the .d exists: Make must generate/reparse it, not
# merely tolerate a dependency file that has never been consumed.
# --deps-only and --deps use separate CMOC marker readers.
(dependency_project / 'child').mkdir()
(dependency_project / 'child/Makefile').write_text('stamp:\n\tprintf child > stamp\n')
(dependency_project / 'Makefile').write_text(
'all: main.bin child/stamp\n'
'child/stamp:\n\t$(MAKE) -C child\n'
'main.bin: main.o\n\tcmoc --coco -o $@ $<\n'
'main.o: main.c\n\tcmoc --coco --deps -c -o $@ $<\n'
'main.d: main.c\n\tcmoc --coco --deps-only main.c\n'
'include main.d\n')
# Same recursion-safe override used by the app: Make's default MAKE
# is its absolute executable path, unsafe unquoted inside recipes.
command = [str(toolchain / 'bin/make'), 'MAKE=make', '--no-builtin-rules']
def make(*options):
subprocess.run(command + list(options), cwd=dependency_project, env=tool_env, check=True)
make()
make('-q')
first = (dependency_project / 'main.bin').read_bytes()
# A changed header must invalidate both the generated dependency
# file and compiled object; no-op rebuilds must remain no-ops.
header.write_text('#define VALUE 43\n')
make()
make('-q')
if first == (dependency_project / 'main.bin').read_bytes():
raise SystemExit('Changed dependency did not rebuild the executable')
dependency_text = (dependency_project / 'main.d').read_text()
if 'VCC\\ Plus.app' not in dependency_text and 'Toolchain' not in dependency_text:
raise SystemExit('Dependency test did not exercise relocated SDK headers')
if '\\#$$' not in dependency_text:
raise SystemExit('Make dependency special characters were not escaped')
check_disk_allocation(moved, Path(folder) / "Disk allocation")
check_dependencies(moved, env, 'Dependency project')
check_perl(moved, env)
# A deliberately unusable developer directory catches accidental xcrun
# or Apple compiler use even on the development machine.
subprocess.run([str(moved / 'bin/make'), '--no-builtin-rules'], cwd=project, env=env, check=True)
compile_disk(moved, env)
for name in ('main.bin', 'main.s', 'main.lst', 'main.map', 'main.link'):
if not (project / name).is_file(): raise SystemExit('Missing build product: ' + name)
binary = (project / 'main.bin').read_bytes()
if len(binary) < 10 or binary[0] != 0 or binary[-5:-2] != b'\xff\0\0':
raise SystemExit('Invalid DECB binary')
# Thin copies force the entire compiler/preprocessor/assembler/linker
# chain onto each architecture, not just a --version smoke test.
for architecture in record['architectures']:
thin = Path(folder) / (architecture + ' Toolchain')
shutil.copytree(runtime, thin)
for relative, architectures in helpers.items():
if architecture not in architectures: continue
binary = thin / relative
temporary = binary.with_name(binary.name + '.thin')
slices = subprocess.check_output(['/usr/bin/lipo', '-archs', str(binary)], text=True).split()
if len(slices) > 1:
subprocess.run(['/usr/bin/lipo', str(binary), '-thin', architecture,
'-output', str(temporary)], check=True)
temporary.replace(binary)
binary.chmod(0o755)
subprocess.run(['/usr/bin/codesign', '--force', '--sign', '-', '--timestamp=none',
str(binary)], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
arch_env = dict(env, PATH=str(thin / 'bin') + ':/usr/bin:/bin')
subprocess.run([str(thin / 'bin/make'), '--no-builtin-rules', '-B'],
cwd=project, env=arch_env, check=True)
if {"ssc", "orch90", "vcccompat"}.issubset(record.get("libraries", {})):
check_guest_example(thin, arch_env, Path(folder) / (architecture + " audio SDK"),
"TalkingPlanetLander", [("LANDER.dsk", "LANDER.bin", "LANDER.BIN")])
if record["libraries"]["vcccompat"].get("reentrantTokenizer"):
check_guest_example(thin, arch_env, Path(folder) / (architecture + " streams SDK"),
"PlanetSurvey", [("SURVEY.dsk", "SURVEY.bin", "SURVEY.BIN"),
("SURVEY.dsk", "generated/PLANETS.TXT", "PLANETS.TXT")])
if "sdc" in record.get("libraries", {}):
check_guest_example(thin, arch_env, Path(folder) / (architecture + " SDC SDK"),
"PlanetExpedition", [
("EXPED/EXPLORE.DSK", "EXPLORE.BIN", "EXPLORE.BIN"),
("EXPED/EARTH.DSK", "generated/earth/LANDING.TXT", "LANDING.TXT"),
("EXPED/MARS.DSK", "generated/mars/LANDING.TXT", "LANDING.TXT")])
libraries = record.get("libraries", {})
if (record.get("decbfile") and {"bank", "gime", "input", "bcontrol", "hirestxt"} <= libraries.keys()
and libraries.get("orch90", {}).get("backgroundPlayback")
and libraries.get("ssc", {}).get("storedSpeechAndEffects")
and libraries.get("vcccompat", {}).get("diskFileManagement")):
check_guest_example(thin, arch_env, Path(folder) / (architecture + " mission SDK"),
"MissionRecorder", [("RECORDER.dsk", "RECORDER.bin", "RECORDER.BIN")])
manual_examples = (
("vcccompat", "diskWriteStreams", "fieldlog.c", ["decbfile"]),
("vcccompat", "integerConversions", "atlas.c", ["decbfile"]),
("vcccompat", "diskStreamClearError", "returntrip.c", ["decbfile"]),
("vcccompat", "printerFormattedOutput", "flightreport.c", []),
("vcccompat", "cassetteTextFiles", "flightrecorder.c", []),
("vcccompat", "diskFileManagement", "atlasbackup.c", ["decbfile"]),
("vcccompat", "fixedDecimal", "descent.c", []),
("orch90", "backgroundPlayback", "flyby.c", ["input", "bcontrol"]),
("ssc", "storedSpeechAndEffects", "announcer.c", ["input", "bcontrol"]),
)
for library, feature, source, dependencies in manual_examples:
if record.get("libraries", {}).get(library, {}).get(feature):
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual" / source
shutil.copyfile(example, project / source)
link_flags = ["-l" + name for name in [library, *dependencies]]
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o",
example.stem + ".bin", source, *link_flags], cwd=project,
env=dict(arch_env, VCC_CMOC_COMPAT="1" if library == "vcccompat" else "0"), check=True)
if record.get("libraries", {}).get("vcccompat", {}).get("boundedHeap"):
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual/crew.c"
shutil.copyfile(example, project / "crew.c")
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o", "crew.bin",
"crew.c", "-lvcccompat", "-lwydealloc"],
cwd=project, env=dict(arch_env, VCC_CMOC_COMPAT="1"), check=True)
if "gime" in record.get("libraries", {}):
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual/lander.c"
shutil.copyfile(example, project / "lander.c")
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o", "lander.bin",
"lander.c", "-lgime", "-lbgraph", "-lbcontrol"],
cwd=project, env=arch_env, check=True)
if "input" in record.get("libraries", {}):
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual/probe.c"
shutil.copyfile(example, project / "probe.c")
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o", "probe.bin",
"probe.c", "-lgime", "-linput", "-lbcontrol"],
cwd=project, env=arch_env, check=True)
if "bank" in record.get("libraries", {}):
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual/ridge.c"
shutil.copyfile(example, project / "ridge.c")
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o", "ridge.bin",
"ridge.c", "-lbank", "-lgime", "-lhirestxt"],
cwd=project, env=arch_env, check=True)
compile_disk(thin, arch_env)
check_perl(thin, arch_env)
check_assembly(thin, Path(folder) / (architecture + " assembly"))
check_disk_allocation(thin, Path(folder) / (architecture + " disk allocation"))
check_dependencies(thin, arch_env, architecture + " dependency project")
# Preprocessor errors must stop Make and retain source-line diagnostics.
(project / 'main.c').write_text('#line 81 "mapped.c"\n#error VCC_EXPECTED_PREPROCESSOR_FAILURE\n')
failure = subprocess.run([str(moved / 'bin/make'), '-B'], cwd=project, env=env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
if failure.returncode == 0 or 'VCC_EXPECTED_PREPROCESSOR_FAILURE' not in failure.stdout or '81' not in failure.stdout:
raise SystemExit('Preprocessor errors/line directives were not preserved: ' + failure.stdout)
print('PASS: architectures, system-only libraries, relocated space-containing bundle, both complete architecture build chains, preprocessor errors, Make dependency regeneration with spaces/quotes/#/$, optional ROM-backed/standalone disk API linking, bundled Perl/core modules, disk allocation and BIN without developer tools')
if __name__ == '__main__': main()
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Verify the checked-in source snapshots against their reviewed import manifests."""
from pathlib import Path
import hashlib
import json
ROOT = Path(__file__).resolve().parents[2]
def main():
total = 0
for manifest in sorted((ROOT / 'third_party/manifests').glob('*.json')):
record = json.loads(manifest.read_text())
directory = ROOT / record['directory']
actual = {p.relative_to(directory).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
for p in directory.rglob('*') if p.is_file()}
expected = record['vendoredFileSHA256']
if actual != expected:
changed = sorted(k for k in actual.keys() | expected.keys() if actual.get(k) != expected.get(k))
raise SystemExit(f"{record['project']}: unexpected files or content: {changed}")
if any((directory / p).exists() for p in record['excludedFiles']):
raise SystemExit(f"{record['project']}: excluded source was reintroduced")
print(f"PASS {record['project']} {record['version']}: {len(actual)} files")
total += len(actual)
print(f'PASS: {total} reviewed source files; no downloaded binaries or ROMs required')
if __name__ == '__main__':
main()