Publish VCC Plus OSS source distribution for 3.6.2 build 60
This commit is contained in:
commit
c652a5276d
1192 files changed
+272368
No files matched your search
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Explicit app build-content selection and schema-only resource staging.
|
||||
|
||||
Normal is the existing maintainer/developer build. Source builds deliberately
|
||||
omit prepared libraries; they never need to probe a maintainer input location.
|
||||
"""
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
|
||||
PROFILES = ('normal', 'source')
|
||||
PROFILE_ENV = 'VCC_BUILD_CONTENT_PROFILE'
|
||||
|
||||
|
||||
def content_profile(value=None, environ=None):
|
||||
"""CLI values override the environment; an unset profile preserves defaults."""
|
||||
selected = value if value is not None else (os.environ if environ is None else environ).get(PROFILE_ENV, 'normal')
|
||||
if selected not in PROFILES:
|
||||
raise ValueError('Build content profile must be normal or source')
|
||||
return selected
|
||||
|
||||
|
||||
def add_content_profile_argument(parser):
|
||||
parser.add_argument('--content-profile', choices=PROFILES,
|
||||
help='App content: normal (default) or source (empty libraries); overrides ' + PROFILE_ENV)
|
||||
|
||||
|
||||
def _check_output(output, expected_name, expected_parent):
|
||||
# Preserve macOS's public /tmp and /var spellings without resolving arbitrary
|
||||
# links supplied inside a build directory. No other ancestor link is allowed.
|
||||
aliases = {Path('/tmp'): Path('/private/tmp'), Path('/var'): Path('/private/var')}
|
||||
absolute = output if output.is_absolute() else Path.cwd() / output
|
||||
for parent in absolute.parents:
|
||||
if parent.is_symlink() and not (parent in aliases and parent.resolve() == aliases[parent]):
|
||||
raise ValueError('Build resource destination must not have a linked ancestor directory')
|
||||
if output.name != expected_name or output.parent.name != expected_parent:
|
||||
raise ValueError('Expected a ' + expected_parent + '/' + expected_name + ' build resource destination')
|
||||
if output.is_symlink() or (output.exists() and (not output.is_file() or output.stat().st_nlink != 1)):
|
||||
raise ValueError('Build resource destination must be a regular, unlinked file')
|
||||
if any(Path(str(output) + suffix).exists() or Path(str(output) + suffix).is_symlink()
|
||||
for suffix in ('-wal', '-shm', '-journal')):
|
||||
raise ValueError('Remove SQLite transaction sidecars from this build resource before regenerating it')
|
||||
|
||||
|
||||
def atomic_empty_database(output, schema, metadata, content_tables, *, expected_name,
|
||||
expected_parent, application_id=None, user_version, fts_integrity=False):
|
||||
"""Validate a new schema-only database before replacing a disposable resource.
|
||||
|
||||
No source database is accepted. Existing identical resource bytes retain their
|
||||
timestamp. Failed schema/integrity checks leave the previous resource intact.
|
||||
"""
|
||||
output = Path(output)
|
||||
_check_output(output, expected_name, expected_parent)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, name = tempfile.mkstemp(prefix='.empty-library-', suffix='.sqlite', dir=output.parent)
|
||||
os.close(descriptor)
|
||||
temporary = Path(name)
|
||||
try:
|
||||
with closing(sqlite3.connect(temporary)) as db:
|
||||
db.execute('PRAGMA page_size=4096')
|
||||
db.execute('PRAGMA foreign_keys=ON')
|
||||
db.executescript(Path(schema).read_text(encoding='utf-8'))
|
||||
db.executemany('INSERT INTO metadata VALUES(?,?)', metadata)
|
||||
if fts_integrity:
|
||||
db.execute("INSERT INTO page_terms(page_terms) VALUES('integrity-check')")
|
||||
db.commit()
|
||||
db.execute('VACUUM')
|
||||
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
|
||||
raise ValueError('Empty library integrity check failed')
|
||||
if db.execute('PRAGMA foreign_key_check').fetchall():
|
||||
raise ValueError('Empty library has broken relationships')
|
||||
if db.execute('PRAGMA user_version').fetchone() != (user_version,):
|
||||
raise ValueError('Unsupported empty library schema version')
|
||||
if application_id is not None and db.execute('PRAGMA application_id').fetchone() != (application_id,):
|
||||
raise ValueError('Unsupported empty library application identity')
|
||||
for table in content_tables:
|
||||
# Table names are authored constants, never command-line values.
|
||||
if db.execute('SELECT count(*) FROM "' + table + '"').fetchone() != (0,):
|
||||
raise ValueError('Library schema unexpectedly inserts content')
|
||||
data = temporary.read_bytes()
|
||||
result = dict(bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
|
||||
os.chmod(temporary, 0o444)
|
||||
_check_output(output, expected_name, expected_parent)
|
||||
if output.is_file() and output.read_bytes() == data:
|
||||
# chmod only a verified regular file; never follow a linked output.
|
||||
os.chmod(output, 0o444)
|
||||
else:
|
||||
os.replace(temporary, output)
|
||||
return result
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
@@ -0,0 +1,44 @@
|
||||
-- SPDX-License-Identifier: GPL-3.0-or-later
|
||||
-- Review/export schema; contains no local cache paths or emulator code.
|
||||
PRAGMA foreign_keys=ON;
|
||||
CREATE TABLE metadata(key TEXT PRIMARY KEY,value TEXT NOT NULL);
|
||||
CREATE TABLE title(id TEXT PRIMARY KEY,name TEXT NOT NULL,identity_status TEXT NOT NULL);
|
||||
CREATE TABLE package(sha256 TEXT PRIMARY KEY,byte_size INTEGER NOT NULL CHECK(byte_size>=0));
|
||||
CREATE TABLE release(
|
||||
id TEXT PRIMARY KEY,title_id TEXT NOT NULL REFERENCES title(id),name TEXT NOT NULL,
|
||||
category TEXT NOT NULL,media_type TEXT NOT NULL,content_role TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL UNIQUE,modified_at TEXT,modified_precision TEXT,modified_timezone TEXT,
|
||||
package_sha256 TEXT NOT NULL REFERENCES package(sha256),language_status TEXT NOT NULL,
|
||||
metadata_json TEXT NOT NULL CHECK(json_valid(metadata_json)));
|
||||
CREATE TABLE member(
|
||||
release_id TEXT NOT NULL REFERENCES release(id),path TEXT NOT NULL,sha256 TEXT NOT NULL,
|
||||
byte_size INTEGER NOT NULL,kind TEXT NOT NULL,role TEXT NOT NULL,
|
||||
PRIMARY KEY(release_id,path));
|
||||
CREATE TABLE download_location(
|
||||
package_sha256 TEXT NOT NULL REFERENCES package(sha256),url TEXT NOT NULL,priority INTEGER NOT NULL,
|
||||
PRIMARY KEY(package_sha256,url));
|
||||
CREATE TABLE asset(id TEXT PRIMARY KEY,name TEXT NOT NULL,kind TEXT NOT NULL,source_url TEXT NOT NULL,
|
||||
sha256 TEXT,byte_size INTEGER,credit TEXT NOT NULL);
|
||||
CREATE TABLE asset_delivery(
|
||||
asset_id TEXT PRIMARY KEY REFERENCES asset(id),manual_url TEXT,thumbnail_url TEXT,
|
||||
thumbnail_sha256 TEXT,thumbnail_byte_size INTEGER,priority INTEGER NOT NULL);
|
||||
CREATE TABLE release_asset(
|
||||
release_id TEXT NOT NULL REFERENCES release(id),asset_id TEXT NOT NULL REFERENCES asset(id),
|
||||
status TEXT NOT NULL,reason TEXT NOT NULL,PRIMARY KEY(release_id,asset_id));
|
||||
CREATE TABLE setup(
|
||||
release_id TEXT PRIMARY KEY REFERENCES release(id),status TEXT NOT NULL,
|
||||
configuration_json TEXT NOT NULL CHECK(json_valid(configuration_json)),
|
||||
selected_members_json TEXT NOT NULL CHECK(json_valid(selected_members_json)),
|
||||
command TEXT,instructions TEXT NOT NULL,reasons_json TEXT NOT NULL CHECK(json_valid(reasons_json)),
|
||||
validation_json TEXT NOT NULL CHECK(json_valid(validation_json)));
|
||||
CREATE TABLE source_hint(
|
||||
id TEXT PRIMARY KEY,release_id TEXT NOT NULL REFERENCES release(id),page_url TEXT NOT NULL,
|
||||
page_sha256 TEXT NOT NULL,source_name TEXT NOT NULL,raw_json TEXT NOT NULL CHECK(json_valid(raw_json)),
|
||||
resolution TEXT NOT NULL);
|
||||
CREATE INDEX release_title ON release(title_id);
|
||||
CREATE INDEX release_category ON release(category);
|
||||
CREATE INDEX setup_status ON setup(status);
|
||||
CREATE VIEW browse AS SELECT r.*,t.name AS title,s.status,s.command,s.instructions,
|
||||
s.configuration_json,s.selected_members_json,s.reasons_json,s.validation_json
|
||||
FROM release r JOIN title t ON t.id=r.title_id JOIN setup s ON s.release_id=r.id;
|
||||
PRAGMA user_version=1;
|
||||
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Stage the reviewed software catalog as a read-only app resource, never media.
|
||||
|
||||
Ordinary source builds get an empty schema if the private catalog is absent.
|
||||
Release packaging passes --require-full and fails rather than shipping it empty.
|
||||
The explicit source content profile generates an empty schema without reading
|
||||
maintainer inputs; normal remains the default.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import struct
|
||||
import subprocess
|
||||
import zipfile
|
||||
import sys
|
||||
from urllib.parse import urlsplit
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
from build_content import add_content_profile_argument, atomic_empty_database, content_profile as resolve_content_profile
|
||||
SCHEMA = Path(__file__).with_name('app-schema.sql')
|
||||
DEFAULT_SOURCE = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite'
|
||||
DEFAULT_ARTWORK_CACHE = ROOT / 'media/software-library-full/cache'
|
||||
MAX_BYTES = 256 * 1024 * 1024
|
||||
PRIVATE = re.compile(r'(?:file://|/Users/|/home/|/private/|(?<![A-Za-z])[A-Za-z]:[\\/]|(?:^|[\s"\'])\.build/|software-library-full/)', re.I)
|
||||
|
||||
|
||||
def schema_signature(db):
|
||||
return {(kind, name): ' '.join(sql.split()) for kind, name, sql in db.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'") if sql}
|
||||
|
||||
|
||||
def validate(path, require_full=False):
|
||||
"""Validate a standalone finalized export without allowing writes or extensions."""
|
||||
path = Path(path)
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError('Catalog input must be a regular, nonsymlink file')
|
||||
if not 0 < path.stat().st_size <= MAX_BYTES:
|
||||
raise ValueError('Catalog input has an invalid size')
|
||||
for suffix in ('-wal', '-journal'):
|
||||
sidecar = Path(str(path) + suffix)
|
||||
if sidecar.exists() and sidecar.stat().st_size:
|
||||
raise ValueError('Catalog has an active transaction sidecar; finalize it before packaging')
|
||||
with closing(sqlite3.connect(path.resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
||||
db.execute('PRAGMA query_only=ON')
|
||||
db.execute('PRAGMA trusted_schema=OFF')
|
||||
if db.execute('PRAGMA user_version').fetchone()[0] != 1:
|
||||
raise ValueError('Unsupported catalog schema version')
|
||||
with closing(sqlite3.connect(':memory:')) as expected:
|
||||
expected.executescript(SCHEMA.read_text())
|
||||
if schema_signature(db) != schema_signature(expected):
|
||||
raise ValueError('Catalog does not match the approved app export schema')
|
||||
# Older macOS SQLite marks json_valid() unsafe with trusted_schema=OFF.
|
||||
# The exact schema was just compared to our trusted, checked-in schema.
|
||||
db.execute('PRAGMA trusted_schema=ON')
|
||||
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
|
||||
raise ValueError('Catalog integrity check failed')
|
||||
if db.execute('PRAGMA foreign_key_check').fetchall():
|
||||
raise ValueError('Catalog contains broken relationships')
|
||||
metadata = dict(db.execute('SELECT key,value FROM metadata'))
|
||||
if metadata.get('schema_version') != '1' or metadata.get('recipe_contract') != '2':
|
||||
raise ValueError('Unsupported catalog metadata or loading contract')
|
||||
releases = db.execute('SELECT count(*) FROM release').fetchone()[0]
|
||||
if require_full and releases == 0:
|
||||
raise ValueError('A populated catalog is required for release packaging')
|
||||
if db.execute('SELECT count(*) FROM release LEFT JOIN setup ON release.id=setup.release_id WHERE setup.release_id IS NULL').fetchone()[0]:
|
||||
raise ValueError('Catalog release is missing loading metadata')
|
||||
for table, in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"):
|
||||
for row in db.execute('SELECT * FROM "' + table + '"'):
|
||||
for value in row:
|
||||
if isinstance(value, bytes):
|
||||
raise ValueError('Catalog must not contain bundled media blobs')
|
||||
if isinstance(value, str) and PRIVATE.search(value):
|
||||
raise ValueError('Catalog contains a private filesystem reference')
|
||||
return {'titles': db.execute('SELECT count(*) FROM title').fetchone()[0],
|
||||
'releases': releases, 'assets': db.execute('SELECT count(*) FROM asset').fetchone()[0]}
|
||||
|
||||
|
||||
def empty_catalog(path):
|
||||
with closing(sqlite3.connect(path)) as db:
|
||||
db.executescript(SCHEMA.read_text())
|
||||
db.executemany('INSERT INTO metadata VALUES(?,?)', [
|
||||
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-development'),
|
||||
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')])
|
||||
db.commit()
|
||||
|
||||
|
||||
def source_catalog(output):
|
||||
result = atomic_empty_database(output, SCHEMA, (
|
||||
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-source'),
|
||||
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')),
|
||||
('title', 'package', 'release', 'member', 'download_location', 'asset', 'asset_delivery',
|
||||
'release_asset', 'setup', 'source_hint'), expected_name='catalog.sqlite',
|
||||
expected_parent='SoftwareLibrary', user_version=1)
|
||||
return dict(result, schema_version=1, recipe_contract=2, stage='empty-source', titles=0,
|
||||
releases=0, assets=0, thumbnails=0, empty=True)
|
||||
|
||||
|
||||
def package(source, output, require_full=False, *, content_profile='normal'):
|
||||
if resolve_content_profile(content_profile) == 'source':
|
||||
if require_full:
|
||||
raise ValueError('--require-full cannot be combined with the source content profile')
|
||||
return source_catalog(output)
|
||||
source, output = Path(source), Path(output)
|
||||
if output.is_symlink():
|
||||
raise ValueError('Catalog destination must not be a symlink')
|
||||
if source.resolve() == output.resolve():
|
||||
raise ValueError('Catalog output must differ from the input')
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix='.catalog-', suffix='.sqlite', dir=output.parent)
|
||||
os.close(fd)
|
||||
temporary = Path(temporary)
|
||||
try:
|
||||
if source.exists() or source.is_symlink():
|
||||
validate(source, require_full=require_full)
|
||||
shutil.copyfile(source, temporary)
|
||||
elif require_full:
|
||||
raise ValueError('Full catalog is missing. Prepare media/software-library-full/prepared/app-catalog.sqlite before packaging.')
|
||||
else:
|
||||
print('warning: Full Software Library catalog is absent; bundling an empty development catalog. See tools/catalog/README.md.')
|
||||
empty_catalog(temporary)
|
||||
counts = validate(temporary, require_full=require_full)
|
||||
digest = hashlib.sha256(temporary.read_bytes()).hexdigest()
|
||||
os.chmod(temporary, 0o444)
|
||||
# Avoid touching a resource that is byte-identical across incremental builds.
|
||||
if output.is_file() and hashlib.sha256(output.read_bytes()).hexdigest() == digest:
|
||||
os.chmod(output, 0o444)
|
||||
else:
|
||||
os.replace(temporary, output)
|
||||
return dict(counts, sha256=digest, empty=counts['releases'] == 0)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
|
||||
def artwork_key(url, sha):
|
||||
# Keep in sync with SoftwareLibraryArtworkCache.cacheKey(for:).
|
||||
return hashlib.sha256(('thumbnail-v1-640\n' + url + '\n' + (sha or 'unverified')).encode()).hexdigest()
|
||||
|
||||
|
||||
def png_thumbnail(path):
|
||||
data = path.read_bytes()
|
||||
if not 24 <= len(data) <= 2 * 1024 * 1024 or data[:8] != b'\x89PNG\r\n\x1a\n' or data[12:16] != b'IHDR':
|
||||
return False
|
||||
width, height = struct.unpack('>II', data[16:24])
|
||||
return 0 < width <= 640 and 0 < height <= 640
|
||||
|
||||
|
||||
def screenshot_member(archive):
|
||||
"""Read one bounded image, never extract archive-supplied paths to disk."""
|
||||
with zipfile.ZipFile(archive) as images:
|
||||
members = images.infolist()
|
||||
if len(members) > 256:
|
||||
raise ValueError('Artwork archive has too many members')
|
||||
candidates = [m for m in members if not m.is_dir()
|
||||
and Path(m.filename).suffix.lower() in ('.png', '.jpg', '.jpeg', '.gif', '.webp')
|
||||
and not any(part.startswith('.') or part == '__MACOSX' for part in Path(m.filename).parts)
|
||||
and not (m.flag_bits & 1)
|
||||
and 0 < m.file_size <= 8 * 1024 * 1024
|
||||
and (m.external_attr >> 16) & 0o170000 != 0o120000]
|
||||
def rank(member):
|
||||
stem = Path(member.filename).stem.lower()
|
||||
preferred = {'cover': 0, 'title': 1, 'intro': 2}
|
||||
return (preferred.get(stem, 3), member.filename.casefold(), member.filename)
|
||||
if not candidates:
|
||||
return None
|
||||
member = min(candidates, key=rank)
|
||||
with images.open(member) as stream:
|
||||
data = stream.read(8 * 1024 * 1024 + 1)
|
||||
if len(data) != member.file_size or len(data) > 8 * 1024 * 1024:
|
||||
raise ValueError('Artwork archive image exceeds its declared size')
|
||||
return data
|
||||
|
||||
|
||||
def package_artwork(catalog, cache, destination):
|
||||
"""Generate maintainer thumbnails from previously verified local art.
|
||||
|
||||
The content-addressed crawl cache is optional for a developer checkout.
|
||||
No network requests, original artwork or documents are copied into the app.
|
||||
"""
|
||||
cache, destination = Path(cache), Path(destination)
|
||||
if destination.is_symlink():
|
||||
raise ValueError('Artwork destination must not be a symlink')
|
||||
with closing(sqlite3.connect(Path(catalog).resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
||||
rows = db.execute("SELECT DISTINCT a.source_url,a.sha256,a.byte_size FROM asset a JOIN release_asset ra ON ra.asset_id=a.id WHERE a.kind IN ('replacement-label','screenshot') ORDER BY a.id").fetchall()
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
def convert(row):
|
||||
url, sha, size = row
|
||||
parts = urlsplit(url)
|
||||
suffix = Path(parts.path).suffix.lower()
|
||||
if (parts.scheme != 'https' or parts.hostname != 'colorcomputerarchive.com' or not parts.path.startswith('/repo/')
|
||||
or suffix not in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.zip') or not sha or not re.fullmatch('[a-f0-9]{64}', sha)):
|
||||
return None
|
||||
source = cache / (sha + suffix)
|
||||
if not source.is_file() or source.is_symlink() or not 0 < source.stat().st_size <= 8 * 1024 * 1024:
|
||||
return None
|
||||
original = source.read_bytes()
|
||||
if len(original) != size or hashlib.sha256(original).hexdigest() != sha:
|
||||
raise ValueError('Cached artwork does not match its catalog fingerprint')
|
||||
name = artwork_key(url, sha) + '.png'
|
||||
target = destination / name
|
||||
# A generated output is already bounded; its input identity is in its name.
|
||||
if target.is_file() and not target.is_symlink() and png_thumbnail(target):
|
||||
return name
|
||||
# ZIP assets retain their original URL/fingerprint and attribution. Only
|
||||
# a single selected image becomes a thumbnail; no archive is shipped.
|
||||
image_data = screenshot_member(source) if suffix == '.zip' else None
|
||||
if suffix == '.zip' and image_data is None:
|
||||
return None
|
||||
fd, temporary = tempfile.mkstemp(prefix='.artwork-', suffix='.png', dir=destination)
|
||||
os.close(fd)
|
||||
temporary = Path(temporary)
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-artwork-') as scratch:
|
||||
input_path = source
|
||||
if image_data is not None:
|
||||
input_path = Path(scratch) / 'image'
|
||||
input_path.write_bytes(image_data)
|
||||
subprocess.run(['/usr/bin/sips', '-s', 'format', 'png', '-Z', '640', str(input_path), '--out', str(temporary)],
|
||||
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
||||
if not png_thumbnail(temporary):
|
||||
raise ValueError('Generated artwork thumbnail is not a bounded PNG')
|
||||
os.chmod(temporary, 0o444)
|
||||
os.replace(temporary, target)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
return name
|
||||
with ThreadPoolExecutor(max_workers=4) as workers:
|
||||
names = {name for name in workers.map(convert, rows) if name}
|
||||
# Clear only our generated PNG namespace after a successful complete pass.
|
||||
for path in destination.iterdir():
|
||||
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and path.name not in names and not path.is_symlink():
|
||||
path.unlink()
|
||||
return len(names)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--source', type=Path)
|
||||
add_content_profile_argument(parser)
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--check-only', action='store_true', help='Validate input without staging a resource')
|
||||
parser.add_argument('--require-full', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if not args.check_only and args.output is None:
|
||||
parser.error('--output is required unless --check-only is used')
|
||||
try:
|
||||
profile = resolve_content_profile(args.content_profile)
|
||||
if profile == 'source':
|
||||
if args.require_full:
|
||||
raise ValueError('--require-full cannot be combined with the source content profile')
|
||||
if args.check_only:
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-source-catalog-check-') as temporary:
|
||||
result = source_catalog(Path(temporary) / 'SoftwareLibrary/catalog.sqlite')
|
||||
else:
|
||||
result = source_catalog(args.output)
|
||||
else:
|
||||
source = args.source if args.source is not None else DEFAULT_SOURCE
|
||||
result = validate(source, args.require_full) if args.check_only else package(source, args.output, args.require_full)
|
||||
if not args.check_only:
|
||||
artwork = args.output.parent / 'Artwork'
|
||||
if artwork.is_symlink(): raise ValueError('Artwork destination must not be a symlink')
|
||||
if artwork.is_dir():
|
||||
for path in artwork.iterdir():
|
||||
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and not path.is_symlink(): path.unlink()
|
||||
result['thumbnails'] = 0
|
||||
except (ValueError, OSError, sqlite3.DatabaseError, subprocess.CalledProcessError) as error:
|
||||
parser.exit(1, 'error: Software Library catalog: ' + str(error) + '\n')
|
||||
print('Software Library catalog: ' + json.dumps(result, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,368 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Fetch and verify pinned source dependencies without executing their code.
|
||||
|
||||
Source archives are retained in ignored storage. Only manifest-selected regular
|
||||
files are materialized, excluded payloads are never extracted, and import patches
|
||||
must reproduce the exact reviewed file hashes. No Git repository is cloned.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
import zipfile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MAX_ARCHIVE_BYTES = 128 * 1024 * 1024
|
||||
MAX_FILE_BYTES = 64 * 1024 * 1024
|
||||
MAX_EXPANDED_BYTES = 1024 * 1024 * 1024
|
||||
MAX_MEMBERS = 100_000
|
||||
HASH = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def sha(path):
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def relative(name):
|
||||
if not isinstance(name, str) or not name or "\\" in name or "\x00" in name:
|
||||
raise ValueError("Invalid dependency path: " + repr(name))
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or ".." in path.parts or ".git" in path.parts or re.match(r"^[A-Za-z]:", name):
|
||||
raise ValueError("Unsafe dependency path: " + name)
|
||||
value = path.as_posix()
|
||||
if value == ".":
|
||||
return ""
|
||||
return value
|
||||
|
||||
|
||||
def validate_record(record):
|
||||
if 'optional' in record and not isinstance(record['optional'], bool):
|
||||
raise ValueError('Dependency optional flag must be boolean')
|
||||
directory = relative(record["directory"])
|
||||
if not directory.startswith("third_party/") or directory == "third_party/manifests":
|
||||
raise ValueError("Dependency directory must be beneath third_party")
|
||||
suffix = directory.removeprefix("third_party/")
|
||||
if suffix.startswith("manifests/"):
|
||||
raise ValueError("Dependency cannot overwrite metadata")
|
||||
for key in ("upstreamFileSHA256", "vendoredFileSHA256"):
|
||||
if not isinstance(record[key], dict) or not record[key]:
|
||||
raise ValueError("Dependency requires nonempty file hash maps")
|
||||
for name, digest in record[key].items():
|
||||
if relative(name) != name or (not isinstance(digest, str) or not HASH.fullmatch(digest)):
|
||||
raise ValueError("Invalid file fingerprint: " + name)
|
||||
if set(record["upstreamFileSHA256"]) != set(record["vendoredFileSHA256"]):
|
||||
raise ValueError("Import patches may change bytes, not the approved file inventory")
|
||||
if record.get("archiveSHA256") is not None and (not isinstance(record["archiveSHA256"], str) or not HASH.fullmatch(record["archiveSHA256"])):
|
||||
raise ValueError("Invalid archive fingerprint")
|
||||
for excluded in record.get("excludedFiles", []):
|
||||
excluded = relative(excluded).rstrip("/")
|
||||
if any(p == excluded or p.startswith(excluded + "/") for p in record["upstreamFileSHA256"]):
|
||||
raise ValueError("Selected dependency file overlaps an exclusion: " + excluded)
|
||||
if record.get("archiveRootPrefix"):
|
||||
if not relative(record["archiveRootPrefix"].rstrip("/")):
|
||||
raise ValueError("Archive root prefix cannot be empty")
|
||||
return suffix
|
||||
|
||||
|
||||
def source_url(record):
|
||||
"""Use a release URL or a full immutable GitHub revision archive."""
|
||||
url = record["sourceURL"]
|
||||
if record.get("sourceRevision"):
|
||||
revision = record["sourceRevision"]
|
||||
parts = urlsplit(url)
|
||||
fields = parts.path.strip("/").removesuffix(".git").split("/")
|
||||
if parts.scheme != "https" or parts.hostname != "github.com" or len(fields) != 2 or not re.fullmatch(r"[0-9a-f]{40}", revision):
|
||||
raise ValueError("Unsupported pinned Git source")
|
||||
return f"https://codeload.github.com/{fields[0]}/{fields[1]}/tar.gz/{revision}"
|
||||
if not record.get("archiveSHA256"):
|
||||
raise ValueError("Release archives require a recorded SHA-256")
|
||||
if urlsplit(url).scheme not in ("http", "https"):
|
||||
raise ValueError("Dependency source must use HTTP or HTTPS")
|
||||
return url
|
||||
|
||||
|
||||
class SafeRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
if urlsplit(newurl).scheme not in ("http", "https"):
|
||||
raise ValueError("Unsupported dependency redirect")
|
||||
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
||||
|
||||
|
||||
def archive_names(name, record):
|
||||
filename = Path(urlsplit(record["sourceURL"]).path).name
|
||||
if record.get("sourceRevision"):
|
||||
filename = name + "-" + record["sourceRevision"] + ".tar.gz"
|
||||
if not filename or filename in (".", ".."):
|
||||
raise ValueError("Invalid source archive filename")
|
||||
digest = record.get("archiveSHA256")
|
||||
return ([digest + "-" + filename, filename, digest + ".tar.gz"] if digest else [filename])
|
||||
|
||||
|
||||
def verify_archive(path, record):
|
||||
if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_ARCHIVE_BYTES:
|
||||
raise ValueError("Source archive must be a bounded regular file: " + str(path))
|
||||
digest = sha(path)
|
||||
if record.get("archiveSHA256") and digest != record["archiveSHA256"]:
|
||||
raise ValueError("Source archive SHA-256 mismatch: " + path.name)
|
||||
return digest
|
||||
|
||||
|
||||
|
||||
def require_unlinked_ancestors(path, description):
|
||||
"""Reject redirected destinations, retaining only macOS's public aliases.
|
||||
|
||||
Do not resolve an arbitrary user path before examining its full ancestry:
|
||||
doing so would erase precisely the linked parent this check must detect.
|
||||
"""
|
||||
aliases = {Path("/tmp"): Path("/private/tmp"), Path("/var"): Path("/private/var")}
|
||||
absolute = Path(path).absolute()
|
||||
for candidate in (absolute, *absolute.parents):
|
||||
if candidate.is_symlink() and not (
|
||||
candidate in aliases and candidate.resolve() == aliases[candidate]):
|
||||
raise ValueError(description + " cannot traverse a symbolic link")
|
||||
|
||||
|
||||
def obtain_archive(name, record, cache, offline=False, timeout=30):
|
||||
require_unlinked_ancestors(cache, "Archive cache")
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
names = archive_names(name, record)
|
||||
for filename in names:
|
||||
cached = cache / filename
|
||||
if cached.exists() or cached.is_symlink():
|
||||
verify_archive(cached, record)
|
||||
return cached
|
||||
if offline:
|
||||
raise ValueError("Offline source archive missing: " + names[0])
|
||||
url = source_url(record)
|
||||
request = Request(url, headers={"User-Agent": "VCCPlus-source-fetch/1"})
|
||||
target = cache / names[0]
|
||||
temporary = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(prefix=".download-", dir=cache, delete=False) as output:
|
||||
temporary = Path(output.name)
|
||||
with build_opener(SafeRedirect()).open(request, timeout=timeout) as response:
|
||||
if urlsplit(response.geturl()).scheme not in ("http", "https"):
|
||||
raise ValueError("Unsupported dependency response URL")
|
||||
size = 0
|
||||
while chunk := response.read(1024 * 1024):
|
||||
size += len(chunk)
|
||||
if size > MAX_ARCHIVE_BYTES:
|
||||
raise ValueError("Dependency archive exceeds download limit")
|
||||
output.write(chunk)
|
||||
verify_archive(temporary, record)
|
||||
require_unlinked_ancestors(cache, "Archive cache")
|
||||
if target.exists() or target.is_symlink():
|
||||
raise ValueError("Archive cache target changed during download")
|
||||
os.replace(temporary, target)
|
||||
return target
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def extract_selected(archive, destination, expected, archive_prefix=""):
|
||||
"""Stream only known regular members; never use extract/extractall."""
|
||||
observed = set()
|
||||
all_names = set()
|
||||
prefix = None
|
||||
expanded = 0
|
||||
count = 0
|
||||
|
||||
def consume(name, size, mode, regular, opener):
|
||||
nonlocal prefix, expanded, count
|
||||
count += 1
|
||||
if count > MAX_MEMBERS:
|
||||
raise ValueError("Source archive has too many members")
|
||||
if not 0 <= size <= MAX_FILE_BYTES:
|
||||
raise ValueError("Source archive member exceeds size limit: " + name)
|
||||
expanded += size
|
||||
if expanded > MAX_EXPANDED_BYTES:
|
||||
raise ValueError("Source archive exceeds expansion limit")
|
||||
normalized = relative(name)
|
||||
if normalized in all_names:
|
||||
raise ValueError("Duplicate source archive member: " + normalized)
|
||||
all_names.add(normalized)
|
||||
parts = normalized.split("/")
|
||||
candidates = [(normalized, "")]
|
||||
if len(parts) > 1:
|
||||
candidates.append(("/".join(parts[1:]), parts[0] + "/"))
|
||||
archive_root = archive_prefix.rstrip("/") + "/" if archive_prefix else ""
|
||||
matches = [(path[len(archive_root):], root) for path, root in candidates
|
||||
if path.startswith(archive_root) and path[len(archive_root):] in expected]
|
||||
if not matches:
|
||||
return
|
||||
if len(matches) != 1:
|
||||
raise ValueError("Ambiguous source archive member: " + normalized)
|
||||
path, root = matches[0]
|
||||
if prefix is not None and root != prefix:
|
||||
raise ValueError("Source archive has inconsistent roots")
|
||||
prefix = root
|
||||
if path in observed or not regular:
|
||||
raise ValueError("Selected member is duplicate, linked or not a regular file: " + normalized)
|
||||
with opener() as stream:
|
||||
data = stream.read(MAX_FILE_BYTES + 1)
|
||||
if len(data) != size or hashlib.sha256(data).hexdigest() != expected[path]:
|
||||
raise ValueError("Upstream source file SHA-256 mismatch: " + path)
|
||||
target = destination / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(data)
|
||||
target.chmod(0o755 if mode & 0o111 else 0o644)
|
||||
observed.add(path)
|
||||
|
||||
if zipfile.is_zipfile(archive):
|
||||
with zipfile.ZipFile(archive) as source:
|
||||
for member in source.infolist():
|
||||
mode = member.external_attr >> 16
|
||||
regular = not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG) and not (member.flag_bits & 1)
|
||||
consume(member.filename, member.file_size, mode, regular, lambda member=member: source.open(member))
|
||||
else:
|
||||
with tarfile.open(archive, mode="r|*") as source:
|
||||
for member in source:
|
||||
consume(member.name, member.size, member.mode, member.isfile(), lambda member=member: source.extractfile(member))
|
||||
if observed != set(expected):
|
||||
raise ValueError("Source archive is missing selected files: " + ", ".join(sorted(set(expected) - observed)[:12]))
|
||||
|
||||
|
||||
def verify_tree(root, expected):
|
||||
if root.is_symlink() or not root.is_dir():
|
||||
raise ValueError("Dependency source must be an ordinary directory")
|
||||
actual = {}
|
||||
for path in root.rglob("*"):
|
||||
if path.is_symlink():
|
||||
raise ValueError("Dependency contains a symbolic link: " + str(path))
|
||||
if path.is_file():
|
||||
actual[path.relative_to(root).as_posix()] = sha(path)
|
||||
elif not path.is_dir():
|
||||
raise ValueError("Dependency contains a special file")
|
||||
if actual != expected:
|
||||
changed = sorted(p for p in actual.keys() | expected.keys() if actual.get(p) != expected.get(p))
|
||||
raise ValueError("Dependency source differs from reviewed import: " + ", ".join(changed[:12]))
|
||||
|
||||
|
||||
def apply_patches(root, record, metadata):
|
||||
allowed = set(record["upstreamFileSHA256"])
|
||||
for filename in record.get("patches", []):
|
||||
if relative(filename) != filename or "/" in filename:
|
||||
raise ValueError("Import patch must be a metadata filename")
|
||||
path = metadata / filename
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("Missing import patch: " + filename)
|
||||
text = path.read_text()
|
||||
targets = []
|
||||
for line in text.splitlines():
|
||||
if line.startswith(("--- ", "+++ ")):
|
||||
name = line[4:].split("\t", 1)[0]
|
||||
if name[:2] not in ("a/", "b/") or relative(name[2:]) not in allowed:
|
||||
raise ValueError("Import patch touches an unapproved path")
|
||||
targets.append(name[2:])
|
||||
if not targets:
|
||||
raise ValueError("Import patch contains no unified file headers")
|
||||
subprocess.run(["/usr/bin/patch", "-u", "-t", "-F", "0", "-p1", "-i", str(path.resolve())],
|
||||
cwd=root, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
# A byte-perfect patch may still leave editor backups on some patch versions.
|
||||
# Do not remove them silently; verify_tree fails on any extra output.
|
||||
|
||||
|
||||
def load_records(project_root=ROOT):
|
||||
"""Read ordinary import records plus supplemental pinned-source metadata.
|
||||
|
||||
Supplemental records fill provenance gaps only; they cannot replace or
|
||||
overlap the existing reviewed dependency inventory.
|
||||
"""
|
||||
ordinary = project_root / "third_party/manifests"
|
||||
supplemental = project_root / "tools/dependencies/manifests"
|
||||
records, locations = {}, {}
|
||||
for metadata in (ordinary, supplemental):
|
||||
if metadata.is_symlink():
|
||||
raise ValueError("Dependency metadata cannot be a symbolic link")
|
||||
for path in sorted(metadata.glob("*.json")):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("Dependency manifest must be an ordinary file")
|
||||
name = path.stem
|
||||
if name in records:
|
||||
raise ValueError("Duplicate dependency manifest: " + name)
|
||||
record = json.loads(path.read_text())
|
||||
suffix = validate_record(record)
|
||||
for existing in records.values():
|
||||
other = validate_record(existing)
|
||||
if suffix == other or suffix.startswith(other + "/") or other.startswith(suffix + "/"):
|
||||
raise ValueError("Overlapping dependency source directories")
|
||||
source_url(record) # Reject unpinned/unsupported sources before any writes.
|
||||
records[name], locations[name] = record, metadata
|
||||
return records, locations
|
||||
|
||||
|
||||
def materialize(name, record, metadata, output, cache, offline=False, verify_only=False):
|
||||
suffix = validate_record(record)
|
||||
destination = output / suffix
|
||||
require_unlinked_ancestors(destination, "Dependency destination")
|
||||
if destination.exists():
|
||||
verify_tree(destination, record["vendoredFileSHA256"])
|
||||
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "verified-existing"}
|
||||
if verify_only:
|
||||
raise ValueError("Dependency source has not been fetched: " + name)
|
||||
archive = obtain_archive(name, record, cache, offline)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="." + name + "-", dir=destination.parent) as temporary:
|
||||
staged = Path(temporary) / "source"
|
||||
staged.mkdir()
|
||||
extract_selected(archive, staged, record["upstreamFileSHA256"], record.get("archiveRootPrefix", ""))
|
||||
apply_patches(staged, record, metadata)
|
||||
verify_tree(staged, record["vendoredFileSHA256"])
|
||||
require_unlinked_ancestors(destination, "Dependency destination")
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise ValueError("Dependency destination changed during preparation")
|
||||
staged.rename(destination)
|
||||
return {"dependency": name, "files": len(record["vendoredFileSHA256"]), "status": "fetched-verified",
|
||||
"archive_sha256": sha(archive)}
|
||||
|
||||
|
||||
def default_selection(records):
|
||||
"""Optional firmware preparation is explicit, never part of ordinary fetches."""
|
||||
return [name for name, record in records.items() if not record.get('optional', False)]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--only", nargs="+", help="Manifest names, for example: --only toolshed")
|
||||
parser.add_argument("--archive-cache", type=Path, default=ROOT / ".build/dependencies/downloads")
|
||||
parser.add_argument("--output", type=Path, default=ROOT / "third_party", help="Ignored dependency source root")
|
||||
parser.add_argument("--offline", action="store_true", help="Use only already cached original source archives")
|
||||
parser.add_argument("--verify-only", action="store_true", help="Verify existing source trees without fetching")
|
||||
args = parser.parse_args()
|
||||
records, locations = load_records()
|
||||
selected = args.only or default_selection(records)
|
||||
unknown = set(selected) - records.keys()
|
||||
if unknown:
|
||||
parser.error("Unknown dependency manifest: " + ", ".join(sorted(unknown)))
|
||||
if not records:
|
||||
parser.error("No dependency manifests found")
|
||||
output = args.output.absolute()
|
||||
canonical_output = output.resolve()
|
||||
metadata_roots = {ROOT / "third_party/manifests", ROOT / "tools/dependencies/manifests"}
|
||||
if canonical_output == ROOT or canonical_output in ROOT.parents or canonical_output in metadata_roots:
|
||||
parser.error("Choose a dependency source root separate from project metadata")
|
||||
for name in dict.fromkeys(selected):
|
||||
result = materialize(name, records[name], locations[name], output, args.archive_cache, args.offline, args.verify_only)
|
||||
print(json.dumps(result, sort_keys=True), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (ValueError, OSError, tarfile.TarError, zipfile.BadZipFile, subprocess.CalledProcessError) as error:
|
||||
raise SystemExit(str(error))
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"archiveRootPrefix": "cocoroms/",
|
||||
"directory": "third_party/cocoroms",
|
||||
"excludedFiles": [
|
||||
"bas13.rom",
|
||||
"extbas11.rom",
|
||||
"disk11.rom",
|
||||
"coco3.rom"
|
||||
],
|
||||
"license": "File-specific Tandy, Microsoft and Microware notices retained in reconstructed firmware and assembly comments.",
|
||||
"optional": true,
|
||||
"patches": [],
|
||||
"project": "toolshed-cocoroms",
|
||||
"romOutputs": {
|
||||
"bas13.rom": {
|
||||
"bytes": 8192,
|
||||
"sha256": "5375b1caef9df815b781d9cd6df81ecdaddd2642b06a01adac521e40f1329751"
|
||||
},
|
||||
"coco3.rom": {
|
||||
"bytes": 32768,
|
||||
"sha256": "ad358430568250eb8330a37f85361d9ce67e5ae6d5a28c17b31802cd3953a5d2"
|
||||
},
|
||||
"disk11.rom": {
|
||||
"bytes": 8192,
|
||||
"sha256": "1c40c5a74e72edaeecc2511fee1aab347eae2c3d5fc6778cb622e3df14d63157"
|
||||
},
|
||||
"extbas11.rom": {
|
||||
"bytes": 8192,
|
||||
"sha256": "a18f270d3495d82b9f515de049f46253a623a2d9b1bb226c9ad338508344c5ee"
|
||||
}
|
||||
},
|
||||
"selection": "Assembly-only inputs, including the referenced authors-picture assembly data, for Color BASIC 1.3, Extended Color BASIC 1.1, Disk BASIC 1.1 and Super Color BASIC 2.0; no preassembled ROM or binary input.",
|
||||
"sourceRevision": "3667a41c9605b7fcb150eace29cc7f42d460be1a",
|
||||
"sourceURL": "https://github.com/nitros9project/toolshed",
|
||||
"upstreamFileSHA256": {
|
||||
"Makefile": "4a29262fc1ba0b6106887425f5b0a4edb8ab81fd343af548cb3a863bda3aea51",
|
||||
"bas.asm": "ca2e0fba27a1c960f56648ab7be6571a89a4aff6c433b9a2649924e308e8bf24",
|
||||
"coco3.asm": "01147f426baf1b48fc6d590f18ca5974bedc3dfac486020e694d9f4b890d74f2",
|
||||
"disk.asm": "735443637ebaddc358f6a12856c495cc92e08f3aaa8bb68cf9a0494d7655faae",
|
||||
"equates.asm": "099d4ae145ee5151aa1eb7e9805388d338b2613fd53e2ba666ab757734d94442",
|
||||
"extbas.asm": "75ce20734d0272f636a961b318fb898c555061194da12d9bfc26782fc4b5e42b",
|
||||
"md5sums.txt": "f0374ad4bdd32f9cd322cfa5eafe04a0b27d2e90698b536aa26c00d1ea7c8eab",
|
||||
"mugateerspic.asm": "2878b1a9fbf7cc3bde25475f5e256f25910fdb44b1b2f8e6f01883bcf13548a1"
|
||||
},
|
||||
"vendoredFileSHA256": {
|
||||
"Makefile": "4a29262fc1ba0b6106887425f5b0a4edb8ab81fd343af548cb3a863bda3aea51",
|
||||
"bas.asm": "ca2e0fba27a1c960f56648ab7be6571a89a4aff6c433b9a2649924e308e8bf24",
|
||||
"coco3.asm": "01147f426baf1b48fc6d590f18ca5974bedc3dfac486020e694d9f4b890d74f2",
|
||||
"disk.asm": "735443637ebaddc358f6a12856c495cc92e08f3aaa8bb68cf9a0494d7655faae",
|
||||
"equates.asm": "099d4ae145ee5151aa1eb7e9805388d338b2613fd53e2ba666ab757734d94442",
|
||||
"extbas.asm": "75ce20734d0272f636a961b318fb898c555061194da12d9bfc26782fc4b5e42b",
|
||||
"md5sums.txt": "f0374ad4bdd32f9cd322cfa5eafe04a0b27d2e90698b536aa26c00d1ea7c8eab",
|
||||
"mugateerspic.asm": "2878b1a9fbf7cc3bde25475f5e256f25910fdb44b1b2f8e6f01883bcf13548a1"
|
||||
},
|
||||
"version": "2.6.1-3667a41"
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"project": "legacy-graphics",
|
||||
"version": "74c571ecf055",
|
||||
"sourceURL": "https://github.com/VCCE/VCC",
|
||||
"sourceRevision": "74c571ecf055e843b48e1e91fd8ae2303e5bca69",
|
||||
"archiveRootPrefix": "3rdparty/",
|
||||
"directory": "third_party/legacy_graphics",
|
||||
"license": "Original file-specific Khronos/OpenGL/stb notices retained",
|
||||
"excludedFiles": [],
|
||||
"patches": [],
|
||||
"upstreamFileSHA256": {
|
||||
"GL/glcorearb.h": "8841702a2934583c3860a19600f55e85e741cb2c1e73cd512e681e7625f10ad4",
|
||||
"GL/glext.h": "d66da97d4030d0a8f2180a9338e4378f63d8343f05b56da52064f67f9493a6a3",
|
||||
"GL/glxext.h": "3ded5604e6d7643632a6f50133e15d52603d1cd95399a7781d0a49d153ade8e9",
|
||||
"GL/wgl.h": "e95270f0e57101478833a23d5f89c38eea91befd323894498e3c3edb199a2805",
|
||||
"GL/wglext.h": "b8f828a41bf545195268aedd6677cc621a5fd87054762eb535b4304de84c2bf2",
|
||||
"KHR/khrplatform.h": "7b1e01aaa7ad8f6fc34b5c7bdf79ebf5189bb09e2c4d2e79fc5d350623d11e83",
|
||||
"stb/stb_image_write.h": "cbd5f0ad7a9cf4468affb36354a1d2338034f2c12473cf1a8e32053cb6914a05"
|
||||
},
|
||||
"vendoredFileSHA256": {
|
||||
"GL/glcorearb.h": "8841702a2934583c3860a19600f55e85e741cb2c1e73cd512e681e7625f10ad4",
|
||||
"GL/glext.h": "d66da97d4030d0a8f2180a9338e4378f63d8343f05b56da52064f67f9493a6a3",
|
||||
"GL/glxext.h": "3ded5604e6d7643632a6f50133e15d52603d1cd95399a7781d0a49d153ade8e9",
|
||||
"GL/wgl.h": "e95270f0e57101478833a23d5f89c38eea91befd323894498e3c3edb199a2805",
|
||||
"GL/wglext.h": "b8f828a41bf545195268aedd6677cc621a5fd87054762eb535b4304de84c2bf2",
|
||||
"KHR/khrplatform.h": "7b1e01aaa7ad8f6fc34b5c7bdf79ebf5189bb09e2c4d2e79fc5d350623d11e83",
|
||||
"stb/stb_image_write.h": "cbd5f0ad7a9cf4468affb36354a1d2338034f2c12473cf1a8e32053cb6914a05"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Authored archive fixtures exercise source selection and filesystem boundaries."""
|
||||
import hashlib
|
||||
import gzip
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
|
||||
|
||||
MODULE = Path(__file__).resolve().parents[1] / "fetch.py"
|
||||
SPEC = importlib.util.spec_from_file_location("fetch_dependencies", MODULE)
|
||||
FETCH = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(FETCH)
|
||||
|
||||
|
||||
def fingerprint(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
class FetchDependenciesTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(prefix="vcc-source-fetch-test-")
|
||||
self.root = Path(self.temp.name)
|
||||
self.metadata = self.root / "metadata"
|
||||
self.metadata.mkdir()
|
||||
self.cache = self.root / "cache"
|
||||
self.cache.mkdir()
|
||||
self.output = self.root / "output"
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def archive(self, entries, name="fixture-1.tar.gz"):
|
||||
path = self.cache / name
|
||||
with tarfile.open(path, "w:gz") as archive:
|
||||
for filename, data, mode, kind in entries:
|
||||
member = tarfile.TarInfo(filename)
|
||||
member.mode = mode
|
||||
if kind == "symlink":
|
||||
member.type = tarfile.SYMTYPE
|
||||
member.linkname = "/private/escaped"
|
||||
archive.addfile(member)
|
||||
else:
|
||||
member.size = len(data)
|
||||
archive.addfile(member, io.BytesIO(data))
|
||||
return path
|
||||
|
||||
def record(self, archive, entries=None):
|
||||
hashes = entries or {"configure": fingerprint(b"#!/bin/sh\nexit 0\n"), "COPYING": fingerprint(b"Fixture notice\n")}
|
||||
return {"directory": "third_party/fixture", "sourceURL": "https://example.invalid/" + archive.name,
|
||||
"archiveSHA256": FETCH.sha(archive), "upstreamFileSHA256": hashes,
|
||||
"vendoredFileSHA256": hashes, "excludedFiles": ["secret.rom"], "patches": []}
|
||||
|
||||
def ordinary_archive(self):
|
||||
return self.archive([("fixture-1/configure", b"#!/bin/sh\nexit 0\n", 0o755, "file"),
|
||||
("fixture-1/COPYING", b"Fixture notice\n", 0o644, "file"),
|
||||
("fixture-1/secret.rom", b"excluded fixture bytes", 0o644, "file")])
|
||||
|
||||
def test_only_manifest_files_materialized_with_safe_modes_and_idempotent_verification(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
first = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual(first["status"], "fetched-verified")
|
||||
result = self.output / "fixture"
|
||||
self.assertEqual(set(p.name for p in result.iterdir()), {"configure", "COPYING"})
|
||||
self.assertEqual((result / "configure").stat().st_mode & 0o7777, 0o755)
|
||||
self.assertEqual((result / "COPYING").stat().st_mode & 0o7777, 0o644)
|
||||
second = FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual(second["status"], "verified-existing")
|
||||
|
||||
def test_corrupt_cached_archive_is_rejected_without_output(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
archive.write_bytes(archive.read_bytes() + b"corrupt")
|
||||
with self.assertRaisesRegex(ValueError, "archive SHA-256 mismatch"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertFalse((self.output / "fixture").exists())
|
||||
|
||||
def test_wrong_selected_bytes_do_not_leave_partial_destination(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
record["upstreamFileSHA256"] = dict(record["upstreamFileSHA256"], COPYING="0" * 64)
|
||||
with self.assertRaisesRegex(ValueError, "source file SHA-256 mismatch"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertFalse((self.output / "fixture").exists())
|
||||
|
||||
def test_traversal_members_are_refused_even_when_not_selected(self):
|
||||
for filename in ("../escaped", "/absolute", "root/../../escaped", "C:/escape", "root\\escape"):
|
||||
with self.subTest(filename=filename):
|
||||
archive = self.archive([(filename, b"x", 0o644, "file")])
|
||||
destination = self.root / "extract"
|
||||
destination.mkdir(exist_ok=True)
|
||||
with self.assertRaises(ValueError):
|
||||
FETCH.extract_selected(archive, destination, {"safe": fingerprint(b"x")})
|
||||
self.assertEqual(list(destination.iterdir()), [])
|
||||
|
||||
def test_selected_links_and_duplicates_are_refused(self):
|
||||
for entries in ([('root/a', b"", 0o777, "symlink")],
|
||||
[('root/a', b"x", 0o644, "file"), ('root/a', b"x", 0o644, "file")]):
|
||||
archive = self.archive(entries)
|
||||
with self.assertRaisesRegex(ValueError, "linked|Duplicate"):
|
||||
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
|
||||
|
||||
def test_unselected_oversized_members_are_refused(self):
|
||||
archive = self.cache / "oversized.tar.gz"
|
||||
member = tarfile.TarInfo("root/unselected")
|
||||
member.size = FETCH.MAX_FILE_BYTES + 1
|
||||
# An authored header is enough: bounds reject before reading payload.
|
||||
with gzip.open(archive, "wb") as source:
|
||||
source.write(member.tobuf() + bytes(1024))
|
||||
with self.assertRaisesRegex(ValueError, "size limit"):
|
||||
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
|
||||
|
||||
def test_zip_traversal_and_unix_symlinks_refused(self):
|
||||
for filename, mode in [("../escape", 0o100644), ("root/a", 0o120777)]:
|
||||
archive = self.cache / "fixture.zip"
|
||||
with zipfile.ZipFile(archive, "w") as target:
|
||||
entry = zipfile.ZipInfo(filename)
|
||||
entry.create_system = 3
|
||||
entry.external_attr = mode << 16
|
||||
target.writestr(entry, b"x")
|
||||
with self.assertRaises(ValueError):
|
||||
FETCH.extract_selected(archive, self.root / "extract", {"a": fingerprint(b"x")})
|
||||
|
||||
def test_pinned_git_archive_prefix_and_per_file_hashes(self):
|
||||
archive = self.archive([("upstream-abc/3rdparty/GL/header.h", b"header", 0o644, "file")],
|
||||
"fixture-" + "a" * 40 + ".tar.gz")
|
||||
record = self.record(archive, {"GL/header.h": fingerprint(b"header")})
|
||||
record.pop("archiveSHA256")
|
||||
record.update(sourceURL="https://github.com/example/project", sourceRevision="a" * 40,
|
||||
archiveRootPrefix="3rdparty/")
|
||||
self.assertEqual(FETCH.source_url(record), "https://codeload.github.com/example/project/tar.gz/" + "a" * 40)
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual((self.output / "fixture/GL/header.h").read_bytes(), b"header")
|
||||
|
||||
def test_import_patch_reproduces_final_hashes(self):
|
||||
archive = self.archive([("fixture-1/a.txt", b"before\n", 0o644, "file")])
|
||||
record = self.record(archive, {"a.txt": fingerprint(b"before\n")})
|
||||
record["vendoredFileSHA256"] = {"a.txt": fingerprint(b"after\n")}
|
||||
record["patches"] = ["change.patch"]
|
||||
(self.metadata / "change.patch").write_text("--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-before\n+after\n")
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual((self.output / "fixture/a.txt").read_bytes(), b"after\n")
|
||||
|
||||
def test_existing_modified_tree_is_not_replaced(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
changed = self.output / "fixture/COPYING"
|
||||
changed.write_bytes(b"local edit")
|
||||
with self.assertRaisesRegex(ValueError, "differs from reviewed import"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual(changed.read_bytes(), b"local edit")
|
||||
|
||||
|
||||
def test_linked_source_destination_and_archive_cache_are_refused(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
target = self.root / "outside"
|
||||
target.mkdir()
|
||||
self.output.symlink_to(target, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertEqual(list(target.iterdir()), [])
|
||||
linked_cache = self.root / "linked-cache"
|
||||
linked_cache.symlink_to(self.cache, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
||||
FETCH.obtain_archive("fixture", record, linked_cache, offline=True)
|
||||
|
||||
def test_linked_ancestor_output_and_cache_never_write_through(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
outside = self.root / "outside"
|
||||
outside.mkdir()
|
||||
sentinel = outside / "keep.txt"
|
||||
sentinel.write_bytes(b"preserved")
|
||||
linked_parent = self.root / "link"
|
||||
linked_parent.symlink_to(outside, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
||||
FETCH.materialize("fixture", record, self.metadata,
|
||||
linked_parent / "new-third-party", self.cache, offline=True)
|
||||
self.assertEqual(sentinel.read_bytes(), b"preserved")
|
||||
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
|
||||
with self.assertRaisesRegex(ValueError, "symbolic link"):
|
||||
FETCH.obtain_archive("fixture", record, linked_parent / "new-cache", offline=True)
|
||||
self.assertEqual(sentinel.read_bytes(), b"preserved")
|
||||
self.assertEqual({p.name for p in outside.iterdir()}, {"keep.txt"})
|
||||
|
||||
def test_only_canonical_macos_aliases_are_allowed_in_ancestry(self):
|
||||
# These public spellings must remain usable by Xcode on macOS. A
|
||||
# similarly named user link is still covered by the preceding test.
|
||||
import os
|
||||
if os.uname().sysname != "Darwin":
|
||||
self.skipTest("macOS public aliases are platform-specific")
|
||||
for public, canonical in [(Path("/tmp"), Path("/private/tmp")),
|
||||
(Path("/var"), Path("/private/var"))]:
|
||||
if public.is_symlink() and public.resolve() == canonical:
|
||||
FETCH.require_unlinked_ancestors(public / "uncreated-dependency-cache", "Fixture")
|
||||
|
||||
def test_verify_only_missing_tree_does_not_create_output_or_fetch(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
with self.assertRaisesRegex(ValueError, "has not been fetched"):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache,
|
||||
offline=True, verify_only=True)
|
||||
self.assertFalse(self.output.exists())
|
||||
|
||||
def test_supplemental_metadata_merges_without_replacing_import_records(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
normal = self.root / "third_party/manifests"
|
||||
supplement = self.root / "tools/dependencies/manifests"
|
||||
normal.mkdir(parents=True)
|
||||
supplement.mkdir(parents=True)
|
||||
first = normal / "fixture.json"
|
||||
first.write_text(json.dumps(record))
|
||||
other = dict(record, directory="third_party/legacy")
|
||||
second = supplement / "legacy.json"
|
||||
second.write_text(json.dumps(other))
|
||||
records, locations = FETCH.load_records(self.root)
|
||||
self.assertEqual(set(records), {"fixture", "legacy"})
|
||||
self.assertEqual(records["fixture"], record)
|
||||
self.assertEqual(locations, {"fixture": normal, "legacy": supplement})
|
||||
(supplement / "fixture.json").write_text(json.dumps(other))
|
||||
with self.assertRaisesRegex(ValueError, "Duplicate"):
|
||||
FETCH.load_records(self.root)
|
||||
self.assertEqual(json.loads(first.read_text()), record)
|
||||
|
||||
def test_overlapping_source_roots_and_linked_metadata_are_refused(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
normal = self.root / "third_party/manifests"
|
||||
normal.mkdir(parents=True)
|
||||
(normal / "fixture.json").write_text(json.dumps(record))
|
||||
other = normal / "other.json"
|
||||
other.write_text(json.dumps(dict(record, directory="third_party/fixture/nested")))
|
||||
with self.assertRaisesRegex(ValueError, "Overlapping"):
|
||||
FETCH.load_records(self.root)
|
||||
other.unlink()
|
||||
other.symlink_to(normal / "fixture.json")
|
||||
with self.assertRaisesRegex(ValueError, "ordinary file"):
|
||||
FETCH.load_records(self.root)
|
||||
|
||||
def test_release_hash_or_full_immutable_revision_is_required(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
record.pop("archiveSHA256")
|
||||
with self.assertRaisesRegex(ValueError, "require a recorded"):
|
||||
FETCH.source_url(record)
|
||||
for revision in ("main", "a" * 12, "a" * 39, "a" * 41):
|
||||
with self.subTest(revision=revision):
|
||||
candidate = dict(record, sourceURL="https://github.com/example/project", sourceRevision=revision)
|
||||
with self.assertRaisesRegex(ValueError, "pinned Git"):
|
||||
FETCH.source_url(candidate)
|
||||
|
||||
def test_unapproved_import_patch_target_leaves_no_destination(self):
|
||||
archive = self.ordinary_archive()
|
||||
record = self.record(archive)
|
||||
record["patches"] = ["escape.patch"]
|
||||
(self.metadata / "escape.patch").write_text("--- a/../escape\n+++ b/../escape\n@@ -1 +1 @@\n-before\n+after\n")
|
||||
with self.assertRaises(ValueError):
|
||||
FETCH.materialize("fixture", record, self.metadata, self.output, self.cache, offline=True)
|
||||
self.assertFalse((self.output / "fixture").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,57 @@
|
||||
-- SPDX-License-Identifier: GPL-3.0-or-later
|
||||
PRAGMA user_version=2;
|
||||
PRAGMA application_id=0x56434341;
|
||||
CREATE TABLE metadata(key TEXT PRIMARY KEY,value TEXT NOT NULL) WITHOUT ROWID;
|
||||
CREATE TABLE document(
|
||||
id INTEGER PRIMARY KEY,
|
||||
sha256 TEXT NOT NULL UNIQUE CHECK(length(sha256)=64 AND sha256 NOT GLOB '*[^a-f0-9]*'),
|
||||
title TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
collection TEXT NOT NULL,
|
||||
format TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL,
|
||||
member_path TEXT,
|
||||
delivery_url TEXT,
|
||||
byte_size INTEGER NOT NULL CHECK(byte_size > 0),
|
||||
page_count INTEGER NOT NULL CHECK(page_count > 0),
|
||||
issue_year INTEGER CHECK(issue_year BETWEEN 1900 AND 2099),
|
||||
member_offset INTEGER,
|
||||
member_compressed_bytes INTEGER,
|
||||
member_method INTEGER,
|
||||
archive_bytes INTEGER
|
||||
);
|
||||
CREATE INDEX document_browse ON document(category,collection,title COLLATE NOCASE,sha256);
|
||||
CREATE INDEX document_issue_year ON document(collection,issue_year,title COLLATE NOCASE,sha256)
|
||||
WHERE issue_year IS NOT NULL;
|
||||
CREATE INDEX document_title ON document(title COLLATE NOCASE,sha256);
|
||||
CREATE TABLE page(
|
||||
id INTEGER PRIMARY KEY,
|
||||
document_id INTEGER NOT NULL REFERENCES document(id),
|
||||
page_number INTEGER NOT NULL CHECK(page_number > 0),
|
||||
page_label TEXT,
|
||||
locator_kind TEXT NOT NULL CHECK(locator_kind IN ('pdf_page','worksheet','image','section')),
|
||||
is_ocr INTEGER NOT NULL CHECK(is_ocr IN (0,1)),
|
||||
UNIQUE(document_id,page_number)
|
||||
);
|
||||
-- Rendered text for complex non-PDF formats. PDF and image OCR stays in the research store.
|
||||
CREATE TABLE reader_section(
|
||||
page_id INTEGER PRIMARY KEY REFERENCES page(id),
|
||||
text TEXT NOT NULL
|
||||
);
|
||||
-- Stored positions refer only to the synthetic alphabetical term bag, never to source order.
|
||||
CREATE VIRTUAL TABLE page_terms USING fts5(
|
||||
terms,title_terms,content='',tokenize='unicode61 remove_diacritics 2',detail=full,columnsize=1
|
||||
);
|
||||
CREATE TABLE reviewed_example(
|
||||
original_sha256 TEXT NOT NULL REFERENCES document(sha256),
|
||||
first_page INTEGER NOT NULL CHECK(first_page > 0),
|
||||
last_page INTEGER NOT NULL CHECK(last_page >= first_page),
|
||||
title TEXT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK(kind IN ('snippet','basic_program','source')),
|
||||
language TEXT NOT NULL,
|
||||
author TEXT NOT NULL,
|
||||
provenance TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
source_sha256 TEXT NOT NULL,
|
||||
PRIMARY KEY(original_sha256,first_page,last_page,title)
|
||||
) WITHOUT ROWID;
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Stage the exported public Archive index as an Xcode resource.
|
||||
|
||||
The explicit source content profile stages only the checked-in empty schema.
|
||||
Normal builds retain the existing prepared-index lookup and verification.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import sys
|
||||
|
||||
NAME = 'ArchiveLibrary.sqlite'
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
from build_content import add_content_profile_argument, atomic_empty_database, content_profile as resolve_content_profile
|
||||
DEFAULT = ROOT / 'media/app-inputs/archive-library' / NAME
|
||||
SCHEMA = Path(__file__).with_name('archive-library.sql')
|
||||
|
||||
|
||||
def source_archive(output):
|
||||
result = atomic_empty_database(output, SCHEMA, (
|
||||
('schema_version', '2'), ('application_id', str(0x56434341)), ('source_snapshot', 'empty-source'),
|
||||
('documents', '0'), ('pages', '0'), ('searchable_pages', '0'), ('verified_deliveries', '0'),
|
||||
('ranged_members', '0'), ('reader_sections', '0'), ('reviewed_examples', '0')),
|
||||
('document', 'page', 'reader_section', 'page_terms', 'reviewed_example'),
|
||||
expected_name=NAME, expected_parent='ArchiveLibrary', application_id=0x56434341,
|
||||
user_version=2, fts_integrity=True)
|
||||
return dict(result, schema_version=2, application_id=0x56434341, source_snapshot='empty-source',
|
||||
documents=0, pages=0, searchable_pages=0, verified_deliveries=0, ranged_members=0,
|
||||
reader_sections=0, reviewed_examples=0, empty=True)
|
||||
|
||||
|
||||
def stage(source, output, required=False, *, content_profile='normal'):
|
||||
if resolve_content_profile(content_profile) == 'source':
|
||||
if required:
|
||||
raise ValueError('--require cannot be combined with the source content profile')
|
||||
if output is not None:
|
||||
return source_archive(output)
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-source-archive-check-') as temporary:
|
||||
return source_archive(Path(temporary) / 'ArchiveLibrary' / NAME)
|
||||
if source is None:
|
||||
source = DEFAULT
|
||||
if not source.exists():
|
||||
source = ROOT / 'media/knowledge-base/archive-library' / NAME
|
||||
source = Path(source)
|
||||
if output is not None and (output.name != NAME or output.parent.name != 'ArchiveLibrary' or output.is_symlink()):
|
||||
raise ValueError('Expected an ArchiveLibrary/ArchiveLibrary.sqlite app resource destination')
|
||||
if not source.exists():
|
||||
if required:
|
||||
raise ValueError('Prepare the Archive Library index before packaging this release.')
|
||||
if output is not None:
|
||||
output.unlink(missing_ok=True)
|
||||
return
|
||||
manifest = json.loads((source.parent / 'archive-library.json').read_bytes())
|
||||
checksum = (source.parent / (NAME + '.sha256')).read_text().split()
|
||||
version = manifest['schema_version']
|
||||
if (checksum != [manifest['sha256'], NAME] or manifest['bytes'] != source.stat().st_size
|
||||
or version not in (1, 2)):
|
||||
raise ValueError('The Archive index manifest and sidecar do not match the export')
|
||||
if required and version < 2:
|
||||
raise ValueError('Re-export the Archive Library index with inline reader support before packaging this release.')
|
||||
with closing(sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
||||
if (db.execute('PRAGMA application_id').fetchone() != (0x56434341,)
|
||||
or db.execute('PRAGMA user_version').fetchone() != (version,)
|
||||
or db.execute('SELECT COUNT(*) FROM document').fetchone() != (manifest['documents'],)
|
||||
or not manifest['documents']):
|
||||
raise ValueError('Incompatible Archive Library input')
|
||||
if version == 2 and (db.execute('SELECT COUNT(*) FROM reader_section').fetchone() != (manifest['reader_sections'],)
|
||||
or db.execute('SELECT COUNT(*) FROM document WHERE member_offset IS NOT NULL').fetchone() != (manifest['ranged_members'],)):
|
||||
raise ValueError('Archive reader coverage differs from the verified export')
|
||||
if output is not None:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Copy and verify the same stream. Replace only after both checks complete.
|
||||
fd = None
|
||||
temporary = None
|
||||
try:
|
||||
if output is not None:
|
||||
fd = tempfile.NamedTemporaryFile(dir=output.parent, delete=False)
|
||||
temporary = Path(fd.name)
|
||||
checksum = hashlib.sha256()
|
||||
with source.open('rb') as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
||||
checksum.update(chunk)
|
||||
if fd is not None:
|
||||
fd.write(chunk)
|
||||
if checksum.hexdigest() != manifest['sha256']:
|
||||
raise ValueError('Archive Library fingerprint differs from the verified export')
|
||||
if fd is not None:
|
||||
fd.close()
|
||||
os.chmod(temporary, 0o444)
|
||||
os.replace(temporary, output)
|
||||
finally:
|
||||
if fd is not None:
|
||||
fd.close()
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--source', type=Path)
|
||||
add_content_profile_argument(parser)
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--require', action='store_true')
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = stage(args.source, args.output, args.require, content_profile=resolve_content_profile(args.content_profile))
|
||||
except (OSError, ValueError, KeyError, sqlite3.DatabaseError) as error:
|
||||
parser.exit(1, 'error: Archive Library: ' + str(error) + '\n')
|
||||
if result is not None:
|
||||
print('Archive Library index: ' + json.dumps(result, sort_keys=True))
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Generate the source content profile lookup from production reference catalogs.
|
||||
|
||||
The reader callback is restricted to explicitly enumerated application inputs;
|
||||
full manual chapters, examples and maintainer collections are never opened.
|
||||
"""
|
||||
from html import escape
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
LOOKUP_ROOT = "docs/manuals/"
|
||||
|
||||
|
||||
def _json(value):
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2)
|
||||
|
||||
|
||||
def _bytes(value):
|
||||
return value.encode("utf-8")
|
||||
|
||||
|
||||
def basic_anchor(entry):
|
||||
if entry["id"].startswith("ERROR."):
|
||||
return "err-" + entry["keyword"].lower().replace("/0", "div-zero")
|
||||
return "term-" + entry["id"].lower().replace(".", "-").replace("$", "-string").replace("@", "-at")
|
||||
|
||||
|
||||
def _page(title, body, home="../index.html"):
|
||||
return _bytes(f'''<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>{escape(title)} · VCC Plus Reference Lookup</title>
|
||||
<style>body{{font:17px/1.6 -apple-system,BlinkMacSystemFont,sans-serif;max-width:78ch;margin:2rem auto;padding:0 1rem;color:#182721;background:#fff}}a{{color:#175b43}}pre{{white-space:pre-wrap;overflow-wrap:anywhere;background:#f1f5f2;padding:1rem}}section{{border-top:1px solid #d5ded8;margin-top:2rem;padding-top:1rem}}input{{font:inherit;max-width:100%;box-sizing:border-box;width:100%;padding:.5rem}}small{{display:block}}h1,h2{{line-height:1.25}}:target{{background:#f2f7ed}}</style>
|
||||
</head><body><nav><a href="{home}">Lookup library</a></nav><main>
|
||||
<h1>{escape(title)}</h1>{body}</main></body></html>\n''')
|
||||
|
||||
|
||||
def _index(book, directory, chapters):
|
||||
return _bytes("(globalThis.manualBooks ??= []).push(" + _json({
|
||||
"book": book, "directory": directory, "chapters": chapters,
|
||||
}) + ");\n")
|
||||
|
||||
|
||||
def build_resources(read_source):
|
||||
"""Return generated lookup reader files; never read full manual content."""
|
||||
catalog = json.loads(read_source("platforms/macos/Resources/BASICReference.json"))
|
||||
assembly = json.loads(read_source("platforms/macos/Resources/AssemblyReference.json"))
|
||||
signature = hashlib.sha256()
|
||||
for suffix in ("", "Instructions", "Topics"):
|
||||
name = f"AssemblyReference{suffix}.swift"
|
||||
data = read_source("platforms/macos/App/" + name).replace(b"\r\n", b"\n")
|
||||
signature.update(name.encode() + b"\0" + data + b"\0")
|
||||
if assembly["sourceSHA256"] != signature.hexdigest():
|
||||
raise ValueError("Assembly lookup snapshot differs from the selected production catalog")
|
||||
|
||||
resources = {}
|
||||
chapters = []
|
||||
for errors in (False, True):
|
||||
page = "errors.html" if errors else "reference.html"
|
||||
title = "BASIC errors" if errors else "BASIC command lookup"
|
||||
rows = [r for r in catalog["entries"] if r["id"].startswith("ERROR.") == errors]
|
||||
sections, html = [], []
|
||||
for row in rows:
|
||||
anchor = basic_anchor(row)
|
||||
paragraphs = [row["summary"], *row.get("parameters", []), *row.get("notes", [])]
|
||||
paragraphs = [p if isinstance(p, str) else _json(p) for p in paragraphs]
|
||||
citation = f"Source: {row['source']}, {row['sourcePage']}."
|
||||
body = "".join("<p>" + escape(p) + "</p>" for p in paragraphs)
|
||||
for syntax in row.get("syntax", []):
|
||||
body += "<pre><code>" + escape(syntax) + "</code></pre>"
|
||||
if row.get("sample"):
|
||||
body += "<pre><code>" + escape(row["sample"]) + "</code></pre>"
|
||||
body += "<p>Dialect: " + escape(row["dialect"]) + ". " + escape(citation) + "</p>"
|
||||
html.append(f'<section id="{escape(anchor)}"><h2>{escape(row["title"])}</h2>{body}</section>')
|
||||
sections.append({"id": anchor, "title": row["title"],
|
||||
"passages": [*paragraphs, *row.get("syntax", []), row.get("sample", ""), citation]})
|
||||
resources[LOOKUP_ROOT + "programming-in-coco-basic/" + page] = _page(title, "\n".join(html))
|
||||
chapters.append({"page": page, "label": "Lookup", "title": title, "sections": sections})
|
||||
resources[LOOKUP_ROOT + "book-index.js"] = _index("BASIC Lookup", "programming-in-coco-basic", chapters)
|
||||
|
||||
sections, html = [], []
|
||||
for i, row in enumerate(assembly["entries"]):
|
||||
anchor = f"assembly-{i:04d}"
|
||||
paragraphs = [str(row[key]) for key in ("category", "operation", "syntax", "flags", "detail") if row.get(key)]
|
||||
if row.get("aliases"):
|
||||
paragraphs.append("Aliases: " + ", ".join(row["aliases"]))
|
||||
if row.get("hd6309"):
|
||||
paragraphs.append("HD6309 only.")
|
||||
paragraphs.extend(row.get("sources", []))
|
||||
sections.append({"id": anchor, "title": row["name"], "passages": paragraphs})
|
||||
html.append(f'<section id="{anchor}"><h2>{escape(row["name"])}</h2>' +
|
||||
"".join("<p>" + escape(p) + "</p>" for p in paragraphs) + "</section>")
|
||||
resources[LOOKUP_ROOT + "assembly-language-programming/instructions.html"] = _page("Assembly lookup", "\n".join(html))
|
||||
resources[LOOKUP_ROOT + "assembly-language-programming/catalog.json"] = read_source("platforms/macos/Resources/AssemblyReference.json")
|
||||
resources[LOOKUP_ROOT + "assembly-language-programming/book-index.js"] = _index(
|
||||
"Assembly Lookup", "assembly-language-programming", [{"page": "instructions.html", "label": "Lookup", "title": "Assembly", "sections": sections}])
|
||||
|
||||
home = '''<p>This source edition includes BASIC and Assembly lookup. The complete native quick-reference browsers are also available from the Develop menu.</p>
|
||||
<ul><li><a href="programming-in-coco-basic/reference.html">BASIC commands</a></li><li><a href="programming-in-coco-basic/errors.html">BASIC errors</a></li><li><a href="assembly-language-programming/instructions.html">Assembly instructions, addressing and directives</a></li></ul>
|
||||
<section id="search"><h2>Search the lookup library</h2><form id="search-form"><label for="manual-query">Commands, instructions or keywords</label><input id="manual-query" type="search" autocomplete="off"></form><p id="search-status" aria-live="polite"></p><ol id="search-results"></ol></section>
|
||||
<script src="book-index.js"></script><script src="assembly-language-programming/book-index.js"></script><script src="lookup-search.js"></script>'''
|
||||
resources[LOOKUP_ROOT + "index.html"] = _page("VCC Plus Reference Lookup", home, "index.html")
|
||||
resources[LOOKUP_ROOT + "lookup-search.js"] = _bytes('''// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
const field = document.getElementById("manual-query"), results = document.getElementById("search-results"), statusLine = document.getElementById("search-status");
|
||||
const topics = globalThis.manualBooks.flatMap(book => book.chapters.flatMap(chapter => chapter.sections.map(section => ({...section, url: book.directory + "/" + chapter.page + "#" + section.id}))));
|
||||
document.getElementById("search-form").addEventListener("submit", event => event.preventDefault());
|
||||
field.addEventListener("input", () => {
|
||||
const words = field.value.trim().toLowerCase().split(/\\s+/).filter(Boolean);
|
||||
const matches = words.length ? topics.filter(topic => words.every(word => (topic.title + " " + topic.passages.join(" ")).toLowerCase().includes(word))) : [];
|
||||
results.replaceChildren();
|
||||
for (const topic of matches) { const item = document.createElement("li"), link = document.createElement("a"); link.href = topic.url; link.textContent = topic.title; item.append(link); results.append(item); }
|
||||
statusLine.textContent = words.length ? matches.length + " matching topics" : "Enter a search term.";
|
||||
});
|
||||
''')
|
||||
return resources
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Check an exported source distribution without private inputs or network access."""
|
||||
import hashlib
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import sqlite3
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from publication_urls import unapproved_app_links
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def artwork_provenance_issues(name, data):
|
||||
"""Inspect artwork descriptions and generator strings, preserving attribution."""
|
||||
if name.endswith('.svg'):
|
||||
root = ET.fromstring(data)
|
||||
text = '\n'.join(''.join(node.itertext()) for node in root.iter()
|
||||
if node.tag.rsplit('}', 1)[-1] in {'metadata', 'desc'})
|
||||
elif name.startswith('platforms/macos/Tools/') and name.endswith(('.py', '.swift')):
|
||||
source = data.decode('utf-8')
|
||||
if name.endswith('.py'):
|
||||
text = '\n'.join(node.value for node in ast.walk(ast.parse(source))
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str))
|
||||
else:
|
||||
text = source
|
||||
else:
|
||||
return []
|
||||
patterns = [r'\buser[- ]supplied\b', r'\brequested by (?:the )?user\b',
|
||||
r'\b(?:supplied|provided) by Brian\b', r'\bIMG_\d+\.(?:jpg|jpeg|png)\b',
|
||||
r'\b\d{12,}_[\w]+_n\.jpg\b',
|
||||
r'\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\.(?:webp|jpg|png)\b',
|
||||
r'\bapproved\s+(?:[\w-]+\s+){0,3}concept\b',
|
||||
r'\brequested\s+(?:TANDY\s+)?branding\b',
|
||||
r'\bno image-generation service\b']
|
||||
return [name + ': private-reference or authoring-session metadata'
|
||||
for pattern in patterns if re.search(pattern, text, re.I)]
|
||||
|
||||
|
||||
def source_mode_matches(mode, expected):
|
||||
"""Git preserves executable status; checkout write bits follow local umask."""
|
||||
return bool(mode & 0o111) == bool(int(expected, 8) & 0o111)
|
||||
|
||||
|
||||
def source_inventory():
|
||||
result = set()
|
||||
for target in ROOT.rglob('*'):
|
||||
relative = target.relative_to(ROOT)
|
||||
if '__pycache__' in relative.parts or target.name in {'.DS_Store'} or target.suffix == '.pyc':
|
||||
continue
|
||||
if relative.parts[0] in {'.git', '.build', 'build', 'dist'} or relative.parts[:3] == ('platforms', 'macos', 'LocalResources'):
|
||||
continue
|
||||
if relative.parts[0] == 'third_party' and len(relative.parts) > 1 and relative.parts[1] not in {'README.md', 'manifests'}:
|
||||
continue
|
||||
if target.is_symlink() or target.is_file():
|
||||
result.add(relative.as_posix())
|
||||
if (ROOT / '.git').exists():
|
||||
result.update(subprocess.check_output(['git', '-C', str(ROOT), 'ls-files', '-z']).decode().strip('\0').split('\0'))
|
||||
result.discard('')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
record = json.loads((ROOT / 'EXPORT.json').read_text())
|
||||
expected_paths = set(record['files']) | {'EXPORT.json'}
|
||||
if source_inventory() != expected_paths:
|
||||
raise ValueError('Unexpected or missing distribution source: ' + str(sorted(source_inventory() ^ expected_paths)))
|
||||
for relative, expected in record['files'].items():
|
||||
path = ROOT / relative
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError('Missing or linked distribution file: ' + relative)
|
||||
if hashlib.sha256(path.read_bytes()).hexdigest() != expected['sha256']:
|
||||
raise ValueError('Distribution file changed: ' + relative)
|
||||
if not source_mode_matches(path.stat().st_mode, expected['mode']):
|
||||
raise ValueError('Distribution file mode changed: ' + relative)
|
||||
# Check tracked/source files rather than generated, ignored build products.
|
||||
for relative in record['files']:
|
||||
path = ROOT / relative
|
||||
if path.suffix.lower() in {'.rom', '.sqlite', '.sqlite3', '.db'}:
|
||||
raise ValueError('Forbidden source payload: ' + relative)
|
||||
if path.read_bytes().startswith(b'SQLite format 3\x00'):
|
||||
raise ValueError('Database payload: ' + relative)
|
||||
if unapproved_app_links(path.read_bytes().decode('utf-8', errors='ignore')):
|
||||
raise ValueError('Unreviewed application repository link: ' + relative)
|
||||
issues = artwork_provenance_issues(relative, path.read_bytes())
|
||||
if issues:
|
||||
raise ValueError('\n'.join(issues))
|
||||
project = (ROOT / 'platforms/macos/VCC.xcodeproj/project.pbxproj').read_text()
|
||||
if '"VCC_BUILD_CONTENT_PROFILE" = "normal";' in project or project.count('"VCC_BUILD_CONTENT_PROFILE" = "source";') != 6:
|
||||
raise ValueError('Distribution must default to the shared source content profile')
|
||||
if '"CODE_SIGN_STYLE" = "Automatic";' in project or '"DEVELOPMENT_TEAM" = "";' not in project:
|
||||
raise ValueError('Distribution must use local ad-hoc signing without a developer team')
|
||||
if project.count('"PRODUCT_NAME" = "VCC Plus OSS";') != 3 or '"PRODUCT_BUNDLE_IDENTIFIER" = "com.brianholman.vccplus.oss.mac";' not in project:
|
||||
raise ValueError('OSS must have an independent app identity')
|
||||
forbidden = {'platforms/macos/App/AppUpdateService.swift', 'platforms/macos/Tests/AppUpdateTests.swift'}
|
||||
if forbidden.intersection(record['files']) or 'AppUpdate' in project or 'AppUpdate' in (ROOT / 'platforms/macos/App/VCCMacApp.swift').read_text():
|
||||
raise ValueError('Commercial updater source, membership or hooks remain')
|
||||
if 'static let bundleID = "com.brianholman.vccplus.oss.mac"' not in (ROOT / 'platforms/macos/App/StartupDefaults.swift').read_text():
|
||||
raise ValueError('OSS must use an independent application-support directory')
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-oss-schema-') as folder:
|
||||
for script, name, table in [('catalog/package.py', 'SoftwareLibrary/catalog.sqlite', 'release'),
|
||||
('knowledge/package_archive.py', 'ArchiveLibrary/ArchiveLibrary.sqlite', 'document')]:
|
||||
output = Path(folder) / name
|
||||
subprocess.run([sys.executable, str(ROOT / 'tools' / script), '--output', str(output), '--content-profile', 'source'], check=True)
|
||||
with sqlite3.connect(output.resolve().as_uri() + '?mode=ro', uri=True) as db:
|
||||
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
|
||||
raise ValueError('Empty dataset is damaged')
|
||||
if db.execute('SELECT count(*) FROM ' + table).fetchone() != (0,):
|
||||
raise ValueError('Dataset unexpectedly contains content')
|
||||
print(f"PASS: {len(record['files'])} source fingerprints; no ROM/database payload; both empty datasets valid")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,31 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Closed public-app link policy shared by both publication checkers.
|
||||
|
||||
An organization name is not proof of repository visibility. This inventory
|
||||
records the specific external app repository reviewed for public references.
|
||||
No network lookup or permission change occurs during publication checks.
|
||||
"""
|
||||
import re
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
|
||||
PUBLIC_HOST = 'vccplus-git.brianholman.com'
|
||||
APPS_NAMESPACE = '/VCCPlus' + 'Apps/'
|
||||
PUBLIC_APP_REPOSITORIES = frozenset({'cococapitalquiz'})
|
||||
|
||||
|
||||
def unapproved_app_links(text):
|
||||
problems = []
|
||||
for match in re.finditer(r'https?://[^\s<>"`\)\]]+', text):
|
||||
value = match.group(0)
|
||||
parsed = urlsplit(value)
|
||||
if parsed.hostname != PUBLIC_HOST:
|
||||
continue
|
||||
path = unquote(parsed.path)
|
||||
if not path.lower().startswith(APPS_NAMESPACE.lower()):
|
||||
continue
|
||||
rest = path[len(APPS_NAMESPACE):]
|
||||
name = rest.split('/', 1)[0].lower()
|
||||
if name not in PUBLIC_APP_REPOSITORIES or '..' in path.split('/'):
|
||||
problems.append(value)
|
||||
return problems
|
||||
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Reconstruct pinned upstream firmware from verified assembly-only inputs.
|
||||
|
||||
Preparation is explicit and offline. Fetch cocoroms/LWTools separately, or supply
|
||||
an existing LWASM. Builds use fresh copies and verify all generated ROM hashes.
|
||||
Only the two boot ROMs, provenance manifest and notice enter an optional bundle.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SPEC = importlib.util.spec_from_file_location('vcc_dependency_fetch', ROOT / 'tools/dependencies/fetch.py')
|
||||
DEPENDENCIES = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(DEPENDENCIES)
|
||||
ROM_NAMES = ('bas13.rom', 'extbas11.rom', 'disk11.rom', 'coco3.rom')
|
||||
BUNDLE_NAMES = ('coco3.rom', 'disk11.rom')
|
||||
SOURCE_NAMES = {'Makefile', 'equates.asm', 'bas.asm', 'extbas.asm', 'disk.asm', 'coco3.asm', 'md5sums.txt', 'mugateerspic.asm'}
|
||||
NOTICE = '''Reconstructed upstream Color Computer firmware
|
||||
|
||||
Source: ToolShed cocoroms, https://github.com/nitros9project/toolshed
|
||||
Pinned revision: 3667a41c9605b7fcb150eace29cc7f42d460be1a
|
||||
|
||||
The original assembly headers identify material copied from Color BASIC
|
||||
Unravelled, Extended Color BASIC Unravelled, Disk Color BASIC Unravelled and
|
||||
Super Color BASIC Unravelled, adjusted for assembly. Their original source
|
||||
comments are retained in the explicitly prepared source dependency.
|
||||
|
||||
Original firmware notices retained in the generated ROM data include:
|
||||
Color BASIC: (C) 1982 TANDY; MICROSOFT
|
||||
Extended Color BASIC: COPYRIGHT (C) 1982 BY TANDY;
|
||||
UNDER LICENSE FROM MICROSOFT
|
||||
Disk Extended Color BASIC: COPYRIGHT (C) 1982 BY TANDY;
|
||||
UNDER LICENSE FROM MICROSOFT
|
||||
Super Color BASIC: COPR. 1982, 1986 BY TANDY;
|
||||
UNDER LICENSE FROM MICROSOFT; AND MICROWARE SYSTEMS CORP.
|
||||
|
||||
These are reconstructed upstream firmware, not newly authored or clean-room
|
||||
replacement ROMs. The emulator's GPL notice does not relabel this firmware.
|
||||
No preassembled ROM or binary input is used by this reconstruction process.
|
||||
'''
|
||||
|
||||
|
||||
def encoded(value):
|
||||
return (json.dumps(value, sort_keys=True, indent=2) + '\n').encode('utf-8')
|
||||
|
||||
|
||||
def sha(path):
|
||||
return DEPENDENCIES.sha(path)
|
||||
|
||||
|
||||
def source_record(records):
|
||||
record = records['cocoroms']
|
||||
if set(record['vendoredFileSHA256']) != SOURCE_NAMES or set(record['upstreamFileSHA256']) != SOURCE_NAMES:
|
||||
raise ValueError('Firmware input inventory must contain exactly the eight assembly/build reference files')
|
||||
if (record.get('sourceRevision') != '3667a41c9605b7fcb150eace29cc7f42d460be1a'
|
||||
or record.get('archiveRootPrefix') != 'cocoroms/' or record.get('optional') is not True):
|
||||
raise ValueError('Unsupported firmware source identity')
|
||||
outputs = record.get('romOutputs')
|
||||
if not isinstance(outputs, dict) or set(outputs) != set(ROM_NAMES):
|
||||
raise ValueError('Firmware output inventory must contain the four qualified ROMs')
|
||||
sizes = {'coco3.rom': 32768, 'disk11.rom': 8192, 'bas13.rom': 8192, 'extbas11.rom': 8192}
|
||||
for name, spec in outputs.items():
|
||||
if (not isinstance(spec, dict) or spec.get('bytes') != sizes[name]
|
||||
or not isinstance(spec.get('sha256'), str) or not DEPENDENCIES.HASH.fullmatch(spec['sha256'])):
|
||||
raise ValueError('Invalid generated firmware fingerprint: ' + name)
|
||||
return record
|
||||
|
||||
|
||||
def copy_verified(source, destination, fingerprints):
|
||||
DEPENDENCIES.verify_tree(source, fingerprints)
|
||||
destination.mkdir(parents=True, exist_ok=False)
|
||||
for name in sorted(fingerprints):
|
||||
original = source / name
|
||||
target = destination / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(original, target)
|
||||
target.chmod(0o755 if original.stat().st_mode & 0o111 else 0o644)
|
||||
DEPENDENCIES.verify_tree(destination, fingerprints)
|
||||
|
||||
|
||||
def validate_outputs(directory, expected):
|
||||
if set(expected) != set(ROM_NAMES):
|
||||
raise ValueError('Expected all four reconstructed ROM outputs')
|
||||
results = {}
|
||||
for name in ROM_NAMES:
|
||||
path = directory / name
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_nlink != 1:
|
||||
raise ValueError('Generated ROM must be an ordinary unlinked file: ' + name)
|
||||
result = {'bytes': path.stat().st_size, 'sha256': sha(path)}
|
||||
if result != expected[name]:
|
||||
raise ValueError('Generated ROM does not match the qualified fingerprint: ' + name)
|
||||
results[name] = result
|
||||
return results
|
||||
|
||||
|
||||
def bundle_manifest(record, metadata, assembler, outputs):
|
||||
return {
|
||||
'schemaVersion': 1,
|
||||
'generator': 'vccplus-rom-build',
|
||||
'source': {
|
||||
'project': 'ToolShed cocoroms', 'url': record['sourceURL'],
|
||||
'revision': record['sourceRevision'], 'manifestSHA256': sha(metadata),
|
||||
'filesSHA256': record['vendoredFileSHA256'], 'preassembledROMInputs': False,
|
||||
},
|
||||
'assembler': assembler,
|
||||
'roms': {name: outputs[name] for name in BUNDLE_NAMES},
|
||||
}
|
||||
|
||||
|
||||
def write_bundle(destination, source, manifest):
|
||||
"""Atomically create a fresh optional payload; never replace a prior bundle."""
|
||||
destination = Path(destination).absolute()
|
||||
DEPENDENCIES.require_unlinked_ancestors(destination, 'ROM bundle destination')
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise ValueError('Choose a fresh ROM bundle destination; existing payloads are never overwritten')
|
||||
for name in BUNDLE_NAMES:
|
||||
path = source / name
|
||||
spec = manifest['roms'][name]
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_nlink != 1 or path.stat().st_size != spec['bytes'] or sha(path) != spec['sha256']:
|
||||
raise ValueError('ROM output changed before optional bundle staging')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix='.rom-bundle-', dir=destination.parent) as temporary:
|
||||
staged = Path(temporary) / 'bundle'
|
||||
staged.mkdir()
|
||||
for name in BUNDLE_NAMES:
|
||||
shutil.copyfile(source / name, staged / name)
|
||||
if (staged / name).stat().st_size != manifest['roms'][name]['bytes'] or sha(staged / name) != manifest['roms'][name]['sha256']:
|
||||
raise ValueError('ROM output changed while copying the optional bundle')
|
||||
(staged / 'ROMBuild.json').write_bytes(encoded(manifest))
|
||||
(staged / 'ROM_SOURCE_NOTICE.txt').write_text(NOTICE, encoding='utf-8')
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise ValueError('ROM bundle destination changed during preparation')
|
||||
staged.rename(destination)
|
||||
|
||||
|
||||
def build(output, dependency_root, lwasm=None, bundle_output=None, jobs=4):
|
||||
output, dependency_root = Path(output).absolute(), Path(dependency_root).absolute()
|
||||
DEPENDENCIES.require_unlinked_ancestors(output, 'Firmware build output')
|
||||
DEPENDENCIES.require_unlinked_ancestors(dependency_root, 'Firmware dependency root')
|
||||
if output.exists() or output.is_symlink():
|
||||
raise ValueError('Choose a fresh build output; previous evidence is never overwritten')
|
||||
if output == ROOT or output in ROOT.parents or output == dependency_root or dependency_root in output.parents:
|
||||
raise ValueError('Choose build output separate from the source/dependency tree')
|
||||
records, locations = DEPENDENCIES.load_records(ROOT)
|
||||
record = source_record(records)
|
||||
source = dependency_root / DEPENDENCIES.validate_record(record)
|
||||
DEPENDENCIES.verify_tree(source, record['vendoredFileSHA256'])
|
||||
if lwasm is None:
|
||||
lwrecord = records['lwtools']
|
||||
lwsource = dependency_root / DEPENDENCIES.validate_record(lwrecord)
|
||||
DEPENDENCIES.verify_tree(lwsource, lwrecord['vendoredFileSHA256'])
|
||||
else:
|
||||
lwasm = Path(lwasm).resolve()
|
||||
if not lwasm.is_file() or not os.access(lwasm, os.X_OK):
|
||||
raise ValueError('The explicitly selected LWASM is unavailable')
|
||||
if bundle_output is not None:
|
||||
bundle = Path(bundle_output).absolute()
|
||||
DEPENDENCIES.require_unlinked_ancestors(bundle, 'ROM bundle destination')
|
||||
# An optional package must not alter verified dependency inputs. Reject
|
||||
# a parent of the build too: creating the build would occupy that bundle
|
||||
# destination before its final atomic installation.
|
||||
canonical_bundle, canonical_dependencies, canonical_output = bundle.resolve(), dependency_root.resolve(), output.resolve()
|
||||
if (canonical_bundle == canonical_dependencies or canonical_dependencies in canonical_bundle.parents
|
||||
or canonical_bundle == canonical_output or canonical_bundle in canonical_output.parents):
|
||||
raise ValueError('ROM bundle must be separate from dependency inputs and the build ancestor path')
|
||||
if bundle.exists() or bundle.is_symlink():
|
||||
raise ValueError('Choose a fresh ROM bundle destination')
|
||||
output.mkdir(parents=True)
|
||||
report = {'schemaVersion': 1, 'status': 'running', 'networkUsed': False,
|
||||
'sourceRevision': record['sourceRevision'], 'preassembledROMInputs': False, 'commands': []}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with (output / 'build.log').open('wb') as log:
|
||||
def run(command, cwd, stdout=None):
|
||||
report['commands'].append({'argv': [str(p) for p in command], 'cwd': str(cwd)})
|
||||
log.write(('$ ' + ' '.join(str(p) for p in command) + '\n').encode())
|
||||
log.flush()
|
||||
subprocess.run(command, cwd=cwd, check=True, stdout=log if stdout is None else stdout, stderr=log)
|
||||
if lwasm is None:
|
||||
lwcopy = output / 'work/lwtools'
|
||||
copy_verified(lwsource, lwcopy, lwrecord['vendoredFileSHA256'])
|
||||
run(['/usr/bin/make', '-j' + str(jobs), 'lwasm'], lwcopy)
|
||||
lwasm = lwcopy / 'lwasm/lwasm'
|
||||
assembler = {'version': subprocess.check_output([str(lwasm), '--version'], stderr=subprocess.STDOUT).decode().strip(),
|
||||
'sha256': sha(lwasm)}
|
||||
report['assembler'] = assembler
|
||||
work = output / 'work/cocoroms'
|
||||
copy_verified(source, work, record['vendoredFileSHA256'])
|
||||
if any(p.suffix.lower() in ('.rom', '.bin') for p in work.rglob('*')):
|
||||
raise ValueError('Preassembled firmware input is forbidden')
|
||||
# Execute the pinned Makefile's assembly sequence directly. A fresh
|
||||
# directory forces every output and --lwasm selects the exact tool,
|
||||
# independent of executable names or the user's PATH.
|
||||
run([str(lwasm), '--raw', '-o', 'bas13.rom', 'equates.asm', 'bas.asm'], work)
|
||||
with (work / 'cb_equates.asm').open('wb') as symbols:
|
||||
run([str(lwasm), '--symbol-dump', '--no-output', 'equates.asm', 'bas.asm'], work, symbols)
|
||||
run([str(lwasm), '--raw', '-o', 'extbas11.rom', 'cb_equates.asm', 'extbas.asm'], work)
|
||||
with (work / 'ecb_equates.asm').open('wb') as symbols:
|
||||
run([str(lwasm), '--symbol-dump', '--no-output', 'equates.asm', 'bas.asm', 'extbas.asm'], work, symbols)
|
||||
run([str(lwasm), '--raw', '--pragma=noindex0tonone', '-o', 'disk11.rom', 'ecb_equates.asm', 'disk.asm'], work)
|
||||
run([str(lwasm), '--raw', '-o', 'coco3.rom', 'equates.asm', 'coco3.asm'], work)
|
||||
outputs = validate_outputs(work, record['romOutputs'])
|
||||
for name in ROM_NAMES:
|
||||
shutil.copyfile(work / name, output / name)
|
||||
manifest = bundle_manifest(record, locations['cocoroms'] / 'cocoroms.json', assembler, outputs)
|
||||
(output / 'ROMBuild.json').write_bytes(encoded(manifest))
|
||||
(output / 'ROM_SOURCE_NOTICE.txt').write_text(NOTICE, encoding='utf-8')
|
||||
if bundle_output is not None:
|
||||
write_bundle(Path(bundle_output), output, manifest)
|
||||
report.update(status='passed', roms=outputs)
|
||||
except Exception as error:
|
||||
report.update(status='failed', failure=str(error))
|
||||
raise
|
||||
finally:
|
||||
report['elapsedSeconds'] = round(time.monotonic() - started, 3)
|
||||
(output / 'BuildReport.json').write_bytes(encoded(report))
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, default=ROOT / '.build/roms')
|
||||
parser.add_argument('--dependency-root', type=Path, default=ROOT / 'third_party')
|
||||
parser.add_argument('--lwasm', type=Path, help='Explicit existing assembler; otherwise build verified LWTools lwasm only')
|
||||
parser.add_argument('--bundle-output', type=Path, help='Fresh optional SourceROMBundle destination')
|
||||
parser.add_argument('--jobs', type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.jobs <= 32:
|
||||
parser.error('--jobs must be between 1 and 32')
|
||||
result = build(args.output, args.dependency_root, args.lwasm, args.bundle_output, args.jobs)
|
||||
print(json.dumps({'status': result['status'], 'roms': result['roms'], 'networkUsed': False}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (ValueError, OSError, subprocess.CalledProcessError) as error:
|
||||
raise SystemExit(str(error))
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Validate an explicitly selected source-built core ROM bundle.
|
||||
|
||||
Use the same bounded directory, manifest and byte checks as source-profile
|
||||
staging. Validation establishes byte identity, not a redistribution license.
|
||||
"""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import runpy
|
||||
|
||||
|
||||
_STAGE = runpy.run_path(str(Path(__file__).with_name('stage.py')))
|
||||
|
||||
|
||||
def validate_bundle(path):
|
||||
source = _STAGE['absolute'](path)
|
||||
_STAGE['directory'](source, 'Source ROM bundle')
|
||||
return _STAGE['read_bundle'](source)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['validate'])
|
||||
parser.add_argument('bundle', type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
validate_bundle(args.bundle)
|
||||
except (OSError, ValueError, UnicodeError) as error:
|
||||
parser.error(str(error))
|
||||
print('PASS: source ROM bundle validated')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Stage an optional explicitly built ROM bundle for source-profile Mac builds.
|
||||
|
||||
Validation establishes byte identity and a safe copy boundary, not redistribution
|
||||
permission or hardware compatibility. No maintainer ROM or media path is probed.
|
||||
Isolated UI-test products omit all bundles without inspecting their input paths.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ROM_SIZES = {'coco3.rom': 32768, 'disk11.rom': 8192}
|
||||
BUNDLE_FILES = set(ROM_SIZES) | {'ROMBuild.json', 'ROM_SOURCE_NOTICE.txt'}
|
||||
STARTER_FILES = ('Hello World.dsk', 'Hello World.cas', 'Blank Disk.dsk',
|
||||
'Hello Hard Disk.dsk', 'Blank OS9 Header.bin')
|
||||
HASH = re.compile(r'^[0-9a-f]{64}$')
|
||||
MANIFEST_LIMIT = 256 * 1024
|
||||
NOTICE_LIMIT = 64 * 1024
|
||||
STARTER_LIMIT = 2 * 1024 * 1024
|
||||
|
||||
|
||||
def absolute(path):
|
||||
value = Path(path)
|
||||
if '..' in value.parts:
|
||||
raise ValueError('Staging paths must not contain parent traversal')
|
||||
return value if value.is_absolute() else Path.cwd() / value
|
||||
|
||||
|
||||
def check_ancestors(path, description):
|
||||
"""Reject redirected inputs/outputs, except the two public macOS aliases."""
|
||||
aliases = {Path('/tmp'): Path('/private/tmp'), Path('/var'): Path('/private/var')}
|
||||
for parent in path.parents:
|
||||
try:
|
||||
info = parent.lstat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if stat.S_ISLNK(info.st_mode):
|
||||
if parent in aliases and parent.resolve() == aliases[parent]:
|
||||
continue
|
||||
raise ValueError(description + ' must not have a linked ancestor')
|
||||
if not stat.S_ISDIR(info.st_mode):
|
||||
raise ValueError(description + ' must have directory ancestors')
|
||||
|
||||
|
||||
def directory(path, description, optional=False):
|
||||
check_ancestors(path, description)
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
if optional:
|
||||
return False
|
||||
raise ValueError(description + ' is missing')
|
||||
if not stat.S_ISDIR(info.st_mode):
|
||||
raise ValueError(description + ' must be an unlinked directory')
|
||||
return True
|
||||
|
||||
|
||||
def signature(info):
|
||||
return (info.st_dev, info.st_ino, info.st_mode, info.st_nlink,
|
||||
info.st_size, info.st_mtime_ns, info.st_ctime_ns)
|
||||
|
||||
|
||||
def read_regular(path, maximum, description, expected_size=None):
|
||||
check_ancestors(path, description)
|
||||
try:
|
||||
before = path.lstat()
|
||||
except FileNotFoundError:
|
||||
raise ValueError(description + ' is missing')
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
|
||||
raise ValueError(description + ' must be a regular file without links')
|
||||
if not 0 < before.st_size <= maximum or (expected_size is not None and before.st_size != expected_size):
|
||||
raise ValueError(description + ' has an invalid size')
|
||||
flags = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
with os.fdopen(os.open(path, flags), 'rb') as stream:
|
||||
if signature(os.fstat(stream.fileno())) != signature(before):
|
||||
raise ValueError(description + ' changed before it could be read')
|
||||
data = stream.read(maximum + 1)
|
||||
after = os.fstat(stream.fileno())
|
||||
if signature(before) != signature(after) or len(data) != before.st_size:
|
||||
raise ValueError(description + ' changed while it was read')
|
||||
# A concurrently replaced pathname must not be accepted as the same input.
|
||||
if signature(path.lstat()) != signature(before):
|
||||
raise ValueError(description + ' was replaced while it was read')
|
||||
return data
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
value = {}
|
||||
for key, item in pairs:
|
||||
if key in value:
|
||||
raise ValueError('ROM build manifest contains a duplicate field')
|
||||
value[key] = item
|
||||
return value
|
||||
|
||||
|
||||
def read_bundle(source):
|
||||
if set(item.name for item in source.iterdir()) != BUNDLE_FILES:
|
||||
raise ValueError('Source ROM bundle must contain exactly the manifest, notice, and two required ROMs')
|
||||
manifest_bytes = read_regular(source / 'ROMBuild.json', MANIFEST_LIMIT, 'ROM build manifest')
|
||||
try:
|
||||
record = json.loads(manifest_bytes.decode('utf-8'), object_pairs_hook=unique_object)
|
||||
except RecursionError:
|
||||
raise ValueError('ROM build manifest nesting is excessive')
|
||||
if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record['schemaVersion'] != 1:
|
||||
raise ValueError('Unsupported ROM build manifest schema')
|
||||
if record.get('generator') != 'vccplus-rom-build':
|
||||
raise ValueError('Unsupported ROM bundle generator')
|
||||
if not isinstance(record.get('source'), dict) or not record['source']:
|
||||
raise ValueError('ROM build manifest requires source metadata')
|
||||
roms = record.get('roms')
|
||||
if not isinstance(roms, dict) or set(roms) != set(ROM_SIZES):
|
||||
raise ValueError('ROM build manifest must identify both required ROMs only')
|
||||
payload = {'ROMBuild.json': manifest_bytes}
|
||||
notice = read_regular(source / 'ROM_SOURCE_NOTICE.txt', NOTICE_LIMIT, 'ROM source notice')
|
||||
if not notice.decode('utf-8').strip() or b'\0' in notice:
|
||||
raise ValueError('ROM source notice must be nonempty UTF-8 text')
|
||||
payload['ROM_SOURCE_NOTICE.txt'] = notice
|
||||
for name, size in ROM_SIZES.items():
|
||||
entry = roms[name]
|
||||
if (not isinstance(entry, dict) or type(entry.get('bytes')) is not int or entry['bytes'] != size
|
||||
or not isinstance(entry.get('sha256'), str) or not HASH.fullmatch(entry['sha256'])):
|
||||
raise ValueError('Invalid ROM manifest entry: ' + name)
|
||||
data = read_regular(source / name, size, name, expected_size=size)
|
||||
if hashlib.sha256(data).hexdigest() != entry['sha256']:
|
||||
raise ValueError('ROM bytes do not match their build manifest: ' + name)
|
||||
payload[name] = data
|
||||
return payload
|
||||
|
||||
|
||||
def check_resources(resources):
|
||||
if (resources.name != 'Resources' or resources.parent.name != 'Contents'
|
||||
or resources.parent.parent.suffix != '.app'):
|
||||
raise ValueError('Destination must be APP.app/Contents/Resources')
|
||||
directory(resources, 'App Resources destination', optional=True)
|
||||
|
||||
|
||||
def check_kit(kit):
|
||||
if not directory(kit, 'Build-owned StarterKit', optional=True):
|
||||
return False
|
||||
# Cleanup is confined to this build-owned directory. Reject redirected files
|
||||
# before either swapping an existing kit or removing an obsolete payload.
|
||||
for folder, names, files in os.walk(kit, followlinks=False):
|
||||
for name in names:
|
||||
directory(Path(folder) / name, 'StarterKit member directory')
|
||||
for name in files:
|
||||
info = (Path(folder) / name).lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
|
||||
raise ValueError('StarterKit members must be regular files without links')
|
||||
return True
|
||||
|
||||
|
||||
def overlaps(first, second):
|
||||
# Ancestors were checked before canonicalizing the accepted /tmp,/var aliases.
|
||||
first, second = first.resolve(), second.resolve()
|
||||
return first == second or first in second.parents or second in first.parents
|
||||
|
||||
|
||||
def atomic_file(destination, data):
|
||||
check_ancestors(destination, 'Build resource')
|
||||
if destination.exists() or destination.is_symlink():
|
||||
info = destination.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
|
||||
raise ValueError('Build resource must be a regular file without links')
|
||||
if info.st_size == len(data) and read_regular(destination, STARTER_LIMIT, 'Existing build resource') == data:
|
||||
return
|
||||
temporary = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(prefix='.source-rom-resource-', dir=destination.parent, delete=False) as stream:
|
||||
temporary = Path(stream.name)
|
||||
stream.write(data)
|
||||
os.chmod(temporary, 0o644)
|
||||
# Recheck the owned output boundary immediately before replacement.
|
||||
check_ancestors(destination, 'Build resource')
|
||||
if destination.exists() or destination.is_symlink():
|
||||
info = destination.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
|
||||
raise ValueError('Build resource changed to a linked or nonregular file')
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def exchange_directories(first, second):
|
||||
"""Atomically exchange complete directories; never expose a partial kit."""
|
||||
library = ctypes.CDLL(None, use_errno=True)
|
||||
if sys.platform == 'darwin':
|
||||
operation = getattr(library, 'renamex_np', None)
|
||||
if operation is not None:
|
||||
operation.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint)
|
||||
operation.restype = ctypes.c_int
|
||||
result = operation(os.fsencode(first), os.fsencode(second), 2) # RENAME_SWAP
|
||||
else:
|
||||
raise OSError(errno.ENOTSUP, 'Atomic directory exchange is unavailable')
|
||||
elif sys.platform.startswith('linux'):
|
||||
operation = getattr(library, 'renameat2', None)
|
||||
if operation is not None:
|
||||
operation.argtypes = (ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint)
|
||||
operation.restype = ctypes.c_int
|
||||
result = operation(-100, os.fsencode(first), -100, os.fsencode(second), 2) # RENAME_EXCHANGE
|
||||
else:
|
||||
raise OSError(errno.ENOTSUP, 'Atomic directory exchange is unavailable')
|
||||
else:
|
||||
raise OSError(errno.ENOTSUP, 'Atomic directory exchange is unavailable')
|
||||
if result:
|
||||
error = ctypes.get_errno()
|
||||
raise OSError(error, os.strerror(error))
|
||||
|
||||
|
||||
def selected_source(srcroot, explicit=None, environ=None):
|
||||
environment = os.environ if environ is None else environ
|
||||
return absolute(explicit if explicit is not None else
|
||||
environment.get('VCC_SOURCE_ROM_DIR') or srcroot / 'LocalResources/SourceROMBundle')
|
||||
|
||||
|
||||
def isolated_ui_testing(environment):
|
||||
if environment.get('CONFIGURATION') == 'UITesting':
|
||||
return True
|
||||
swift_conditions = environment.get('SWIFT_ACTIVE_COMPILATION_CONDITIONS', '').split()
|
||||
if 'VCC_UI_TESTING' in swift_conditions:
|
||||
return True
|
||||
definitions = environment.get('GCC_PREPROCESSOR_DEFINITIONS', '').split()
|
||||
return any(value in ('VCC_UI_TESTING', 'VCC_UI_TESTING=1') for value in definitions)
|
||||
|
||||
|
||||
def remove_interrupted_staging(resources):
|
||||
# After an exchange, the temporary pathname contains the former StarterKit.
|
||||
# A killed build must not leave that old profile's payload in the next app.
|
||||
# Match only this helper's generated eight-character temporary names.
|
||||
for path in resources.iterdir():
|
||||
if re.fullmatch(r'\.source-rom-starter-[a-z0-9_]{8}', path.name):
|
||||
check_kit(path)
|
||||
shutil.rmtree(path)
|
||||
elif re.fullmatch(r'\.source-rom-resource-[a-z0-9_]{8}', path.name):
|
||||
info = path.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
|
||||
raise ValueError('Interrupted build resource must be a regular file without links')
|
||||
path.unlink()
|
||||
|
||||
|
||||
def stage_source_roms(srcroot, resources, source=None, environ=None):
|
||||
srcroot, resources = absolute(srcroot), absolute(resources)
|
||||
directory(srcroot, 'Mac source directory')
|
||||
check_resources(resources)
|
||||
environment = os.environ if environ is None else environ
|
||||
ui_testing = isolated_ui_testing(environment)
|
||||
# Do not even resolve or inspect an explicit/generated ROM location for the
|
||||
# isolated UI app. Only its build-owned output and authored header are used.
|
||||
bundle = None if ui_testing else selected_source(srcroot, source, environment)
|
||||
present = directory(bundle, 'Source ROM bundle', optional=True) if bundle is not None else False
|
||||
starter = srcroot / 'Resources/StarterMedia'
|
||||
directory(starter, 'Authored starter media')
|
||||
if (bundle is not None and overlaps(bundle, resources)) or overlaps(starter, resources):
|
||||
raise ValueError('Build resources must not overlap source inputs')
|
||||
kit = resources / 'StarterKit'
|
||||
check_kit(kit)
|
||||
# Validate all selected inputs before creating or changing a build resource.
|
||||
payload = read_bundle(bundle) if present else None
|
||||
header = read_regular(starter / 'Blank OS9 Header.bin', STARTER_LIMIT, 'Authored OS-9 header', expected_size=8192)
|
||||
if payload is not None:
|
||||
for name in STARTER_FILES:
|
||||
payload[name] = read_regular(starter / name, STARTER_LIMIT, 'Authored starter media: ' + name)
|
||||
resources.mkdir(parents=True, exist_ok=True)
|
||||
check_resources(resources)
|
||||
remove_interrupted_staging(resources)
|
||||
atomic_file(resources / 'Blank OS9 Header.bin', header)
|
||||
if payload is None:
|
||||
if check_kit(kit):
|
||||
shutil.rmtree(kit)
|
||||
return {'staged': False, 'roms': {}, 'ui_testing': ui_testing}
|
||||
staging = Path(tempfile.mkdtemp(prefix='.source-rom-starter-', dir=resources))
|
||||
try:
|
||||
for name, data in payload.items():
|
||||
(staging / name).write_bytes(data)
|
||||
os.chmod(staging / name, 0o644)
|
||||
os.chmod(staging, 0o755)
|
||||
check_resources(resources)
|
||||
if check_kit(kit):
|
||||
exchange_directories(staging, kit)
|
||||
# The staging pathname now contains only the previous owned kit.
|
||||
else:
|
||||
os.rename(staging, kit)
|
||||
return {'staged': True, 'roms': {name: {'bytes': len(payload[name]), 'sha256': hashlib.sha256(payload[name]).hexdigest()}
|
||||
for name in ROM_SIZES}}
|
||||
finally:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--srcroot', type=Path, required=True, help='Mac project directory (platforms/macos)')
|
||||
parser.add_argument('--resources', type=Path, required=True, help='Build-owned APP.app/Contents/Resources')
|
||||
parser.add_argument('--source', type=Path, help='Explicit source ROM bundle; overrides VCC_SOURCE_ROM_DIR')
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = stage_source_roms(args.srcroot, args.resources, args.source)
|
||||
except (OSError, ValueError, UnicodeError) as error:
|
||||
parser.error(str(error))
|
||||
if result['staged']:
|
||||
print('Included explicitly built source ROMs, original source notice, and authored starter media.')
|
||||
elif result.get('ui_testing'):
|
||||
print('Isolated UI-test build omits ROM bundles and retains only the authored standalone header.')
|
||||
else:
|
||||
print('Optional source ROM bundle is absent; building without bundled ROMs.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,182 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Authored regressions for explicit firmware preparation and bundle boundaries."""
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
MODULE = Path(__file__).resolve().parents[1] / 'build.py'
|
||||
SPEC = importlib.util.spec_from_file_location('rom_build', MODULE)
|
||||
BUILD = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(BUILD)
|
||||
|
||||
|
||||
def fingerprint(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
class ROMBuildTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory(prefix='vcc-rom-builder-test-')
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = Path(self.temporary.name)
|
||||
self.records, self.locations = BUILD.DEPENDENCIES.load_records()
|
||||
self.record = json.loads(json.dumps(self.records['cocoroms']))
|
||||
|
||||
def fixture_outputs(self):
|
||||
directory = self.root / 'roms'
|
||||
directory.mkdir()
|
||||
expected = {}
|
||||
for name in BUILD.ROM_NAMES:
|
||||
data = bytes(32768 if name == 'coco3.rom' else 8192)
|
||||
(directory / name).write_bytes(data)
|
||||
expected[name] = {'bytes': len(data), 'sha256': fingerprint(data)}
|
||||
return directory, expected
|
||||
|
||||
def fixture_inputs(self):
|
||||
dependencies = self.root / 'dependencies'
|
||||
source = dependencies / 'cocoroms'
|
||||
source.mkdir(parents=True)
|
||||
hashes = {}
|
||||
for name in BUILD.SOURCE_NAMES:
|
||||
data = ('Authored source fixture: ' + name + '\n').encode()
|
||||
(source / name).write_bytes(data)
|
||||
hashes[name] = fingerprint(data)
|
||||
self.record['upstreamFileSHA256'] = self.record['vendoredFileSHA256'] = hashes
|
||||
_, self.record['romOutputs'] = self.fixture_outputs()
|
||||
return dependencies
|
||||
|
||||
def fixture_assembler(self, fail=False):
|
||||
tool = self.root / 'explicit-assembler-with-another-name'
|
||||
tool.write_text('''#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
import sys
|
||||
if '--version' in sys.argv: print('Authored assembler fixture'); raise SystemExit(0)
|
||||
''' + ("raise SystemExit(7)\n" if fail else '''if '--raw' in sys.argv:
|
||||
name = sys.argv[sys.argv.index('-o') + 1]
|
||||
Path(name).write_bytes(bytes(32768 if name == 'coco3.rom' else 8192))
|
||||
else: print('fixture EQU 0')
|
||||
'''))
|
||||
tool.chmod(0o755)
|
||||
return tool
|
||||
|
||||
def test_firmware_fetch_is_explicit_and_manifest_contains_only_source(self):
|
||||
record = BUILD.source_record(self.records)
|
||||
self.assertEqual(set(record['upstreamFileSHA256']), BUILD.SOURCE_NAMES)
|
||||
self.assertNotIn('cocoroms', BUILD.DEPENDENCIES.default_selection(self.records))
|
||||
self.assertEqual(len(BUILD.DEPENDENCIES.default_selection(self.records)), 17)
|
||||
self.assertNotIn('.rom', ''.join(record['upstreamFileSHA256']))
|
||||
self.assertEqual(record['archiveRootPrefix'], 'cocoroms/')
|
||||
|
||||
def test_unreviewed_firmware_input_is_refused(self):
|
||||
self.record['vendoredFileSHA256']['supplied.rom'] = '0' * 64
|
||||
with self.assertRaisesRegex(ValueError, 'exactly the eight'):
|
||||
BUILD.source_record({'cocoroms': self.record})
|
||||
|
||||
def test_optional_flag_requires_a_boolean(self):
|
||||
self.record['optional'] = 'true'
|
||||
with self.assertRaisesRegex(ValueError, 'optional flag'):
|
||||
BUILD.DEPENDENCIES.validate_record(self.record)
|
||||
|
||||
def test_malformed_output_metadata_is_rejected_explicitly(self):
|
||||
for spec in [None, {'bytes': 32768, 'sha256': None}, {'bytes': 32768, 'sha256': 'not-a-hash'}]:
|
||||
record = json.loads(json.dumps(self.records['cocoroms']))
|
||||
record['romOutputs']['coco3.rom'] = spec
|
||||
with self.assertRaisesRegex(ValueError, 'firmware fingerprint'):
|
||||
BUILD.source_record({'cocoroms': record})
|
||||
record = json.loads(json.dumps(self.records['cocoroms']))
|
||||
record['romOutputs'] = None
|
||||
with self.assertRaisesRegex(ValueError, 'output inventory'):
|
||||
BUILD.source_record({'cocoroms': record})
|
||||
|
||||
def test_generated_hash_size_links_and_hardlinks_are_checked(self):
|
||||
source, expected = self.fixture_outputs()
|
||||
self.assertEqual(BUILD.validate_outputs(source, expected), expected)
|
||||
target = source / 'disk11.rom'
|
||||
target.write_bytes(b'changed')
|
||||
with self.assertRaisesRegex(ValueError, 'qualified fingerprint'):
|
||||
BUILD.validate_outputs(source, expected)
|
||||
target.unlink()
|
||||
target.symlink_to(source / 'bas13.rom')
|
||||
with self.assertRaisesRegex(ValueError, 'unlinked'):
|
||||
BUILD.validate_outputs(source, expected)
|
||||
target.unlink()
|
||||
os.link(source / 'bas13.rom', target)
|
||||
with self.assertRaisesRegex(ValueError, 'unlinked'):
|
||||
BUILD.validate_outputs(source, expected)
|
||||
|
||||
def test_bundle_has_exactly_two_roms_manifest_notice_and_no_private_paths(self):
|
||||
source, expected = self.fixture_outputs()
|
||||
manifest = BUILD.bundle_manifest(self.record, self.locations['cocoroms'] / 'cocoroms.json',
|
||||
{'version': 'fixture', 'sha256': 'a' * 64}, expected)
|
||||
destination = self.root / 'bundle'
|
||||
BUILD.write_bundle(destination, source, manifest)
|
||||
self.assertEqual({p.name for p in destination.iterdir()},
|
||||
{'ROMBuild.json', 'ROM_SOURCE_NOTICE.txt', 'coco3.rom', 'disk11.rom'})
|
||||
published = json.loads((destination / 'ROMBuild.json').read_text())
|
||||
self.assertEqual(published['schemaVersion'], 1)
|
||||
self.assertEqual(published['generator'], 'vccplus-rom-build')
|
||||
self.assertEqual(set(published['roms']), set(BUILD.BUNDLE_NAMES))
|
||||
self.assertNotIn(str(self.root), (destination / 'ROMBuild.json').read_text())
|
||||
self.assertIn('MICROWARE', (destination / 'ROM_SOURCE_NOTICE.txt').read_text())
|
||||
with self.assertRaisesRegex(ValueError, 'fresh ROM bundle'):
|
||||
BUILD.write_bundle(destination, source, manifest)
|
||||
|
||||
def test_cold_build_uses_exact_explicit_assembler_and_no_existing_roms(self):
|
||||
dependencies = self.fixture_inputs()
|
||||
records = dict(self.records, cocoroms=self.record)
|
||||
tool = self.fixture_assembler()
|
||||
output, bundle = self.root / 'output', self.root / 'bundle'
|
||||
with patch.object(BUILD.DEPENDENCIES, 'load_records', return_value=(records, self.locations)):
|
||||
report = BUILD.build(output, dependencies, tool, bundle)
|
||||
self.assertEqual(report['status'], 'passed')
|
||||
self.assertEqual(len(report['commands']), 6)
|
||||
self.assertTrue(all(command['argv'][0] == str(tool.resolve()) for command in report['commands']))
|
||||
self.assertFalse(report['preassembledROMInputs'])
|
||||
self.assertEqual(report['roms'], self.record['romOutputs'])
|
||||
self.assertTrue((bundle / 'coco3.rom').is_file())
|
||||
|
||||
def test_preassembled_dependency_payload_is_refused_before_execution(self):
|
||||
dependencies = self.fixture_inputs()
|
||||
(dependencies / 'cocoroms/preassembled.rom').write_bytes(b'fixture')
|
||||
records = dict(self.records, cocoroms=self.record)
|
||||
with patch.object(BUILD.DEPENDENCIES, 'load_records', return_value=(records, self.locations)), \
|
||||
patch.object(BUILD.subprocess, 'run') as execute:
|
||||
with self.assertRaisesRegex(ValueError, 'differs from reviewed import'):
|
||||
BUILD.build(self.root / 'output', dependencies, self.fixture_assembler())
|
||||
execute.assert_not_called()
|
||||
self.assertFalse((self.root / 'output').exists())
|
||||
|
||||
def test_failed_assembly_retains_evidence_without_creating_bundle(self):
|
||||
dependencies = self.fixture_inputs()
|
||||
records = dict(self.records, cocoroms=self.record)
|
||||
output, bundle = self.root / 'output', self.root / 'bundle'
|
||||
with patch.object(BUILD.DEPENDENCIES, 'load_records', return_value=(records, self.locations)):
|
||||
with self.assertRaises(BUILD.subprocess.CalledProcessError):
|
||||
BUILD.build(output, dependencies, self.fixture_assembler(fail=True), bundle)
|
||||
self.assertEqual(json.loads((output / 'BuildReport.json').read_text())['status'], 'failed')
|
||||
self.assertTrue((output / 'build.log').is_file())
|
||||
self.assertFalse(bundle.exists())
|
||||
|
||||
def test_bundle_cannot_contaminate_dependencies_or_occupy_build_ancestor(self):
|
||||
dependencies = self.fixture_inputs()
|
||||
records = dict(self.records, cocoroms=self.record)
|
||||
tool = self.fixture_assembler()
|
||||
for bundle in [dependencies / 'cocoroms/package', dependencies.resolve() / 'cocoroms/canonical-package', self.root / 'new-build-parent']:
|
||||
output = self.root / 'new-build-parent/output'
|
||||
with patch.object(BUILD.DEPENDENCIES, 'load_records', return_value=(records, self.locations)), \
|
||||
patch.object(BUILD.subprocess, 'run') as execute:
|
||||
with self.assertRaisesRegex(ValueError, 'separate from dependency'):
|
||||
BUILD.build(output, dependencies, tool, bundle)
|
||||
execute.assert_not_called()
|
||||
self.assertFalse(output.exists())
|
||||
self.assertFalse(bundle.exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,468 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Source-ROM staging checks using authored synthetic bytes, never real ROMs."""
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / 'tools/roms/stage.py'
|
||||
SPEC = importlib.util.spec_from_file_location('source_rom_stage', SCRIPT)
|
||||
stage = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(stage)
|
||||
|
||||
|
||||
class SourceROMStageTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory(prefix='vcc-source-rom-stage-')
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.root = Path(temporary.name)
|
||||
self.srcroot = self.root / 'Source With Spaces/platforms/macos'
|
||||
self.starter = self.srcroot / 'Resources/StarterMedia'
|
||||
self.starter.mkdir(parents=True)
|
||||
self.media = {
|
||||
'Hello World.dsk': b'AUTHORED HELLO DISK',
|
||||
'Hello World.cas': b'AUTHORED HELLO CASSETTE',
|
||||
'Blank Disk.dsk': b'AUTHORED BLANK DISK',
|
||||
'Hello Hard Disk.dsk': b'AUTHORED HARD DISK',
|
||||
'Blank OS9 Header.bin': b'\x00' * 8192,
|
||||
}
|
||||
for name, data in self.media.items():
|
||||
(self.starter / name).write_bytes(data)
|
||||
self.resources = self.root / 'Build With Spaces/VCC Plus.app/Contents/Resources'
|
||||
self.bundle = self.root / 'Source ROM Bundle'
|
||||
self.create_bundle(self.bundle)
|
||||
|
||||
def create_bundle(self, path, seed=0x12):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
roms = {}
|
||||
for index, (name, size) in enumerate(stage.ROM_SIZES.items()):
|
||||
data = bytes([(seed + index) & 255]) * size
|
||||
(path / name).write_bytes(data)
|
||||
roms[name] = {'bytes': size, 'sha256': hashlib.sha256(data).hexdigest()}
|
||||
record = {'schemaVersion': 1, 'generator': 'vccplus-rom-build',
|
||||
'source': {'project': 'authored unit fixture', 'revision': '0' * 40},
|
||||
'assembler': {'version': 'authored fixture'}, 'roms': roms}
|
||||
(path / 'ROMBuild.json').write_text(json.dumps(record, indent=2) + '\n')
|
||||
(path / 'ROM_SOURCE_NOTICE.txt').write_text('Authored synthetic test bytes. No hardware ROM payload.\n')
|
||||
return record
|
||||
|
||||
def apply(self, source=None, environ=None):
|
||||
return stage.stage_source_roms(self.srcroot, self.resources, source,
|
||||
environ={} if environ is None else environ)
|
||||
|
||||
def tree(self, root):
|
||||
if not root.exists():
|
||||
return None
|
||||
return {str(path.relative_to(root)): (path.read_bytes(), path.stat().st_mode & 0o777,
|
||||
path.stat().st_mtime_ns)
|
||||
for path in root.rglob('*') if path.is_file()}
|
||||
|
||||
def alter_manifest(self, operation):
|
||||
path = self.bundle / 'ROMBuild.json'
|
||||
record = json.loads(path.read_text())
|
||||
operation(record)
|
||||
path.write_text(json.dumps(record))
|
||||
|
||||
def seed_old_kit(self):
|
||||
kit = self.resources / 'StarterKit'
|
||||
kit.mkdir(parents=True)
|
||||
(kit / 'stale-private-marker').write_bytes(b'OLDER BUILD ONLY')
|
||||
(self.resources / 'Blank OS9 Header.bin').write_bytes(self.media['Blank OS9 Header.bin'])
|
||||
return kit
|
||||
|
||||
def test_valid_bundle_copies_owned_bytes_and_preserves_all_inputs(self):
|
||||
bundle_before, starter_before = self.tree(self.bundle), self.tree(self.starter)
|
||||
result = self.apply(self.bundle)
|
||||
self.assertTrue(result['staged'])
|
||||
kit = self.resources / 'StarterKit'
|
||||
self.assertEqual(set(path.name for path in kit.iterdir()), stage.BUNDLE_FILES | set(stage.STARTER_FILES))
|
||||
for name in stage.BUNDLE_FILES:
|
||||
self.assertEqual((kit / name).read_bytes(), (self.bundle / name).read_bytes())
|
||||
self.assertNotEqual((kit / name).stat().st_ino, (self.bundle / name).stat().st_ino)
|
||||
for name, data in self.media.items():
|
||||
self.assertEqual((kit / name).read_bytes(), data)
|
||||
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
|
||||
self.assertEqual(self.tree(self.bundle), bundle_before)
|
||||
self.assertEqual(self.tree(self.starter), starter_before)
|
||||
self.assertEqual(result['roms']['coco3.rom']['bytes'], 32768)
|
||||
self.assertEqual(result['roms']['disk11.rom']['bytes'], 8192)
|
||||
|
||||
def test_missing_default_removes_only_owned_stale_kit(self):
|
||||
kit = self.seed_old_kit()
|
||||
untouched = self.resources / 'unrelated-resource.txt'
|
||||
untouched.write_bytes(b'PRESERVE')
|
||||
self.assertFalse(self.apply()['staged'])
|
||||
self.assertFalse(kit.exists())
|
||||
self.assertEqual(untouched.read_bytes(), b'PRESERVE')
|
||||
self.assertTrue((self.resources / 'Blank OS9 Header.bin').is_file())
|
||||
|
||||
def test_explicit_and_environment_selection_never_fall_back(self):
|
||||
default = self.srcroot / 'LocalResources/SourceROMBundle'
|
||||
self.create_bundle(default, seed=0x44)
|
||||
chosen = stage.selected_source(self.srcroot, self.bundle, {'VCC_SOURCE_ROM_DIR': str(default)})
|
||||
self.assertEqual(chosen, self.bundle)
|
||||
self.assertEqual(stage.selected_source(self.srcroot, environ={'VCC_SOURCE_ROM_DIR': str(self.bundle)}), self.bundle)
|
||||
self.assertEqual(stage.selected_source(self.srcroot, environ={'VCC_SOURCE_ROM_DIR': ''}), default)
|
||||
self.assertTrue(self.apply(environ={'VCC_SOURCE_ROM_DIR': str(self.bundle)})['staged'])
|
||||
self.assertEqual((self.resources / 'StarterKit/coco3.rom').read_bytes(), (self.bundle / 'coco3.rom').read_bytes())
|
||||
self.assertFalse(self.apply(self.root / 'Absent Explicit Bundle')['staged'])
|
||||
self.assertFalse((self.resources / 'StarterKit').exists())
|
||||
|
||||
def test_default_bundle_is_used_only_from_source_specific_directory(self):
|
||||
destination = self.srcroot / 'LocalResources/SourceROMBundle'
|
||||
shutil.copytree(self.bundle, destination)
|
||||
self.assertTrue(self.apply()['staged'])
|
||||
shutil.rmtree(destination)
|
||||
private = self.srcroot / 'LocalResources/ROMs'
|
||||
private.mkdir()
|
||||
(private / 'coco3.rom').write_bytes(b'PRIVATE PATH MUST NOT BE READ')
|
||||
media = self.srcroot / 'LocalResources/Media'
|
||||
media.mkdir()
|
||||
(media / 'NitrOS-9 L2 V3.3.0.vhd').write_bytes(b'NO PRIVATE MEDIA DISCOVERY')
|
||||
self.assertFalse(self.apply()['staged'])
|
||||
self.assertFalse((self.resources / 'StarterKit').exists())
|
||||
self.assertEqual((private / 'coco3.rom').read_bytes(), b'PRIVATE PATH MUST NOT BE READ')
|
||||
|
||||
def test_isolated_ui_product_never_selects_or_reads_prepared_rom_bundles(self):
|
||||
default = self.srcroot / 'LocalResources/SourceROMBundle'
|
||||
shutil.copytree(self.bundle, default)
|
||||
original, default_original = self.tree(self.bundle), self.tree(default)
|
||||
configurations = (
|
||||
{'CONFIGURATION': 'UITesting'},
|
||||
{'CONFIGURATION': 'Debug', 'SWIFT_ACTIVE_COMPILATION_CONDITIONS': 'DEBUG VCC_UI_TESTING'},
|
||||
{'CONFIGURATION': 'Release', 'GCC_PREPROCESSOR_DEFINITIONS': 'VCC_UI_TESTING=1'},
|
||||
)
|
||||
for environment in configurations:
|
||||
with self.subTest(environment=environment):
|
||||
self.seed_old_kit()
|
||||
environment['VCC_SOURCE_ROM_DIR'] = str(self.bundle)
|
||||
with mock.patch.object(stage, 'selected_source', side_effect=AssertionError('UI build selected a ROM input')):
|
||||
with mock.patch.object(stage, 'read_bundle', side_effect=AssertionError('UI build read a ROM input')):
|
||||
result = self.apply(self.bundle, environment)
|
||||
self.assertFalse(result['staged'])
|
||||
self.assertTrue(result['ui_testing'])
|
||||
self.assertFalse((self.resources / 'StarterKit').exists())
|
||||
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
|
||||
self.assertEqual(self.tree(self.bundle), original)
|
||||
self.assertEqual(self.tree(default), default_original)
|
||||
|
||||
def test_isolated_ui_product_does_not_probe_unsafe_explicit_or_environment_paths(self):
|
||||
alias = self.root / 'Unsafe ROM Alias'
|
||||
alias.symlink_to(self.bundle, target_is_directory=True)
|
||||
original = self.tree(self.bundle)
|
||||
self.seed_old_kit()
|
||||
# Parent traversal and linked input locations would both be refused for
|
||||
# an ordinary source app, but neither location is probed by a UI build.
|
||||
result = self.apply(alias / '..' / 'missing', {'CONFIGURATION': 'UITesting',
|
||||
'VCC_SOURCE_ROM_DIR': str(alias)})
|
||||
self.assertFalse(result['staged'])
|
||||
self.assertFalse((self.resources / 'StarterKit').exists())
|
||||
self.assertEqual(self.tree(self.bundle), original)
|
||||
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
|
||||
|
||||
def test_atomic_replacement_removes_all_previous_profile_payload(self):
|
||||
old = self.seed_old_kit()
|
||||
nested = old / 'old-nested'
|
||||
nested.mkdir()
|
||||
(nested / 'payload').write_bytes(b'OLD')
|
||||
self.assertTrue(self.apply(self.bundle)['staged'])
|
||||
self.assertFalse((old / 'stale-private-marker').exists())
|
||||
self.assertFalse(nested.exists())
|
||||
self.assertEqual(set(path.name for path in old.iterdir()), stage.BUNDLE_FILES | set(stage.STARTER_FILES))
|
||||
self.assertFalse(list(self.resources.glob('.source-rom-starter-*')))
|
||||
|
||||
def test_failed_atomic_exchange_preserves_previous_kit(self):
|
||||
old = self.seed_old_kit()
|
||||
before = self.tree(old)
|
||||
with mock.patch.object(stage, 'exchange_directories', side_effect=OSError('authored exchange refusal')):
|
||||
with self.assertRaises(OSError):
|
||||
self.apply(self.bundle)
|
||||
self.assertEqual(self.tree(old), before)
|
||||
self.assertFalse(list(self.resources.glob('.source-rom-starter-*')))
|
||||
|
||||
def test_next_build_removes_only_interrupted_owned_staging(self):
|
||||
self.resources.mkdir(parents=True)
|
||||
stale = self.resources / '.source-rom-starter-abcdefgh'
|
||||
stale.mkdir()
|
||||
(stale / 'prior-private-rom-marker').write_bytes(b'PRIOR PROFILE')
|
||||
(self.resources / '.source-rom-resource-12345678').write_bytes(b'PARTIAL HEADER')
|
||||
unrelated = self.resources / '.source-rom-starter-not-a-generated-name'
|
||||
unrelated.mkdir()
|
||||
(unrelated / 'preserve').write_bytes(b'KEEP')
|
||||
self.assertFalse(self.apply()['staged'])
|
||||
self.assertFalse(stale.exists())
|
||||
self.assertFalse((self.resources / '.source-rom-resource-12345678').exists())
|
||||
self.assertEqual((unrelated / 'preserve').read_bytes(), b'KEEP')
|
||||
# Even a helper-shaped name cannot redirect cleanup through a link.
|
||||
stale.symlink_to(unrelated, target_is_directory=True)
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply()
|
||||
self.assertEqual((unrelated / 'preserve').read_bytes(), b'KEEP')
|
||||
|
||||
def test_invalid_manifest_fields_refuse_before_changing_build_output(self):
|
||||
cases = (
|
||||
lambda r: r.update(schemaVersion=2),
|
||||
lambda r: r.update(schemaVersion=True),
|
||||
lambda r: r.update(generator='different-generator'),
|
||||
lambda r: r.update(source={}),
|
||||
lambda r: r.update(roms=[]),
|
||||
lambda r: r['roms'].pop('disk11.rom'),
|
||||
lambda r: r['roms'].update({'extra.rom': {'bytes': 1, 'sha256': '0' * 64}}),
|
||||
lambda r: r['roms']['coco3.rom'].update(bytes=32767),
|
||||
lambda r: r['roms']['coco3.rom'].update(bytes=True),
|
||||
lambda r: r['roms']['disk11.rom'].update(sha256='A' * 64),
|
||||
lambda r: r['roms']['disk11.rom'].update(sha256='0' * 64),
|
||||
)
|
||||
old = self.seed_old_kit()
|
||||
before = self.tree(self.resources)
|
||||
for operation in cases:
|
||||
with self.subTest(operation=operation):
|
||||
self.create_bundle(self.bundle)
|
||||
self.alter_manifest(operation)
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
self.assertEqual(self.tree(self.resources), before)
|
||||
self.assertTrue((old / 'stale-private-marker').is_file())
|
||||
|
||||
def test_invalid_json_duplicate_fields_and_nonobject_manifest_are_refused(self):
|
||||
for data in (b'{', b'[]', b'{"schemaVersion":1,"schemaVersion":1}', b'\xff', b' ' * (stage.MANIFEST_LIMIT + 1)):
|
||||
with self.subTest(data=data[:25]):
|
||||
(self.bundle / 'ROMBuild.json').write_bytes(data)
|
||||
with self.assertRaises((ValueError, UnicodeError)):
|
||||
self.apply(self.bundle)
|
||||
self.assertFalse(self.resources.exists())
|
||||
|
||||
def test_missing_extra_or_nonregular_bundle_members_are_refused(self):
|
||||
for name in stage.BUNDLE_FILES:
|
||||
with self.subTest(missing=name):
|
||||
self.create_bundle(self.bundle)
|
||||
(self.bundle / name).unlink()
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
self.create_bundle(self.bundle)
|
||||
(self.bundle / '.unexpected').write_bytes(b'NO EXTRA FILES')
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
(self.bundle / '.unexpected').unlink()
|
||||
(self.bundle / 'disk11.rom').unlink()
|
||||
(self.bundle / 'disk11.rom').mkdir()
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
self.assertFalse(self.resources.exists())
|
||||
|
||||
def test_rom_payload_sizes_and_manifest_identity_are_enforced(self):
|
||||
for name, size in stage.ROM_SIZES.items():
|
||||
for length in (0, size - 1, size + 1):
|
||||
with self.subTest(name=name, length=length):
|
||||
self.create_bundle(self.bundle)
|
||||
(self.bundle / name).write_bytes(bytes(length))
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
self.create_bundle(self.bundle)
|
||||
(self.bundle / name).write_bytes(bytes([0x77]) * size)
|
||||
with self.assertRaisesRegex(ValueError, 'do not match'):
|
||||
self.apply(self.bundle)
|
||||
self.assertFalse(self.resources.exists())
|
||||
|
||||
def test_notice_is_required_bounded_nonempty_utf8(self):
|
||||
for notice in (b'', b' \n\t', b'\xff', b'notice\0', b'x' * (stage.NOTICE_LIMIT + 1)):
|
||||
with self.subTest(notice=notice[:25]):
|
||||
(self.bundle / 'ROM_SOURCE_NOTICE.txt').write_bytes(notice)
|
||||
with self.assertRaises((ValueError, UnicodeError)):
|
||||
self.apply(self.bundle)
|
||||
self.assertFalse(self.resources.exists())
|
||||
|
||||
def test_source_bundle_leaf_and_ancestor_symlinks_are_refused(self):
|
||||
alias = self.root / 'Linked Bundle'
|
||||
alias.symlink_to(self.bundle, target_is_directory=True)
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(alias)
|
||||
parent = self.root / 'Linked Parent'
|
||||
parent.symlink_to(self.root, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, 'linked ancestor'):
|
||||
self.apply(parent / self.bundle.name)
|
||||
dangling = self.root / 'Dangling Bundle'
|
||||
dangling.symlink_to(self.root / 'missing')
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(dangling)
|
||||
self.assertFalse(self.resources.exists())
|
||||
|
||||
def test_source_members_must_not_be_symbolic_or_hard_links(self):
|
||||
for name in stage.BUNDLE_FILES:
|
||||
for kind in ('symbolic', 'hard'):
|
||||
with self.subTest(name=name, kind=kind):
|
||||
self.create_bundle(self.bundle)
|
||||
original = self.bundle / name
|
||||
victim = self.root / 'Input Victim'
|
||||
victim.write_bytes(original.read_bytes())
|
||||
before = victim.read_bytes()
|
||||
original.unlink()
|
||||
if kind == 'symbolic':
|
||||
original.symlink_to(victim)
|
||||
else:
|
||||
os.link(victim, original)
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
self.assertEqual(victim.read_bytes(), before)
|
||||
original.unlink()
|
||||
victim.unlink()
|
||||
self.assertFalse(self.resources.exists())
|
||||
|
||||
def test_output_shape_and_linked_ancestors_cannot_redirect_writes(self):
|
||||
with self.assertRaisesRegex(ValueError, 'APP.app'):
|
||||
stage.stage_source_roms(self.srcroot, self.root / 'Not An App/Resources', self.bundle, {})
|
||||
victim = self.root / 'Untouched Output Victim'
|
||||
victim.mkdir()
|
||||
build = self.root / 'Linked Build'
|
||||
build.symlink_to(victim, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, 'linked ancestor'):
|
||||
stage.stage_source_roms(self.srcroot, build / 'VCC.app/Contents/Resources', self.bundle, {})
|
||||
self.assertEqual(list(victim.iterdir()), [])
|
||||
|
||||
def test_existing_kit_links_refuse_cleanup_or_replacement(self):
|
||||
kit = self.seed_old_kit()
|
||||
victim = self.root / 'Kit Victim'
|
||||
victim.write_bytes(b'UNCHANGED')
|
||||
for kind in ('symbolic', 'hard', 'directory'):
|
||||
with self.subTest(kind=kind):
|
||||
link = kit / 'unsafe-member'
|
||||
if kind == 'symbolic':
|
||||
link.symlink_to(victim)
|
||||
elif kind == 'hard':
|
||||
os.link(victim, link)
|
||||
else:
|
||||
link.symlink_to(self.root, target_is_directory=True)
|
||||
for source in (None, self.bundle):
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(source)
|
||||
self.assertEqual(victim.read_bytes(), b'UNCHANGED')
|
||||
self.assertTrue((kit / 'stale-private-marker').is_file())
|
||||
link.unlink()
|
||||
shutil.rmtree(kit)
|
||||
kit.symlink_to(self.bundle, target_is_directory=True)
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply()
|
||||
self.assertEqual(set(path.name for path in self.bundle.iterdir()), stage.BUNDLE_FILES)
|
||||
|
||||
def test_linked_standalone_header_is_never_replaced(self):
|
||||
self.resources.mkdir(parents=True)
|
||||
victim = self.root / 'Header Victim'
|
||||
victim.write_bytes(b'UNCHANGED')
|
||||
for kind in ('symbolic', 'hard'):
|
||||
with self.subTest(kind=kind):
|
||||
destination = self.resources / 'Blank OS9 Header.bin'
|
||||
if kind == 'symbolic':
|
||||
destination.symlink_to(victim)
|
||||
else:
|
||||
os.link(victim, destination)
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
self.assertEqual(victim.read_bytes(), b'UNCHANGED')
|
||||
self.assertFalse((self.resources / 'StarterKit').exists())
|
||||
destination.unlink()
|
||||
|
||||
def test_authored_input_links_and_bad_header_refuse_before_mutation(self):
|
||||
header = self.starter / 'Blank OS9 Header.bin'
|
||||
header.write_bytes(b'SHORT')
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
header.write_bytes(self.media[header.name])
|
||||
target = self.starter / 'Hello World.cas'
|
||||
victim = self.root / 'Authored Input Victim'
|
||||
victim.write_bytes(target.read_bytes())
|
||||
target.unlink()
|
||||
target.symlink_to(victim)
|
||||
with self.assertRaises(ValueError):
|
||||
self.apply(self.bundle)
|
||||
self.assertFalse(self.resources.exists())
|
||||
|
||||
def test_inputs_and_build_output_must_not_overlap(self):
|
||||
self.resources.mkdir(parents=True)
|
||||
embedded = self.resources / 'StarterKit'
|
||||
shutil.copytree(self.bundle, embedded)
|
||||
before = self.tree(embedded)
|
||||
with self.assertRaisesRegex(ValueError, 'overlap'):
|
||||
self.apply(embedded)
|
||||
self.assertEqual(self.tree(embedded), before)
|
||||
with self.assertRaisesRegex(ValueError, 'parent traversal'):
|
||||
self.apply(self.bundle / '..' / self.bundle.name)
|
||||
|
||||
def run_build_phase(self, *, profile, configuration='Release', source=None):
|
||||
helper = self.srcroot.parents[1] / 'tools/roms/stage.py'
|
||||
helper.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(SCRIPT, helper)
|
||||
environment = os.environ.copy()
|
||||
environment.update(SRCROOT=str(self.srcroot), TARGET_BUILD_DIR=str(self.resources.parents[2]),
|
||||
UNLOCALIZED_RESOURCES_FOLDER_PATH='VCC Plus.app/Contents/Resources',
|
||||
VCC_BUILD_CONTENT_PROFILE=profile, CONFIGURATION=configuration)
|
||||
environment.pop('VCC_SOURCE_ROM_DIR', None)
|
||||
if source is not None:
|
||||
environment['VCC_SOURCE_ROM_DIR'] = str(source)
|
||||
return subprocess.run(['/bin/sh', str(ROOT / 'platforms/macos/Packaging/copy-development-starter.sh')],
|
||||
env=environment, capture_output=True, text=True)
|
||||
|
||||
def test_source_build_phase_accepts_bundle_in_debug_and_release_without_private_discovery(self):
|
||||
private = self.srcroot / 'LocalResources/ROMs'
|
||||
private.mkdir(parents=True)
|
||||
(private / 'coco3.rom').write_bytes(b'INVALID NORMAL PROFILE INPUT')
|
||||
for configuration in ('Debug', 'Release'):
|
||||
with self.subTest(configuration=configuration):
|
||||
result = self.run_build_phase(profile='source', configuration=configuration, source=self.bundle)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual((self.resources / 'StarterKit/coco3.rom').read_bytes(), (self.bundle / 'coco3.rom').read_bytes())
|
||||
self.assertEqual((private / 'coco3.rom').read_bytes(), b'INVALID NORMAL PROFILE INPUT')
|
||||
|
||||
def test_source_ui_testing_build_phase_omits_prepared_bundle_and_stale_kit(self):
|
||||
default = self.srcroot / 'LocalResources/SourceROMBundle'
|
||||
shutil.copytree(self.bundle, default)
|
||||
original, default_original = self.tree(self.bundle), self.tree(default)
|
||||
self.seed_old_kit()
|
||||
result = self.run_build_phase(profile='source', configuration='UITesting', source=self.bundle)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn('Isolated UI-test build', result.stdout)
|
||||
self.assertFalse((self.resources / 'StarterKit').exists())
|
||||
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
|
||||
self.assertEqual(self.tree(self.bundle), original)
|
||||
self.assertEqual(self.tree(default), default_original)
|
||||
|
||||
def test_source_build_phase_rejects_malformed_present_bundle(self):
|
||||
(self.bundle / 'coco3.rom').write_bytes(b'INVALID')
|
||||
result = self.run_build_phase(profile='source', source=self.bundle)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn('invalid size', result.stderr)
|
||||
self.assertFalse((self.resources / 'StarterKit').exists())
|
||||
|
||||
def test_normal_debug_phase_preserves_existing_private_workflow(self):
|
||||
private = self.srcroot / 'LocalResources/ROMs'
|
||||
private.mkdir(parents=True)
|
||||
expected = {'coco3.rom': bytes([0x33]) * 32768, 'disk11.rom': bytes([0x44]) * 8192,
|
||||
'orch90.rom': bytes([0x55]) * 8192}
|
||||
for name, data in expected.items():
|
||||
(private / name).write_bytes(data)
|
||||
result = self.run_build_phase(profile='normal', configuration='Debug', source=self.bundle)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
kit = self.resources / 'StarterKit'
|
||||
for name, data in expected.items():
|
||||
self.assertEqual((kit / name).read_bytes(), data)
|
||||
self.assertFalse((kit / 'ROMBuild.json').exists())
|
||||
for name, data in self.media.items():
|
||||
self.assertEqual((kit / name).read_bytes(), data)
|
||||
|
||||
def test_normal_release_phase_still_excludes_optional_development_roms(self):
|
||||
self.seed_old_kit()
|
||||
result = self.run_build_phase(profile='normal', configuration='Release', source=self.bundle)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse((self.resources / 'StarterKit').exists())
|
||||
self.assertEqual((self.resources / 'Blank OS9 Header.bin').read_bytes(), self.media['Blank OS9 Header.bin'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,248 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Offline content-profile checks using authored disposable inputs only."""
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
RESOURCES = (
|
||||
('tools/catalog/package.py', 'tools/catalog/app-schema.sql', 'SoftwareLibrary/catalog.sqlite'),
|
||||
('tools/knowledge/package_archive.py', 'tools/knowledge/archive-library.sql', 'ArchiveLibrary/ArchiveLibrary.sqlite'),
|
||||
)
|
||||
|
||||
|
||||
class BuildContentTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory(prefix='vcc-content-profile-')
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.root = Path(temporary.name)
|
||||
self.source = self.root / 'Source With Spaces'
|
||||
self.build = self.root / 'Build With Spaces'
|
||||
for relative in ['tools/build_content.py', *(p for r in RESOURCES for p in r[:2])]:
|
||||
destination = self.source / relative
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(ROOT / relative, destination)
|
||||
self.tempting = []
|
||||
for relative in ('media/software-library-full/prepared/app-catalog.sqlite',
|
||||
'media/app-inputs/archive-library/ArchiveLibrary.sqlite',
|
||||
'media/knowledge-base/archive-library/ArchiveLibrary.sqlite'):
|
||||
target = self.source / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(b'PRIVATE-DATA-TEST-MARKER\x00')
|
||||
self.tempting.append(target)
|
||||
|
||||
def run_script(self, resource, *extra, profile='source', env_profile=None, output=True, check=True):
|
||||
command = [sys.executable, str(self.source / resource[0])]
|
||||
if profile is not None:
|
||||
command += ['--content-profile', profile]
|
||||
if output:
|
||||
command += ['--output', str(self.build / resource[2])]
|
||||
environment = os.environ.copy()
|
||||
environment.pop('VCC_BUILD_CONTENT_PROFILE', None)
|
||||
if env_profile is not None:
|
||||
environment['VCC_BUILD_CONTENT_PROFILE'] = env_profile
|
||||
completed = subprocess.run([*command, *extra], cwd=self.source,
|
||||
env=environment, capture_output=True, text=True)
|
||||
if check:
|
||||
self.assertEqual(completed.returncode, 0, completed.stderr)
|
||||
return completed
|
||||
|
||||
def metadata(self, resource, **kwargs):
|
||||
return json.loads(self.run_script(resource, **kwargs).stdout.split(': ', 1)[1])
|
||||
|
||||
def test_software_schema_and_reader_contract_have_no_content(self):
|
||||
resource = RESOURCES[0]
|
||||
info = self.metadata(resource)
|
||||
self.assertTrue(info['empty'])
|
||||
self.assertEqual((info['titles'], info['releases'], info['assets'], info['thumbnails']), (0, 0, 0, 0))
|
||||
path = self.build / resource[2]
|
||||
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
|
||||
self.assertEqual(db.execute('PRAGMA user_version').fetchone(), (1,))
|
||||
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['stage'], 'empty-source')
|
||||
for table in ('title', 'package', 'release', 'member', 'download_location',
|
||||
'asset', 'asset_delivery', 'release_asset', 'setup', 'source_hint'):
|
||||
self.assertEqual(db.execute('SELECT count(*) FROM ' + table).fetchone(), (0,))
|
||||
self.assertEqual(db.execute('SELECT * FROM browse').fetchall(), [])
|
||||
self.assertEqual(db.execute('SELECT ra.release_id,a.*,d.manual_url FROM release_asset ra '
|
||||
'JOIN asset a ON a.id=ra.asset_id LEFT JOIN asset_delivery d '
|
||||
'ON d.asset_id=a.id').fetchall(), [])
|
||||
self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(), [('ok',)])
|
||||
|
||||
def test_archive_schema_supports_empty_browse_search_and_reader(self):
|
||||
resource = RESOURCES[1]
|
||||
info = self.metadata(resource)
|
||||
self.assertTrue(info['empty'])
|
||||
path = self.build / resource[2]
|
||||
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
|
||||
db.execute('PRAGMA query_only=ON')
|
||||
db.execute('PRAGMA trusted_schema=OFF')
|
||||
self.assertEqual(db.execute('PRAGMA application_id').fetchone(), (0x56434341,))
|
||||
self.assertEqual(db.execute('PRAGMA user_version').fetchone(), (2,))
|
||||
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['source_snapshot'], 'empty-source')
|
||||
for table in ('document', 'page', 'reader_section', 'page_terms', 'reviewed_example'):
|
||||
self.assertEqual(db.execute('SELECT count(*) FROM ' + table).fetchone(), (0,))
|
||||
self.assertEqual(db.execute('SELECT d.* FROM document d ORDER BY d.title COLLATE NOCASE,d.sha256 LIMIT ? OFFSET ?', (50, 0)).fetchall(), [])
|
||||
self.assertEqual(db.execute('SELECT d.*,p.page_number FROM page_terms JOIN page p '
|
||||
'ON p.id=page_terms.rowid JOIN document d ON d.id=p.document_id '
|
||||
'WHERE page_terms MATCH ? ORDER BY bm25(page_terms,1.0,4.0),p.id '
|
||||
'LIMIT ? OFFSET ?', ('"basic"', 50, 0)).fetchall(), [])
|
||||
self.assertEqual(db.execute('SELECT p.page_label,s.text FROM document d JOIN page p '
|
||||
'ON p.document_id=d.id JOIN reader_section s ON s.page_id=p.id '
|
||||
'WHERE d.sha256=? AND p.page_number=?', ('0'*64, 1)).fetchall(), [])
|
||||
self.assertEqual(db.execute('PRAGMA foreign_key_check').fetchall(), [])
|
||||
|
||||
def test_repeat_build_keeps_identical_bytes_and_timestamp(self):
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
first = self.metadata(resource)
|
||||
path = self.build / resource[2]
|
||||
original, stamp = path.read_bytes(), path.stat().st_mtime_ns
|
||||
second = self.metadata(resource)
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
self.assertEqual(path.stat().st_mtime_ns, stamp)
|
||||
self.assertEqual(path.stat().st_mode & 0o777, 0o444)
|
||||
self.assertEqual(first['sha256'], hashlib.sha256(original).hexdigest())
|
||||
self.assertNotIn(b'PRIVATE-DATA-TEST-MARKER', original)
|
||||
|
||||
def test_source_profile_ignores_explicit_and_default_maintainer_inputs(self):
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
self.run_script(resource, '--source', str(self.tempting[0]))
|
||||
for source in self.tempting:
|
||||
self.assertEqual(source.read_bytes(), b'PRIVATE-DATA-TEST-MARKER\x00')
|
||||
self.assertNotIn(b'PRIVATE-DATA-TEST-MARKER', (self.build / resource[2]).read_bytes())
|
||||
|
||||
def test_old_resource_is_replaced_but_invalid_schema_preserves_it(self):
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
path = self.build / resource[2]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(b'OLD-BUILD-PRIVATE-CONTENT')
|
||||
self.metadata(resource)
|
||||
original = path.read_bytes()
|
||||
self.assertNotIn(b'OLD-BUILD-PRIVATE-CONTENT', original)
|
||||
with (self.source / resource[1]).open('a') as stream:
|
||||
stream.write('\nINVALID SQL INTENTIONALLY AUTHORED FOR TEST;\n')
|
||||
self.assertNotEqual(self.run_script(resource, check=False).returncode, 0)
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
self.assertFalse(list(path.parent.glob('.empty-library-*.sqlite')))
|
||||
|
||||
def test_linked_outputs_never_change_the_target(self):
|
||||
for resource in RESOURCES:
|
||||
for kind in ('symlink', 'hardlink', 'dangling'):
|
||||
with self.subTest(resource=resource[0], kind=kind):
|
||||
path = self.build / resource[2]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
target = self.root / 'link-victim'
|
||||
if kind != 'dangling':
|
||||
target.write_bytes(b'unchanged')
|
||||
if kind == 'hardlink':
|
||||
os.link(target, path)
|
||||
else:
|
||||
path.symlink_to(target)
|
||||
self.assertNotEqual(self.run_script(resource, check=False).returncode, 0)
|
||||
if kind != 'dangling':
|
||||
self.assertEqual(target.read_bytes(), b'unchanged')
|
||||
target.unlink()
|
||||
path.unlink()
|
||||
|
||||
def test_linked_output_ancestor_cannot_redirect_a_build_resource(self):
|
||||
target = self.root / 'untouched-directory'
|
||||
target.mkdir()
|
||||
self.build.symlink_to(target, target_is_directory=True)
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
rejected = self.run_script(resource, check=False)
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertIn('linked ancestor', rejected.stderr)
|
||||
self.assertEqual(list(target.iterdir()), [])
|
||||
|
||||
def test_standard_temporary_path_alias_remains_usable(self):
|
||||
if not Path('/tmp').is_symlink() or Path('/tmp').resolve() != Path('/private/tmp'):
|
||||
self.skipTest('macOS temporary-directory alias is absent')
|
||||
with tempfile.TemporaryDirectory(dir='/tmp', prefix='vcc-content-alias-') as temporary:
|
||||
self.build = Path(temporary)
|
||||
for resource in RESOURCES:
|
||||
self.assertTrue(self.metadata(resource)['empty'])
|
||||
|
||||
def test_transaction_sidecars_are_rejected_including_empty_and_dangling(self):
|
||||
for resource in RESOURCES:
|
||||
self.metadata(resource)
|
||||
path = self.build / resource[2]
|
||||
original = path.read_bytes()
|
||||
for suffix in ('-wal', '-shm', '-journal'):
|
||||
for kind in ('empty', 'data', 'dangling'):
|
||||
with self.subTest(resource=resource[0], suffix=suffix, kind=kind):
|
||||
sidecar = Path(str(path) + suffix)
|
||||
if kind == 'dangling':
|
||||
sidecar.symlink_to(self.root / 'missing-sidecar-target')
|
||||
else:
|
||||
sidecar.write_bytes(b'' if kind == 'empty' else b'old transaction')
|
||||
rejected = self.run_script(resource, check=False)
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertIn('sidecars', rejected.stderr)
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
sidecar.unlink()
|
||||
|
||||
def test_source_profile_rejects_full_release_requirement(self):
|
||||
for resource, flag in zip(RESOURCES, ('--require-full', '--require')):
|
||||
self.assertNotEqual(self.run_script(resource, flag, check=False).returncode, 0)
|
||||
self.assertFalse((self.build / resource[2]).exists())
|
||||
|
||||
def test_source_check_only_builds_schema_without_input_or_output(self):
|
||||
self.run_script(RESOURCES[0], '--check-only', output=False)
|
||||
self.run_script(RESOURCES[1], output=False)
|
||||
self.assertFalse(self.build.exists())
|
||||
|
||||
def test_environment_selects_source_and_explicit_cli_overrides_it(self):
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
self.assertTrue(self.metadata(resource, profile=None, env_profile='source')['empty'])
|
||||
# The normal path must read and reject the authored corrupt maintainer input.
|
||||
rejected = self.run_script(resource, profile='normal', env_profile='source', check=False)
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertNotEqual(self.run_script(resource, profile=None, env_profile='unknown', check=False).returncode, 0)
|
||||
self.assertEqual(self.run_script(resource, profile='source', env_profile='unknown').returncode, 0)
|
||||
|
||||
def test_normal_archive_retains_verified_populated_input_exactly(self):
|
||||
source = self.tempting[1]
|
||||
source.unlink()
|
||||
with closing(sqlite3.connect(source)) as db:
|
||||
db.executescript((self.source / RESOURCES[1][1]).read_text())
|
||||
db.execute("INSERT INTO document(id,sha256,title,category,collection,format,source_url,byte_size,page_count) "
|
||||
"VALUES(1,?,'Authored fixture','Books','Books','pdf','https://example.test/fixture.pdf',20,1)", ('a'*64,))
|
||||
db.commit()
|
||||
original = source.read_bytes()
|
||||
digest = hashlib.sha256(original).hexdigest()
|
||||
(source.parent / 'archive-library.json').write_text(json.dumps(dict(schema_version=2,
|
||||
sha256=digest, bytes=len(original), documents=1, reader_sections=0, ranged_members=0)))
|
||||
(source.parent / 'ArchiveLibrary.sqlite.sha256').write_text(digest + ' ArchiveLibrary.sqlite\n')
|
||||
self.run_script(RESOURCES[1], '--require', profile=None)
|
||||
self.assertEqual((self.build / RESOURCES[1][2]).read_bytes(), original)
|
||||
self.assertEqual(source.read_bytes(), original)
|
||||
|
||||
def test_default_normal_behavior_still_uses_existing_fallbacks(self):
|
||||
for path in self.tempting:
|
||||
path.unlink()
|
||||
self.assertIn('absent', self.run_script(RESOURCES[0], profile=None).stdout)
|
||||
path = self.build / RESOURCES[0][2]
|
||||
with closing(sqlite3.connect(path)) as db:
|
||||
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['stage'], 'empty-development')
|
||||
archive = self.build / RESOURCES[1][2]
|
||||
archive.parent.mkdir(parents=True, exist_ok=True)
|
||||
archive.write_bytes(b'old index')
|
||||
self.run_script(RESOURCES[1], profile=None)
|
||||
self.assertFalse(archive.exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Build the reviewed CoCo Letters source and a DOS-bootable DSK without downloading."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
ROOT=Path(__file__).resolve().parents[2]
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--toolchain', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, default=ROOT/'.build/coco-letters')
|
||||
args=parser.parse_args()
|
||||
prefix=args.toolchain.resolve();out=args.output.resolve()
|
||||
features=json.loads((prefix/'VCC-toolchain.json').read_text())
|
||||
if not features['standaloneDisk']:
|
||||
raise SystemExit('CoCo Letters requires the optional private standalone disk source; see docs/development/TOOLCHAIN.md')
|
||||
if out.exists(): raise SystemExit('Choose a fresh --output directory')
|
||||
out.mkdir(parents=True)
|
||||
env=dict(os.environ, PATH=str(prefix/'bin')+os.pathsep+os.environ['PATH'])
|
||||
env.setdefault('DEVELOPER_DIR','/Applications/Xcode.app/Contents/Developer')
|
||||
for name in ('decbfile','bcontrol','bsound'):
|
||||
shutil.copytree(ROOT/'third_party/cmoc-libraries'/name,out/name)
|
||||
subprocess.run(['make','-j4',f'lib{name}.a'],cwd=out/name,env=env,check=True)
|
||||
game=out/'game';shutil.copytree(ROOT/'third_party/examples/coco-letters',game)
|
||||
# Upstream generates two outputs in one rule; serial make avoids duplicate writes.
|
||||
subprocess.run(['make','DECBFILE_LIB_DIR=../decbfile','BCONTROL_LIB_DIR=../bcontrol','BSOUND_LIB_DIR=../bsound'],cwd=game,env=env,check=True)
|
||||
disk=game/'coco-letters-0.1.0.dsk'
|
||||
if disk.stat().st_size!=35*18*256: raise SystemExit('Unexpected disk geometry')
|
||||
print(f'Built disk: {disk}\nMount in FD0, select RGB, then type DOS. Requires CoCo 3, 128 KB or more.')
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Build pinned Perl for Make recipes; no host Perl is required after packaging."""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VERSION = '5.44.0'
|
||||
|
||||
|
||||
def build(output, architectures, jobs):
|
||||
source = ROOT / 'third_party/perl'
|
||||
env = dict(os.environ)
|
||||
env.setdefault('DEVELOPER_DIR', '/Applications/Xcode.app/Contents/Developer')
|
||||
prefixes = {}
|
||||
for arch in architectures:
|
||||
work = output / arch / 'work'
|
||||
prefix = output / arch / 'install'
|
||||
shutil.copytree(source, work)
|
||||
flags = f'-arch {arch} -mmacosx-version-min=14.0'
|
||||
commands = [
|
||||
['sh', 'Configure', '-des', f'-Dprefix={prefix}', '-Dcc=clang',
|
||||
f'-Dccflags={flags}', f'-Dldflags={flags}',
|
||||
f'-Dlddlflags=-bundle -undefined dynamic_lookup {flags}',
|
||||
'-Duserelocatableinc', '-Uuseshrplib', '-Uuseithreads',
|
||||
'-Dman1dir=none', '-Dman3dir=none'],
|
||||
['/usr/bin/make', f'-j{jobs}'], ['/usr/bin/make', 'install']]
|
||||
for command in commands:
|
||||
print('+', ' '.join(map(str, command)), flush=True)
|
||||
subprocess.run(command, cwd=work, env=env, check=True)
|
||||
prefixes[arch] = prefix
|
||||
return prefixes
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--architectures', nargs='+', choices=['arm64', 'x86_64'], default=['arm64', 'x86_64'])
|
||||
parser.add_argument('--jobs', type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
if output.exists(): parser.error('Choose a fresh output directory')
|
||||
output.mkdir(parents=True)
|
||||
build(output, list(dict.fromkeys(args.architectures)), max(1, args.jobs))
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Build a relocatable macOS CMOC runtime; host SDK tools are build-time only."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, default=ROOT / '.build/cmoc-runtime')
|
||||
parser.add_argument('--architectures', nargs='+', choices=['arm64', 'x86_64'], default=['arm64', 'x86_64'])
|
||||
parser.add_argument('--jobs', type=int, default=4)
|
||||
parser.add_argument('--standalone-disk-source', type=Path, help='Optional private DECB-derived disk driver; never downloaded automatically')
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
if output.exists(): parser.error('Choose a fresh output directory')
|
||||
output.mkdir(parents=True)
|
||||
architectures = list(dict.fromkeys(args.architectures))
|
||||
base = dict(os.environ)
|
||||
base.setdefault('DEVELOPER_DIR', '/Applications/Xcode.app/Contents/Developer')
|
||||
def run(*command, cwd=ROOT, env=base):
|
||||
print('+', ' '.join(map(str, command)), flush=True)
|
||||
subprocess.run(list(map(str, command)), cwd=cwd, env=env, check=True)
|
||||
run(sys.executable, ROOT / 'tools/toolchain/verify-sources.py')
|
||||
roots = []
|
||||
for architecture in architectures:
|
||||
arch_root = output / architecture
|
||||
run(sys.executable, ROOT / 'tools/toolchain/build.py', '--output', arch_root,
|
||||
'--host-arch', architecture, '--jobs', args.jobs)
|
||||
prefix = arch_root / 'install'
|
||||
env = dict(base, CC='clang', CXX='clang++',
|
||||
CFLAGS=f'-O2 -arch {architecture} -mmacosx-version-min=14.0 -Wno-error=implicit-function-declaration',
|
||||
CXXFLAGS=f'-O2 -arch {architecture} -mmacosx-version-min=14.0',
|
||||
LDFLAGS=f'-arch {architecture} -mmacosx-version-min=14.0')
|
||||
for name in ('mcpp', 'make'):
|
||||
source = arch_root / 'work' / name
|
||||
shutil.copytree(ROOT / 'third_party' / name, source)
|
||||
for file in source.rglob('*'):
|
||||
if file.is_file(): os.utime(file, (946684800, 946684800))
|
||||
options = [f'--host={architecture}-apple-darwin', '--without-guile', '--disable-dependency-tracking'] if name == 'make' else []
|
||||
run('./configure', f'--prefix={prefix}', *options, cwd=source, env=env)
|
||||
run('/usr/bin/make', f'-j{args.jobs}', cwd=source, env=env)
|
||||
run('/usr/bin/make', 'install', cwd=source, env=env)
|
||||
run('xcrun', 'clang', '-O2', '-arch', architecture, '-mmacosx-version-min=14.0',
|
||||
ROOT / 'tools/toolchain/cpp-mcpp.c', '-o', prefix / 'bin/cpp')
|
||||
roots.append(prefix)
|
||||
bundle = output / 'Toolchain'
|
||||
for name in ('bin', 'libexec', 'share', 'LICENSES'): (bundle / name).mkdir(parents=True)
|
||||
shutil.copytree(roots[0] / 'share/cmoc', bundle / 'share/cmoc')
|
||||
shutil.copyfile(ROOT / 'tools/toolchain/cmoc-runtime.sh', bundle / 'bin/cmoc')
|
||||
(bundle / 'bin/cmoc').chmod(0o755)
|
||||
for tool in ('cmoc', 'cpp', 'mcpp', 'make', 'lwasm', 'lwlink', 'lwar', 'writecocofile'):
|
||||
target = bundle / ('libexec' if tool == 'cmoc' else 'bin') / tool
|
||||
inputs = [prefix / 'bin' / tool for prefix in roots]
|
||||
if len(inputs) > 1: run('xcrun', 'lipo', '-create', *inputs, '-output', target)
|
||||
else: shutil.copyfile(inputs[0], target)
|
||||
target.chmod(0o755)
|
||||
notices = {'CMOC-COPYING.txt': 'cmoc/COPYING', 'CMOC-README.txt': 'cmoc/README',
|
||||
'LWTools-COPYING.txt': 'lwtools/COPYING', 'LWTools-GPL3.txt': 'lwtools/GPL3',
|
||||
'MCPP-LICENSE.txt': 'mcpp/LICENSE', 'MAKE-COPYING.txt': 'make/COPYING'}
|
||||
for target, source in notices.items(): shutil.copyfile(ROOT / 'third_party' / source, bundle / 'LICENSES' / target)
|
||||
shutil.copyfile(ROOT / 'tools/toolchain/patches/README.md', bundle / 'LICENSES/CMOC-VCC-EXTENSIONS.txt')
|
||||
record = {'schemaVersion': 1, 'cmoc': '0.1.100', 'lwtools': '4.25', 'mcpp': '2.7.2',
|
||||
'make': '4.4.1', 'vccDebugMetadata': 1, 'vccAddZeroFlagsFix': 1, 'vccIndexedByteSubtractionFix': 1, 'vccIndexedArgumentLoadFix': 1, 'vccWriteCoCoFileBeforeDir': 1, 'vccLWASMProjectInputs': 1, 'architectures': architectures, 'privateRuntimeInputs': False,
|
||||
'minimumMacOS': '14.0', 'appleDeveloperToolsRequiredAtRuntime': False}
|
||||
(bundle / 'manifest.json').write_text(json.dumps(record, indent=2) + '\n')
|
||||
perl_output = output / 'perl'
|
||||
run(sys.executable, ROOT / 'tools/toolchain/build-perl.py', '--output', perl_output,
|
||||
'--architectures', *architectures, '--jobs', args.jobs)
|
||||
perl_prefixes = [argument for arch in architectures
|
||||
for argument in ('--prefix', f'{arch}={perl_output / arch / "install"}')]
|
||||
run(sys.executable, ROOT / 'tools/toolchain/stage-perl.py', bundle, *perl_prefixes)
|
||||
run(sys.executable, ROOT / 'tools/toolchain/stage-libraries.py', bundle, '--jobs', args.jobs)
|
||||
run(sys.executable, ROOT / 'tools/toolchain/stage-decbfile.py', bundle, '--jobs', args.jobs)
|
||||
if args.standalone_disk_source:
|
||||
run(sys.executable, ROOT / 'tools/toolchain/stage-standalone-disk.py', bundle, '--source', args.standalone_disk_source)
|
||||
run(sys.executable, ROOT / 'tools/toolchain/verify-runtime.py', bundle)
|
||||
print(f'Runtime ready: {bundle}\nStage at platforms/macos/LocalResources/ToolchainBundle for app builds.')
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Build the pinned host toolchain in ignored storage; never download or install globally."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, default=ROOT / '.build/toolchain')
|
||||
parser.add_argument('--mc6839-rom', type=Path, help='Optional private Float09.bin')
|
||||
parser.add_argument('--standalone-disk-source', type=Path, help='Optional private upstream dskcon-standalone.c')
|
||||
parser.add_argument('--jobs', type=int, default=4)
|
||||
parser.add_argument('--host-arch', choices=['native', 'arm64', 'x86_64'], default='native')
|
||||
args = parser.parse_args()
|
||||
out = args.output.resolve()
|
||||
if out == ROOT or ROOT / 'third_party' in out.parents or out == ROOT / 'third_party':
|
||||
parser.error('Choose a separate build output directory')
|
||||
subprocess.run([sys.executable, str(ROOT / 'tools/toolchain/verify-sources.py')], check=True)
|
||||
prefix = out / 'install'
|
||||
work = out / 'work'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
env = dict(os.environ)
|
||||
env['PATH'] = str(prefix / 'bin') + os.pathsep + env['PATH']
|
||||
env.setdefault('DEVELOPER_DIR', '/Applications/Xcode.app/Contents/Developer')
|
||||
if args.host_arch != 'native':
|
||||
if sys.platform != 'darwin':
|
||||
parser.error('--host-arch is currently a macOS build option')
|
||||
flags = f'-O2 -arch {args.host_arch} -mmacosx-version-min=14.0'
|
||||
env.update(CC='clang', CXX='clang++', CFLAGS=flags, CXXFLAGS=flags, LDFLAGS=f'-arch {args.host_arch} -mmacosx-version-min=14.0')
|
||||
def run(*command, cwd):
|
||||
print('+', ' '.join(map(str, command)), flush=True)
|
||||
subprocess.run(list(map(str, command)), cwd=cwd, env=env, check=True)
|
||||
# Copies protect pristine vendored snapshots from generated makefiles/objects.
|
||||
for name in ('lwtools', 'cmoc'):
|
||||
target = work / name
|
||||
if target.exists():
|
||||
raise SystemExit(f'{target} already exists; choose a fresh --output for a reproducible build')
|
||||
shutil.copytree(ROOT / 'third_party' / name, target)
|
||||
# Git does not retain mtimes. Equal times keep shipped generated files
|
||||
# from unnecessarily invoking version-specific Autotools generators.
|
||||
for item in target.rglob('*'):
|
||||
if item.is_file():
|
||||
os.utime(item, (946684800, 946684800))
|
||||
# Retain pristine upstream snapshots; apply the reviewed patches only to the build copies.
|
||||
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/cmoc-debug-metadata.patch', cwd=work / 'cmoc')
|
||||
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/cmoc-add-zero-flags.patch', cwd=work / 'cmoc')
|
||||
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/cmoc-indexed-byte-subtraction.patch', cwd=work / 'cmoc')
|
||||
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/cmoc-indexed-argument-load.patch', cwd=work / 'cmoc')
|
||||
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/writecocofile-before-dir.patch', cwd=work / 'cmoc')
|
||||
run('/usr/bin/patch', '-p1', '-i', ROOT / 'tools/toolchain/patches/lwtools-project-inputs.patch', cwd=work / 'lwtools')
|
||||
run('make', f'-j{args.jobs}', f'PREFIX={prefix}', cwd=work / 'lwtools')
|
||||
run('make', f'PREFIX={prefix}', 'install', cwd=work / 'lwtools')
|
||||
# Optional private inputs are checked against the reviewed upstream release.
|
||||
record = json.loads((ROOT / 'third_party/manifests/cmoc.json').read_text())
|
||||
for supplied, relative in ((args.mc6839_rom, 'src/float/Float09.bin'),
|
||||
(args.standalone_disk_source, 'src/stdlib/dskcon-standalone.c')):
|
||||
if supplied:
|
||||
if hashlib.sha256(supplied.read_bytes()).hexdigest() != record['excludedFileSHA256'][relative]:
|
||||
raise SystemExit(f'Private input does not match reviewed CMOC release: {supplied}')
|
||||
shutil.copyfile(supplied, work / 'cmoc' / relative)
|
||||
for extension in ('am', 'in'):
|
||||
if args.mc6839_rom:
|
||||
path = work / 'cmoc/src/float' / ('Makefile.' + extension)
|
||||
text = path.read_text().replace('cmoclib_DATA = ', 'cmoclib_DATA = float-ctor.mc6839_o float-ctor.mc6839_os9_o libcmoc-float-mc6839.a libcmoc-float-mc6839_os9.a ')
|
||||
path.write_text(text); os.utime(path, (946684800, 946684800))
|
||||
if args.standalone_disk_source:
|
||||
path = work / 'cmoc/src/stdlib' / ('Makefile.' + extension)
|
||||
text = path.read_text()
|
||||
# Restore the omitted implementation beside the other C runtime sources.
|
||||
assert 'C_COCO_BASIC =' in text
|
||||
text = text.replace('C_COCO_BASIC =', 'C_COCO_BASIC = dskcon-standalone.c')
|
||||
path.write_text(text); os.utime(path, (946684800, 946684800))
|
||||
if sys.platform == 'darwin':
|
||||
wrapper = prefix / 'bin/cpp'
|
||||
shutil.copyfile(ROOT / 'tools/toolchain/cpp-macos.sh', wrapper)
|
||||
wrapper.chmod(0o755)
|
||||
run('./configure', f'--prefix={prefix}', '--with-writecocofile', cwd=work / 'cmoc')
|
||||
run('make', f'-j{args.jobs}', cwd=work / 'cmoc')
|
||||
run('make', 'install', cwd=work / 'cmoc')
|
||||
(prefix / 'VCC-toolchain.json').write_text(json.dumps({'mc6839': bool(args.mc6839_rom), 'standaloneDisk': bool(args.standalone_disk_source)}, indent=2) + '\n')
|
||||
for tool in ('cmoc', 'lwasm', 'lwlink', 'lwar'):
|
||||
run(prefix / 'bin' / tool, '--version', cwd=out)
|
||||
print(f'Toolchain ready in {prefix}; no app or global installation changed.')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Compile the pinned libraries and a CoCo program; optionally execute it in VCC."""
|
||||
import argparse
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--toolchain', type=Path, default=ROOT / '.build/toolchain/install')
|
||||
p.add_argument('--output', type=Path, default=ROOT / '.build/toolchain-check')
|
||||
p.add_argument('--rom', type=Path, help='Optional private 32 KiB CoCo 3 ROM; read only')
|
||||
p.add_argument('--disk-rom', type=Path, help='Private Disk BASIC ROM for the optional game check')
|
||||
p.add_argument('--game-disk', type=Path, help='Source-built CoCo Letters DSK; read only')
|
||||
args = p.parse_args()
|
||||
if (args.disk_rom or args.game_disk) and not (args.rom and args.disk_rom and args.game_disk):
|
||||
p.error('The game check needs --rom, --disk-rom and --game-disk together')
|
||||
out=args.output.resolve();out.mkdir(parents=True,exist_ok=True)
|
||||
prefix=args.toolchain.resolve()
|
||||
env=dict(os.environ, PATH=str(prefix/'bin')+os.pathsep+os.environ['PATH'])
|
||||
env.setdefault('DEVELOPER_DIR','/Applications/Xcode.app/Contents/Developer')
|
||||
def run(*command, cwd=out, **kw):
|
||||
print('+', ' '.join(map(str,command)),flush=True)
|
||||
return subprocess.run(list(map(str,command)),cwd=cwd,env=env,check=True,**kw)
|
||||
run('python3',ROOT/'tools/toolchain/verify-sources.py')
|
||||
for source in sorted((ROOT/'third_party/cmoc-libraries').iterdir()):
|
||||
if not source.is_dir(): continue
|
||||
dest=out/source.name
|
||||
if dest.exists(): raise SystemExit(f'{dest} already exists; use a fresh --output')
|
||||
shutil.copytree(source,dest)
|
||||
run('make','-j4',f'lib{source.name}.a',cwd=dest)
|
||||
assert (dest/f'lib{source.name}.a').stat().st_size > 0
|
||||
run(prefix/'bin/cmoc','--coco','--intermediate','-o',out/'HELLO.BIN',ROOT/'tests/toolchain/hello.c')
|
||||
data=(out/'HELLO.BIN').read_bytes()
|
||||
offset=0;blocks=0
|
||||
while offset<len(data) and data[offset]==0:
|
||||
size=int.from_bytes(data[offset+1:offset+3],'big');offset+=5+size;blocks+=1
|
||||
assert blocks and offset+5==len(data) and data[offset:offset+3]==b'\xff\0\0', 'Invalid DECB executable'
|
||||
# A small known assembly result verifies the checked-in assembler independently.
|
||||
(out/'known.asm').write_text(' org $4000\nstart lda #$42\n sta $0400\n rts\n end start\n')
|
||||
run(prefix/'bin/lwasm','--format=raw','--output=known.bin','--list=known.lst','known.asm')
|
||||
assert (out/'known.bin').read_bytes()==bytes.fromhex('8642b7040039')
|
||||
features=json.loads((prefix/'VCC-toolchain.json').read_text())
|
||||
if features['mc6839']:
|
||||
run(prefix/'bin/cmoc','--coco','--mc6839','-o',out/'FLOAT.BIN',ROOT/'tests/toolchain/float.c')
|
||||
if args.rom:
|
||||
build=out/'xcode'
|
||||
run('xcodebuild','-project',ROOT/'platforms/macos/VCC.xcodeproj','-scheme','VCCCoreChecks','-configuration','Release','-derivedDataPath',build,'build')
|
||||
run('xcrun','clang++','-std=c++17','-O2','-mmacosx-version-min=14.0',ROOT/'tests/toolchain/run-compiled.cpp',build/'Build/Products/Release/libVCCCore.a','-o',out/'run-compiled')
|
||||
run(out/'run-compiled',args.rom.resolve(),out/'HELLO.BIN')
|
||||
for optimization in ('-O0', '-O2'):
|
||||
binary = out / f'logical-negation{optimization}.bin'
|
||||
run(prefix/'bin/cmoc', '--coco', '--org=2800', optimization, '-Werror', '-o', binary,
|
||||
ROOT/'tests/toolchain/logical-negation.c')
|
||||
run(out/'run-compiled', args.rom.resolve(), binary, 'NOT 010 1000 END')
|
||||
run('python3', ROOT/'tests/toolchain/indexed-subtraction/check.py', prefix,
|
||||
build/'Build/Products/Release/libVCCCore.a', args.rom.resolve())
|
||||
run('python3', ROOT/'tests/toolchain/indexed-arguments/check.py', prefix,
|
||||
build/'Build/Products/Release/libVCCCore.a', args.rom.resolve())
|
||||
if features['mc6839']:
|
||||
run(out/'run-compiled',args.rom.resolve(),out/'FLOAT.BIN','MC6839 MATH OK')
|
||||
if args.game_disk:
|
||||
run('xcrun','clang++','-std=c++17','-O2','-mmacosx-version-min=14.0',ROOT/'tests/toolchain/run-coco-letters.cpp',build/'Build/Products/Release/libVCCCore.a','-o',out/'run-coco-letters')
|
||||
run(out/'run-coco-letters',args.rom.resolve(),args.disk_rom.resolve(),args.game_disk.resolve(),out/'captures')
|
||||
print('PASS: nine library archives, CMOC compile/link, DECB format, assembler bytes; optional MC6839 arithmetic')
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
# The pinned CMOC release interpolates runtime-library paths into shell commands.
|
||||
# A private, space-free alias makes a relocated 'VCC Plus.app' safe while
|
||||
# dependency-file escaping is handled by the maintained compiler patch. Source/output names are supplied relative to
|
||||
# the project's build directory by the prototype.
|
||||
set -eu
|
||||
runtime=$(CDPATH= cd -- "$(/usr/bin/dirname -- "$0")/.." && pwd -P)
|
||||
alias_dir=$(/usr/bin/mktemp -d /tmp/vcc-cmoc.XXXXXXXX)
|
||||
trap '/bin/rm -f "$alias_dir/cmoc"; /bin/rmdir "$alias_dir"' EXIT
|
||||
/bin/ln -s "$runtime/share/cmoc" "$alias_dir/cmoc"
|
||||
export CMOCPKGDATADIR="$alias_dir/cmoc"
|
||||
export CMOCFLOATLIBDIR="$alias_dir/cmoc/lib"
|
||||
export PATH="$runtime/bin:${PATH:-/usr/bin:/bin}"
|
||||
if [ "${VCC_CMOC_COMPAT:-0}" = 1 ]; then
|
||||
set -- "-I$alias_dir/cmoc/include/vcc/compat" "$@"
|
||||
fi
|
||||
"$runtime/libexec/cmoc" "$@"
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
# CMOC expects modern preprocessing. Apple's legacy cpp shim forces traditional
|
||||
# mode, which misreads valid macros in hirestxt. Use Clang directly.
|
||||
exec xcrun clang -E -Wno-builtin-macro-redefined "$@"
|
||||
@@ -0,0 +1,174 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// Adapt CMOC's GCC-style preprocessor invocation to bundled mcpp. Ordinary C
|
||||
// keeps mcpp's standard token separation. Only explicit asm { ... } bodies use
|
||||
// mcpp's documented assembler expansion, which preserves #MACRO and -MACRO,x.
|
||||
#include <ctype.h>
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <mach-o/dyld.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
typedef struct { size_t start, end; } Block;
|
||||
typedef struct { char *text; size_t size; Block *blocks; size_t count; } Output;
|
||||
|
||||
static int preprocess(const char *executable, char **args, Output *out) {
|
||||
FILE *capture = tmpfile();
|
||||
if (!capture) { perror("preprocessor capture"); return 1; }
|
||||
pid_t child = fork();
|
||||
if (child < 0) { perror("preprocessor fork"); fclose(capture); return 1; }
|
||||
if (!child) {
|
||||
if (dup2(fileno(capture), STDOUT_FILENO) < 0) _exit(126);
|
||||
execv(executable, args); perror("bundled mcpp"); _exit(127);
|
||||
}
|
||||
int status;
|
||||
while (waitpid(child, &status, 0) < 0) {
|
||||
if (errno != EINTR) { fclose(capture); return 1; }
|
||||
}
|
||||
off_t length = ftello(capture);
|
||||
// Guard malformed tools/unbounded inputs; no truncated source is compiled.
|
||||
if (length < 0 || length > 64 * 1024 * 1024 || fseeko(capture, 0, SEEK_SET)) {
|
||||
fputs("Preprocessor output is too large or unreadable\n", stderr);
|
||||
fclose(capture); return 1;
|
||||
}
|
||||
out->size = (size_t)length;
|
||||
out->text = malloc(out->size + 1);
|
||||
if (!out->text || fread(out->text, 1, out->size, capture) != out->size) {
|
||||
fclose(capture); return 1;
|
||||
}
|
||||
out->text[out->size] = 0;
|
||||
fclose(capture);
|
||||
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
|
||||
}
|
||||
|
||||
static int identifier(char c) { return isalnum((unsigned char)c) || c == '_'; }
|
||||
static size_t quote_end(const char *s, size_t i, size_t length) {
|
||||
char quote = s[i++];
|
||||
while (i < length) {
|
||||
if (s[i] == '\\' && i + 1 < length) i += 2;
|
||||
else if (s[i++] == quote) break;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
static int find_blocks(Output *out) {
|
||||
const char *s = out->text;
|
||||
size_t i = 0;
|
||||
while (i < out->size) {
|
||||
// Line markers contain filenames and must never count as source tokens.
|
||||
if ((i == 0 || s[i - 1] == '\n') && s[i] == '#') {
|
||||
while (i < out->size && s[i] != '\n') ++i;
|
||||
} else if (s[i] == '"' || s[i] == '\'') i = quote_end(s, i, out->size);
|
||||
else if (s[i] == '/' && i + 1 < out->size && s[i + 1] == '/') {
|
||||
while (i < out->size && s[i] != '\n') ++i;
|
||||
} else if (s[i] == '/' && i + 1 < out->size && s[i + 1] == '*') {
|
||||
i += 2;
|
||||
while (i + 1 < out->size && !(s[i] == '*' && s[i + 1] == '/')) ++i;
|
||||
i = i + 2 < out->size ? i + 2 : out->size;
|
||||
} else if (i + 3 <= out->size && !memcmp(s + i, "asm", 3)
|
||||
&& (i == 0 || !identifier(s[i - 1]))
|
||||
&& (i + 3 == out->size || !identifier(s[i + 3]))) {
|
||||
i += 3;
|
||||
while (i < out->size && isspace((unsigned char)s[i])) ++i;
|
||||
if (i == out->size || s[i] != '{') continue;
|
||||
size_t start = ++i;
|
||||
unsigned depth = 1;
|
||||
while (i < out->size && depth) {
|
||||
// Match CMOC's balanced-brace inline-assembly token, retaining
|
||||
// line markers and assembler comments byte for byte.
|
||||
if (s[i] == '{') ++depth;
|
||||
else if (s[i] == '}') --depth;
|
||||
++i;
|
||||
}
|
||||
if (depth) return 0;
|
||||
Block *grown = realloc(out->blocks, (out->count + 1) * sizeof(Block));
|
||||
if (!grown) return 0;
|
||||
out->blocks = grown;
|
||||
out->blocks[out->count++] = (Block){ start, i - 1 };
|
||||
} else ++i;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void emit(const char *text, size_t length) {
|
||||
size_t start = 0;
|
||||
while (start < length) {
|
||||
size_t end = start;
|
||||
while (end < length && text[end] != '\n') ++end;
|
||||
if (end < length) ++end;
|
||||
// mcpp's cur_file() prints raw path bytes inside quotes. Escape them
|
||||
// for CMOC's C-style marker parser; paths may themselves contain quotes.
|
||||
// CMOC accepts GCC markers (# NUMBER), not mcpp's #line NUMBER.
|
||||
size_t number = start;
|
||||
if (end - start >= 6 && !memcmp(text + start, "#line ", 6)) number += 6;
|
||||
else if (end - start >= 2 && text[start] == '#' && text[start + 1] == ' ') number += 2;
|
||||
size_t p = number;
|
||||
while (p < end && isdigit((unsigned char)text[p])) ++p;
|
||||
while (p < end && text[p] == ' ') ++p;
|
||||
size_t last = end;
|
||||
while (last > p && text[last - 1] != '"') --last;
|
||||
if (number != start && p < end && text[p] == '"' && last > p + 1) {
|
||||
fputs("# ", stdout);
|
||||
fwrite(text + number, 1, p + 1 - number, stdout);
|
||||
for (size_t i = p + 1; i < last - 1; ++i) {
|
||||
if (text[i] == '"' || text[i] == '\\') fputc('\\', stdout);
|
||||
fputc(text[i], stdout);
|
||||
}
|
||||
fwrite(text + last - 1, 1, end - last + 1, stdout);
|
||||
} else fwrite(text + start, 1, end - start, stdout);
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
char executable[PATH_MAX], resolved[PATH_MAX];
|
||||
uint32_t size = sizeof executable;
|
||||
if (_NSGetExecutablePath(executable, &size) || !realpath(executable, resolved)) {
|
||||
perror("Cannot locate bundled preprocessor"); return 1;
|
||||
}
|
||||
char *last = strrchr(resolved, '/');
|
||||
if (!last || (size_t)(last - resolved) + sizeof "/mcpp" > sizeof resolved) return 1;
|
||||
strcpy(last + 1, "mcpp");
|
||||
char **args = calloc((size_t)argc + 9, sizeof *args);
|
||||
if (!args) return 1;
|
||||
int count = 0;
|
||||
args[count++] = resolved;
|
||||
args[count++] = "-e"; args[count++] = "utf8";
|
||||
args[count++] = "-W0";
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (!strcmp(argv[i], "-xc++")) args[count++] = "-+";
|
||||
else if (!strcmp(argv[i], "-nostdinc")) args[count++] = "-I-";
|
||||
else if (!strcmp(argv[i], "-undef")) args[count++] = "-N";
|
||||
else if (!strcmp(argv[i], "-Wno-builtin-macro-redefined")) continue;
|
||||
else args[count++] = argv[i];
|
||||
}
|
||||
Output standard = {0}, assembly = {0};
|
||||
int result = preprocess(resolved, args, &standard);
|
||||
if (!result && !find_blocks(&standard)) {
|
||||
fputs("Cannot identify CMOC inline assembly blocks\n", stderr); result = 1;
|
||||
}
|
||||
if (!result && standard.count) {
|
||||
args[count++] = "-a";
|
||||
result = preprocess(resolved, args, &assembly);
|
||||
if (!result && (!find_blocks(&assembly) || assembly.count != standard.count)) {
|
||||
fputs("CMOC inline assembly differs between preprocessing modes\n", stderr); result = 1;
|
||||
}
|
||||
if (!result) {
|
||||
size_t offset = 0;
|
||||
for (size_t i = 0; i < standard.count; ++i) {
|
||||
Block a = standard.blocks[i], b = assembly.blocks[i];
|
||||
emit(standard.text + offset, a.start - offset);
|
||||
emit(assembly.text + b.start, b.end - b.start);
|
||||
offset = a.end;
|
||||
}
|
||||
emit(standard.text + offset, standard.size - offset);
|
||||
}
|
||||
} else if (standard.text) emit(standard.text, standard.size);
|
||||
free(standard.text); free(standard.blocks);
|
||||
free(assembly.text); free(assembly.blocks); free(args);
|
||||
if (ferror(stdout) || fflush(stdout)) return 1;
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
# Local CMOC compiler patches
|
||||
|
||||
`cmoc-debug-metadata.patch` is an original VCC Plus extension to CMOC 0.1.100,
|
||||
modified November 3, 2026 and licensed GPL-3.0-or-later. It is applied to the ignored build copy by
|
||||
`tools/toolchain/build.py`; the pinned upstream snapshot and its checksums stay
|
||||
unchanged.
|
||||
|
||||
With `-fno-omit-frame-pointer`, the compiler writes `* VCCDBG1 {…}` JSON comments
|
||||
into retained assembly/listings. Metadata comes from resolved compiler types and
|
||||
declarations, not a C-source text parser. It contains:
|
||||
|
||||
- function entry, frame-ready body and epilogue boundaries;
|
||||
- lexical block ranges, including `for`-statement scopes;
|
||||
- signedness, storage size, array dimensions, member offsets and pointer types;
|
||||
- explicit `floatingFormat` tags: `coco-basic-40` for Color BASIC/native 5-byte floats and `ieee754-binary32-be` for MC6839 single precision;
|
||||
- linker symbols for globals/static locals and U-relative offsets for parameters
|
||||
and ordinary locals.
|
||||
|
||||
The boundaries are zero-byte assembler labels. They can inhibit some assembly
|
||||
optimizations in debug builds; default/release compiler invocations emit neither
|
||||
the comments nor these labels. No instructions or data bytes are introduced by
|
||||
the metadata. The compiler runtime manifest records `vccDebugMetadata: 1`.
|
||||
|
||||
The app accepts metadata only from objects/listings/linker maps verified against
|
||||
the same frozen sources and final executable. Inspection additionally checks
|
||||
live mapped code and uses the debugger's atomic, non-I/O memory snapshot. Frame
|
||||
locals require `-O0` as well as the preserved frame pointer and are unavailable in assembly-only functions, alternative calling
|
||||
conventions, prologues/epilogues, or builds without a preserved U frame. Unsupported
|
||||
floating-point formats and recursively truncated types remain explicitly
|
||||
unavailable. Legacy metadata without `floatingFormat` is accepted but not decoded by guessing from storage size. Known format tags must agree with storage size; a genuine double-precision type is not tagged as supported. CMOC 0.1.100 currently lowers source-level `double` to `float`, so that alias carries the selected single-precision format. Expressions do not execute guest code or modify memory.
|
||||
|
||||
To regenerate the patch, edit a separate copy of the upstream source and produce
|
||||
a unified diff with `a/src/…` and `b/src/…` paths. Verify that applying it to a fresh
|
||||
copy succeeds and that ordinary optimized builds retain their original bytes.
|
||||
|
||||
## Indexed byte arithmetic and comparisons
|
||||
|
||||
`cmoc-indexed-byte-subtraction.patch` corrects three peephole rules in the build copy of CMOC 0.1.100 (modified November 7, 2026, GPL-3.0-or-later). On the 6809, B is the low byte of D. A byte load or negation can change the effective address of a later B- or D-indexed operand, including an indirect one. The subtraction and comparison rules now keep the original instruction order in those cases. The rules that remove a stack push also guard operands whose address would change with the stack pointer or an index side effect. Ordinary immediate substitutions remain available.
|
||||
|
||||
The [guest regression](../../../tests/toolchain/indexed-subtraction/qualification.c) exercises `184 - heights[i]` with runtime indices, variable bases, byte wraparound, both operand orders and equality/inequality against an array element. The [runner](../../../tests/toolchain/indexed-subtraction/check.py) checks the output against host-calculated expectations on MC6809 and HD6309 at `-O0` and `-O2`. It also runs as part of `tools/toolchain/check.py --rom`. Packaged runtimes identify the change with `vccIndexedByteSubtractionFix: 1` in their manifest.
|
||||
|
||||
## Indexed byte loads in function arguments
|
||||
|
||||
`cmoc-indexed-argument-load.patch` (November 11, 2026, GPL-3.0-or-later)
|
||||
corrects `removeUselessLdaOrLdb` in the CMOC 0.1.100 build copy. The rule used
|
||||
to remove an index reload from `LDB index; LDX #array; LDB D,X`, although B is
|
||||
the low byte of D and is needed to select the loaded byte. Consecutive indexed
|
||||
function arguments could therefore use the preceding argument's value as an
|
||||
array subscript. The rule now uses the existing instruction-effects analysis
|
||||
to preserve B dependencies in both intervening and final loads, including
|
||||
indirect B/D-indexed operands. It also checks the actual operand for
|
||||
predecrement/postincrement side effects. Independent overwritten loads remain
|
||||
eligible for removal.
|
||||
|
||||
The [target regression](../../../tests/toolchain/indexed-arguments/qualification.c)
|
||||
passes adjacent elements from two arrays at runtime indices 0 and 2, using
|
||||
byte indices, word indices and pointer bases, with distinct surrounding
|
||||
sentinels. Its [runner](../../../tests/toolchain/indexed-arguments/check.py)
|
||||
executes `--coco --no-relocate --switch=jump` binaries at `-O0` and `-O2` on
|
||||
MC6809 and HD6309; it is included in `tools/toolchain/check.py --rom`.
|
||||
The [rule check](../../../tests/toolchain/indexed-arguments/check-rule.py)
|
||||
links the actual built compiler objects and checks dependency/side-effect guards
|
||||
as well as safe removals. See the [qualification record](../../../docs/development/CMOC_INDEXED_ARGUMENT_LOAD.md).
|
||||
Packaged runtimes identify the change with `vccIndexedArgumentLoadFix: 1`.
|
||||
|
||||
## Logical negation of integer results
|
||||
|
||||
`cmoc-add-zero-flags.patch` (November 7, 2026, GPL-3.0-or-later) corrects CMOC 0.1.100's general peephole optimizer. CMOC emits `ADDD #0` to set the flags for a word-sized comparison with zero. A function can return 1 in D with Z still set from clearing A. Removing the add before the `tfrZtoB` helper made `!getReady()` return 1 in that case. The patch removes the general rule's unproven add-zero deletions and byte-test substitutions. The separate optimizations for recognized load/compare/branch sequences still run.
|
||||
|
||||
The [guest regression](../../../tests/toolchain/logical-negation.c) prints actual results for zero, widened unsigned bytes and 16-bit values. Run `tools/toolchain/check.py --rom` with a local CoCo 3 ROM to check those results at `-O0` and `-O2` on MC6809 and HD6309. Rebuilt app toolchains record `vccAddZeroFlagsFix: 1` in the manifest.
|
||||
|
||||
## Explicit before-directory disk allocation
|
||||
|
||||
`writecocofile-before-dir.patch` (November 4, 2026, GPL-3.0-or-later) adds
|
||||
`--allocate-before-dir` as an explicit choice of the existing forward allocator.
|
||||
It starts at granule zero, skips occupied granules, and continues above the
|
||||
directory track if necessary. It does not imply a strict below-directory limit.
|
||||
The last allocation option wins, as with the existing choices.
|
||||
|
||||
`build.py` applies it only to the build copy. `build-runtime.py` records
|
||||
`vccWriteCoCoFileBeforeDir: 1`. The original CoCo Letters Makefile uses this option;
|
||||
its former recipe-removal patch is retired and the vendored game matches upstream.
|
||||
Runtime verification checks allocation order, byte-identical default behavior,
|
||||
file readback, and spillover beyond the lower half of the disk.
|
||||
|
||||
## LWASM project input boundary
|
||||
|
||||
The native Assembly workspace passes `--project-inputs` when invoking LWASM.
|
||||
`lwtools-project-inputs.patch` (November 5, 2026, GPL-3.0-or-later) constrains
|
||||
source and binary input reads to regular files inside the physical working
|
||||
directory. It checks each open, including binary data reopened during emission.
|
||||
The companion `--project-map=FILE` records physical source paths, physical
|
||||
one-based lines, sections, offsets and original instruction bytes. Macro
|
||||
expansions, data directives, synthesized 6800/convenience operations and
|
||||
OS-9 calls cannot become source breakpoints. The Swift client verifies final
|
||||
DECB bytes after relocation. A project map requested with `--6809` also
|
||||
rejects source pragmas that change that target.
|
||||
The app supplies an isolated, symlink-free copy of the source snapshot. Other
|
||||
invocations retain ordinary LWASM path behavior. Custom Make recipes execute
|
||||
as user-supplied build commands and have a separate trust boundary.
|
||||
|
||||
Rebuild and stage the native runtime with `build-runtime.py` before packaging
|
||||
the workspace. The runtime manifest records `vccLWASMProjectInputs: 1`;
|
||||
older assemblers reject the required option.
|
||||
@@ -0,0 +1,57 @@
|
||||
--- a/src/ASMText2.cpp
|
||||
+++ b/src/ASMText2.cpp
|
||||
@@ -1517,8 +1517,9 @@
|
||||
}
|
||||
|
||||
|
||||
-// Remove AND, OR, MUL or ADD operations if we can show that
|
||||
-// they will not change performance of the program.
|
||||
+// ADD #0 can be an explicit condition-code test. Leave it to rules that
|
||||
+// prove which flags the surrounding instructions produce and consume.
|
||||
+// In particular, a called runtime helper may read the flags on entry.
|
||||
bool
|
||||
ASMText::removeAndOrMulAddSub(size_t index)
|
||||
{
|
||||
@@ -1588,42 +1589,6 @@
|
||||
commentOut(index, "optim: removeAndOrMulAddSub case 5");
|
||||
madeChanges = true;
|
||||
}
|
||||
- } else if ((instr == "ADDA") || (instr == "ADDB") || (instr == "ADDD")) {
|
||||
- if ((instr[3] == 'A' && simulator.regs.accum.a.known && simulator.regs.accum.a.val == 0) &&
|
||||
- (instr[3] == 'B' && simulator.regs.accum.b.known && simulator.regs.accum.b.val == 0) &&
|
||||
- (instr[3] == 'D' && simulator.regs.accum.dknown() && simulator.regs.accum.dval() == 0)) {
|
||||
- // We know that A, B or D are zero, so change to a LD
|
||||
- const Element &e1 = elements[index + 1];
|
||||
- if (e1.type != INSTR || !isBasicBlockEndingInstruction(e1)) {
|
||||
- char newInstr[8];
|
||||
- snprintf(newInstr, sizeof(newInstr), "LD%s", instr.c_str() + 3);
|
||||
- replaceWithInstr(index, newInstr, oper, "optim: removeAndOrMulAddSub case 6");
|
||||
- madeChanges = true;
|
||||
- }
|
||||
- } else if (extractConstantLiteral(e.fields[1], val)) {
|
||||
- // We know that the operand is a constant 0 and the side effects are not valued,
|
||||
- // so we can comment it out
|
||||
- const Element &e1 = elements[index + 1];
|
||||
- if (val == 0 && (e1.type == INSTR && !isBasicBlockEndingInstruction(e1))) {
|
||||
- InsEffects effects(e1);
|
||||
- if (!(effects.read & CC)) {
|
||||
- commentOut(index, "optim: removeAndOrMulAddSub case 7");
|
||||
- madeChanges = true;
|
||||
- }
|
||||
- } else if (val == 0 && e1.type == INSTR && isBasicBlockEndingInstruction(e1) &&
|
||||
- (((instr[3] == 'D') && (simulator.regs.accum.a.known && simulator.regs.accum.a.val == 0)) ||
|
||||
- (instr[3] == 'B'))) {
|
||||
- // ADDD #0 and A is definitely zero or ADDB #0 so replace with TSTB
|
||||
- replaceWithInstr(index, "TSTB", "", "optim: removeAndOrMulAddSub case 8");
|
||||
- madeChanges = true;
|
||||
- } else if (val == 0 && e1.type == INSTR && isBasicBlockEndingInstruction(e1) &&
|
||||
- (((instr[3] == 'D') && (simulator.regs.accum.b.known && simulator.regs.accum.b.val == 0)) ||
|
||||
- (instr[3] == 'A'))) {
|
||||
- // ADDD #0 and B is definitely zero or ADDA #0 so replace with TSTA
|
||||
- replaceWithInstr(index, "TSTA", "", "optim: removeAndOrMulAddSub case 9");
|
||||
- madeChanges = true;
|
||||
- }
|
||||
- }
|
||||
} else if ((instr == "LEAX") && (oper == "D,X")) {
|
||||
if (simulator.regs.accum.dknown() && simulator.regs.accum.dval() == 0) {
|
||||
commentOut(index, "optim: removeAndOrMulAddSub case 10");
|
||||
@@ -0,0 +1,230 @@
|
||||
--- a/src/Declaration.cpp
|
||||
+++ b/src/Declaration.cpp
|
||||
@@ -15,6 +15,7 @@
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
+#include "VCCDebug.h"
|
||||
#include "Declaration.h"
|
||||
|
||||
#include "TranslationUnit.h"
|
||||
@@ -334,6 +335,8 @@
|
||||
bool canBeStaticInit = (initializationExpr && initializationExpr->isStaticallyInitializable(*getTypeDesc()));
|
||||
if (canBeStaticInit != withStaticInitializer)
|
||||
return true; // nothing to do
|
||||
+
|
||||
+ if (isGlobal()) VCCDebug::variable(out, *this, "", "");
|
||||
|
||||
if (withStaticInitializer)
|
||||
{
|
||||
--- a/src/ForStmt.cpp
|
||||
+++ b/src/ForStmt.cpp
|
||||
@@ -15,6 +15,7 @@
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
+#include "VCCDebug.h"
|
||||
#include "ForStmt.h"
|
||||
|
||||
#include "TranslationUnit.h"
|
||||
@@ -115,6 +116,8 @@
|
||||
|
||||
//cerr << "pushing ForStmt scope:\n";
|
||||
pushScopeIfExists();
|
||||
+ const string debugBegin = getScope() && VCCDebug::enabled() ? VCCDebug::label() : "", debugEnd = getScope() && VCCDebug::enabled() ? VCCDebug::label() : "";
|
||||
+ VCCDebug::scope(out, getScope(), debugBegin, debugEnd);
|
||||
TranslationUnit &tu = TranslationUnit::instance();
|
||||
|
||||
string bodyLabel = TranslationUnit::instance().generateLabel('L');
|
||||
@@ -128,6 +131,7 @@
|
||||
|
||||
tu.popBreakableLabels();
|
||||
//cerr << "popping ForStmt scope:\n";
|
||||
+ if (getScope() && VCCDebug::enabled()) out.emitLabel(debugEnd);
|
||||
popScopeIfExists();
|
||||
return cs;
|
||||
}
|
||||
--- a/src/FunctionDef.cpp
|
||||
+++ b/src/FunctionDef.cpp
|
||||
@@ -15,6 +15,7 @@
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
+#include "VCCDebug.h"
|
||||
#include "FunctionDef.h"
|
||||
|
||||
#include "Scope.h"
|
||||
@@ -853,11 +854,23 @@
|
||||
functor.emitComments(out);
|
||||
}
|
||||
|
||||
+ // VCC Plus: exact compiler-resolved body/scope boundaries, before epilogue.
|
||||
+ const string debugBody = VCCDebug::enabled() ? VCCDebug::label() : "", debugEnd = VCCDebug::enabled() ? VCCDebug::label() : "";
|
||||
+ if (VCCDebug::enabled()) out.emitComment("VCCDBG1 {\"kind\":\"function\",\"name\":" + VCCDebug::quote(functionId)
|
||||
+ + ",\"location\":" + VCCDebug::quote(getLineNo())
|
||||
+ + ",\"symbol\":" + VCCDebug::quote(functionLabel)
|
||||
+ + ",\"begin\":" + VCCDebug::quote(debugBody) + ",\"end\":" + VCCDebug::quote(debugEnd)
|
||||
+ + ",\"frame\":" + (needStackFrame && !isAssemblyOnly() && callConvention == DEFAULT_CMOC_CALL_CONV
|
||||
+ && TranslationUnit::instance().getFramePointerOption() == FramePointerOption::KEEP_ALL
|
||||
+ && TranslationUnit::instance().hasUnoptimizedDebugStorage() ? "true" : "false") + "}");
|
||||
+
|
||||
// Generate code for the body:
|
||||
|
||||
TranslationUnit::instance().setCurrentFunctionEndLabel(endLabel);
|
||||
TranslationUnit::instance().pushScope(const_cast<Scope *>(getScope())); // const_cast should be removed...
|
||||
+ VCCDebug::scope(out, getScope(), debugBody, debugEnd);
|
||||
bool success = bodyStmts->emitCode(out, false);
|
||||
+ if (VCCDebug::enabled()) out.emitLabel(debugEnd);
|
||||
out.emitLabel(endLabel, "end of " + functionId + "()");
|
||||
|
||||
if (!asmOnly && callConvention == GCC6809_CALL_CONV && getTypeSize() == 2)
|
||||
--- a/src/TranslationUnit.cpp
|
||||
+++ b/src/TranslationUnit.cpp
|
||||
@@ -2573,3 +2573,6 @@
|
||||
irFile << "}\n"; // main JSON object
|
||||
return true;
|
||||
}
|
||||
+
|
||||
+// VCC Plus: optimized stores/register reuse cannot claim stable C locals.
|
||||
+bool TranslationUnit::hasUnoptimizedDebugStorage() const { return params.optimizationLevel == 0; }
|
||||
--- a/src/TranslationUnit.h
|
||||
+++ b/src/TranslationUnit.h
|
||||
@@ -194,6 +194,7 @@
|
||||
bool isRelocatabilitySupported() const;
|
||||
|
||||
FramePointerOption getFramePointerOption() const;
|
||||
+ bool hasUnoptimizedDebugStorage() const;
|
||||
|
||||
// Called during parsing, right after the parser recognized the #pragma directive.
|
||||
// May change some fields of this TranslationUnit object.
|
||||
--- a/src/TreeSequence.cpp
|
||||
+++ b/src/TreeSequence.cpp
|
||||
@@ -15,6 +15,7 @@
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
+#include "VCCDebug.h"
|
||||
#include "TreeSequence.h"
|
||||
|
||||
#include "TranslationUnit.h"
|
||||
@@ -141,6 +142,8 @@
|
||||
const bool isCommaExpr = !!dynamic_cast<const CommaExpr *>(this);
|
||||
|
||||
pushScopeIfExists();
|
||||
+ const string debugBegin = getScope() && VCCDebug::enabled() ? VCCDebug::label() : "", debugEnd = getScope() && VCCDebug::enabled() ? VCCDebug::label() : "";
|
||||
+ VCCDebug::scope(out, getScope(), debugBegin, debugEnd);
|
||||
|
||||
bool success = true;
|
||||
for (vector<Tree *>::const_iterator it = sequence.begin(); it != sequence.end(); it++)
|
||||
@@ -178,6 +181,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
+ if (getScope() && VCCDebug::enabled()) out.emitLabel(debugEnd);
|
||||
popScopeIfExists();
|
||||
|
||||
return success;
|
||||
--- /dev/null
|
||||
+++ b/src/VCCDebug.h
|
||||
@@ -0,0 +1,101 @@
|
||||
+// VCC Plus compiler debug extension. SPDX-License-Identifier: GPL-3.0-or-later
|
||||
+// Metadata comments and labels occupy no guest memory. Never infer C storage
|
||||
+// or type layout from source text: use the compiler's resolved declarations.
|
||||
+#ifndef VCC_DEBUG_H
|
||||
+#define VCC_DEBUG_H
|
||||
+#include "ASMText.h"
|
||||
+#include "Declaration.h"
|
||||
+#include "Scope.h"
|
||||
+#include "TranslationUnit.h"
|
||||
+#include "ClassDef.h"
|
||||
+#include <sstream>
|
||||
+#include <iomanip>
|
||||
+namespace VCCDebug {
|
||||
+inline bool enabled() { return TranslationUnit::instance().getFramePointerOption() == FramePointerOption::KEEP_ALL; }
|
||||
+inline std::string arrayType(const TypeDesc *, const std::vector<uint16_t> &, unsigned);
|
||||
+inline std::string quote(const std::string &s) {
|
||||
+ std::ostringstream o; o << '"';
|
||||
+ for (unsigned char c : s) {
|
||||
+ if (c == '"' || c == '\\') o << '\\' << c;
|
||||
+ else if (c < 32) o << "\\u00" << std::hex << std::setw(2) << std::setfill('0') << unsigned(c) << std::dec;
|
||||
+ else o << c;
|
||||
+ }
|
||||
+ o << '"'; return o.str();
|
||||
+}
|
||||
+inline std::string type(const TypeDesc *td, unsigned depth = 0) {
|
||||
+ std::ostringstream o;
|
||||
+ const int size = td->type == FUNCTION_TYPE || td->type == VOID_TYPE || (td->type == ARRAY_TYPE && td->numArrayElements == TypeDesc::NO_DIMENSION) ? 0 : TranslationUnit::instance().getTypeSize(*td);
|
||||
+ o << "{\"name\":" << quote(td->toString()) << ",\"size\":" << (size < 0 ? 0 : size);
|
||||
+ if (depth > 5) o << ",\"kind\":\"unsupported\"";
|
||||
+ else if (td->type == BYTE_TYPE || td->type == WORD_TYPE || td->isLong())
|
||||
+ o << ",\"kind\":\"integer\",\"signed\":" << (td->isSigned ? "true" : "false");
|
||||
+ else if (td->isReal()) {
|
||||
+ auto &tu = TranslationUnit::instance();
|
||||
+ const char *format = "unsupported";
|
||||
+ if (!td->isDouble() && tu.isFloatingPointSupported()) {
|
||||
+ if (tu.getFloatingPointLibrary() == FloatingPointLibrary::MC6839_LIB && size == 4)
|
||||
+ format = "ieee754-binary32-be";
|
||||
+ else if (size == 5 && (tu.getFloatingPointLibrary() == FloatingPointLibrary::NATIVE_LIB
|
||||
+ || tu.getTargetPlatform() == COCO_BASIC || tu.getTargetPlatform() == DRAGON))
|
||||
+ format = "coco-basic-40";
|
||||
+ }
|
||||
+ o << ",\"kind\":\"floating\",\"floatingFormat\":" << quote(format);
|
||||
+ }
|
||||
+ else if (td->type == POINTER_TYPE)
|
||||
+ o << ",\"kind\":\"pointer\",\"element\":" << type(td->pointedTypeDesc, depth + 1);
|
||||
+ else if (td->type == ARRAY_TYPE && td->numArrayElements != TypeDesc::NO_DIMENSION)
|
||||
+ o << ",\"kind\":\"array\",\"count\":" << td->numArrayElements << ",\"element\":" << type(td->pointedTypeDesc, depth + 1);
|
||||
+ else if (td->type == CLASS_TYPE) {
|
||||
+ const ClassDef *cl = TranslationUnit::instance().getClassDef(td->className);
|
||||
+ o << ",\"kind\":\"aggregate\",\"members\":[";
|
||||
+ if (cl) for (size_t i=0;i<cl->getNumDataMembers();++i) {
|
||||
+ auto m = cl->getDataMember(i);
|
||||
+ if (i) o << ',';
|
||||
+ auto dims = m->getArrayDimensions(); m->getTypeDesc()->appendDimensions(dims);
|
||||
+ o << "{\"name\":" << quote(m->getName()) << ",\"offset\":" << cl->getDataMemberOffset(i, NULL)
|
||||
+ << ",\"type\":" << (dims.empty() ? type(m->getTypeDesc(), depth+1) : arrayType(m->getTypeDesc(), dims, depth+1)) << '}';
|
||||
+ }
|
||||
+ o << ']';
|
||||
+ } else o << ",\"kind\":\"unsupported\"";
|
||||
+ o << '}'; return o.str();
|
||||
+}
|
||||
+inline std::string arrayType(const TypeDesc *original, const std::vector<uint16_t> &dims, unsigned depth) {
|
||||
+ const TypeDesc *td=original;
|
||||
+ while (td->type == ARRAY_TYPE) td=td->pointedTypeDesc;
|
||||
+ std::string result=type(td, depth+1);
|
||||
+ size_t size=TranslationUnit::instance().getTypeSize(*td);
|
||||
+ for (auto it=dims.rbegin();it!=dims.rend();++it) {
|
||||
+ size *= *it;
|
||||
+ std::ostringstream o; o << "{\"name\":" << quote(original->toString())
|
||||
+ << ",\"kind\":\"array\",\"size\":" << size << ",\"count\":" << *it << ",\"element\":" << result << '}';
|
||||
+ result=o.str();
|
||||
+ }
|
||||
+ return result;
|
||||
+}
|
||||
+inline std::string declarationType(const Declaration &decl) {
|
||||
+ return decl.getTypeDesc()->type == ARRAY_TYPE ? arrayType(decl.getTypeDesc(), decl.arrayDimensions, 0) : type(decl.getTypeDesc());
|
||||
+}
|
||||
+inline std::string label() { return ".vcc.debug." + TranslationUnit::instance().generateLabel('L'); }
|
||||
+inline void variable(ASMText &out, const Declaration &d, const std::string &begin, const std::string &end) {
|
||||
+ if (!enabled() || d.isExtern || d.getVariableId().empty() || d.getVariableId()[0] == '$') return;
|
||||
+ uint16_t size=0; if (!d.getVariableSizeInBytes(size) || !size) return;
|
||||
+ std::ostringstream o; o << "VCCDBG1 {\"kind\":\"variable\",\"name\":" << quote(d.getVariableId())
|
||||
+ << ",\"location\":" << quote(d.getLineNo()) << ",\"type\":" << declarationType(d)
|
||||
+ << ",\"begin\":" << quote(begin) << ",\"end\":" << quote(end)
|
||||
+ << ",\"static\":" << (d.isStatic() ? "true" : "false");
|
||||
+ if (d.isGlobal() || d.isLocalStatic()) o << ",\"symbol\":" << quote(d.getLabel());
|
||||
+ else o << ",\"offset\":" << d.getFrameDisplacement();
|
||||
+ o << '}'; out.emitComment(o.str());
|
||||
+}
|
||||
+inline void scope(ASMText &out, const Scope *s, const std::string &begin, const std::string &end) {
|
||||
+ if (!enabled() || !s) return;
|
||||
+ class Visitor : public Scope::DeclarationFunctor {
|
||||
+ ASMText &out; const std::string &begin, &end;
|
||||
+ public:
|
||||
+ Visitor(ASMText &o, const std::string &b, const std::string &e):out(o),begin(b),end(e){}
|
||||
+ bool operator()(Declaration &d) override { variable(out,d,begin,end); return true; }
|
||||
+ } visit(out,begin,end);
|
||||
+ out.emitLabel(begin); s->iterateDeclarations(visit,false);
|
||||
+}
|
||||
+}
|
||||
+#endif
|
||||
@@ -0,0 +1,45 @@
|
||||
--- a/src/ASMText2.cpp
|
||||
+++ b/src/ASMText2.cpp
|
||||
@@ -4952,29 +4952,31 @@
|
||||
|
||||
|
||||
// Look for this:
|
||||
-// LDB {foo} ; {bar} involves not pre/post-increment, e.g., LDB ,S+
|
||||
-// LDX/LEAX {bar} ; {bar} does not involve B, e.g., LDX B,U
|
||||
-// LDB {baz}
|
||||
-// Remove the first LDB.
|
||||
+// LDB {foo} ; no pre/post-increment side effects
|
||||
+// LDX/LEAX {bar} ; must not read B, including D-indexed operands
|
||||
+// LDB {baz} ; must not read the previous B in its effective address
|
||||
+// Remove the first LDB only when its value and side effects are unused.
|
||||
//
|
||||
bool
|
||||
ASMText::removeUselessLdaOrLdb(size_t index)
|
||||
{
|
||||
if (! isInstrAnyArg(index, "LDB"))
|
||||
return false;
|
||||
- const string &arg = elements[index].fields[0];
|
||||
- if (startsWith(arg, ",-") || endsWith(arg, "+")) // if pre/post increment
|
||||
- return false;
|
||||
+ if (isInstrWithPreDecrOrPostIncr(index))
|
||||
+ return false; // Removing the load must not remove an index/stack update.
|
||||
size_t otherLoad = findNextInstrBeforeLabel(index + 1);
|
||||
if (! isInstrAnyArg(otherLoad, "LDX") && ! isInstrAnyArg(otherLoad, "LEAX"))
|
||||
return false;
|
||||
- const string &otherLoadArg = elements[otherLoad].fields[1];
|
||||
- if (startsWith(otherLoadArg, "B,"))
|
||||
- return false; // we have "B,reg" as the arg to LDX/LEAX, so previous LDB is useful
|
||||
+ // B is also the low byte of D. Use the common instruction-effects
|
||||
+ // analysis for both direct and indirect accumulator-indexed operands.
|
||||
+ if (InsEffects(elements[otherLoad]).read & B)
|
||||
+ return false;
|
||||
size_t secondLdAcc = findNextInstrBeforeLabel(otherLoad + 1);
|
||||
if (! isInstrAnyArg(secondLdAcc, "LDB"))
|
||||
return false;
|
||||
-
|
||||
+ if (InsEffects(elements[secondLdAcc]).read & B)
|
||||
+ return false; // The overwritten value may still select the byte to load.
|
||||
+
|
||||
// Apply the optimization.
|
||||
commentOut(index, "optim: removeUselessLdaOrLdb (removed LDB b/c later LDB)");
|
||||
return true;
|
||||
@@ -0,0 +1,68 @@
|
||||
--- a/src/ASMText2.cpp
|
||||
+++ b/src/ASMText2.cpp
|
||||
@@ -4159,16 +4159,27 @@
|
||||
}
|
||||
|
||||
|
||||
+// B is the low half of D. Both direct and indirect accumulator-indexed
|
||||
+// addresses can therefore depend on the value of B before a rewrite.
|
||||
+static bool
|
||||
+usesBInAddress(const string &operand)
|
||||
+{
|
||||
+ return startsWith(operand, "B,") || startsWith(operand, "D,")
|
||||
+ || startsWith(operand, "[B,") || startsWith(operand, "[D,");
|
||||
+}
|
||||
+
|
||||
+
|
||||
// Look for: LDB foo; PSHS B; LDB bar; CMPB ,S+.
|
||||
// Change to: LDB bar; CMPB foo.
|
||||
-// LDB args must not involve S.
|
||||
+// Swapping the two reads must preserve their offsets and index registers.
|
||||
//
|
||||
bool
|
||||
ASMText::removePushBFromLdbPushLdbCmp(size_t index)
|
||||
{
|
||||
if (!isInstrAnyArg(index, "LDB"))
|
||||
return false;
|
||||
- if (elements[index].fields[1].find(",S") != string::npos)
|
||||
+ if (usesBInAddress(elements[index].fields[1]) || isInstrWithPreDecrOrPostIncr(index)
|
||||
+ || elements[index].fields[1].find(",S") != string::npos)
|
||||
return false;
|
||||
|
||||
size_t pushIndex = findNextInstrBeforeLabel(index + 1);
|
||||
@@ -4178,7 +4189,8 @@
|
||||
size_t secondLoadIndex = findNextInstrBeforeLabel(pushIndex + 1);
|
||||
if (!isInstrAnyArg(secondLoadIndex, "LDB"))
|
||||
return false;
|
||||
- if (elements[secondLoadIndex].fields[1].find(",S") != string::npos)
|
||||
+ if (usesBInAddress(elements[secondLoadIndex].fields[1]) || isInstrWithPreDecrOrPostIncr(secondLoadIndex)
|
||||
+ || elements[secondLoadIndex].fields[1].find(",S") != string::npos)
|
||||
return false;
|
||||
|
||||
size_t cmpIndex = findNextInstrBeforeLabel(secondLoadIndex + 1);
|
||||
@@ -4440,6 +4452,13 @@
|
||||
if (!isInstr(subIndex, "SUBB", ",S+"))
|
||||
return false;
|
||||
|
||||
+ // NEGB changes B before the load's address is used, and removing the
|
||||
+ // push changes S. Keep the original sequence when either matters.
|
||||
+ const string &operand = elements[ldbIndex].fields[1];
|
||||
+ if (usesBInAddress(operand) || operand.find(",S") != string::npos
|
||||
+ || isInstrWithPreDecrOrPostIncr(ldbIndex))
|
||||
+ return false;
|
||||
+
|
||||
// Apply the optimization.
|
||||
commentOut(index, "optim: removePushBBeforeSubB"); // remove PSHS
|
||||
elements[subIndex].fields[0] = "ADDB";
|
||||
@@ -4833,6 +4852,11 @@
|
||||
if (isInstrWithPreDecrOrPostIncr(addb))
|
||||
return false;
|
||||
|
||||
+ // Swapping the operands also changes which value of B is available
|
||||
+ // when each effective address is evaluated.
|
||||
+ if (usesBInAddress(elements[index].fields[1]) || usesBInAddress(elements[addb].fields[1]))
|
||||
+ return false;
|
||||
+
|
||||
// Apply the optimization.
|
||||
const string ldbArg = elements[index].fields[1];
|
||||
elements[index].fields[1] = elements[addb].fields[1];
|
||||
@@ -0,0 +1,362 @@
|
||||
--- a/lwasm/input.c
|
||||
+++ b/lwasm/input.c
|
||||
@@ -26,6 +26,9 @@
|
||||
*/
|
||||
|
||||
#include <errno.h>
|
||||
+#include <limits.h>
|
||||
+#include <stdlib.h>
|
||||
+#include <sys/stat.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
@@ -65,8 +68,40 @@
|
||||
void *data;
|
||||
int data2;
|
||||
char *filespec;
|
||||
+ const char *source_path;
|
||||
+ int source_line;
|
||||
struct input_stack_node *stack;
|
||||
};
|
||||
+
|
||||
+/* VCC Plus native assembly builds run in a private, immutable snapshot.
|
||||
+ * Resolve the target before opening it, including the second binary-include
|
||||
+ * read during emission. Ordinary LWASM invocations retain upstream behavior. */
|
||||
+static char *project_root;
|
||||
+
|
||||
+void input_restrict_to_project(void)
|
||||
+{
|
||||
+ project_root = realpath(".", NULL);
|
||||
+ if (!project_root)
|
||||
+ lw_error("Cannot resolve assembly project folder.\n");
|
||||
+}
|
||||
+
|
||||
+FILE *input_fopen(const char *path)
|
||||
+{
|
||||
+ if (project_root)
|
||||
+ {
|
||||
+ char resolved[PATH_MAX];
|
||||
+ struct stat info;
|
||||
+ size_t length = strlen(project_root);
|
||||
+ if (!realpath(path, resolved))
|
||||
+ return NULL;
|
||||
+ if (strncmp(project_root, resolved, length) || resolved[length] != '/')
|
||||
+ lw_error("Assembly input leaves the project snapshot: %s\n", path);
|
||||
+ if (stat(resolved, &info) != 0 || !S_ISREG(info.st_mode))
|
||||
+ lw_error("Assembly input is not a regular file: %s\n", path);
|
||||
+ return fopen(resolved, "rb");
|
||||
+ }
|
||||
+ return fopen(path, "rb");
|
||||
+}
|
||||
|
||||
static char *make_filename(char *p, char *f)
|
||||
{
|
||||
@@ -110,7 +145,7 @@
|
||||
}
|
||||
|
||||
/* this adds real filenames that were opened to a list */
|
||||
-void input_add_to_resource_list(asmstate_t *as, const char *s)
|
||||
+static const char *input_add_to_resource_list(asmstate_t *as, const char *s)
|
||||
{
|
||||
struct ifl *ifl;
|
||||
|
||||
@@ -127,6 +162,7 @@
|
||||
ifl_head = ifl;
|
||||
ifl -> fn = lw_strdup(s);
|
||||
}
|
||||
+ return ifl -> fn;
|
||||
}
|
||||
|
||||
void input_init(asmstate_t *as)
|
||||
@@ -155,6 +191,12 @@
|
||||
char *dn, *dp;
|
||||
// int o;
|
||||
|
||||
+ if (as->project_map_file && project_root)
|
||||
+ {
|
||||
+ char resolved[PATH_MAX];
|
||||
+ if (realpath(fn, resolved))
|
||||
+ IS->source_path = input_add_to_resource_list(as, resolved + strlen(project_root) + 1);
|
||||
+ }
|
||||
dn = lw_strdup(fn);
|
||||
lw_stack_push(as -> includelist, dn);
|
||||
|
||||
@@ -186,6 +228,8 @@
|
||||
struct input_stack *t;
|
||||
|
||||
t = lw_alloc(sizeof(struct input_stack));
|
||||
+ t -> source_path = NULL;
|
||||
+ t -> source_line = 0;
|
||||
t -> filespec = lw_strdup(s);
|
||||
|
||||
t -> type = input_type_string;
|
||||
@@ -204,6 +248,8 @@
|
||||
char *p, *p2;
|
||||
|
||||
t = lw_alloc(sizeof(struct input_stack));
|
||||
+ t -> source_path = NULL;
|
||||
+ t -> source_line = 0;
|
||||
t -> filespec = lw_strdup(s);
|
||||
|
||||
for (s2 = s; *s2 && (*s2 != ':'); s2++)
|
||||
@@ -238,7 +284,7 @@
|
||||
if (input_isabsolute(s))
|
||||
{
|
||||
/* absolute path */
|
||||
- IS -> data = fopen(s, "rb");
|
||||
+ IS -> data = input_fopen(s);
|
||||
debug_message(as, 1, "Opening (abs) %s", s);
|
||||
if (!IS -> data && !IGNOREERROR)
|
||||
{
|
||||
@@ -257,7 +303,7 @@
|
||||
p = lw_stack_top(as -> file_dir);
|
||||
p2 = make_filename(p, s);
|
||||
debug_message(as, 1, "Open: (cd) %s\n", p2);
|
||||
- IS -> data = fopen(p2, "rb");
|
||||
+ IS -> data = input_fopen(p2);
|
||||
if (IS -> data)
|
||||
{
|
||||
input_pushpath(as, p2);
|
||||
@@ -274,7 +320,7 @@
|
||||
{
|
||||
p2 = make_filename(p, s);
|
||||
debug_message(as, 1, "Open (sp): %s\n", p2);
|
||||
- IS -> data = fopen(p2, "rb");
|
||||
+ IS -> data = input_fopen(p2);
|
||||
if (IS -> data)
|
||||
{
|
||||
input_pushpath(as, p2);
|
||||
@@ -300,7 +346,7 @@
|
||||
if (s[0] == '-' && s[1] == '\0')
|
||||
IS -> data = stdin;
|
||||
else
|
||||
- IS -> data = fopen(s, "rb");
|
||||
+ IS -> data = input_fopen(s);
|
||||
|
||||
if (!IS -> data)
|
||||
{
|
||||
@@ -328,7 +374,7 @@
|
||||
debug_message(as, 2, "Open file (st abs) %s", s);
|
||||
if (as -> flags & FLAG_DEPEND)
|
||||
printf("%s\n", s);
|
||||
- fp = fopen(s, "rb");
|
||||
+ fp = input_fopen(s);
|
||||
if (!fp)
|
||||
{
|
||||
return NULL;
|
||||
@@ -343,7 +389,7 @@
|
||||
p = lw_stack_top(as -> file_dir);
|
||||
p2 = make_filename(p ? p : "", s);
|
||||
debug_message(as, 2, "Open file (st cd) %s", p2);
|
||||
- fp = fopen(p2, "rb");
|
||||
+ fp = input_fopen(p2);
|
||||
if (fp)
|
||||
{
|
||||
if (as -> flags & FLAG_DEPEND)
|
||||
@@ -362,7 +408,7 @@
|
||||
{
|
||||
p2 = make_filename(p, s);
|
||||
debug_message(as, 2, "Open file (st ip) %s", p2);
|
||||
- fp = fopen(p2, "rb");
|
||||
+ fp = input_fopen(p2);
|
||||
if (fp)
|
||||
{
|
||||
if (as -> flags & FLAG_DEPEND)
|
||||
@@ -471,6 +517,7 @@
|
||||
if (eol)
|
||||
{
|
||||
s = lw_strdup(linebuff);
|
||||
+ IS->source_line++;
|
||||
return s;
|
||||
}
|
||||
}
|
||||
@@ -537,6 +584,15 @@
|
||||
default:
|
||||
lw_error("Problem reading from unknown input type\n");
|
||||
return NULL;
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+void input_project_location(asmstate_t *as, line_t *line)
|
||||
+{
|
||||
+ if (IS && IS->type != input_type_string)
|
||||
+ {
|
||||
+ line->project_source = IS->source_path;
|
||||
+ line->project_line = IS->source_line;
|
||||
}
|
||||
}
|
||||
|
||||
--- a/lwasm/input.h
|
||||
+++ b/lwasm/input.h
|
||||
@@ -32,6 +32,9 @@
|
||||
void input_stack_push(asmstate_t *as, input_stack_entry *se);
|
||||
input_stack_entry *input_stack_pop(asmstate_t *as, int magic, int (*fn)(input_stack_entry *e, void *data), void *data);
|
||||
|
||||
+void input_restrict_to_project(void);
|
||||
+FILE *input_fopen(const char *path);
|
||||
+void input_project_location(asmstate_t *as, line_t *line);
|
||||
void input_init(asmstate_t *as);
|
||||
void input_openstring(asmstate_t *as, char *s, char *str);
|
||||
void input_open(asmstate_t *as, char *s);
|
||||
--- a/lwasm/list.c
|
||||
+++ b/lwasm/list.c
|
||||
@@ -24,6 +24,7 @@
|
||||
#include <string.h>
|
||||
|
||||
#include <lw_alloc.h>
|
||||
+#include <lw_error.h>
|
||||
#include <lw_string.h>
|
||||
|
||||
#include "lwasm.h"
|
||||
@@ -489,3 +490,43 @@
|
||||
if (of && of != stdout)
|
||||
fclose(of);
|
||||
}
|
||||
+
|
||||
+/* VCC Plus project source map. Keep the normal human-readable listing intact.
|
||||
+ * Physical file lines are captured at input; macro strings have no file identity.
|
||||
+ * The client still validates original object bytes, link relocations and the BIN. */
|
||||
+void do_project_map(asmstate_t *as)
|
||||
+{
|
||||
+ FILE *out = fopen(as->project_map_file, "w");
|
||||
+ line_t *line;
|
||||
+ const int synthetic = lwasm_insn_is6800 | lwasm_insn_is6809conv |
|
||||
+ lwasm_insn_is6309conv | lwasm_insn_isemuext;
|
||||
+ if (!out) lw_error("Cannot write project instruction map.\n");
|
||||
+ for (line = as->line_head; line; line = line->next)
|
||||
+ {
|
||||
+ const char *section = line->csect ? line->csect->name : "";
|
||||
+ lw_expr_t address;
|
||||
+ int i;
|
||||
+ if (!line->project_source || !line->emitted_opcode || line->insn < 0 ||
|
||||
+ (instab[line->insn].flags & synthetic) || !strcmp(instab[line->insn].opcode, "os9") ||
|
||||
+ line->outputl < 1 || line->outputl > 8 || line->outputl != line->len ||
|
||||
+ strpbrk(line->project_source, "\t\r\n") || strpbrk(section, "\t\r\n"))
|
||||
+ continue;
|
||||
+ address = lw_expr_copy(line->addr);
|
||||
+ as->exportcheck = 1;
|
||||
+ as->csect = line->csect;
|
||||
+ lwasm_reduce_expr(as, address);
|
||||
+ as->exportcheck = 0;
|
||||
+ if (lw_expr_istype(address, lw_expr_type_int))
|
||||
+ {
|
||||
+ int value = lw_expr_intval(address);
|
||||
+ if (value >= 0 && value <= 65535)
|
||||
+ {
|
||||
+ fprintf(out, "%s\t%d\t%s\t%04X\t", line->project_source, line->project_line, section, value);
|
||||
+ for (i = 0; i < line->outputl; ++i) fprintf(out, "%02X", line->output[i]);
|
||||
+ fputc('\n', out);
|
||||
+ }
|
||||
+ }
|
||||
+ lw_expr_destroy(address);
|
||||
+ }
|
||||
+ if (fclose(out)) lw_error("Cannot finish writing project instruction map.\n");
|
||||
+}
|
||||
--- a/lwasm/lwasm.c
|
||||
+++ b/lwasm/lwasm.c
|
||||
@@ -504,6 +504,7 @@
|
||||
|
||||
void lwasm_emitop(line_t *cl, int opc)
|
||||
{
|
||||
+ cl->emitted_opcode = 1;
|
||||
if (cl->cycle_base == 0)
|
||||
lwasm_cycle_update_count(cl, opc); /* only call first time, never on postbyte */
|
||||
|
||||
--- a/lwasm/lwasm.h
|
||||
+++ b/lwasm/lwasm.h
|
||||
@@ -278,6 +278,9 @@
|
||||
|
||||
struct line_s
|
||||
{
|
||||
+ const char *project_source;
|
||||
+ int project_line;
|
||||
+ int emitted_opcode;
|
||||
lw_expr_t addr; // assembly address of the line
|
||||
lw_expr_t daddr; // data address of the line (os9 only)
|
||||
lw_expr_t phase; // the "phase" setting (assumed address for assembly)
|
||||
@@ -440,6 +443,8 @@
|
||||
sectiontab_t *sections; // section table
|
||||
exportlist_t *exportlist; // list of exported symbols
|
||||
importlist_t *importlist; // list of imported symbols
|
||||
+ char *project_map_file;
|
||||
+ int project_6809_only;
|
||||
char *list_file; // name of file to list to
|
||||
char *symbol_dump_file; // name of file to dump symbol table to
|
||||
int tabwidth; // tab width in list file
|
||||
--- a/lwasm/main.c
|
||||
+++ b/lwasm/main.c
|
||||
@@ -71,6 +71,8 @@
|
||||
{ "6800compat", 0x200, 0, 0, "Enable 6800 compatibility instructions, equivalent to --pragma=6800compat" },
|
||||
{ "no-output", 0x105, 0, 0, "Inhibit creation of output file" },
|
||||
{ "no-warn", 0x109, "FLAG", 0, "Suppress warnings of the specified type" },
|
||||
+ { "project-map", 0x10d, "FILE", 0, "Write physical source instruction locations for a project snapshot" },
|
||||
+ { "project-inputs", 0x10c, 0, 0, "Restrict all assembly inputs to the current project folder" },
|
||||
{ "error-format", 0x10b, "STYLE", 0, "Error output style: lwasm (default), gcc, vs" },
|
||||
{ 0 }
|
||||
};
|
||||
@@ -82,6 +84,12 @@
|
||||
|
||||
switch (key)
|
||||
{
|
||||
+ case 0x10d:
|
||||
+ as->project_map_file = lw_strdup(arg);
|
||||
+ break;
|
||||
+ case 0x10c:
|
||||
+ input_restrict_to_project();
|
||||
+ break;
|
||||
case 'I':
|
||||
lw_stringlist_addstring(as -> include_list, arg);
|
||||
break;
|
||||
@@ -333,6 +341,7 @@
|
||||
void do_output(asmstate_t *as);
|
||||
void do_symdump(asmstate_t *as);
|
||||
void do_list(asmstate_t *as);
|
||||
+void do_project_map(asmstate_t *as);
|
||||
void do_map(asmstate_t *as);
|
||||
lw_expr_t lwasm_evaluate_special(int t, void *ptr, void *priv);
|
||||
lw_expr_t lwasm_evaluate_var(char *var, void *priv);
|
||||
@@ -391,6 +400,7 @@
|
||||
asmstate.output_file = lw_strdup("a.out");
|
||||
}
|
||||
|
||||
+ asmstate.project_6809_only = asmstate.project_map_file && (asmstate.pragmas & PRAGMA_6809);
|
||||
input_init(&asmstate);
|
||||
|
||||
for (passnum = 0; passlist[passnum].fn; passnum++)
|
||||
@@ -450,6 +460,7 @@
|
||||
}
|
||||
do_symdump(&asmstate);
|
||||
do_list(&asmstate);
|
||||
+ if (asmstate.project_map_file) do_project_map(&asmstate);
|
||||
do_map(&asmstate);
|
||||
|
||||
if (asmstate.testmode_errorcount > 0) exit(1);
|
||||
--- a/lwasm/pass1.c
|
||||
+++ b/lwasm/pass1.c
|
||||
@@ -115,12 +115,15 @@
|
||||
memset(cl, 0, sizeof(line_t));
|
||||
cl -> outputl = -1;
|
||||
cl -> linespec = lw_strdup(input_curspec(as));
|
||||
+ if (as->project_map_file) input_project_location(as, cl);
|
||||
cl -> prev = as -> line_tail;
|
||||
cl -> insn = -1;
|
||||
cl -> as = as;
|
||||
cl -> inmod = as -> inmod;
|
||||
cl -> csect = as -> csect;
|
||||
cl -> pragmas = as -> pragmas;
|
||||
+ if (as->project_6809_only && !(cl->pragmas & PRAGMA_6809))
|
||||
+ lwasm_register_error2(as, cl, E_6309_INVALID, "Source changes the selected 6809 project target.");
|
||||
cl -> context = as -> context;
|
||||
cl -> hasoperand = 1; // default: assume operand present; cleared by inherent insns
|
||||
cl -> ltext = lw_strdup(line);
|
||||
--- a/lwasm/pseudo.c
|
||||
+++ b/lwasm/pseudo.c
|
||||
@@ -1662,7 +1662,7 @@
|
||||
FILE *fp;
|
||||
int c, i;
|
||||
|
||||
- fp = fopen(l -> lstr, "rb");
|
||||
+ fp = input_fopen(l -> lstr);
|
||||
if (!fp)
|
||||
{
|
||||
lwasm_register_error2(as, l, E_FILE_OPEN, "%s", "(emit)!");
|
||||
@@ -0,0 +1,32 @@
|
||||
--- a/src/writecocofile.cpp
|
||||
+++ b/src/writecocofile.cpp
|
||||
@@ -1,3 +1,8 @@
|
||||
+/* VCC Plus modification, November 4, 2026: support an explicit
|
||||
+ --allocate-before-dir option using the existing forward allocator.
|
||||
+ This modification is licensed GPL-3.0-or-later.
|
||||
+*/
|
||||
+
|
||||
/* writecocofile.cpp - A tool to write native files to a CoCo DECB disk image.
|
||||
Copyright (C) 2003-2026 Pierre Sarrazin <http://sarrazip.com/>
|
||||
|
||||
@@ -874,6 +879,8 @@
|
||||
"--kill|-k Kill the designated file.\n"
|
||||
"--read|-r Read instead of writing. Refuses to overwrite existing native file.\n"
|
||||
"--stdout|-s Like --read, but print file contents on standard out.\n"
|
||||
+ "--allocate-before-dir Allocate from granule zero (the default), preferring space\n"
|
||||
+ " before the directory, then continuing above it if needed.\n"
|
||||
"--allocate-after-dir When adding a file, allocate granules after the directory track,\n"
|
||||
" then continue allocating below that track if needed.\n"
|
||||
"--allocate-from-end When adding a file, allocate granules from the end.\n"
|
||||
@@ -1376,6 +1383,11 @@
|
||||
granuleAllocator.reset(new BackwardGranuleAllocator());
|
||||
continue;
|
||||
}
|
||||
+ if (curopt == "--allocate-before-dir")
|
||||
+ {
|
||||
+ granuleAllocator.reset(new ForwardGranuleAllocator());
|
||||
+ continue;
|
||||
+ }
|
||||
if (curopt == "--allocate-after-dir")
|
||||
{
|
||||
granuleAllocator.reset(new PostDirTrackGranuleAllocator());
|
||||
@@ -0,0 +1,30 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// Select the matching relocatable Perl installation. Architecture-specific
|
||||
// Config.pm and XS modules remain paired with the interpreter that built them.
|
||||
#include <mach-o/dyld.h>
|
||||
#include <limits.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
int main(int argc, char **argv) {
|
||||
(void)argc;
|
||||
char path[PATH_MAX], resolved[PATH_MAX]; uint32_t size = sizeof path;
|
||||
if (_NSGetExecutablePath(path, &size) || !realpath(path, resolved)) {
|
||||
perror("Cannot locate bundled Perl"); return 1;
|
||||
}
|
||||
char *last = strrchr(resolved, '/'); if (!last) return 1; *last = 0;
|
||||
last = strrchr(resolved, '/'); if (!last) return 1; *last = 0;
|
||||
#if defined(__aarch64__)
|
||||
const char *relative = "/libexec/perl/arm64/bin/perl";
|
||||
#elif defined(__x86_64__)
|
||||
const char *relative = "/libexec/perl/x86_64/bin/perl";
|
||||
#else
|
||||
#error Unsupported Perl host architecture
|
||||
#endif
|
||||
if (strlen(resolved) + strlen(relative) >= sizeof path) return 1;
|
||||
snprintf(path, sizeof path, "%s%s", resolved, relative);
|
||||
argv[0] = path;
|
||||
execv(path, argv);
|
||||
perror("Cannot execute bundled Perl"); return 1;
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Prepare the real CoCo Letters Make project for local VCC Plus qualification.
|
||||
|
||||
This is a test fixture, not an application resource. The explicitly supplied
|
||||
private standalone disk source and generated game stay in the chosen local
|
||||
output folder and are never copied into the public toolchain or Git snapshot.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--standalone-disk-source', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
if output.exists(): parser.error('Choose a fresh project directory')
|
||||
if output == ROOT or ROOT / 'third_party' == output or ROOT / 'third_party' in output.parents:
|
||||
parser.error('Choose local test storage outside the source snapshots')
|
||||
record = json.loads((ROOT / 'third_party/manifests/cmoc.json').read_text())
|
||||
expected = record['excludedFileSHA256']['src/stdlib/dskcon-standalone.c']
|
||||
if hashlib.sha256(args.standalone_disk_source.read_bytes()).hexdigest() != expected:
|
||||
parser.error('Private disk source does not match the reviewed CMOC release')
|
||||
subprocess.run([sys.executable, ROOT / 'tools/toolchain/verify-sources.py'], check=True)
|
||||
shutil.copytree(ROOT / 'third_party/examples/coco-letters', output)
|
||||
# A grouped target makes the upstream two-output dictionary recipe safe when
|
||||
# GNU Make runs in parallel. The vendored source itself remains unchanged.
|
||||
upstream = (output / 'Makefile').read_text()
|
||||
original = '$(DATA_FILENAME) $(WORD_LENGTH_FILENAME): $(BUILD_DATA)'
|
||||
if upstream.count(original) != 1: raise SystemExit('Review updated dictionary generation rule')
|
||||
upstream = upstream.replace(original, '$(DATA_FILENAME) $(WORD_LENGTH_FILENAME) &: $(BUILD_DATA)')
|
||||
(output / 'Game.mk').write_text(upstream)
|
||||
for name in ('decbfile', 'bcontrol', 'bsound'):
|
||||
shutil.copytree(ROOT / 'third_party/cmoc-libraries' / name, output / 'libraries' / name)
|
||||
(output / 'private').mkdir()
|
||||
shutil.copyfile(args.standalone_disk_source, output / 'private/dskcon-standalone.c')
|
||||
(output / 'Makefile').write_text('''# VCC Plus local qualification wrapper. GPL-3.0-or-later.
|
||||
# Game.mk is the upstream game Makefile, with its dictionary recipe grouped.
|
||||
override DECBFILE_LIB_DIR = libraries/decbfile
|
||||
override BCONTROL_LIB_DIR = libraries/bcontrol
|
||||
override BSOUND_LIB_DIR = libraries/bsound
|
||||
include Game.mk
|
||||
.DEFAULT_GOAL := all
|
||||
INTERMEDIATE = --intermediate
|
||||
LIBS += private/dskcon-standalone.o
|
||||
|
||||
$(BIN): private/dskcon-standalone.o
|
||||
private/dskcon-standalone.o: private/dskcon-standalone.c
|
||||
\t$(CMOC) --coco --intermediate -O2 -c -o $@ $<
|
||||
|
||||
libraries/decbfile/libdecbfile.a: $(wildcard libraries/decbfile/*.c libraries/decbfile/*.h) libraries/decbfile/Makefile
|
||||
\t$(MAKE) -C libraries/decbfile libdecbfile.a
|
||||
libraries/bcontrol/libbcontrol.a: $(wildcard libraries/bcontrol/*.c libraries/bcontrol/*.h) libraries/bcontrol/Makefile
|
||||
\t$(MAKE) -C libraries/bcontrol libbcontrol.a
|
||||
libraries/bsound/libbsound.a: $(wildcard libraries/bsound/*.c libraries/bsound/*.h) libraries/bsound/Makefile
|
||||
\t$(MAKE) -C libraries/bsound libbsound.a
|
||||
|
||||
.PHONY: clean-libraries
|
||||
clean: clean-libraries
|
||||
clean-libraries:
|
||||
\t$(MAKE) -C libraries/decbfile clean
|
||||
\t$(MAKE) -C libraries/bcontrol clean
|
||||
\t$(MAKE) -C libraries/bsound clean
|
||||
\trm -f private/*.o private/*.s private/*.lst $(DSK)
|
||||
''')
|
||||
(output / 'vccplus-project.json').write_text(json.dumps({
|
||||
'entrySource': 'main.c', 'makefile': 'Makefile', 'buildTarget': 'all',
|
||||
'cleanTarget': 'clean', 'diskTarget': 'all', 'binary': 'letters.bin',
|
||||
'diskImage': 'coco-letters-0.1.0.dsk', 'launchMode': 'disk',
|
||||
'launchCommands': ['DOS'], 'jobs': 4,
|
||||
'configurations': [{'name': 'Debug', 'makeArguments': ['INTERMEDIATE=--intermediate']},
|
||||
{'name': 'Release', 'makeArguments': []}]
|
||||
}, indent=2) + '\n')
|
||||
(output / '.gitignore').write_text('private/\n*.bin\n*.dsk\n*.dat\n*.o\n*.s\n*.lst\n*.map\n*.link\n*.d\nword-length-index.c\n')
|
||||
(output / 'VCC-QUALIFICATION.txt').write_text(
|
||||
'Local qualification fixture. Includes separately supplied private disk source.\n'
|
||||
'Do not distribute this folder as part of VCC Plus or its public toolchain.\n'
|
||||
'Open the folder in C Development, build the all target, then mount/boot its disk.\n'
|
||||
'Requires CoCo 3, at least 128 KB, floppy controller with Disk BASIC, and RGB video.\n'
|
||||
'The original optimization flags are retained to fit its memory map.\n'
|
||||
'Source breakpoints are available only where exact executable mappings are verified.\n')
|
||||
print('Prepared local Make project:', output)
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Validate the explicitly staged private disk-driver extension."""
|
||||
import hashlib
|
||||
|
||||
STANDALONE_FILES = {
|
||||
'share/cmoc/lib/libdskcon-standalone.a',
|
||||
'share/cmoc/include/dskcon-standalone.h',
|
||||
'LICENSES/DSKCON-STANDALONE-NOTICE.txt',
|
||||
}
|
||||
|
||||
|
||||
def validate_private_inputs(runtime, record):
|
||||
disk = record.get('standaloneDisk')
|
||||
if disk is None:
|
||||
if record.get('privateRuntimeInputs') is not False:
|
||||
raise ValueError('Private runtime input declaration is missing')
|
||||
if any((runtime / path).exists() for path in STANDALONE_FILES - {'share/cmoc/include/dskcon-standalone.h'}):
|
||||
raise ValueError('Undeclared standalone disk payload')
|
||||
return
|
||||
if record.get('privateRuntimeInputs') is not True or disk.get('profile') != 'decb-derived-standalone':
|
||||
raise ValueError('Unsupported private runtime profile')
|
||||
hashes = disk.get('filesSHA256', {})
|
||||
if set(hashes) != STANDALONE_FILES:
|
||||
raise ValueError('Incomplete standalone disk payload declaration')
|
||||
if disk.get('license') != 'public-domain-except-DECB-derived-portions':
|
||||
raise ValueError('Missing standalone disk license distinction')
|
||||
if not isinstance(disk.get('sourceSHA256'), str) or len(disk['sourceSHA256']) != 64:
|
||||
raise ValueError('Missing standalone disk source identity')
|
||||
for relative, expected in hashes.items():
|
||||
path = runtime / relative
|
||||
if path.is_symlink() or not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected:
|
||||
raise ValueError('Standalone disk payload does not match its manifest: ' + relative)
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Prebuild the public-domain, ROM-backed DECB file API into an existing runtime.
|
||||
|
||||
The same bundled compiler/preprocessor builds libraries and end-user programs;
|
||||
Apple developer tools are not required for this staging operation.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from runtime_inputs import validate_private_inputs
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SOURCE = ROOT / 'third_party/cmoc-libraries/decbfile'
|
||||
VERSION = '0.1.13'
|
||||
EXCLUDED = {
|
||||
'drivewire.c', 'dwReadBeckerPort.c', 'dwWriteBeckerPort.c',
|
||||
'dwReadBitBangerSerialPort.c', 'dwWriteBitBangerSerialPort.c',
|
||||
'FileChoiceDialog.c', 'runDOSLoader.c', 'binfile.c', 'writeMemoryToFile.c',
|
||||
'readFileToMemory.c', 'readFileToDynamicMemory.c', 'decbutil.c', 'crc16.c',
|
||||
}
|
||||
|
||||
|
||||
def sha(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('runtime', type=Path)
|
||||
parser.add_argument('--jobs', type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
runtime = args.runtime.resolve()
|
||||
manifest_path = runtime / 'manifest.json'
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
validate_private_inputs(runtime, manifest)
|
||||
source_manifest = ROOT / 'third_party/manifests/decbfile.json'
|
||||
source_record = json.loads(source_manifest.read_text())
|
||||
if source_record['version'] != VERSION:
|
||||
parser.error('Review the ROM-backed library selection before changing upstream version')
|
||||
subprocess.run([sys.executable, ROOT / 'tools/toolchain/verify-sources.py'], check=True)
|
||||
source_list = re.search(r'^LIBSRC = \\\n(.*?)\n\nLIBHEADERS', (SOURCE / 'Makefile').read_text(), re.M | re.S)
|
||||
if not source_list:
|
||||
parser.error('Unexpected upstream source list')
|
||||
sources = source_list.group(1).replace('\\', '').split()
|
||||
if not EXCLUDED.issubset(sources):
|
||||
parser.error('Unexpected upstream optional components')
|
||||
selected = [name for name in sources if name not in EXCLUDED]
|
||||
env = dict(os.environ, PATH=str(runtime / 'bin') + ':/usr/bin:/bin')
|
||||
env['DEVELOPER_DIR'] = '/VCC-no-developer-tools'
|
||||
env['SDKROOT'] = '/VCC-no-sdk'
|
||||
# A simple temporary path also avoids upstream CMOC's shell-quoting limits.
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-decbfile-', dir='/tmp') as folder:
|
||||
work = Path(folder)
|
||||
for name in selected + ['decbfile.h', 'decbfile_private.h']:
|
||||
shutil.copyfile(SOURCE / name, work / name)
|
||||
def compile_source(name):
|
||||
subprocess.run([str(runtime / 'bin/cmoc'), '--coco', '-O2',
|
||||
'-DNDEBUG', '-Wfor-condition-sizes', '-Werror', '-I.', '-c',
|
||||
'-o', name[:-2] + '.o', name], cwd=work, env=env, check=True)
|
||||
with ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool:
|
||||
list(pool.map(compile_source, selected))
|
||||
archive = work / 'libdecbfile.a'
|
||||
subprocess.run([str(runtime / 'bin/lwar'), '--create', str(archive),
|
||||
*[name[:-2] + '.o' for name in selected]], cwd=work, env=env, check=True)
|
||||
archive_listing = subprocess.check_output([str(runtime / 'bin/lwar'), '--list', str(archive)], text=True)
|
||||
for name in EXCLUDED:
|
||||
if name[:-2] + '.o' in archive_listing:
|
||||
raise SystemExit('Unexpected excluded archive member: ' + name)
|
||||
target_archive = runtime / 'share/cmoc/lib/libdecbfile.a'
|
||||
target_header = runtime / 'share/cmoc/include/decbfile.h'
|
||||
shutil.copyfile(archive, target_archive)
|
||||
shutil.copyfile(SOURCE / 'decbfile.h', target_header)
|
||||
# Retain the original declaration verbatim instead of assigning a GPL label
|
||||
# to a component whose author explicitly released it into the public domain.
|
||||
header_notice = (SOURCE / 'decbfile.h').read_text().split('*/', 1)[0] + '*/\n'
|
||||
notice = ('DECB File Library ' + VERSION + ' — ROM-backed core\n\n' + header_notice +
|
||||
'\nUses the Disk BASIC ROM already mapped in the emulated machine.\n'
|
||||
'No Disk BASIC ROM bytes or standalone Disk BASIC-derived driver are included in this archive.\n'
|
||||
'Optional standalone, DriveWire, UI and binary-loader extensions are not included.\n\n'
|
||||
'Included upstream source files:\n' + ''.join(' ' + name + '\n' for name in selected))
|
||||
(runtime / 'LICENSES/DECBFILE-NOTICE.txt').write_text(notice)
|
||||
manifest['decbfile'] = {'version': VERSION, 'profile': 'rom-backed-core',
|
||||
'license': 'public-domain', 'requiresDiskBASICROM': True,
|
||||
'standaloneDriverIncluded': False, 'sourceFiles': selected,
|
||||
'sourceManifestSHA256': sha(source_manifest),
|
||||
'archiveSHA256': sha(target_archive), 'headerSHA256': sha(target_header)}
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
print(f'PASS: staged {len(selected)} public-domain DECB core objects; link with -ldecbfile')
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Compile the original VCC Plus guest libraries with the installed CMOC."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SDK = ROOT / "sdk/cmoc"
|
||||
LIBRARIES = {"ssc": "ssc", "orch90": "orch90", "gime": "gime", "input": "input", "bank": "bank", "sdc": "sdc", "vcccompat": "compat"}
|
||||
|
||||
|
||||
def stage(runtime):
|
||||
manifest_file = runtime / "manifest.json"
|
||||
manifest = json.loads(manifest_file.read_text())
|
||||
env = dict(os.environ, PATH=str(runtime / "bin") + ":/usr/bin:/bin", LC_ALL="C",
|
||||
VCC_CMOC_COMPAT="0")
|
||||
with tempfile.TemporaryDirectory(dir="/tmp", prefix="vcc-sdk-") as tmp:
|
||||
work = Path(tmp)
|
||||
include = work / "include"
|
||||
shutil.copytree(SDK / "include", include)
|
||||
shutil.copytree(SDK / "compat/include", include / "vcc/compat")
|
||||
# Build against the pinned public header even when the ROM library has
|
||||
# not yet been staged in a fresh runtime. Only SDK headers ship here.
|
||||
shutil.copyfile(ROOT / "third_party/cmoc-libraries/decbfile/decbfile.h", include / "decbfile.h")
|
||||
payload = {}
|
||||
for name, source in LIBRARIES.items():
|
||||
folder = work / source
|
||||
folder.mkdir()
|
||||
objects = []
|
||||
flags = ["-I" + str(include)]
|
||||
if name == "vcccompat":
|
||||
flags.insert(0, "-I" + str(include / "vcc/compat"))
|
||||
for header in (SDK / source).glob("*.h"):
|
||||
shutil.copyfile(header, folder / header.name)
|
||||
for unit in sorted((SDK / source).glob("*.c")):
|
||||
shutil.copyfile(unit, folder / unit.name)
|
||||
obj = unit.stem + ".o"
|
||||
subprocess.run([runtime / "bin/cmoc", "--coco", "-O2", "-Werror",
|
||||
*flags, "-c", "-o", obj, unit.name],
|
||||
cwd=folder, env=env, check=True)
|
||||
objects.append(obj)
|
||||
if not objects:
|
||||
raise ValueError("No compilation units for " + name)
|
||||
archive = folder / ("lib" + name + ".a")
|
||||
subprocess.run([runtime / "bin/lwar", "--create", archive.name, *objects],
|
||||
cwd=folder, env=env, check=True)
|
||||
if name == "vcccompat":
|
||||
headers = [*sorted((include / "vcc/compat").glob("*.h")),
|
||||
include / "vcc/printer.h", include / "vcc/cassette.h", include / "vcc/decimal.h"]
|
||||
else:
|
||||
headers = [include / ("vcc/" + name + ".h")]
|
||||
files = {"share/cmoc/lib/" + archive.name: archive, "LICENSES/VCC-Guest-SDK.txt": ROOT / "COPYING"}
|
||||
files.update({"share/cmoc/include/" + h.relative_to(include).as_posix(): h for h in headers})
|
||||
version = {"vcccompat": "0.14.0", "ssc": "0.2.0", "orch90": "0.2.0"}.get(name, "0.1.0")
|
||||
manifest.setdefault("libraries", {})[name] = dict(version=version, profile="coco",
|
||||
license="GPL-3.0-or-later", filesSHA256={k: hashlib.sha256(p.read_bytes()).hexdigest() for k, p in files.items()})
|
||||
if name == "vcccompat":
|
||||
manifest["libraries"][name].update(diskReadStreams=True, diskWriteStreams=True, diskStreamPositioning=True,
|
||||
boundedHeap=True, integerConversions=True, reentrantTokenizer=True,
|
||||
diskUpdateStreams=True, createAppendUpdateStreams=True, diskFormattedOutput=True,
|
||||
diskStreamClearError=True, printerFormattedOutput=True,
|
||||
cassetteTextFiles=True, diskFileManagement=True, fixedDecimal=True)
|
||||
if name == "ssc":
|
||||
manifest["libraries"][name]["storedSpeechAndEffects"] = True
|
||||
if name == "orch90":
|
||||
manifest["libraries"][name]["backgroundPlayback"] = True
|
||||
payload.update(files)
|
||||
for path, original in payload.items():
|
||||
target = runtime / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(original, target)
|
||||
shutil.copyfile(ROOT / "tools/toolchain/cmoc-runtime.sh", runtime / "bin/cmoc")
|
||||
temporary = manifest_file.with_suffix(".json.new")
|
||||
temporary.write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
temporary.replace(manifest_file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("runtime", type=Path)
|
||||
stage(parser.parse_args().runtime.resolve())
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Build and package reviewed CMOC libraries with the standalone compiler chain."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from runtime_inputs import validate_private_inputs
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
LIBRARIES = ('astar', 'bcontrol', 'bgraph', 'bsound', 'cardgame', 'ebitor', 'hirestxt', 'wydealloc')
|
||||
|
||||
|
||||
def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('runtime', type=Path)
|
||||
parser.add_argument('--jobs', type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
runtime = args.runtime.resolve()
|
||||
manifest_path = runtime / 'manifest.json'
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
validate_private_inputs(runtime, manifest)
|
||||
subprocess.run([sys.executable, ROOT / 'tools/toolchain/verify-sources.py'], check=True)
|
||||
env = dict(os.environ, PATH=str(runtime / 'bin') + ':/usr/bin:/bin', LC_ALL='C',
|
||||
DEVELOPER_DIR='/VCC-no-developer-tools', SDKROOT='/VCC-no-sdk')
|
||||
records = {}
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-cmoc-libraries-', dir='/tmp') as folder:
|
||||
for name in LIBRARIES:
|
||||
source = ROOT / 'third_party/cmoc-libraries' / name
|
||||
work = Path(folder) / name
|
||||
shutil.copytree(source, work)
|
||||
subprocess.run([str(runtime / 'bin/make'), f'-j{max(1, args.jobs)}', f'lib{name}.a'],
|
||||
cwd=work, env=env, check=True)
|
||||
output = subprocess.check_output([str(runtime / 'bin/make'), '--no-print-directory',
|
||||
'--eval', 'vcc-list-headers:;@echo VCC_HEADERS $(LIBHEADERS)', 'vcc-list-headers'],
|
||||
cwd=work, env=env, text=True)
|
||||
matches = [line for line in output.splitlines() if line.startswith('VCC_HEADERS ')]
|
||||
if len(matches) != 1: raise SystemExit('Cannot determine library headers: ' + name)
|
||||
headers = matches[0].split()[1:]
|
||||
archive = runtime / 'share/cmoc/lib' / f'lib{name}.a'
|
||||
shutil.copyfile(work / f'lib{name}.a', archive)
|
||||
files = {archive.relative_to(runtime).as_posix(): sha(archive)}
|
||||
for header in headers:
|
||||
if Path(header).name != header or not header.endswith('.h'):
|
||||
raise SystemExit('Unexpected library header: ' + header)
|
||||
target = runtime / 'share/cmoc/include' / header
|
||||
if target.exists() and target.read_bytes() != (source / header).read_bytes():
|
||||
raise SystemExit('Conflicting library header: ' + header)
|
||||
shutil.copyfile(source / header, target)
|
||||
files[target.relative_to(runtime).as_posix()] = sha(target)
|
||||
record = json.loads((ROOT / 'third_party/manifests' / (name + '.json')).read_text())
|
||||
notice = runtime / 'LICENSES' / (name.upper() + '-NOTICE.txt')
|
||||
text = f"{name} {record['version']}\n\n{record['license']}\n\n"
|
||||
# Keep declarations/acknowledgments from every compiled source and
|
||||
# public header. Full corresponding source remains in third_party.
|
||||
for file in sorted(source.glob('*')):
|
||||
if not file.is_file(): continue
|
||||
if file.name in ('COPYING', 'README.md'):
|
||||
text += '\n--- ' + file.name + ' ---\n' + file.read_text()
|
||||
elif file.suffix in ('.c', '.h'):
|
||||
content = file.read_text()
|
||||
if content.startswith('/*'):
|
||||
notice_end = content.find('*/')
|
||||
excerpt = content[:notice_end + 2] if notice_end >= 0 else ''
|
||||
else:
|
||||
lines = []
|
||||
for line in content.splitlines():
|
||||
if line.startswith('//') or not line.strip(): lines.append(line)
|
||||
else: break
|
||||
excerpt = '\n'.join(lines).strip()
|
||||
if excerpt: text += '\n--- ' + file.name + ' ---\n' + excerpt + '\n'
|
||||
notice.write_text(text)
|
||||
records[name] = {'version': record['version'], 'license': record['license'],
|
||||
'profile': 'coco', 'filesSHA256': files}
|
||||
manifest['libraries'] = records
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
subprocess.run([sys.executable, ROOT / 'tools/toolchain/stage-guest-sdk.py', runtime], check=True)
|
||||
print('PASS: guest SDK and eight optional library archives and headers built with bundled tools')
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Package locally built Perl, preserving separate architecture-dependent cores."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VERSION = '5.44.0'
|
||||
MACHO_MAGIC = {b'\xcf\xfa\xed\xfe', b'\xfe\xed\xfa\xcf', b'\xca\xfe\xba\xbe', b'\xbe\xba\xfe\xca'}
|
||||
|
||||
|
||||
def native(path):
|
||||
with path.open('rb') as stream: return stream.read(4) in MACHO_MAGIC
|
||||
|
||||
|
||||
def stage(runtime, prefixes):
|
||||
manifest_path = runtime / 'manifest.json'
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
if set(prefixes) != set(manifest['architectures']):
|
||||
raise SystemExit('Perl builds must cover every declared runtime architecture')
|
||||
helpers = {p: arches for p, arches in manifest.get('nativeHelpers', {}).items()
|
||||
if not p.startswith('libexec/perl/') and p != 'bin/perl'}
|
||||
target = runtime / 'libexec/perl'
|
||||
if target.exists(): shutil.rmtree(target)
|
||||
for arch, prefix in prefixes.items():
|
||||
actual = subprocess.check_output([str(prefix / 'bin/perl'), '-e', 'print $^V'], text=True)
|
||||
if actual != 'v' + VERSION: raise SystemExit('Unexpected Perl version: ' + actual)
|
||||
installation = target / arch
|
||||
(installation / 'bin').mkdir(parents=True)
|
||||
shutil.copyfile(prefix / 'bin/perl', installation / 'bin/perl')
|
||||
(installation / 'bin/perl').chmod(0o755)
|
||||
# Core modules and their data/XS binaries, without SDK headers, static
|
||||
# development archives, manuals, command-line maintenance tools or CPAN
|
||||
# download caches. Original module notices remain in their source text.
|
||||
def ignored(folder, names):
|
||||
return [name for name in names if name == 'CORE' or name == 'pods'
|
||||
or name.endswith(('.pod', '.a', '.h')) or name == '.packlist']
|
||||
shutil.copytree(prefix / 'lib', installation / 'lib', ignore=ignored)
|
||||
for path in installation.rglob('*'):
|
||||
if path.is_symlink(): raise SystemExit('Unexpected symlink in Perl runtime: ' + str(path))
|
||||
if path.is_file() and native(path):
|
||||
slices = subprocess.check_output(['/usr/bin/lipo', '-archs', str(path)], text=True).split()
|
||||
if arch not in slices: raise SystemExit('Incorrect Perl architecture: ' + str(path))
|
||||
helpers[path.relative_to(runtime).as_posix()] = [arch]
|
||||
env = dict(os.environ)
|
||||
env.setdefault('DEVELOPER_DIR', '/Applications/Xcode.app/Contents/Developer')
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-perl-launcher-') as folder:
|
||||
slices = []
|
||||
for arch in prefixes:
|
||||
binary = Path(folder) / arch
|
||||
subprocess.run(['xcrun', 'clang', '-O2', '-arch', arch, '-mmacosx-version-min=14.0',
|
||||
str(ROOT / 'tools/toolchain/perl-launcher.c'), '-o', str(binary)], env=env, check=True)
|
||||
slices.append(binary)
|
||||
launcher = runtime / 'bin/perl'
|
||||
if len(slices) == 1: shutil.copyfile(slices[0], launcher)
|
||||
else: subprocess.run(['/usr/bin/lipo', '-create', *map(str, slices), '-output', str(launcher)], check=True)
|
||||
launcher.chmod(0o755)
|
||||
helpers['bin/perl'] = list(prefixes)
|
||||
# Upstream script's /usr/bin/perl shebang would bypass our bundled runtime.
|
||||
# Preserve the script unchanged behind a relocatable wrapper instead.
|
||||
shutil.copyfile(ROOT / 'third_party/cmoc/src/install-coco-boot-loader',
|
||||
runtime / 'libexec/install-coco-boot-loader.pl')
|
||||
wrapper = runtime / 'bin/install-coco-boot-loader'
|
||||
wrapper.write_text('#!/bin/sh\n# SPDX-License-Identifier: GPL-3.0-or-later\n'
|
||||
'set -eu\nruntime=$(CDPATH= cd -- "$(/usr/bin/dirname -- "$0")/.." && pwd -P)\n'
|
||||
'exec "$runtime/bin/perl" "$runtime/libexec/install-coco-boot-loader.pl" "$@"\n')
|
||||
wrapper.chmod(0o755)
|
||||
notices = runtime / 'LICENSES'
|
||||
for source, name in [('README', 'PERL-README.txt'), ('Copying', 'PERL-COPYING.txt'),
|
||||
('Artistic', 'PERL-ARTISTIC.txt'),
|
||||
('cpan/Compress-Raw-Bzip2/bzip2-src/LICENSE', 'PERL-BZIP2-LICENSE.txt')]:
|
||||
shutil.copyfile(ROOT / 'third_party/perl' / source, notices / name)
|
||||
manifest['nativeHelpers'] = dict(sorted(helpers.items()))
|
||||
manifest['perl'] = {'version': VERSION, 'license': 'GPL-1.0-or-later OR Artistic-1.0-Perl',
|
||||
'profile': 'interpreter-and-core-modules', 'architectures': list(prefixes),
|
||||
'sourceManifestSHA256': hashlib.sha256((ROOT / 'third_party/manifests/perl.json').read_bytes()).hexdigest()}
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
for arch in prefixes:
|
||||
subprocess.run(['/usr/bin/arch', '-' + arch, str(runtime / 'bin/perl'), '-MGetopt::Long',
|
||||
'-MFile::Spec', '-MDigest::SHA', '-e',
|
||||
'use strict; use warnings; print "Bundled Perl $^V OK\\n";'], check=True)
|
||||
print('PASS: staged independent Perl/core-module runtime for ' + ', '.join(prefixes))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('runtime', type=Path)
|
||||
parser.add_argument('--prefix', action='append', required=True, help='ARCH=installed-prefix (repeat per architecture)')
|
||||
args = parser.parse_args()
|
||||
prefixes = {}
|
||||
for item in args.prefix:
|
||||
arch, separator, value = item.partition('=')
|
||||
if not separator or arch not in ('arm64', 'x86_64') or arch in prefixes:
|
||||
parser.error('Expected one ARCH=prefix per architecture')
|
||||
prefixes[arch] = Path(value).resolve()
|
||||
stage(args.runtime.resolve(), prefixes)
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Build an explicitly supplied, private DECB-derived driver as an optional library.
|
||||
|
||||
The source remains outside Git. The ROM-backed decbfile archive is unchanged.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from runtime_inputs import validate_private_inputs
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def sha(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('runtime', type=Path)
|
||||
parser.add_argument('--source', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
runtime = args.runtime.resolve()
|
||||
manifest = runtime / 'manifest.json'
|
||||
record = json.loads(manifest.read_text())
|
||||
validate_private_inputs(runtime, record)
|
||||
cmoc = json.loads((ROOT / 'third_party/manifests/cmoc.json').read_text())
|
||||
expected = cmoc['excludedFileSHA256']['src/stdlib/dskcon-standalone.c']
|
||||
if sha(args.source) != expected:
|
||||
parser.error('Private driver must match the reviewed CMOC source release')
|
||||
header = ROOT / 'third_party/cmoc/src/stdlib/dskcon-standalone.h'
|
||||
if sha(header) != cmoc['upstreamFileSHA256']['src/stdlib/dskcon-standalone.h']:
|
||||
parser.error('Standalone header differs from the reviewed source')
|
||||
env = dict(os.environ, PATH=str(runtime / 'bin') + ':/usr/bin:/bin',
|
||||
DEVELOPER_DIR='/VCC-no-developer-tools', SDKROOT='/VCC-no-sdk')
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-dskcon-', dir='/tmp') as folder:
|
||||
work = Path(folder)
|
||||
shutil.copyfile(args.source, work / 'dskcon-standalone.c')
|
||||
shutil.copyfile(header, work / header.name)
|
||||
subprocess.run([str(runtime / 'bin/cmoc'), '--coco', '-O2', '-I.', '-c',
|
||||
'-o', 'dskcon-standalone.o', 'dskcon-standalone.c'], cwd=work, env=env, check=True)
|
||||
subprocess.run([str(runtime / 'bin/lwar'), '--create', 'libdskcon-standalone.a',
|
||||
'dskcon-standalone.o'], cwd=work, env=env, check=True)
|
||||
shutil.copyfile(work / 'libdskcon-standalone.a', runtime / 'share/cmoc/lib/libdskcon-standalone.a')
|
||||
shutil.copyfile(header, runtime / 'share/cmoc/include/dskcon-standalone.h')
|
||||
upstream_notice = args.source.read_text().split('*/', 1)[0] + '*/\n'
|
||||
notice = ('Standalone CoCo floppy disk driver — optional linked library\n\n' + upstream_notice +
|
||||
'\nThis library contains portions taken from Disk Extended Color BASIC (DECB).\n'
|
||||
'Those portions are separate third-party code, not covered by Pierre Sarrazin\'s\n'
|
||||
'public-domain dedication or VCC Plus\'s GPL license. Original rights remain\n'
|
||||
'with their respective owners; this notice grants no additional rights.\n\n'
|
||||
'The ROM-backed decbfile core remains a separate public-domain library.\n'
|
||||
'Use -ldskcon-standalone to explicitly link this driver into a CoCo program.\n'
|
||||
'Its code runs from RAM and accesses the emulated floppy controller.\n'
|
||||
'It does not require Disk BASIC to remain mapped during disk operations.\n\n'
|
||||
'Rights-holder inquiries or removal requests: [email protected]\n')
|
||||
(runtime / 'LICENSES/DSKCON-STANDALONE-NOTICE.txt').write_text(notice)
|
||||
paths = ['share/cmoc/lib/libdskcon-standalone.a', 'share/cmoc/include/dskcon-standalone.h',
|
||||
'LICENSES/DSKCON-STANDALONE-NOTICE.txt']
|
||||
record['privateRuntimeInputs'] = True
|
||||
record['standaloneDisk'] = {
|
||||
'profile': 'decb-derived-standalone', 'cmocVersion': cmoc['version'],
|
||||
'license': 'public-domain-except-DECB-derived-portions', 'sourceSHA256': expected,
|
||||
'filesSHA256': {p: sha(runtime / p) for p in paths},
|
||||
}
|
||||
validate_private_inputs(runtime, record)
|
||||
manifest.write_text(json.dumps(record, indent=2) + '\n')
|
||||
print('Staged optional standalone disk API; explicitly link with -ldskcon-standalone.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Audit a staged runtime and compile through Make without Apple developer tools."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from runtime_inputs import validate_private_inputs
|
||||
|
||||
TOOLS = ('libexec/cmoc', 'bin/cpp', 'bin/mcpp', 'bin/make', 'bin/lwasm', 'bin/lwlink', 'bin/lwar', 'bin/writecocofile')
|
||||
|
||||
def check_assembly(runtime, folder):
|
||||
"""Both native slices must resolve source identity and reject escaping inputs."""
|
||||
folder.mkdir()
|
||||
assembler = str(runtime / 'bin/lwasm')
|
||||
(folder / 'part.inc').write_text(' lda #$8f\n fcb $86,$8f\n')
|
||||
source = folder / 'main.asm'
|
||||
source.write_text(' org $3f00\n include "part.inc"\n end $3f00\n')
|
||||
command = [assembler, '--project-inputs', '--project-map=lines.tsv', '--6809', '--decb', '-o', 'first.bin', 'main.asm']
|
||||
subprocess.run(command, cwd=folder, check=True)
|
||||
if (folder / 'lines.tsv').read_text() != 'part.inc\t1\t\t3F00\t868F\n':
|
||||
raise SystemExit('LWASM physical source map is missing or includes data')
|
||||
if (folder / 'first.bin').read_bytes() != bytes.fromhex('00 0004 3f00 868f 868f ff 0000 3f00'):
|
||||
raise SystemExit('LWASM project image differs')
|
||||
outside = folder.with_suffix('.inc')
|
||||
outside.write_text(' nop\n')
|
||||
(folder / 'linked.inc').symlink_to(outside)
|
||||
for directive, path in [('include', str(outside)), ('include', 'linked.inc'), ('includebin', '../' + outside.name)]:
|
||||
source.write_text(f" org $3f00\n {directive} '{path}'\n end $3f00\n")
|
||||
failure = subprocess.run(command, cwd=folder, capture_output=True, text=True)
|
||||
if failure.returncode == 0 or 'leaves the project snapshot' not in failure.stderr:
|
||||
raise SystemExit('Unexpected LWASM result for escaping ' + directive + ': ' + failure.stderr)
|
||||
|
||||
def check_disk_allocation(runtime, folder):
|
||||
"""The explicit before-dir option must match the existing forward allocator."""
|
||||
folder.mkdir()
|
||||
tool = str(runtime / 'bin/writecocofile')
|
||||
payload = folder / 'DATA.DAT'
|
||||
payload.write_bytes(bytes(range(256)) * (35 * 9)) # 35 granules: spills above track 17.
|
||||
images = []
|
||||
for name, options, first in [('default', [], 0), ('before', ['--allocate-before-dir'], 0),
|
||||
('after', ['--allocate-after-dir'], 34),
|
||||
('last-option', ['--allocate-after-dir', '--allocate-before-dir'], 0)]:
|
||||
image = folder / (name + '.dsk')
|
||||
image.write_bytes(b'\xff' * (35 * 18 * 256))
|
||||
subprocess.run([tool, *options, str(image), str(payload)], check=True, stdout=subprocess.DEVNULL)
|
||||
content = image.read_bytes()
|
||||
entry = 17 * 18 * 256 + 2 * 256
|
||||
if content[entry + 13] != first:
|
||||
raise SystemExit('Incorrect disk allocation start: ' + name)
|
||||
restored = subprocess.check_output([tool, '--stdout', str(image), 'DATA.DAT'])
|
||||
if restored != payload.read_bytes():
|
||||
raise SystemExit('Disk allocation readback mismatch: ' + name)
|
||||
images.append(content)
|
||||
if images[0] != images[1] or images[0] != images[3]:
|
||||
raise SystemExit('Before-directory option changed default allocation semantics')
|
||||
|
||||
def check_guest_example(runtime, env, folder, project_name, payloads):
|
||||
"""Build an editable example and compare its guest disk files with the build outputs."""
|
||||
source = Path(__file__).resolve().parents[2] / "examples/cmoc" / project_name
|
||||
shutil.copytree(source, folder, ignore=shutil.ignore_patterns("obj", "generated", "*.bin", "*.BIN", "*.dsk", "*.DSK", "*.map", "*.link"))
|
||||
subprocess.run([runtime / "bin/make", "-B", "disk"], cwd=folder, env=env, check=True)
|
||||
for disk_name, output, guest_name in payloads:
|
||||
expected = (folder / output).read_bytes()
|
||||
extracted = subprocess.check_output([runtime / "bin/writecocofile", "--stdout", disk_name, guest_name],
|
||||
cwd=folder, env=env)
|
||||
if not expected or extracted != expected:
|
||||
raise SystemExit(f"{project_name}: {guest_name} differs from {output}")
|
||||
subprocess.run([runtime / "bin/make", "-q", "build"], cwd=folder, env=env, check=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('runtime', type=Path)
|
||||
args = parser.parse_args()
|
||||
runtime = args.runtime.resolve()
|
||||
record = json.loads((runtime / 'manifest.json').read_text())
|
||||
validate_private_inputs(runtime, record)
|
||||
disk = record.get('decbfile')
|
||||
if disk:
|
||||
if disk.get('profile') != 'rom-backed-core' or disk.get('standaloneDriverIncluded') is not False:
|
||||
raise SystemExit('Unsupported disk library profile')
|
||||
for relative, key in [('share/cmoc/lib/libdecbfile.a', 'archiveSHA256'), ('share/cmoc/include/decbfile.h', 'headerSHA256')]:
|
||||
if hashlib.sha256((runtime / relative).read_bytes()).hexdigest() != disk[key]:
|
||||
raise SystemExit('Disk library manifest hash mismatch: ' + relative)
|
||||
|
||||
helpers = {relative: record['architectures'] for relative in TOOLS}
|
||||
helpers.update(record.get('nativeHelpers', {}))
|
||||
for library in record.get('libraries', {}).values():
|
||||
for relative, expected in library['filesSHA256'].items():
|
||||
if hashlib.sha256((runtime / relative).read_bytes()).hexdigest() != expected:
|
||||
raise SystemExit('Library manifest hash mismatch: ' + relative)
|
||||
for relative, architectures in helpers.items():
|
||||
binary = runtime / relative
|
||||
info = subprocess.check_output(['/usr/bin/lipo', '-archs', str(binary)], text=True).split()
|
||||
if not set(architectures).issubset(info): raise SystemExit(f'Missing architecture: {relative}')
|
||||
links = subprocess.check_output(['/usr/bin/otool', '-L', str(binary)], text=True)
|
||||
for line in links.splitlines():
|
||||
if line.startswith('\t') and not line.strip().startswith(('/usr/lib/', '/System/Library/')):
|
||||
raise SystemExit(f'Non-system runtime dependency: {relative}: {line}')
|
||||
with tempfile.TemporaryDirectory(prefix='VCC runtime #$ quote" relocation ') as folder:
|
||||
moved = Path(folder) / 'VCC Plus.app/Contents/Resources/Toolchain'
|
||||
shutil.copytree(runtime, moved)
|
||||
project = Path(folder) / 'C project with spaces'; project.mkdir()
|
||||
(project / 'main.c').write_text('#include <cmoc.h>\n#define JOIN(a,b) a##b\n#define STRING(x) #x\nint main(void) { unsigned JOIN(co,unt)=7; printf(STRING(HELLO) " %u\\n", count); return 0; }\n')
|
||||
if disk:
|
||||
(project / 'disk.c').write_text('''#include <decbfile.h>
|
||||
DECBDrive drives[1]; DECBFile file; byte written[256], readback[256];
|
||||
int main(void) {
|
||||
initCoCoSupport();
|
||||
if (decb_init(drives, 1) || decb_registerDrive(0)) return 1;
|
||||
memset(written, 42, 256);
|
||||
if (decb_createSectorFile(&file, 0, "ROUND.DAT", DECB_TYPE_BASIC_DATA, DECB_FORMAT_BINARY)) return 2;
|
||||
if (decb_writeSector(&file, written, 0) || decb_setNumBytesUsedInLastSector(&file, 256) || decb_closeSectorFile(&file)) return 3;
|
||||
if (decb_openSectorFile(&file, 0, "ROUND.DAT") || decb_readSector(&file, readback, 0) || decb_closeSectorFile(&file)) return 4;
|
||||
if (decb_rename(0, "ROUND.DAT", "RESULT.DAT") || decb_kill(0, "RESULT.DAT") || decb_shutdown()) return 5;
|
||||
return memcmp(written, readback, 256) != 0;
|
||||
}
|
||||
''')
|
||||
def compile_disk(toolchain, tool_env):
|
||||
if disk:
|
||||
subprocess.run([str(toolchain / 'bin/cmoc'), '--coco', '--intermediate', '-O0',
|
||||
'-o', 'disk.bin', 'disk.c', '-ldecbfile'], cwd=project, env=tool_env, check=True)
|
||||
image = (project / 'disk.bin').read_bytes()
|
||||
if len(image) < 10 or image[0] != 0 or image[-5:-2] != b'\xff\0\0':
|
||||
raise SystemExit('Invalid linked disk example binary')
|
||||
if record.get('standaloneDisk'):
|
||||
(project / 'standalone.c').write_text('#include <dskcon-standalone.h>\nint main(void) { unsigned long saved = dskcon_init(dskcon_nmiService); DCOPC=2; DCDRV=0; DCTRK=17; DCSEC=3; DCBPT=(void *)0x500; dskcon_processSector(); dskcon_irqService(); dskcon_shutdown(saved); return DCSTA; }\n')
|
||||
subprocess.run([str(toolchain / 'bin/cmoc'), '--coco', '-o', 'standalone.bin',
|
||||
'standalone.c', '-ldskcon-standalone'], cwd=project, env=tool_env, check=True)
|
||||
(project / 'Makefile').write_text('all: main.bin\nmain.bin: main.c\n\tcmoc --coco --intermediate -O0 -o main.bin main.c\n')
|
||||
env = dict(os.environ, PATH=str(moved / 'bin') + ':/usr/bin:/bin',
|
||||
DEVELOPER_DIR='/VCC-validation-no-developer-tools', SDKROOT='/VCC-validation-no-sdk', LC_ALL='C')
|
||||
for key in ('PERL5LIB', 'PERLLIB', 'PERL5OPT'): env.pop(key, None)
|
||||
def check_perl(toolchain, tool_env):
|
||||
if not record.get('perl'): return
|
||||
result = subprocess.check_output([str(toolchain / 'bin/perl'), '-MGetopt::Long',
|
||||
'-MDigest::SHA', '-e', 'use strict; use warnings; print $^X, \"\\n\", Digest::SHA::sha256_hex(\"VCC\");'],
|
||||
cwd=project, env=tool_env, text=True)
|
||||
if str(toolchain) not in result or hashlib.sha256(b'VCC').hexdigest() not in result:
|
||||
raise SystemExit('Perl did not use the relocated bundled interpreter and modules')
|
||||
subprocess.run([str(toolchain / 'bin/install-coco-boot-loader'), '--help'],
|
||||
cwd=project, env=tool_env, check=True, stdout=subprocess.DEVNULL)
|
||||
def check_dependencies(toolchain, tool_env, name):
|
||||
dependency_project = Path(folder) / name
|
||||
dependency_project.mkdir()
|
||||
header = dependency_project / 'headers #$ spaces' / 'value.h'
|
||||
header.parent.mkdir()
|
||||
header.write_text('#define VALUE 42\n')
|
||||
(dependency_project / 'main.c').write_text('#include <cmoc.h>\n#include "headers #$ spaces/value.h"\nint main(void) { return VALUE; }\n')
|
||||
# Include before the .d exists: Make must generate/reparse it, not
|
||||
# merely tolerate a dependency file that has never been consumed.
|
||||
# --deps-only and --deps use separate CMOC marker readers.
|
||||
(dependency_project / 'child').mkdir()
|
||||
(dependency_project / 'child/Makefile').write_text('stamp:\n\tprintf child > stamp\n')
|
||||
(dependency_project / 'Makefile').write_text(
|
||||
'all: main.bin child/stamp\n'
|
||||
'child/stamp:\n\t$(MAKE) -C child\n'
|
||||
'main.bin: main.o\n\tcmoc --coco -o $@ $<\n'
|
||||
'main.o: main.c\n\tcmoc --coco --deps -c -o $@ $<\n'
|
||||
'main.d: main.c\n\tcmoc --coco --deps-only main.c\n'
|
||||
'include main.d\n')
|
||||
# Same recursion-safe override used by the app: Make's default MAKE
|
||||
# is its absolute executable path, unsafe unquoted inside recipes.
|
||||
command = [str(toolchain / 'bin/make'), 'MAKE=make', '--no-builtin-rules']
|
||||
def make(*options):
|
||||
subprocess.run(command + list(options), cwd=dependency_project, env=tool_env, check=True)
|
||||
make()
|
||||
make('-q')
|
||||
first = (dependency_project / 'main.bin').read_bytes()
|
||||
# A changed header must invalidate both the generated dependency
|
||||
# file and compiled object; no-op rebuilds must remain no-ops.
|
||||
header.write_text('#define VALUE 43\n')
|
||||
make()
|
||||
make('-q')
|
||||
if first == (dependency_project / 'main.bin').read_bytes():
|
||||
raise SystemExit('Changed dependency did not rebuild the executable')
|
||||
dependency_text = (dependency_project / 'main.d').read_text()
|
||||
if 'VCC\\ Plus.app' not in dependency_text and 'Toolchain' not in dependency_text:
|
||||
raise SystemExit('Dependency test did not exercise relocated SDK headers')
|
||||
if '\\#$$' not in dependency_text:
|
||||
raise SystemExit('Make dependency special characters were not escaped')
|
||||
check_disk_allocation(moved, Path(folder) / "Disk allocation")
|
||||
check_dependencies(moved, env, 'Dependency project')
|
||||
check_perl(moved, env)
|
||||
# A deliberately unusable developer directory catches accidental xcrun
|
||||
# or Apple compiler use even on the development machine.
|
||||
subprocess.run([str(moved / 'bin/make'), '--no-builtin-rules'], cwd=project, env=env, check=True)
|
||||
compile_disk(moved, env)
|
||||
for name in ('main.bin', 'main.s', 'main.lst', 'main.map', 'main.link'):
|
||||
if not (project / name).is_file(): raise SystemExit('Missing build product: ' + name)
|
||||
binary = (project / 'main.bin').read_bytes()
|
||||
if len(binary) < 10 or binary[0] != 0 or binary[-5:-2] != b'\xff\0\0':
|
||||
raise SystemExit('Invalid DECB binary')
|
||||
# Thin copies force the entire compiler/preprocessor/assembler/linker
|
||||
# chain onto each architecture, not just a --version smoke test.
|
||||
for architecture in record['architectures']:
|
||||
thin = Path(folder) / (architecture + ' Toolchain')
|
||||
shutil.copytree(runtime, thin)
|
||||
for relative, architectures in helpers.items():
|
||||
if architecture not in architectures: continue
|
||||
binary = thin / relative
|
||||
temporary = binary.with_name(binary.name + '.thin')
|
||||
slices = subprocess.check_output(['/usr/bin/lipo', '-archs', str(binary)], text=True).split()
|
||||
if len(slices) > 1:
|
||||
subprocess.run(['/usr/bin/lipo', str(binary), '-thin', architecture,
|
||||
'-output', str(temporary)], check=True)
|
||||
temporary.replace(binary)
|
||||
binary.chmod(0o755)
|
||||
subprocess.run(['/usr/bin/codesign', '--force', '--sign', '-', '--timestamp=none',
|
||||
str(binary)], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
arch_env = dict(env, PATH=str(thin / 'bin') + ':/usr/bin:/bin')
|
||||
subprocess.run([str(thin / 'bin/make'), '--no-builtin-rules', '-B'],
|
||||
cwd=project, env=arch_env, check=True)
|
||||
if {"ssc", "orch90", "vcccompat"}.issubset(record.get("libraries", {})):
|
||||
check_guest_example(thin, arch_env, Path(folder) / (architecture + " audio SDK"),
|
||||
"TalkingPlanetLander", [("LANDER.dsk", "LANDER.bin", "LANDER.BIN")])
|
||||
if record["libraries"]["vcccompat"].get("reentrantTokenizer"):
|
||||
check_guest_example(thin, arch_env, Path(folder) / (architecture + " streams SDK"),
|
||||
"PlanetSurvey", [("SURVEY.dsk", "SURVEY.bin", "SURVEY.BIN"),
|
||||
("SURVEY.dsk", "generated/PLANETS.TXT", "PLANETS.TXT")])
|
||||
if "sdc" in record.get("libraries", {}):
|
||||
check_guest_example(thin, arch_env, Path(folder) / (architecture + " SDC SDK"),
|
||||
"PlanetExpedition", [
|
||||
("EXPED/EXPLORE.DSK", "EXPLORE.BIN", "EXPLORE.BIN"),
|
||||
("EXPED/EARTH.DSK", "generated/earth/LANDING.TXT", "LANDING.TXT"),
|
||||
("EXPED/MARS.DSK", "generated/mars/LANDING.TXT", "LANDING.TXT")])
|
||||
libraries = record.get("libraries", {})
|
||||
if (record.get("decbfile") and {"bank", "gime", "input", "bcontrol", "hirestxt"} <= libraries.keys()
|
||||
and libraries.get("orch90", {}).get("backgroundPlayback")
|
||||
and libraries.get("ssc", {}).get("storedSpeechAndEffects")
|
||||
and libraries.get("vcccompat", {}).get("diskFileManagement")):
|
||||
check_guest_example(thin, arch_env, Path(folder) / (architecture + " mission SDK"),
|
||||
"MissionRecorder", [("RECORDER.dsk", "RECORDER.bin", "RECORDER.BIN")])
|
||||
manual_examples = (
|
||||
("vcccompat", "diskWriteStreams", "fieldlog.c", ["decbfile"]),
|
||||
("vcccompat", "integerConversions", "atlas.c", ["decbfile"]),
|
||||
("vcccompat", "diskStreamClearError", "returntrip.c", ["decbfile"]),
|
||||
("vcccompat", "printerFormattedOutput", "flightreport.c", []),
|
||||
("vcccompat", "cassetteTextFiles", "flightrecorder.c", []),
|
||||
("vcccompat", "diskFileManagement", "atlasbackup.c", ["decbfile"]),
|
||||
("vcccompat", "fixedDecimal", "descent.c", []),
|
||||
("orch90", "backgroundPlayback", "flyby.c", ["input", "bcontrol"]),
|
||||
("ssc", "storedSpeechAndEffects", "announcer.c", ["input", "bcontrol"]),
|
||||
)
|
||||
for library, feature, source, dependencies in manual_examples:
|
||||
if record.get("libraries", {}).get(library, {}).get(feature):
|
||||
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual" / source
|
||||
shutil.copyfile(example, project / source)
|
||||
link_flags = ["-l" + name for name in [library, *dependencies]]
|
||||
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o",
|
||||
example.stem + ".bin", source, *link_flags], cwd=project,
|
||||
env=dict(arch_env, VCC_CMOC_COMPAT="1" if library == "vcccompat" else "0"), check=True)
|
||||
if record.get("libraries", {}).get("vcccompat", {}).get("boundedHeap"):
|
||||
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual/crew.c"
|
||||
shutil.copyfile(example, project / "crew.c")
|
||||
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o", "crew.bin",
|
||||
"crew.c", "-lvcccompat", "-lwydealloc"],
|
||||
cwd=project, env=dict(arch_env, VCC_CMOC_COMPAT="1"), check=True)
|
||||
if "gime" in record.get("libraries", {}):
|
||||
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual/lander.c"
|
||||
shutil.copyfile(example, project / "lander.c")
|
||||
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o", "lander.bin",
|
||||
"lander.c", "-lgime", "-lbgraph", "-lbcontrol"],
|
||||
cwd=project, env=arch_env, check=True)
|
||||
if "input" in record.get("libraries", {}):
|
||||
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual/probe.c"
|
||||
shutil.copyfile(example, project / "probe.c")
|
||||
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o", "probe.bin",
|
||||
"probe.c", "-lgime", "-linput", "-lbcontrol"],
|
||||
cwd=project, env=arch_env, check=True)
|
||||
if "bank" in record.get("libraries", {}):
|
||||
example = Path(__file__).resolve().parents[2] / "examples/cmoc/manual/ridge.c"
|
||||
shutil.copyfile(example, project / "ridge.c")
|
||||
subprocess.run([thin / "bin/cmoc", "--coco", "-O2", "-Werror", "-o", "ridge.bin",
|
||||
"ridge.c", "-lbank", "-lgime", "-lhirestxt"],
|
||||
cwd=project, env=arch_env, check=True)
|
||||
compile_disk(thin, arch_env)
|
||||
check_perl(thin, arch_env)
|
||||
check_assembly(thin, Path(folder) / (architecture + " assembly"))
|
||||
check_disk_allocation(thin, Path(folder) / (architecture + " disk allocation"))
|
||||
check_dependencies(thin, arch_env, architecture + " dependency project")
|
||||
# Preprocessor errors must stop Make and retain source-line diagnostics.
|
||||
(project / 'main.c').write_text('#line 81 "mapped.c"\n#error VCC_EXPECTED_PREPROCESSOR_FAILURE\n')
|
||||
failure = subprocess.run([str(moved / 'bin/make'), '-B'], cwd=project, env=env,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
if failure.returncode == 0 or 'VCC_EXPECTED_PREPROCESSOR_FAILURE' not in failure.stdout or '81' not in failure.stdout:
|
||||
raise SystemExit('Preprocessor errors/line directives were not preserved: ' + failure.stdout)
|
||||
print('PASS: architectures, system-only libraries, relocated space-containing bundle, both complete architecture build chains, preprocessor errors, Make dependency regeneration with spaces/quotes/#/$, optional ROM-backed/standalone disk API linking, bundled Perl/core modules, disk allocation and BIN without developer tools')
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Verify the checked-in source snapshots against their reviewed import manifests."""
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
def main():
|
||||
total = 0
|
||||
for manifest in sorted((ROOT / 'third_party/manifests').glob('*.json')):
|
||||
record = json.loads(manifest.read_text())
|
||||
directory = ROOT / record['directory']
|
||||
actual = {p.relative_to(directory).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in directory.rglob('*') if p.is_file()}
|
||||
expected = record['vendoredFileSHA256']
|
||||
if actual != expected:
|
||||
changed = sorted(k for k in actual.keys() | expected.keys() if actual.get(k) != expected.get(k))
|
||||
raise SystemExit(f"{record['project']}: unexpected files or content: {changed}")
|
||||
if any((directory / p).exists() for p in record['excludedFiles']):
|
||||
raise SystemExit(f"{record['project']}: excluded source was reintroduced")
|
||||
print(f"PASS {record['project']} {record['version']}: {len(actual)} files")
|
||||
total += len(actual)
|
||||
print(f'PASS: {total} reviewed source files; no downloaded binaries or ROMs required')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user