Publish VCC Plus OSS source distribution for 3.6.2 build 60
This commit is contained in:
commit
c652a5276d
1192 files changed
+272368
No files matched your search
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Reconstruct pinned upstream firmware from verified assembly-only inputs.
|
||||
|
||||
Preparation is explicit and offline. Fetch cocoroms/LWTools separately, or supply
|
||||
an existing LWASM. Builds use fresh copies and verify all generated ROM hashes.
|
||||
Only the two boot ROMs, provenance manifest and notice enter an optional bundle.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SPEC = importlib.util.spec_from_file_location('vcc_dependency_fetch', ROOT / 'tools/dependencies/fetch.py')
|
||||
DEPENDENCIES = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(DEPENDENCIES)
|
||||
ROM_NAMES = ('bas13.rom', 'extbas11.rom', 'disk11.rom', 'coco3.rom')
|
||||
BUNDLE_NAMES = ('coco3.rom', 'disk11.rom')
|
||||
SOURCE_NAMES = {'Makefile', 'equates.asm', 'bas.asm', 'extbas.asm', 'disk.asm', 'coco3.asm', 'md5sums.txt', 'mugateerspic.asm'}
|
||||
NOTICE = '''Reconstructed upstream Color Computer firmware
|
||||
|
||||
Source: ToolShed cocoroms, https://github.com/nitros9project/toolshed
|
||||
Pinned revision: 3667a41c9605b7fcb150eace29cc7f42d460be1a
|
||||
|
||||
The original assembly headers identify material copied from Color BASIC
|
||||
Unravelled, Extended Color BASIC Unravelled, Disk Color BASIC Unravelled and
|
||||
Super Color BASIC Unravelled, adjusted for assembly. Their original source
|
||||
comments are retained in the explicitly prepared source dependency.
|
||||
|
||||
Original firmware notices retained in the generated ROM data include:
|
||||
Color BASIC: (C) 1982 TANDY; MICROSOFT
|
||||
Extended Color BASIC: COPYRIGHT (C) 1982 BY TANDY;
|
||||
UNDER LICENSE FROM MICROSOFT
|
||||
Disk Extended Color BASIC: COPYRIGHT (C) 1982 BY TANDY;
|
||||
UNDER LICENSE FROM MICROSOFT
|
||||
Super Color BASIC: COPR. 1982, 1986 BY TANDY;
|
||||
UNDER LICENSE FROM MICROSOFT; AND MICROWARE SYSTEMS CORP.
|
||||
|
||||
These are reconstructed upstream firmware, not newly authored or clean-room
|
||||
replacement ROMs. The emulator's GPL notice does not relabel this firmware.
|
||||
No preassembled ROM or binary input is used by this reconstruction process.
|
||||
'''
|
||||
|
||||
|
||||
def encoded(value):
|
||||
return (json.dumps(value, sort_keys=True, indent=2) + '\n').encode('utf-8')
|
||||
|
||||
|
||||
def sha(path):
|
||||
return DEPENDENCIES.sha(path)
|
||||
|
||||
|
||||
def source_record(records):
|
||||
record = records['cocoroms']
|
||||
if set(record['vendoredFileSHA256']) != SOURCE_NAMES or set(record['upstreamFileSHA256']) != SOURCE_NAMES:
|
||||
raise ValueError('Firmware input inventory must contain exactly the eight assembly/build reference files')
|
||||
if (record.get('sourceRevision') != '3667a41c9605b7fcb150eace29cc7f42d460be1a'
|
||||
or record.get('archiveRootPrefix') != 'cocoroms/' or record.get('optional') is not True):
|
||||
raise ValueError('Unsupported firmware source identity')
|
||||
outputs = record.get('romOutputs')
|
||||
if not isinstance(outputs, dict) or set(outputs) != set(ROM_NAMES):
|
||||
raise ValueError('Firmware output inventory must contain the four qualified ROMs')
|
||||
sizes = {'coco3.rom': 32768, 'disk11.rom': 8192, 'bas13.rom': 8192, 'extbas11.rom': 8192}
|
||||
for name, spec in outputs.items():
|
||||
if (not isinstance(spec, dict) or spec.get('bytes') != sizes[name]
|
||||
or not isinstance(spec.get('sha256'), str) or not DEPENDENCIES.HASH.fullmatch(spec['sha256'])):
|
||||
raise ValueError('Invalid generated firmware fingerprint: ' + name)
|
||||
return record
|
||||
|
||||
|
||||
def copy_verified(source, destination, fingerprints):
|
||||
DEPENDENCIES.verify_tree(source, fingerprints)
|
||||
destination.mkdir(parents=True, exist_ok=False)
|
||||
for name in sorted(fingerprints):
|
||||
original = source / name
|
||||
target = destination / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(original, target)
|
||||
target.chmod(0o755 if original.stat().st_mode & 0o111 else 0o644)
|
||||
DEPENDENCIES.verify_tree(destination, fingerprints)
|
||||
|
||||
|
||||
def validate_outputs(directory, expected):
|
||||
if set(expected) != set(ROM_NAMES):
|
||||
raise ValueError('Expected all four reconstructed ROM outputs')
|
||||
results = {}
|
||||
for name in ROM_NAMES:
|
||||
path = directory / name
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_nlink != 1:
|
||||
raise ValueError('Generated ROM must be an ordinary unlinked file: ' + name)
|
||||
result = {'bytes': path.stat().st_size, 'sha256': sha(path)}
|
||||
if result != expected[name]:
|
||||
raise ValueError('Generated ROM does not match the qualified fingerprint: ' + name)
|
||||
results[name] = result
|
||||
return results
|
||||
|
||||
|
||||
def bundle_manifest(record, metadata, assembler, outputs):
|
||||
return {
|
||||
'schemaVersion': 1,
|
||||
'generator': 'vccplus-rom-build',
|
||||
'source': {
|
||||
'project': 'ToolShed cocoroms', 'url': record['sourceURL'],
|
||||
'revision': record['sourceRevision'], 'manifestSHA256': sha(metadata),
|
||||
'filesSHA256': record['vendoredFileSHA256'], 'preassembledROMInputs': False,
|
||||
},
|
||||
'assembler': assembler,
|
||||
'roms': {name: outputs[name] for name in BUNDLE_NAMES},
|
||||
}
|
||||
|
||||
|
||||
def write_bundle(destination, source, manifest):
|
||||
"""Atomically create a fresh optional payload; never replace a prior bundle."""
|
||||
destination = Path(destination).absolute()
|
||||
DEPENDENCIES.require_unlinked_ancestors(destination, 'ROM bundle destination')
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise ValueError('Choose a fresh ROM bundle destination; existing payloads are never overwritten')
|
||||
for name in BUNDLE_NAMES:
|
||||
path = source / name
|
||||
spec = manifest['roms'][name]
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_nlink != 1 or path.stat().st_size != spec['bytes'] or sha(path) != spec['sha256']:
|
||||
raise ValueError('ROM output changed before optional bundle staging')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix='.rom-bundle-', dir=destination.parent) as temporary:
|
||||
staged = Path(temporary) / 'bundle'
|
||||
staged.mkdir()
|
||||
for name in BUNDLE_NAMES:
|
||||
shutil.copyfile(source / name, staged / name)
|
||||
if (staged / name).stat().st_size != manifest['roms'][name]['bytes'] or sha(staged / name) != manifest['roms'][name]['sha256']:
|
||||
raise ValueError('ROM output changed while copying the optional bundle')
|
||||
(staged / 'ROMBuild.json').write_bytes(encoded(manifest))
|
||||
(staged / 'ROM_SOURCE_NOTICE.txt').write_text(NOTICE, encoding='utf-8')
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise ValueError('ROM bundle destination changed during preparation')
|
||||
staged.rename(destination)
|
||||
|
||||
|
||||
def build(output, dependency_root, lwasm=None, bundle_output=None, jobs=4):
|
||||
output, dependency_root = Path(output).absolute(), Path(dependency_root).absolute()
|
||||
DEPENDENCIES.require_unlinked_ancestors(output, 'Firmware build output')
|
||||
DEPENDENCIES.require_unlinked_ancestors(dependency_root, 'Firmware dependency root')
|
||||
if output.exists() or output.is_symlink():
|
||||
raise ValueError('Choose a fresh build output; previous evidence is never overwritten')
|
||||
if output == ROOT or output in ROOT.parents or output == dependency_root or dependency_root in output.parents:
|
||||
raise ValueError('Choose build output separate from the source/dependency tree')
|
||||
records, locations = DEPENDENCIES.load_records(ROOT)
|
||||
record = source_record(records)
|
||||
source = dependency_root / DEPENDENCIES.validate_record(record)
|
||||
DEPENDENCIES.verify_tree(source, record['vendoredFileSHA256'])
|
||||
if lwasm is None:
|
||||
lwrecord = records['lwtools']
|
||||
lwsource = dependency_root / DEPENDENCIES.validate_record(lwrecord)
|
||||
DEPENDENCIES.verify_tree(lwsource, lwrecord['vendoredFileSHA256'])
|
||||
else:
|
||||
lwasm = Path(lwasm).resolve()
|
||||
if not lwasm.is_file() or not os.access(lwasm, os.X_OK):
|
||||
raise ValueError('The explicitly selected LWASM is unavailable')
|
||||
if bundle_output is not None:
|
||||
bundle = Path(bundle_output).absolute()
|
||||
DEPENDENCIES.require_unlinked_ancestors(bundle, 'ROM bundle destination')
|
||||
# An optional package must not alter verified dependency inputs. Reject
|
||||
# a parent of the build too: creating the build would occupy that bundle
|
||||
# destination before its final atomic installation.
|
||||
canonical_bundle, canonical_dependencies, canonical_output = bundle.resolve(), dependency_root.resolve(), output.resolve()
|
||||
if (canonical_bundle == canonical_dependencies or canonical_dependencies in canonical_bundle.parents
|
||||
or canonical_bundle == canonical_output or canonical_bundle in canonical_output.parents):
|
||||
raise ValueError('ROM bundle must be separate from dependency inputs and the build ancestor path')
|
||||
if bundle.exists() or bundle.is_symlink():
|
||||
raise ValueError('Choose a fresh ROM bundle destination')
|
||||
output.mkdir(parents=True)
|
||||
report = {'schemaVersion': 1, 'status': 'running', 'networkUsed': False,
|
||||
'sourceRevision': record['sourceRevision'], 'preassembledROMInputs': False, 'commands': []}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with (output / 'build.log').open('wb') as log:
|
||||
def run(command, cwd, stdout=None):
|
||||
report['commands'].append({'argv': [str(p) for p in command], 'cwd': str(cwd)})
|
||||
log.write(('$ ' + ' '.join(str(p) for p in command) + '\n').encode())
|
||||
log.flush()
|
||||
subprocess.run(command, cwd=cwd, check=True, stdout=log if stdout is None else stdout, stderr=log)
|
||||
if lwasm is None:
|
||||
lwcopy = output / 'work/lwtools'
|
||||
copy_verified(lwsource, lwcopy, lwrecord['vendoredFileSHA256'])
|
||||
run(['/usr/bin/make', '-j' + str(jobs), 'lwasm'], lwcopy)
|
||||
lwasm = lwcopy / 'lwasm/lwasm'
|
||||
assembler = {'version': subprocess.check_output([str(lwasm), '--version'], stderr=subprocess.STDOUT).decode().strip(),
|
||||
'sha256': sha(lwasm)}
|
||||
report['assembler'] = assembler
|
||||
work = output / 'work/cocoroms'
|
||||
copy_verified(source, work, record['vendoredFileSHA256'])
|
||||
if any(p.suffix.lower() in ('.rom', '.bin') for p in work.rglob('*')):
|
||||
raise ValueError('Preassembled firmware input is forbidden')
|
||||
# Execute the pinned Makefile's assembly sequence directly. A fresh
|
||||
# directory forces every output and --lwasm selects the exact tool,
|
||||
# independent of executable names or the user's PATH.
|
||||
run([str(lwasm), '--raw', '-o', 'bas13.rom', 'equates.asm', 'bas.asm'], work)
|
||||
with (work / 'cb_equates.asm').open('wb') as symbols:
|
||||
run([str(lwasm), '--symbol-dump', '--no-output', 'equates.asm', 'bas.asm'], work, symbols)
|
||||
run([str(lwasm), '--raw', '-o', 'extbas11.rom', 'cb_equates.asm', 'extbas.asm'], work)
|
||||
with (work / 'ecb_equates.asm').open('wb') as symbols:
|
||||
run([str(lwasm), '--symbol-dump', '--no-output', 'equates.asm', 'bas.asm', 'extbas.asm'], work, symbols)
|
||||
run([str(lwasm), '--raw', '--pragma=noindex0tonone', '-o', 'disk11.rom', 'ecb_equates.asm', 'disk.asm'], work)
|
||||
run([str(lwasm), '--raw', '-o', 'coco3.rom', 'equates.asm', 'coco3.asm'], work)
|
||||
outputs = validate_outputs(work, record['romOutputs'])
|
||||
for name in ROM_NAMES:
|
||||
shutil.copyfile(work / name, output / name)
|
||||
manifest = bundle_manifest(record, locations['cocoroms'] / 'cocoroms.json', assembler, outputs)
|
||||
(output / 'ROMBuild.json').write_bytes(encoded(manifest))
|
||||
(output / 'ROM_SOURCE_NOTICE.txt').write_text(NOTICE, encoding='utf-8')
|
||||
if bundle_output is not None:
|
||||
write_bundle(Path(bundle_output), output, manifest)
|
||||
report.update(status='passed', roms=outputs)
|
||||
except Exception as error:
|
||||
report.update(status='failed', failure=str(error))
|
||||
raise
|
||||
finally:
|
||||
report['elapsedSeconds'] = round(time.monotonic() - started, 3)
|
||||
(output / 'BuildReport.json').write_bytes(encoded(report))
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, default=ROOT / '.build/roms')
|
||||
parser.add_argument('--dependency-root', type=Path, default=ROOT / 'third_party')
|
||||
parser.add_argument('--lwasm', type=Path, help='Explicit existing assembler; otherwise build verified LWTools lwasm only')
|
||||
parser.add_argument('--bundle-output', type=Path, help='Fresh optional SourceROMBundle destination')
|
||||
parser.add_argument('--jobs', type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.jobs <= 32:
|
||||
parser.error('--jobs must be between 1 and 32')
|
||||
result = build(args.output, args.dependency_root, args.lwasm, args.bundle_output, args.jobs)
|
||||
print(json.dumps({'status': result['status'], 'roms': result['roms'], 'networkUsed': False}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (ValueError, OSError, subprocess.CalledProcessError) as error:
|
||||
raise SystemExit(str(error))
|
||||
Reference in new issue
Block a user