Harden release packaging and recoverable installation after review

This commit is contained in:
bholman committed 2026-11-18 16:26:25 -07:00
1 parent cb40fed3b9
commit 8088c1fbf6
14 files changed
+933 -101

No files matched your search

+238
View File
@@ -0,0 +1,238 @@
"""Check launcher-last installation and recoverable interruption on a mock target.
Copyright (c) 2026 Brian K. Holman. BSD-3-Clause; see LICENSES/.
The target is a disposable local filesystem, never an SSH/device connection.
"""
import json
import os
from pathlib import Path, PurePosixPath
import shutil
import sys
import tempfile
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tools'))
import install_mister as installer
class DisposableTarget:
def __init__(self, root, package, *, interrupt_on=None, fail_preflight=False):
self.root = root
self.package = package
self.interrupt_on = interrupt_on
self.fail_preflight = fail_preflight
self.events = []
def local(self, remote):
relative = PurePosixPath(remote).relative_to('/media/fat')
return self.root.joinpath(*relative.parts)
def preflight(self):
self.events.append('preflight')
if self.fail_preflight:
raise OSError('Deliberate mock preflight failure')
return 'Disposable target only\n'
def prepare(self, stage, backup):
# The recovery plan must already be durable before this first target write.
receipt = json.loads((self.package / 'install-receipt.json').read_text())
assert receipt['stage'] == stage and receipt['backup'] == backup
assert receipt['phase'] == 'staging'
assert Path(receipt['receipt']).is_file()
self.events.append('prepare')
self.local(stage).mkdir(parents=True)
self.local(backup).mkdir(parents=True)
def upload(self, fat, stage):
self.events.append('upload')
shutil.copytree(fat, self.local(stage) / 'fat')
def verify(self, files):
self.events.append('verify')
for name, digest in files.items():
assert installer.sha(self.local(name)) == digest, name
def enter_menu(self):
self.events.append('menu')
def install_file(self, source, destination, backup, entry, temporary):
if self.interrupt_on and destination.endswith('/' + self.interrupt_on):
self.events.append('interrupted')
raise KeyboardInterrupt('Deliberate interruption before launcher activation')
target = self.local(destination)
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists():
if entry['policy'] == 'create-if-missing':
self.events.append('preserved:' + destination)
return 'preserved'
old = self.local(backup)
old.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(target, old)
pending = self.local(temporary)
assert not pending.exists()
shutil.copyfile(self.local(source), pending)
assert installer.sha(pending) == entry['sha256']
os.replace(pending, target)
assert installer.sha(target) == entry['sha256']
self.events.append('installed:' + destination)
return 'installed'
def activate_launcher(self, source, destination, backup, entry, temporary):
self.events.append('activate')
return self.install_file(source, destination, backup, entry, temporary)
def sync(self):
self.events.append('sync')
def load(self):
self.events.append('load')
class InstallerTransactionTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(dir=ROOT / 'build')
self.addCleanup(self.temporary.cleanup)
self.base = Path(self.temporary.name)
self.package = self.base / 'package'
self.device = self.base / 'device'
self.package.mkdir()
self.device.mkdir()
self.launcher = installer.LAUNCHER
self.boot0 = 'games/coco3plus/boot0.rom'
self.boot1 = 'games/coco3plus/boot1.rom'
self.config = 'config/coco3plus_v51.CFG'
self.rbf = '_Computer/.coco3plus/fixture/core.rbf'
contents = {
# Deliberately put the launcher first: manifest ordering cannot activate it early.
self.launcher: b'new launcher fixture\n',
self.boot0: b'new boot0 fixture\n',
self.config: b'new default configuration\n',
self.boot1: b'new boot1 fixture\n',
self.rbf: b'new bitstream fixture\n',
}
entries = {}
for name, data in contents.items():
path = self.package / 'fat' / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
entries[name] = dict(
bytes=len(data), sha256=installer.sha(path),
policy='create-if-missing' if name == self.config else 'backup-and-replace')
self.manifest = dict(files=entries, rbf_sha256=entries[self.rbf]['sha256'])
(self.package / 'manifest.json').write_text(json.dumps(self.manifest))
for name, data in ((self.launcher, b'old launcher remains selectable\n'),
(self.boot0, b'old boot0 recoverable\n'),
(self.config, b'owner saved configuration\n')):
path = self.device / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
def install(self, target, stamp='fixture-attempt', load=False):
return installer.install_release(
self.package, 'disposable.invalid', load=load, target=target, stamp=stamp)
def test_support_interruption_keeps_old_launcher_and_recovery_evidence(self):
target = DisposableTarget(self.device, self.package, interrupt_on=self.boot1)
original_launcher = (self.device / self.launcher).read_bytes()
original_boot0 = (self.device / self.boot0).read_bytes()
with self.assertRaises(KeyboardInterrupt):
self.install(target, load=True)
receipt = json.loads((self.package / 'install-receipt.json').read_text())
self.assertEqual(receipt['status'], 'failed')
self.assertEqual(receipt['phase'], 'supporting-assets')
self.assertEqual(receipt['active_file'], self.boot1)
self.assertIn(self.launcher, receipt['planned_files'])
self.assertFalse(receipt['activation_attempted'])
self.assertIs(receipt['launcher_activated'], False)
self.assertFalse(receipt['loaded'])
self.assertEqual(receipt['error']['type'], 'KeyboardInterrupt')
self.assertEqual((self.device / self.launcher).read_bytes(), original_launcher)
self.assertNotIn('activate', target.events)
self.assertNotIn('load', target.events)
self.assertEqual((self.device / self.boot0).read_bytes(),
(self.package / 'fat' / self.boot0).read_bytes())
self.assertEqual(target.local(receipt['backup'] + '/' + self.boot0).read_bytes(),
original_boot0)
self.assertEqual(target.local(receipt['stage'] + '/fat/' + self.launcher).read_bytes(),
(self.package / 'fat' / self.launcher).read_bytes())
self.assertTrue(Path(receipt['receipt']).is_file())
self.assertIn(self.boot0, {item['path'] for item in receipt['completed_files']})
def test_success_installs_support_then_atomically_activates_and_preserves_config(self):
target = DisposableTarget(self.device, self.package)
original_launcher = (self.device / self.launcher).read_bytes()
original_config = (self.device / self.config).read_bytes()
receipt = self.install(target, load=True)
self.assertEqual(receipt['status'], 'complete')
self.assertTrue(receipt['launcher_activated'])
self.assertTrue(receipt['loaded'])
self.assertEqual((self.device / self.config).read_bytes(), original_config)
self.assertEqual((self.device / self.launcher).read_bytes(),
(self.package / 'fat' / self.launcher).read_bytes())
self.assertEqual(target.local(receipt['backup'] + '/' + self.launcher).read_bytes(),
original_launcher)
activation = target.events.index('activate')
for i, event in enumerate(target.events):
if event.startswith('installed:') and not event.endswith('/' + self.launcher):
self.assertLess(i, activation)
self.assertEqual(target.events[activation - 1], 'sync')
self.assertGreater(target.events.index('load'), activation)
self.assertFalse(target.local(receipt['launcher_temporary']).exists())
self.assertTrue(target.local(receipt['stage'] + '/fat/' + self.launcher).exists())
def test_preflight_failure_keeps_plan_and_does_not_mutate_target(self):
target = DisposableTarget(self.device, self.package, fail_preflight=True)
before = {p.relative_to(self.device): p.read_bytes()
for p in self.device.rglob('*') if p.is_file()}
with self.assertRaisesRegex(OSError, 'mock preflight failure'):
self.install(target)
receipt = json.loads((self.package / 'install-receipt.json').read_text())
self.assertEqual(receipt['status'], 'failed')
self.assertEqual(receipt['phase'], 'preflight')
self.assertEqual(target.events, ['preflight'])
self.assertFalse(target.local(receipt['stage']).exists())
self.assertEqual(before, {p.relative_to(self.device): p.read_bytes()
for p in self.device.rglob('*') if p.is_file()})
def test_failed_retry_preserves_prior_complete_receipt(self):
first = self.install(DisposableTarget(self.device, self.package), stamp='first')
original_receipt = Path(first['receipt']).read_bytes()
with self.assertRaises(OSError):
self.install(DisposableTarget(self.device, self.package, fail_preflight=True),
stamp='second')
failed = json.loads((self.package / 'install-receipt.json').read_text())
self.assertEqual(failed['status'], 'failed')
self.assertEqual(Path(first['receipt']).read_bytes(), original_receipt)
self.assertEqual(Path(failed['previous_receipt']).read_bytes(), original_receipt)
def test_remote_launcher_primitive_verifies_temporary_file_before_rename(self):
target = installer.OpenSshTarget('disposable.invalid', 'root')
source = '/media/fat/.coco3plus/stage/' + self.launcher
destination = '/media/fat/' + self.launcher
temporary = destination + '.fixture.tmp'
backup = '/media/fat/.coco3plus/backup/' + self.launcher
with patch.object(target, 'run', return_value='installed\n') as run:
result = target.activate_launcher(
source, destination, backup, self.manifest['files'][self.launcher], temporary)
self.assertEqual(result, 'installed')
script = run.call_args.args[0]
copy = 'cp ' + installer.q(source) + ' ' + installer.q(temporary)
check = 'sha256sum ' + installer.q(temporary)
rename = 'mv -f ' + installer.q(temporary) + ' ' + installer.q(destination)
self.assertLess(script.index(copy), script.index(check))
self.assertLess(script.index(check), script.index(rename))
self.assertNotIn('cp ' + installer.q(source) + ' ' + installer.q(destination) + '\n', script)
if __name__ == '__main__':
unittest.main()
+54 -7
View File
@@ -1,10 +1,20 @@
"""Check release integrity boundaries against a real completed local build."""
import json,sys,tempfile,unittest,zipfile
import hashlib,json,os,sys,tempfile,unittest,zipfile
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tools'))
from release import default_config,make_release,SSC
from release import (default_config,make_release,qualify_inputs,committed_tree,
committed_blob,source_member,SSC)
def selected_path(variable,default):
path=Path(os.environ.get(variable,str(default))).expanduser()
return (path if path.is_absolute() else ROOT/path).resolve()
BUILD=selected_path('COCO3PLUS_TEST_BUILD',ROOT/'build/fpga')
RELEASE=selected_path('COCO3PLUS_TEST_RELEASE',ROOT/'dist/coco3plus')
def digest(data):return hashlib.sha256(data).hexdigest()
class ReleaseGuards(unittest.TestCase):
def require_files(self,*paths):
missing=[str(p) for p in paths if not p.is_file()]
if missing:self.skipTest('Actual local dependency unavailable: '+', '.join(missing))
def test_fresh_profile_preserves_status_contract(self):
data=default_config(True);self.assertEqual(len(data),16)
value=int.from_bytes(data,'little')
@@ -13,6 +23,16 @@ class ReleaseGuards(unittest.TestCase):
self.assertEqual([(value>>(43+3*i))&7 for i in range(4)],[6,3,1,2])
self.assertEqual(value^int.from_bytes(default_config(False),'little'),1<<42)
def test_changed_ssc_fails_before_creating_output(self):
# A preserved historical fit is not automatically a release-qualified
# receipt. Do not modify it merely to reach a later ROM-input guard.
self.require_files(BUILD/'build.json',BUILD/'project/output_files/CoCo3.rbf')
record=json.loads((BUILD/'build.json').read_text())
if 'project_inventory_sha256' not in record:
with self.assertRaisesRegex(RuntimeError,'Legacy FPGA receipt lacks'):
qualify_inputs(record)
self.skipTest('SSC packaging test requires a new inventory-qualified fit; legacy evidence preserved')
self.require_files(*(ROOT/'roms/ssc'/name for name in SSC),
ROOT/'build/roms/boot0.rom',ROOT/'build/roms/boot1.rom')
with tempfile.TemporaryDirectory(dir=ROOT/'build') as folder:
base=Path(folder);roms=base/'roms';roms.mkdir()
for name in SSC:
@@ -21,13 +41,40 @@ class ReleaseGuards(unittest.TestCase):
chip=roms/'pic-7040-510.bin';data=bytearray(chip.read_bytes());data[-1]^=1;chip.write_bytes(data)
out=base/'release'
with self.assertRaisesRegex(RuntimeError,'incorrect input'):
make_release(out,ROOT/'build/fpga',ROOT/'build/roms',roms)
make_release(out,BUILD,ROOT/'build/roms',roms)
self.assertFalse(out.exists())
def test_corresponding_source_contains_no_local_roms(self):
release=ROOT/'dist/coco3plus'
with zipfile.ZipFile(release/'corresponding-source.zip') as archive:
names=archive.namelist();self.assertTrue(any(n.endswith('/rtl/coco3fpga.sv') for n in names))
self.assertFalse(any('/roms/ssc/' in n or '/.deps/' in n or '/build/' in n for n in names))
release=RELEASE
outer=release.with_name(release.name+'.zip')
self.require_files(release/'manifest.json',release/'corresponding-source.zip',outer)
record=json.loads((release/'manifest.json').read_text())
self.assertIn('source_files',record,'Select a newly qualified package; historical package predates source inventory')
tree=committed_tree(record['source_revision'])
expected=sorted(name for name in tree if source_member(name))
self.assertEqual(record['source_files'],expected)
self.assertEqual(record['corresponding_source_sha256'],digest((release/'corresponding-source.zip').read_bytes()))
greg='LICENSES/Greg-Miller-BSD-3-Clause.txt'
self.assertIn(greg,tree,'Selected release revision lacks the committed Greg Miller notice')
notice=committed_blob(tree[greg])
with zipfile.ZipFile(release/'corresponding-source.zip') as archive:
names=archive.namelist();self.assertEqual(set(names),{'coco3plus/'+n for n in expected})
self.assertTrue(any(n.endswith('/rtl/coco3fpga.sv') for n in names))
self.assertFalse(any('/roms/ssc/' in n or '/.deps/' in n or '/build/' in n for n in names))
self.assertFalse(any(n.endswith(('.rbf','.pdf','.docx','.xlsx','.bin','.rom')) for n in names))
self.assertEqual(archive.read('coco3plus/'+greg),notice)
notice_key='games/coco3plus/licenses/Greg-Miller-BSD-3-Clause.txt'
self.assertIn(notice_key,record['files'])
self.assertEqual((release/'fat'/notice_key).read_bytes(),notice)
with zipfile.ZipFile(outer) as archive:
self.assertEqual(set(archive.namelist()),
{p.relative_to(release).as_posix() for p in release.rglob('*') if p.is_file()})
self.assertEqual(archive.read('fat/'+notice_key),notice)
self.assertEqual(archive.read('corresponding-source.zip'),(release/'corresponding-source.zip').read_bytes())
self.assertIn('git checkout --detach '+record['source_revision'],archive.read('SOURCE-CHECKOUT.txt').decode())
self.assertEqual(json.loads(archive.read('manifest.json')),record)
for name,identity in record['files'].items():
data=archive.read('fat/'+name)
self.assertEqual(len(data),identity['bytes'],name)
self.assertEqual(digest(data),identity['sha256'],name)
self.assertEqual(record['files']['config/coco3plus_v51.CFG']['policy'],'create-if-missing')
if __name__=='__main__':unittest.main()
+60 -1
View File
@@ -118,6 +118,13 @@ class PublicSourceReleaseGuards(unittest.TestCase):
'LICENSES/Fixture.txt': b'Fixture license text.\n',
'rtl/ssc/LICENSE': b'Fixture SSC notice only; no ROM contents.\n',
'rtl/example.sv': b'module example; endmodule\n',
'tools/project_inputs.json': json.dumps({'files': [
'.gitignore', 'README.md', 'changes.txt', 'rtl/example.sv'
]}).encode(),
'releases/old.rbf': b'Historic binary fixture; not source payload',
'docs/old.pdf': b'Historic PDF fixture; not source payload',
'docs/old.docx': b'Historic Office fixture; not source payload',
'unused.bin': b'Historic binary fixture; not source payload',
}
for name, data in files.items():
path = self.root / name
@@ -140,6 +147,7 @@ class PublicSourceReleaseGuards(unittest.TestCase):
'passed': True,
'rbf_bytes': rbf.stat().st_size,
'rbf_sha256': digest(rbf),
'project_inventory_sha256': digest(self.root / 'tools/project_inputs.json'),
'inputs': {'rtl/example.sv': digest(self.root / 'rtl/example.sv')},
}
(self.build / 'build.json').write_text(json.dumps(record))
@@ -163,6 +171,7 @@ class PublicSourceReleaseGuards(unittest.TestCase):
def test_untracked_private_files_do_not_enter_public_release(self):
(self.root / '.env').write_text('fixture-private-value=not-for-export\n')
(self.root / 'private-notes.txt').write_text('private fixture notes\n')
(self.root / 'LICENSES/private-notice.txt').write_text('PRIVATE NOTICE FIXTURE MUST NOT LEAK\n')
untracked = self.git('ls-files', '--others', '--exclude-standard').decode()
self.assertIn('.env', untracked)
self.assertIn('private-notes.txt', untracked)
@@ -183,12 +192,62 @@ class PublicSourceReleaseGuards(unittest.TestCase):
with zipfile.ZipFile(self.output / 'corresponding-source.zip') as archive:
source_files = {n for n in archive.namelist() if not n.endswith('/')}
self.assertEqual(source_files,
{'coco3plus/' + n for n in self.tracked})
{'coco3plus/' + n for n in self.tracked
if release.source_member(n)})
self.assertFalse(any('.env' in n or 'private-notes' in n
for n in source_files))
self.assertNotIn('coco3plus/LICENSES/private-notice.txt', source_files)
self.assertFalse(any(n.endswith(('.rbf', '.pdf', '.docx', '.bin'))
for n in source_files))
with zipfile.ZipFile(self.output.with_name('public-release.zip')) as archive:
self.assertNotIn('.env', archive.namelist())
self.assertNotIn('private-notes.txt', archive.namelist())
self.assertNotIn('fat/games/coco3plus/licenses/private-notice.txt',
archive.namelist())
for name in archive.namelist():
self.assertNotIn(b'PRIVATE NOTICE FIXTURE MUST NOT LEAK',
archive.read(name))
checkout = archive.read('SOURCE-CHECKOUT.txt').decode()
self.assertIn('git clone ' + release.SOURCE_REPOSITORY, checkout)
self.assertIn('git checkout --detach ' + self.revision, checkout)
self.assertIn('ZIP alone is not a standalone build checkout', checkout)
self.assertEqual(record['project_inventory_sha256'],
digest(self.root / 'tools/project_inputs.json'))
self.assertNotIn('games/coco3plus/licenses/private-notice.txt', record['files'])
def rewrite_receipt(self, transform):
receipt = self.build / 'build.json'
record = json.loads(receipt.read_text())
transform(record)
receipt.write_text(json.dumps(record))
def assert_rejected_without_output(self, pattern):
with self.assertRaisesRegex(RuntimeError, pattern):
self.package()
self.assertFalse(self.output.exists())
self.assertFalse(self.output.with_name('public-release.zip').exists())
def test_legacy_receipt_is_rejected_without_backfilling_evidence(self):
self.rewrite_receipt(lambda r: r.pop('project_inventory_sha256'))
original = (self.build / 'build.json').read_bytes()
self.assert_rejected_without_output('Legacy FPGA receipt lacks')
self.assertEqual((self.build / 'build.json').read_bytes(), original)
def test_stale_inventory_identity_is_rejected(self):
self.rewrite_receipt(lambda r: r.update(project_inventory_sha256='0' * 64))
self.assert_rejected_without_output('does not qualify the current')
def test_subset_input_receipt_is_rejected(self):
self.rewrite_receipt(lambda r: r['inputs'].pop('rtl/example.sv'))
self.assert_rejected_without_output('exact current project input set')
def test_extra_input_receipt_is_rejected(self):
self.rewrite_receipt(lambda r: r['inputs'].update({'NOTICE.md': digest(self.root / 'NOTICE.md')}))
self.assert_rejected_without_output('exact current project input set')
def test_matching_set_with_changed_input_bytes_is_rejected(self):
self.rewrite_receipt(lambda r: r['inputs'].update({'rtl/example.sv': '0' * 64}))
self.assert_rejected_without_output('Source differs from built input')
def test_dirty_tracked_source_rejects_before_output_creation(self):
notice = self.root / 'NOTICE.md'