From 8088c1fbf6435162faa2be1521034d2b7c1ca260 Mon Sep 17 00:00:00 2001 From: Brian Holman Date: Wed, 18 Nov 2026 16:26:25 -0700 Subject: [PATCH] Harden release packaging and recoverable installation after review --- CHANGELOG.md | 16 ++ COPYRIGHT.md | 2 + LICENSES/Greg-Miller-BSD-3-Clause.txt | 23 ++ LICENSING.md | 25 +- NOTICE.md | 4 +- docs/BUILD_WINDOWS.md | 55 ++++- docs/INSTALL.md | 60 ++++- docs/THIRD_PARTY_PROVENANCE.md | 48 ++++ docs/VALIDATION.md | 14 +- tests/test_install_mister.py | 238 ++++++++++++++++++ tests/test_release_guards.py | 61 ++++- tests/test_tooling_guards.py | 61 ++++- tools/install_mister.py | 332 ++++++++++++++++++++++---- tools/release.py | 95 +++++++- 14 files changed, 933 insertions(+), 101 deletions(-) create mode 100644 LICENSES/Greg-Miller-BSD-3-Clause.txt create mode 100644 docs/THIRD_PARTY_PROVENANCE.md create mode 100644 tests/test_install_mister.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a9a43a6..7590d63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,3 +21,19 @@ Based on complete upstream history through See docs/BUG_FIXES.md for causes/checks, docs/FEATURES.md for behavior, docs/MENUS.md for the complete tree and docs/VALIDATION.md for acceptance scope. + +## Release tooling and attribution review + +- Selected Greg Miller's original Standard BSD license option, preserving its + complete notice and documenting pinned provenance instead of an unverified + public-domain assumption. Other inherited-asset review remains explicit. +- Export notices from committed Git blobs only; exclude untracked local files. + Limit downloadable source selections and provide a pinned full-clone build route. +- Require a current inventory identity and complete qualified build-input set + before packaging, rejecting legacy/stale receipts without rewriting evidence. +- Stage and verify all supporting files before atomic launcher activation; + preserve planned/failed installation receipts and rollback locations. +- Separate measured timing paths from empty asynchronous checks and identify + the earlier extensive hardware test image separately from final startup checks. + +These changes do not modify the FPGA hardware implementation or timing constraints. diff --git a/COPYRIGHT.md b/COPYRIGHT.md index b7bea95..1cd4ed3 100644 --- a/COPYRIGHT.md +++ b/COPYRIGHT.md @@ -146,3 +146,5 @@ references and third-party excerpts retain their own terms. Copyright notices also accompany synthesized releases through NOTICE.md, LICENSING.md, this inventory and the original component license texts. +- `docs/THIRD_PARTY_PROVENANCE.md` +- `tests/test_install_mister.py` diff --git a/LICENSES/Greg-Miller-BSD-3-Clause.txt b/LICENSES/Greg-Miller-BSD-3-Clause.txt new file mode 100644 index 0000000..5b055b5 --- /dev/null +++ b/LICENSES/Greg-Miller-BSD-3-Clause.txt @@ -0,0 +1,23 @@ +Copyright (c) 2016, Greg Miller +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL GREG MILLER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSING.md b/LICENSING.md index 302b7dc..a833381 100644 --- a/LICENSING.md +++ b/LICENSING.md @@ -17,7 +17,7 @@ licenses of any combined or derived material. | New SSC controller/speech/wiring derived from reference models | BSD-3-Clause; retain rtl/ssc/LICENSE and component attribution to Tim Lindner, Joseph Zbiciak and hap. | | New MPI and original Plus helpers/tests/tooling | BSD-3-Clause for original contributions, with preserved reference-source terms where extracted code is involved. | | Generated Intel/Altera IP and device primitives | Their original vendor notices/terms; a project-level BSD/GPL declaration does not replace these. | -| Greg Miller mc6809glb CPU | Original copyright-only header retained. No explicit upstream license grant found in that file. See assumption below. | +| Greg Miller mc6809glb CPU | Standard BSD option offered by the original CPU project, selected for this distribution. Original copyright retained; complete terms in LICENSES/Greg-Miller-BSD-3-Clause.txt. See provenance below. | For a combined distribution containing GPL-covered material, retain the applicable GPL terms and provide the corresponding source, build scripts and notices as @@ -28,15 +28,22 @@ and license obligations that apply to its components. See the original This repository's permissive contribution license is not a blanket assertion that the complete synthesized design is BSD-only. -## Maintainer assumption: Greg Miller CPU +## Greg Miller CPU: selected Standard BSD license -The Coco 3 Plus FPGA Maintainer currently proceeds on the assumption that Greg -Miller's original CPU implementation is public domain. This is a **maintainer -assumption, not a confirmed upstream dedication or permission grant**. The -original copyright notice remains in rtl/mc6809glb/mc6809glb.v. If contrary or -clarifying information is received, amend this notice and distribution practice. -New Plus modifications are identified separately; this assumption does not -erase another author's rights or claim their work as original Plus code. +This distribution selects the **Standard BSD license** offered by Greg Miller's +original [mc6809 license document at commit 17e94a6](https://github.com/cavnex/mc6809/blob/17e94a6ef163be8b79a9b15b2e814847b6062f0f/documentation/LICENSE.md). +That document offers a standard source-and-binary redistribution option and a +different binary-only option; this project selects the former. The complete +selected copyright notice, conditions and disclaimer are retained in +LICENSES/Greg-Miller-BSD-3-Clause.txt and included in release notice materials. +The original 2016 Greg Miller attribution in rtl/mc6809glb/mc6809glb.v remains. + +The MiSTer file is a derivative of Miller's mc6809i.v, with later modifications +identified by their own notices/history. An independent source comparison found +99.4% agreement of normalized noncomment tokens. That comparison supports the +provenance identification; it is not a claim that every later contributor's work +is Miller's original code. The selected terms supersede the earlier unverified +maintainer public-domain assumption. See docs/THIRD_PARTY_PROVENANCE.md. ## ROMs, software libraries and external tools diff --git a/NOTICE.md b/NOTICE.md index 64730ab..ebe6ec1 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -6,7 +6,9 @@ Original Coco 3 Plus contributions and modifications: This project derives from MiSTer-devel/CoCo3_MiSTer, the MiSTer port of Gary Becker's CoCo3FPGA by Stan Hodge, Gary Becker and Alan Steremberg. Original histories and source notices are retained. CPU implementations credit Greg -Miller and John E. Kent in their respective source files. MiSTer framework work +Miller and John E. Kent in their respective source files. Greg Miller's offered +Standard BSD option is selected; its complete notice is in +LICENSES/Greg-Miller-BSD-3-Clause.txt. MiSTer framework work retains attribution including Alexey Melnikov (Sorgelig), Till Harbaum and other file-specific contributors. diff --git a/docs/BUILD_WINDOWS.md b/docs/BUILD_WINDOWS.md index c485d86..ebbdc49 100644 --- a/docs/BUILD_WINDOWS.md +++ b/docs/BUILD_WINDOWS.md @@ -6,6 +6,24 @@ Use 64-bit Windows, Git for Windows and Python **3.10 or newer**. Scripts use th Python standard library. PowerShell wrappers accept `-Python ` when `python` is not on PATH. All commands below run from the repository root. +Use a full Git checkout. An extracted source-review ZIP has no `.git` metadata +and deliberately omits inherited binary/manual artifacts still present in the +historical project inventory; it is not the supported build checkout. A release's +`SOURCE-CHECKOUT.txt` gives the exact immutable revision and clone commands: + +```powershell +git clone https://vccplus-git.brianholman.com/VCCPlus/CoCo3_Plus_MiSTer.git coco3plus +cd coco3plus +# Replace RELEASE_SOURCE_REVISION with the full hash in SOURCE-CHECKOUT.txt. +git checkout --detach RELEASE_SOURCE_REVISION +git rev-parse HEAD +``` + +Do not pass `--depth`; the clone preserves the original upstream history. Access +to the repository is required. Build tools invoke Git to record the source +revision; merely installing the Git executable does not make an archive a Git +checkout. Keep the pinned checkout unchanged while building and qualifying it. + Install **Quartus Prime Lite 17.0.2, Build 602**, including Cyclone V device files. The device is **5CSEBA6U23I7**. The project originated in Quartus 17.0; its pinned settings record 17.0.2. Install the 17.0 base package and matching device package, @@ -71,6 +89,9 @@ the maintained `tools/project_inputs.json` inventory too. The builder rejects unlisted HDL/constraint/IP inputs in the source directories before copying. `validation/source-import.json` is historical provenance, not the build inventory. The receipt derives the seed from the actual single QSF assignment. +It also records the SHA-256 of the maintained project inventory. Packaging +requires this identity, the exact current hashed-input set, and matching bytes +for every input. A subset receipt cannot qualify additional or omitted inputs. Generated caches/IP upgrades stay in build copies. Outputs are preserved: use `-Output build/fpga-next` for another build, with matching `--build` arguments when packaging/auditing it, rather than deleting prior results. @@ -120,10 +141,42 @@ Packaging verifies the RBF and boot/SSC ROM identities, writes a manifest and SHA256SUMS, and creates a copyable `fat/` overlay. No files from build caches or source-reference downloads are installed. Packaging requires reviewed changes committed; corresponding source comes only from that committed Git tree, never -untracked local files. `-PublicPackage` omits all separately obtained/assembled ROMs +untracked local files. Installed notices are also read from committed Git blobs, +so an untracked file in `LICENSES/` is excluded from both archive layers. The +source-review payload uses a maintained path/type allowlist in `tools/release.py`; +historical RBFs, PDFs and Office files remain in full Git history but are omitted +from that payload. `SOURCE-CHECKOUT.txt` records the pinned full-clone build route. +`-PublicPackage` omits all separately obtained/assembled ROMs and produces a core/source distribution requiring user ROM preparation. It is distinct from a complete local install package. Details: docs/INSTALL.md. +Legacy fitted receipts without `project_inventory_sha256` are rejected rather +than silently upgraded. Preserve the old build directory, RBF and evidence. The +supported independent qualification route is a fresh `Build-FPGA.ps1 -Output +build/fpga-qualified` run from the pinned checkout, followed by its own functional, +detailed timing and hardware checks appropriate to release acceptance. A +prepare-only comparison establishes input equality; it does not create fit or +timing evidence. Do not add an inventory hash to an old receipt and present it as +newly qualified. Packaging-only changes need not alter RTL, but do not waive this +receipt requirement. + +The real local-package guards can target a new qualified build and package +without moving or editing historical evidence. From the checkout in PowerShell: + +```powershell +$env:COCO3PLUS_TEST_BUILD = 'build/fpga-audit-receipt' +$env:COCO3PLUS_TEST_RELEASE = 'dist/coco3plus-audit' +python -B tests/test_release_guards.py +``` + +Paths are resolved relative to the repository root unless absolute. Defaults +remain `build/fpga` and `dist/coco3plus` when the overrides are absent. Missing +actual local artifacts cause explicit skips; these are separate from portable +fixture tests in `tests/test_tooling_guards.py`. A selected complete package must +have the current source inventory, outer ZIP, exact committed source revision, +and committed Greg Miller notice; an existing historical package lacking those +properties is a failure rather than a substitute for checking the new package. + Portable Python entry points under tools/ support native macOS/Linux tooling where available. Quartus itself has no native macOS build; use the Windows or supported Linux toolchain. The automatic prebuilt-tool download is Windows x64. diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 0382aac..5b756d1 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -27,11 +27,30 @@ activity. Installation returns to MENU, so the current guest session is ended. ``` If no address is given, the script prompts for one. It first validates every -local input, then checks remote access, stages files, verifies remote SHA-256, -returns to MENU, backs up replaced files, copies the overlay, checks installed -hashes, syncs the volume and optionally loads the launcher. A timestamped receipt -in ignored dist storage records the device and rollback path. It does not change -accounts, networking, Samba, the Linux installation or nonvolatile FPGA storage. +local input and writes a local per-attempt receipt naming the planned stage, +backup, destinations and temporary files. It then checks remote access, stages +and verifies every payload, and returns to MENU. All supporting assets are +installed and checked before activation: the uniquely addressed RBF, ROMs, +configuration/path files and notices. Existing saved configuration is preserved. +The volume is synced before the launcher is changed. + +The launcher is the last asset installed. Its staged bytes are copied to a +temporary sibling beside the destination, SHA-256 checked, and atomically renamed +to _Computer/Coco 3 Plus.mgl. The installer checks the final bytes, syncs again +and optionally loads the launcher. This avoids exposing a new menu entry before +its supporting files are installed. Filesystem rename is not a claim of +power-loss certification or a transaction covering the entire SD card. + +Receipts are retained in ignored release storage as +`install-receipt-.json`; `install-receipt.json` holds the latest attempt's +complete JSON record. An existing latest receipt is also preserved before a new +attempt. Each phase and completed file is recorded. On failure or keyboard +interruption, the receipt retains the active phase/file, error and planned +recovery paths. Stage and backup files are not automatically deleted; an +already-existing remote attempt directory is rejected rather than merged. + +The installer does not change accounts, networking, Samba, the Linux installation +or nonvolatile FPGA storage. Only the Plus launcher, its uniquely addressed RBF, two boot ROMs and optional SSC ROM/path records are installed. The stock core and other guest media remain. @@ -82,6 +101,12 @@ artifact that omits all boot and SSC ROMs. Supply the required ROMs before manua copy/launch. The SSH installer rejects this incomplete-ROM package and directs you to make a complete local package from your own verified inputs. +The included `corresponding-source.zip` is a source-review payload, not a Git +checkout. It excludes historical RBFs, PDFs and Office files retained in the full +repository. Build scripts need Git metadata and the full project inventory. +Follow `SOURCE-CHECKOUT.txt` beside the archive to clone the repository and check +out the exact release revision before following `docs/BUILD_WINDOWS.md`. + ## First launch and recovery Choose Computer → Coco 3 Plus. It should reach Disk Extended Color BASIC. @@ -96,10 +121,21 @@ welcome/help strings no longer expose the retired development name. The native profile override remains `coco3plus`; changing the display identity does not require abandoning the existing saved settings or media. -For rollback, return to MENU and restore `_Computer/Coco 3 Plus.mgl` from the -receipt's remote backup directory. Its previously addressed RBF remains in place. -Restore backed-up boot/SSC/path files if those inputs also need rollback. Existing -saved CFG was not overwritten. On a first installation, select the preserved -stock CoCo3 entry to use the original environment. Keep backup/staging folders -until physical cold-start and audio checks pass; failed installs retain their -staging evidence instead of deleting it automatically. +For rollback, return to MENU and use the receipt's stage/backup paths. If +failure happened before activation_attempted, the old launcher is unchanged, +but some supporting files may already have been replaced: restore their backed-up +versions if needed before loading the previous core. If activation was attempted, +launcher_activated: null means a transport failure left its outcome uncertain; +inspect the installed launcher rather than assuming it was rolled back. A null +loaded value likewise means the load command was attempted but not confirmed. + +When the old launcher needs restoration, restore _Computer/Coco 3 Plus.mgl from +the remote backup directory. Its previously addressed RBF remains in place. +Restore backed-up boot/SSC/path files when those inputs also need rollback. +Existing saved CFG was not overwritten. On a first installation, select the +preserved stock CoCo3 entry to use the original environment. Keep backup/staging +folders until physical cold-start and audio checks pass. + +Run python tests/test_install_mister.py to exercise launcher-last activation and +deliberate pre-activation interruption on a disposable local mock target. These +tests do not access a MiSTer or establish physical power-loss behavior. diff --git a/docs/THIRD_PARTY_PROVENANCE.md b/docs/THIRD_PARTY_PROVENANCE.md new file mode 100644 index 0000000..065b485 --- /dev/null +++ b/docs/THIRD_PARTY_PROVENANCE.md @@ -0,0 +1,48 @@ +# Third-party provenance and release review + +Component-specific notices and terms apply. Hosting in an upstream repository is +provenance evidence, not by itself a new redistribution permission grant. + +## Greg Miller CPU + +- Original project: https://github.com/cavnex/mc6809 +- Pinned original license: commit + `17e94a6ef163be8b79a9b15b2e814847b6062f0f`, `documentation/LICENSE.md`. +- Retrieved primary document SHA-256: `11d10d396eb75ce78dd49eccc4f575379106f2c5291e863eb653842532694d0a`. +- Selected option: **Standard BSD license**, permitting source and binary + redistribution subject to the stated conditions. The binary-only alternative + is not the selected option. +- Complete selected terms: [Greg-Miller-BSD-3-Clause.txt](../LICENSES/Greg-Miller-BSD-3-Clause.txt). +- Inherited derivative: `rtl/mc6809glb/mc6809glb.v`; original header retained. +- Independent review reports 99.4% normalized noncomment-token agreement with + the original mc6809i.v. This is an ancestry/provenance check, not a new claim of + ownership over original or later work. + +## Additional inherited material under review + +Before public source or binary release, complete the component review for: + +- Generated Intel/Altera IP and device primitives: retain original vendor notices, + identify applicable free-source IP/programming-output terms and target-device + restrictions. The installed Quartus Prime 17.0 agreement's section 2.5 offers + source-IP redistribution terms for the defined free-source component category, + limited to programming Altera devices. Verify each generated family/version + against that category and preserve its notices; do not substitute the Plus + contribution license for those terms. +- `rtl/rom/coco3_char_rom.vhd`: 4,096 initialized character-font bytes are embedded + in HDL and wired into the fitted design. The romgen generator attribution does + not establish the font's separate provenance or distribution terms. +- Inherited reference PDFs and historical RBFs retained in original Git history: + review their distribution basis independently of source-code licenses. In + particular, `releases/Tandy CoCo3 Overview.pdf` is a community MiSTer reference + sheet added by Alan Steremberg in commit a8014eae0b9ee90928389d9ecb2a9a7962edf761, + not an identified original Tandy publication. Its artwork credits/terms are not + specified in the sheet. + Excluding a file from a downloadable source ZIP does not remove historical Git + objects or resolve their publication status. +- GPL-covered framework modules: retain notices, applicable GPL texts and build + materials/corresponding source obligations for combined distributions. + +This list identifies release-review work. It does not revoke or replace original +component terms, establish a blanket prohibition, or claim that review is complete. +External boot and SSC ROMs remain separately supplied and ignored by Git. diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 03d9d1a..fd9a7d4 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -12,12 +12,15 @@ seed5 compile completes in **14:33**, **0 errors / 73 warnings**. It uses 768ee9db2923bf7b7acf7f57473c6c56f92d14ffc6d6bd88719148f80c9a2d42 ``` -Minimum constrained slack: setup **+0.089 ns**, hold **+0.212 ns**, recovery +Quartus summary minimum constrained slack: setup **+0.089 ns**, hold **+0.212 ns**, recovery **+0.691 ns**, removal **+0.701 ns**, pulse width **+0.396 ns**. All 15 functional jobs pass through the documented Windows entry point using the standalone checkout's own verified/extracted simulator. The actual Main name-reader fixture additionally confirms the spaced display name and unchanged canonical -profile override. All **368 fitted report scopes** pass, with remaining optional +profile override. All **368 fitted report scopes** are classified: **288 contain +measured paths that pass**, and **80 are explicitly empty recovery/removal checks**. +The smallest enumerated detailed-path slack is **+0.065 ns**, distinct from the +summary setup minimum above. Remaining optional pin contracts explicitly classified in validation/standalone.json. No timing exceptions or physical delay assumptions were relaxed for the rename. @@ -75,8 +78,9 @@ CPU/video data; audio; pixel; I2C; original Main startup parser; original Main m parser; original/candidate video comparison. Negative controls include incorrect buffer cadence, pixel duty/address corruption and old initialization behavior. -All seven native detailed timing audits pass **368 report scopes** at both slow -and fast corners. This includes 34 MPI reports, 144 live SSC path reports, +All seven native detailed timing audits classify **368 report scopes** at both slow +and fast corners: **288 measured scopes pass**, and **80 synchronous-only +recovery/removal scopes are explicitly empty**. This includes 34 MPI reports, 144 live SSC path reports, CPU/reset/NMI/UART/video scopes, and 48 substantive conditional external-I/O reports. Explicitly empty synchronous-only recovery/removal subscopes are listed as empty, not counted as an observed path with positive slack. @@ -86,7 +90,7 @@ canonical-name fixture mismatch, and a video-runner argument mismatch. These were export-harness defects; corrected fixtures pass in a fresh output folder. No RTL/timing exceptions were changed to satisfy the exported tests. -## Hardware evidence on these exact RBF bytes +## Earlier hardware evidence: RBF a4b6ef0b (not final RBF 768ee9db) Previous physical validation of the same image includes normal Disk BASIC startup, saved non-default socket layout, Slot1/Slot4 ROM autoload, independent socket diff --git a/tests/test_install_mister.py b/tests/test_install_mister.py new file mode 100644 index 0000000..17e7293 --- /dev/null +++ b/tests/test_install_mister.py @@ -0,0 +1,238 @@ +"""Check launcher-last installation and recoverable interruption on a mock target. + +Copyright (c) 2026 Brian K. Holman. BSD-3-Clause; see LICENSES/. +The target is a disposable local filesystem, never an SSH/device connection. +""" +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tools')) +import install_mister as installer + + +class DisposableTarget: + def __init__(self, root, package, *, interrupt_on=None, fail_preflight=False): + self.root = root + self.package = package + self.interrupt_on = interrupt_on + self.fail_preflight = fail_preflight + self.events = [] + + def local(self, remote): + relative = PurePosixPath(remote).relative_to('/media/fat') + return self.root.joinpath(*relative.parts) + + def preflight(self): + self.events.append('preflight') + if self.fail_preflight: + raise OSError('Deliberate mock preflight failure') + return 'Disposable target only\n' + + def prepare(self, stage, backup): + # The recovery plan must already be durable before this first target write. + receipt = json.loads((self.package / 'install-receipt.json').read_text()) + assert receipt['stage'] == stage and receipt['backup'] == backup + assert receipt['phase'] == 'staging' + assert Path(receipt['receipt']).is_file() + self.events.append('prepare') + self.local(stage).mkdir(parents=True) + self.local(backup).mkdir(parents=True) + + def upload(self, fat, stage): + self.events.append('upload') + shutil.copytree(fat, self.local(stage) / 'fat') + + def verify(self, files): + self.events.append('verify') + for name, digest in files.items(): + assert installer.sha(self.local(name)) == digest, name + + def enter_menu(self): + self.events.append('menu') + + def install_file(self, source, destination, backup, entry, temporary): + if self.interrupt_on and destination.endswith('/' + self.interrupt_on): + self.events.append('interrupted') + raise KeyboardInterrupt('Deliberate interruption before launcher activation') + target = self.local(destination) + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + if entry['policy'] == 'create-if-missing': + self.events.append('preserved:' + destination) + return 'preserved' + old = self.local(backup) + old.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(target, old) + pending = self.local(temporary) + assert not pending.exists() + shutil.copyfile(self.local(source), pending) + assert installer.sha(pending) == entry['sha256'] + os.replace(pending, target) + assert installer.sha(target) == entry['sha256'] + self.events.append('installed:' + destination) + return 'installed' + + def activate_launcher(self, source, destination, backup, entry, temporary): + self.events.append('activate') + return self.install_file(source, destination, backup, entry, temporary) + + def sync(self): + self.events.append('sync') + + def load(self): + self.events.append('load') + + +class InstallerTransactionTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(dir=ROOT / 'build') + self.addCleanup(self.temporary.cleanup) + self.base = Path(self.temporary.name) + self.package = self.base / 'package' + self.device = self.base / 'device' + self.package.mkdir() + self.device.mkdir() + self.launcher = installer.LAUNCHER + self.boot0 = 'games/coco3plus/boot0.rom' + self.boot1 = 'games/coco3plus/boot1.rom' + self.config = 'config/coco3plus_v51.CFG' + self.rbf = '_Computer/.coco3plus/fixture/core.rbf' + contents = { + # Deliberately put the launcher first: manifest ordering cannot activate it early. + self.launcher: b'new launcher fixture\n', + self.boot0: b'new boot0 fixture\n', + self.config: b'new default configuration\n', + self.boot1: b'new boot1 fixture\n', + self.rbf: b'new bitstream fixture\n', + } + entries = {} + for name, data in contents.items(): + path = self.package / 'fat' / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + entries[name] = dict( + bytes=len(data), sha256=installer.sha(path), + policy='create-if-missing' if name == self.config else 'backup-and-replace') + self.manifest = dict(files=entries, rbf_sha256=entries[self.rbf]['sha256']) + (self.package / 'manifest.json').write_text(json.dumps(self.manifest)) + for name, data in ((self.launcher, b'old launcher remains selectable\n'), + (self.boot0, b'old boot0 recoverable\n'), + (self.config, b'owner saved configuration\n')): + path = self.device / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + def install(self, target, stamp='fixture-attempt', load=False): + return installer.install_release( + self.package, 'disposable.invalid', load=load, target=target, stamp=stamp) + + def test_support_interruption_keeps_old_launcher_and_recovery_evidence(self): + target = DisposableTarget(self.device, self.package, interrupt_on=self.boot1) + original_launcher = (self.device / self.launcher).read_bytes() + original_boot0 = (self.device / self.boot0).read_bytes() + + with self.assertRaises(KeyboardInterrupt): + self.install(target, load=True) + + receipt = json.loads((self.package / 'install-receipt.json').read_text()) + self.assertEqual(receipt['status'], 'failed') + self.assertEqual(receipt['phase'], 'supporting-assets') + self.assertEqual(receipt['active_file'], self.boot1) + self.assertIn(self.launcher, receipt['planned_files']) + self.assertFalse(receipt['activation_attempted']) + self.assertIs(receipt['launcher_activated'], False) + self.assertFalse(receipt['loaded']) + self.assertEqual(receipt['error']['type'], 'KeyboardInterrupt') + self.assertEqual((self.device / self.launcher).read_bytes(), original_launcher) + self.assertNotIn('activate', target.events) + self.assertNotIn('load', target.events) + self.assertEqual((self.device / self.boot0).read_bytes(), + (self.package / 'fat' / self.boot0).read_bytes()) + self.assertEqual(target.local(receipt['backup'] + '/' + self.boot0).read_bytes(), + original_boot0) + self.assertEqual(target.local(receipt['stage'] + '/fat/' + self.launcher).read_bytes(), + (self.package / 'fat' / self.launcher).read_bytes()) + self.assertTrue(Path(receipt['receipt']).is_file()) + self.assertIn(self.boot0, {item['path'] for item in receipt['completed_files']}) + + def test_success_installs_support_then_atomically_activates_and_preserves_config(self): + target = DisposableTarget(self.device, self.package) + original_launcher = (self.device / self.launcher).read_bytes() + original_config = (self.device / self.config).read_bytes() + + receipt = self.install(target, load=True) + + self.assertEqual(receipt['status'], 'complete') + self.assertTrue(receipt['launcher_activated']) + self.assertTrue(receipt['loaded']) + self.assertEqual((self.device / self.config).read_bytes(), original_config) + self.assertEqual((self.device / self.launcher).read_bytes(), + (self.package / 'fat' / self.launcher).read_bytes()) + self.assertEqual(target.local(receipt['backup'] + '/' + self.launcher).read_bytes(), + original_launcher) + activation = target.events.index('activate') + for i, event in enumerate(target.events): + if event.startswith('installed:') and not event.endswith('/' + self.launcher): + self.assertLess(i, activation) + self.assertEqual(target.events[activation - 1], 'sync') + self.assertGreater(target.events.index('load'), activation) + self.assertFalse(target.local(receipt['launcher_temporary']).exists()) + self.assertTrue(target.local(receipt['stage'] + '/fat/' + self.launcher).exists()) + + def test_preflight_failure_keeps_plan_and_does_not_mutate_target(self): + target = DisposableTarget(self.device, self.package, fail_preflight=True) + before = {p.relative_to(self.device): p.read_bytes() + for p in self.device.rglob('*') if p.is_file()} + + with self.assertRaisesRegex(OSError, 'mock preflight failure'): + self.install(target) + + receipt = json.loads((self.package / 'install-receipt.json').read_text()) + self.assertEqual(receipt['status'], 'failed') + self.assertEqual(receipt['phase'], 'preflight') + self.assertEqual(target.events, ['preflight']) + self.assertFalse(target.local(receipt['stage']).exists()) + self.assertEqual(before, {p.relative_to(self.device): p.read_bytes() + for p in self.device.rglob('*') if p.is_file()}) + + def test_failed_retry_preserves_prior_complete_receipt(self): + first = self.install(DisposableTarget(self.device, self.package), stamp='first') + original_receipt = Path(first['receipt']).read_bytes() + + with self.assertRaises(OSError): + self.install(DisposableTarget(self.device, self.package, fail_preflight=True), + stamp='second') + + failed = json.loads((self.package / 'install-receipt.json').read_text()) + self.assertEqual(failed['status'], 'failed') + self.assertEqual(Path(first['receipt']).read_bytes(), original_receipt) + self.assertEqual(Path(failed['previous_receipt']).read_bytes(), original_receipt) + + def test_remote_launcher_primitive_verifies_temporary_file_before_rename(self): + target = installer.OpenSshTarget('disposable.invalid', 'root') + source = '/media/fat/.coco3plus/stage/' + self.launcher + destination = '/media/fat/' + self.launcher + temporary = destination + '.fixture.tmp' + backup = '/media/fat/.coco3plus/backup/' + self.launcher + with patch.object(target, 'run', return_value='installed\n') as run: + result = target.activate_launcher( + source, destination, backup, self.manifest['files'][self.launcher], temporary) + self.assertEqual(result, 'installed') + script = run.call_args.args[0] + copy = 'cp ' + installer.q(source) + ' ' + installer.q(temporary) + check = 'sha256sum ' + installer.q(temporary) + rename = 'mv -f ' + installer.q(temporary) + ' ' + installer.q(destination) + self.assertLess(script.index(copy), script.index(check)) + self.assertLess(script.index(check), script.index(rename)) + self.assertNotIn('cp ' + installer.q(source) + ' ' + installer.q(destination) + '\n', script) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_release_guards.py b/tests/test_release_guards.py index 9528c9f..976a8c4 100644 --- a/tests/test_release_guards.py +++ b/tests/test_release_guards.py @@ -1,10 +1,20 @@ """Check release integrity boundaries against a real completed local build.""" -import json,sys,tempfile,unittest,zipfile +import hashlib,json,os,sys,tempfile,unittest,zipfile from pathlib import Path ROOT=Path(__file__).resolve().parents[1] sys.path.insert(0,str(ROOT/'tools')) -from release import default_config,make_release,SSC +from release import (default_config,make_release,qualify_inputs,committed_tree, + committed_blob,source_member,SSC) +def selected_path(variable,default): + path=Path(os.environ.get(variable,str(default))).expanduser() + return (path if path.is_absolute() else ROOT/path).resolve() +BUILD=selected_path('COCO3PLUS_TEST_BUILD',ROOT/'build/fpga') +RELEASE=selected_path('COCO3PLUS_TEST_RELEASE',ROOT/'dist/coco3plus') +def digest(data):return hashlib.sha256(data).hexdigest() class ReleaseGuards(unittest.TestCase): + def require_files(self,*paths): + missing=[str(p) for p in paths if not p.is_file()] + if missing:self.skipTest('Actual local dependency unavailable: '+', '.join(missing)) def test_fresh_profile_preserves_status_contract(self): data=default_config(True);self.assertEqual(len(data),16) value=int.from_bytes(data,'little') @@ -13,6 +23,16 @@ class ReleaseGuards(unittest.TestCase): self.assertEqual([(value>>(43+3*i))&7 for i in range(4)],[6,3,1,2]) self.assertEqual(value^int.from_bytes(default_config(False),'little'),1<<42) def test_changed_ssc_fails_before_creating_output(self): + # A preserved historical fit is not automatically a release-qualified + # receipt. Do not modify it merely to reach a later ROM-input guard. + self.require_files(BUILD/'build.json',BUILD/'project/output_files/CoCo3.rbf') + record=json.loads((BUILD/'build.json').read_text()) + if 'project_inventory_sha256' not in record: + with self.assertRaisesRegex(RuntimeError,'Legacy FPGA receipt lacks'): + qualify_inputs(record) + self.skipTest('SSC packaging test requires a new inventory-qualified fit; legacy evidence preserved') + self.require_files(*(ROOT/'roms/ssc'/name for name in SSC), + ROOT/'build/roms/boot0.rom',ROOT/'build/roms/boot1.rom') with tempfile.TemporaryDirectory(dir=ROOT/'build') as folder: base=Path(folder);roms=base/'roms';roms.mkdir() for name in SSC: @@ -21,13 +41,40 @@ class ReleaseGuards(unittest.TestCase): chip=roms/'pic-7040-510.bin';data=bytearray(chip.read_bytes());data[-1]^=1;chip.write_bytes(data) out=base/'release' with self.assertRaisesRegex(RuntimeError,'incorrect input'): - make_release(out,ROOT/'build/fpga',ROOT/'build/roms',roms) + make_release(out,BUILD,ROOT/'build/roms',roms) self.assertFalse(out.exists()) def test_corresponding_source_contains_no_local_roms(self): - release=ROOT/'dist/coco3plus' - with zipfile.ZipFile(release/'corresponding-source.zip') as archive: - names=archive.namelist();self.assertTrue(any(n.endswith('/rtl/coco3fpga.sv') for n in names)) - self.assertFalse(any('/roms/ssc/' in n or '/.deps/' in n or '/build/' in n for n in names)) + release=RELEASE + outer=release.with_name(release.name+'.zip') + self.require_files(release/'manifest.json',release/'corresponding-source.zip',outer) record=json.loads((release/'manifest.json').read_text()) + self.assertIn('source_files',record,'Select a newly qualified package; historical package predates source inventory') + tree=committed_tree(record['source_revision']) + expected=sorted(name for name in tree if source_member(name)) + self.assertEqual(record['source_files'],expected) + self.assertEqual(record['corresponding_source_sha256'],digest((release/'corresponding-source.zip').read_bytes())) + greg='LICENSES/Greg-Miller-BSD-3-Clause.txt' + self.assertIn(greg,tree,'Selected release revision lacks the committed Greg Miller notice') + notice=committed_blob(tree[greg]) + with zipfile.ZipFile(release/'corresponding-source.zip') as archive: + names=archive.namelist();self.assertEqual(set(names),{'coco3plus/'+n for n in expected}) + self.assertTrue(any(n.endswith('/rtl/coco3fpga.sv') for n in names)) + self.assertFalse(any('/roms/ssc/' in n or '/.deps/' in n or '/build/' in n for n in names)) + self.assertFalse(any(n.endswith(('.rbf','.pdf','.docx','.xlsx','.bin','.rom')) for n in names)) + self.assertEqual(archive.read('coco3plus/'+greg),notice) + notice_key='games/coco3plus/licenses/Greg-Miller-BSD-3-Clause.txt' + self.assertIn(notice_key,record['files']) + self.assertEqual((release/'fat'/notice_key).read_bytes(),notice) + with zipfile.ZipFile(outer) as archive: + self.assertEqual(set(archive.namelist()), + {p.relative_to(release).as_posix() for p in release.rglob('*') if p.is_file()}) + self.assertEqual(archive.read('fat/'+notice_key),notice) + self.assertEqual(archive.read('corresponding-source.zip'),(release/'corresponding-source.zip').read_bytes()) + self.assertIn('git checkout --detach '+record['source_revision'],archive.read('SOURCE-CHECKOUT.txt').decode()) + self.assertEqual(json.loads(archive.read('manifest.json')),record) + for name,identity in record['files'].items(): + data=archive.read('fat/'+name) + self.assertEqual(len(data),identity['bytes'],name) + self.assertEqual(digest(data),identity['sha256'],name) self.assertEqual(record['files']['config/coco3plus_v51.CFG']['policy'],'create-if-missing') if __name__=='__main__':unittest.main() diff --git a/tests/test_tooling_guards.py b/tests/test_tooling_guards.py index dcef5ee..1560607 100644 --- a/tests/test_tooling_guards.py +++ b/tests/test_tooling_guards.py @@ -118,6 +118,13 @@ class PublicSourceReleaseGuards(unittest.TestCase): 'LICENSES/Fixture.txt': b'Fixture license text.\n', 'rtl/ssc/LICENSE': b'Fixture SSC notice only; no ROM contents.\n', 'rtl/example.sv': b'module example; endmodule\n', + 'tools/project_inputs.json': json.dumps({'files': [ + '.gitignore', 'README.md', 'changes.txt', 'rtl/example.sv' + ]}).encode(), + 'releases/old.rbf': b'Historic binary fixture; not source payload', + 'docs/old.pdf': b'Historic PDF fixture; not source payload', + 'docs/old.docx': b'Historic Office fixture; not source payload', + 'unused.bin': b'Historic binary fixture; not source payload', } for name, data in files.items(): path = self.root / name @@ -140,6 +147,7 @@ class PublicSourceReleaseGuards(unittest.TestCase): 'passed': True, 'rbf_bytes': rbf.stat().st_size, 'rbf_sha256': digest(rbf), + 'project_inventory_sha256': digest(self.root / 'tools/project_inputs.json'), 'inputs': {'rtl/example.sv': digest(self.root / 'rtl/example.sv')}, } (self.build / 'build.json').write_text(json.dumps(record)) @@ -163,6 +171,7 @@ class PublicSourceReleaseGuards(unittest.TestCase): def test_untracked_private_files_do_not_enter_public_release(self): (self.root / '.env').write_text('fixture-private-value=not-for-export\n') (self.root / 'private-notes.txt').write_text('private fixture notes\n') + (self.root / 'LICENSES/private-notice.txt').write_text('PRIVATE NOTICE FIXTURE MUST NOT LEAK\n') untracked = self.git('ls-files', '--others', '--exclude-standard').decode() self.assertIn('.env', untracked) self.assertIn('private-notes.txt', untracked) @@ -183,12 +192,62 @@ class PublicSourceReleaseGuards(unittest.TestCase): with zipfile.ZipFile(self.output / 'corresponding-source.zip') as archive: source_files = {n for n in archive.namelist() if not n.endswith('/')} self.assertEqual(source_files, - {'coco3plus/' + n for n in self.tracked}) + {'coco3plus/' + n for n in self.tracked + if release.source_member(n)}) self.assertFalse(any('.env' in n or 'private-notes' in n for n in source_files)) + self.assertNotIn('coco3plus/LICENSES/private-notice.txt', source_files) + self.assertFalse(any(n.endswith(('.rbf', '.pdf', '.docx', '.bin')) + for n in source_files)) with zipfile.ZipFile(self.output.with_name('public-release.zip')) as archive: self.assertNotIn('.env', archive.namelist()) self.assertNotIn('private-notes.txt', archive.namelist()) + self.assertNotIn('fat/games/coco3plus/licenses/private-notice.txt', + archive.namelist()) + for name in archive.namelist(): + self.assertNotIn(b'PRIVATE NOTICE FIXTURE MUST NOT LEAK', + archive.read(name)) + checkout = archive.read('SOURCE-CHECKOUT.txt').decode() + self.assertIn('git clone ' + release.SOURCE_REPOSITORY, checkout) + self.assertIn('git checkout --detach ' + self.revision, checkout) + self.assertIn('ZIP alone is not a standalone build checkout', checkout) + self.assertEqual(record['project_inventory_sha256'], + digest(self.root / 'tools/project_inputs.json')) + self.assertNotIn('games/coco3plus/licenses/private-notice.txt', record['files']) + + def rewrite_receipt(self, transform): + receipt = self.build / 'build.json' + record = json.loads(receipt.read_text()) + transform(record) + receipt.write_text(json.dumps(record)) + + def assert_rejected_without_output(self, pattern): + with self.assertRaisesRegex(RuntimeError, pattern): + self.package() + self.assertFalse(self.output.exists()) + self.assertFalse(self.output.with_name('public-release.zip').exists()) + + def test_legacy_receipt_is_rejected_without_backfilling_evidence(self): + self.rewrite_receipt(lambda r: r.pop('project_inventory_sha256')) + original = (self.build / 'build.json').read_bytes() + self.assert_rejected_without_output('Legacy FPGA receipt lacks') + self.assertEqual((self.build / 'build.json').read_bytes(), original) + + def test_stale_inventory_identity_is_rejected(self): + self.rewrite_receipt(lambda r: r.update(project_inventory_sha256='0' * 64)) + self.assert_rejected_without_output('does not qualify the current') + + def test_subset_input_receipt_is_rejected(self): + self.rewrite_receipt(lambda r: r['inputs'].pop('rtl/example.sv')) + self.assert_rejected_without_output('exact current project input set') + + def test_extra_input_receipt_is_rejected(self): + self.rewrite_receipt(lambda r: r['inputs'].update({'NOTICE.md': digest(self.root / 'NOTICE.md')})) + self.assert_rejected_without_output('exact current project input set') + + def test_matching_set_with_changed_input_bytes_is_rejected(self): + self.rewrite_receipt(lambda r: r['inputs'].update({'rtl/example.sv': '0' * 64})) + self.assert_rejected_without_output('Source differs from built input') def test_dirty_tracked_source_rejects_before_output_creation(self): notice = self.root / 'NOTICE.md' diff --git a/tools/install_mister.py b/tools/install_mister.py index b435278..ec6aa72 100644 --- a/tools/install_mister.py +++ b/tools/install_mister.py @@ -1,54 +1,282 @@ #!/usr/bin/env python3 -"""Install a verified overlay using existing passwordless OpenSSH access.""" -import argparse,datetime,json,re,shlex,subprocess -from pathlib import Path,PurePosixPath -from release import ROOT,make_release,sha -def q(s):return shlex.quote(str(s)) -def main(): - p=argparse.ArgumentParser(description=__doc__) - p.add_argument('--host',help='IP or DNS hostname; prompted when omitted') - p.add_argument('--user',default='root') - p.add_argument('--release',type=Path,help='Existing tools/release.py output; otherwise package defaults') - p.add_argument('--without-ssc',action='store_true') - p.add_argument('--load',action='store_true',help='Return to MENU then load the new launcher') - a=p.parse_args();a.host=a.host or input('MiSTer IP address or hostname: ').strip() - if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*',a.host) or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]*',a.user):p.error('Use a plain hostname/IP and Unix username') - stamp=datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%S%fZ') - release=a.release.resolve() if a.release else ROOT/'dist'/('install-'+stamp) - if not a.release:make_release(release,ROOT/'build/fpga',ROOT/'build/roms',ROOT/'roms/ssc',a.without_ssc) - manifest=json.loads((release/'manifest.json').read_text());fat=release/'fat' - if manifest.get('public_package'):raise RuntimeError('Public core/source package omits required ROMs. Build a complete local package with your verified ROM inputs; see docs/INSTALL.md.') - for n,v in manifest['files'].items(): - path=PurePosixPath(n) - if path.is_absolute() or '..' in path.parts or '\\' in n:raise RuntimeError('Unsafe manifest path') - file=fat/n - if not file.is_file() or file.stat().st_size!=v['bytes'] or sha(file)!=v['sha256']:raise RuntimeError('Release changed: '+n) - args=['-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','ConnectTimeout=10'] - target=a.user+'@'+a.host - def ssh(script):return subprocess.check_output(['ssh',*args,target,'sh -s'],input=script.encode(),stderr=subprocess.STDOUT).decode() - # No target writes before all local hashes and this read-only preflight pass. - print(ssh('set -eu\ntest -d /media/fat\ntest -w /media/fat\ntest -e /dev/MiSTer_cmd\ncommand -v sha256sum\nid\n').strip()) - remote='/media/fat/.coco3plus/install-'+stamp;backup=remote+'/backup';stage=remote+'/stage' - ssh('set -eu\nmkdir -p '+q(stage)+' '+q(backup)+'\n') - # Raw argv paths are intentional: OpenSSH SFTP handles spaces without shell quoting. - subprocess.run(['scp',*args,'-r',str(fat),target+':'+stage+'/'],check=True) - lines=['set -eu'] - for n,v in manifest['files'].items():lines.append('test "$(sha256sum '+q(stage+'/fat/'+n)+' | cut -d " " -f 1)" = '+q(v['sha256'])) - lines.append("printf 'load_core /media/fat/menu.rbf\\n' > /dev/MiSTer_cmd") - lines.append('sleep 2') - for n,v in manifest['files'].items(): - src=stage+'/fat/'+n;dest='/media/fat/'+n;old=backup+'/'+n - lines.append('mkdir -p '+q(str(PurePosixPath(dest).parent))+' '+q(str(PurePosixPath(old).parent))) - if v['policy']=='create-if-missing':lines.append('if [ ! -e '+q(dest)+' ]; then cp '+q(src)+' '+q(dest)+'; fi') +"""Install a verified overlay using existing passwordless OpenSSH access. + +Original Coco 3 Plus contributions: Copyright (c) 2026 Brian K. Holman. +Original component notices/terms retained; see COPYRIGHT.md and LICENSING.md. +""" +import argparse +import datetime +import json +import os +from pathlib import Path, PurePosixPath +import re +import shlex +import subprocess +import sys +import uuid + +from release import ROOT, make_release, sha + +LAUNCHER = '_Computer/Coco 3 Plus.mgl' + + +def q(value): + return shlex.quote(str(value)) + + +def atomic_json(path, value): + """Keep the last complete receipt if a local update itself is interrupted.""" + temporary = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp') + with temporary.open('x', encoding='utf-8', newline='\n') as file: + json.dump(value, file, indent=2) + file.write('\n') + file.flush() + os.fsync(file.fileno()) + os.replace(temporary, path) + + +class OpenSshTarget: + def __init__(self, host, user): + self.options = ['-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', + '-o', 'ConnectTimeout=10'] + self.target = user + '@' + host + + def run(self, script): + return subprocess.check_output( + ['ssh', *self.options, self.target, 'sh -s'], + input=('set -eu\n' + script + '\n').encode(), + stderr=subprocess.STDOUT).decode() + + def preflight(self): + return self.run('test -d /media/fat\ntest -w /media/fat\n' + 'test -e /dev/MiSTer_cmd\ncommand -v sha256sum\nid') + + def prepare(self, stage, backup): + self.run('test ! -e ' + q(PurePosixPath(stage).parent) + + '\nmkdir -p ' + q(stage) + ' ' + q(backup)) + + def upload(self, fat, stage): + # Raw argv paths are intentional: SFTP handles spaces without shell quotes. + subprocess.run(['scp', *self.options, '-r', str(fat), + self.target + ':' + stage + '/'], check=True) + + def verify(self, files): + lines = [] + for name, digest in files.items(): + lines.append('test "$(sha256sum ' + q(name) + + ' | cut -d " " -f 1)" = ' + q(digest)) + self.run('\n'.join(lines)) + + def enter_menu(self): + self.run("printf 'load_core /media/fat/menu.rbf\\n' > /dev/MiSTer_cmd\nsleep 2") + + def install_file(self, source, destination, backup, entry, temporary): + # Copy/verify beside the destination, then rename within the same FAT + # filesystem. The staged original remains available for recovery. + lines = [ + 'mkdir -p ' + q(PurePosixPath(destination).parent) + ' ' + + q(PurePosixPath(backup).parent), + 'if [ -e ' + q(destination) + ' ]; then', + ' test -f ' + q(destination), + ] + if entry['policy'] == 'create-if-missing': + lines += [" printf 'preserved\\n'", ' exit 0'] else: - lines.append('if [ -e '+q(dest)+' ]; then cp '+q(dest)+' '+q(old)+'; fi') - lines.append('cp '+q(src)+' '+q(dest)) - lines.append('test "$(sha256sum '+q(dest)+' | cut -d " " -f 1)" = '+q(v['sha256'])) - lines.append('sync') - if a.load:lines.append("printf 'load_core /media/fat/_Computer/Coco 3 Plus.mgl\\n' > /dev/MiSTer_cmd") - result=ssh('\n'.join(lines)+'\n') - receipt=dict(host=a.host,user=a.user,backup=backup,manifest=manifest,loaded=a.load,remote_output=result) - (release/'install-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n') - print('Installed Coco 3 Plus; backup: '+backup+'; RBF '+manifest['rbf_sha256']) - print('Receipt: '+str(release/'install-receipt.json')) -if __name__=='__main__':main() + lines += [' cp ' + q(destination) + ' ' + q(backup)] + lines += [ + 'fi', + 'test ! -e ' + q(temporary), + 'cp ' + q(source) + ' ' + q(temporary), + 'test "$(sha256sum ' + q(temporary) + ' | cut -d " " -f 1)" = ' + + q(entry['sha256']), + 'mv -f ' + q(temporary) + ' ' + q(destination), + 'test "$(sha256sum ' + q(destination) + ' | cut -d " " -f 1)" = ' + + q(entry['sha256']), + "printf 'installed\\n'", + ] + result = self.run('\n'.join(lines)).splitlines() + if not result or result[-1] not in ('installed', 'preserved'): + raise RuntimeError('Missing remote install result: ' + destination) + return result[-1] + + def activate_launcher(self, source, destination, backup, entry, temporary): + return self.install_file(source, destination, backup, entry, temporary) + + def sync(self): + self.run('sync') + + def load(self): + self.run("printf 'load_core /media/fat/_Computer/Coco 3 Plus.mgl\\n' > /dev/MiSTer_cmd") + + +def validated_manifest(package): + manifest = json.loads((package / 'manifest.json').read_text()) + if manifest.get('public_package'): + raise RuntimeError('Public core/source package omits required ROMs. ' + 'Build a complete local package with your verified ' + 'ROM inputs; see docs/INSTALL.md.') + for name, entry in manifest['files'].items(): + path = PurePosixPath(name) + if path.is_absolute() or '..' in path.parts or '\\' in name: + raise RuntimeError('Unsafe manifest path') + if entry['policy'] not in ('create-if-missing', 'backup-and-replace'): + raise RuntimeError('Unsupported install policy: ' + name) + file = package / 'fat' / name + if (not file.is_file() or file.stat().st_size != entry['bytes'] or + sha(file) != entry['sha256']): + raise RuntimeError('Release changed: ' + name) + if (LAUNCHER not in manifest['files'] or + manifest['files'][LAUNCHER]['policy'] != 'backup-and-replace'): + raise RuntimeError('A replaceable Coco 3 Plus launcher is required') + return manifest + + +def install_release(package, host, user='root', load=False, *, target=None, stamp=None): + """Install support first and activate last; target injection is for tests.""" + package = Path(package).resolve() + manifest = validated_manifest(package) + if (not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*', host) or + not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]*', user)): + raise ValueError('Use a plain hostname/IP and Unix username') + stamp = stamp or datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%S%fZ') + if not re.fullmatch(r'[A-Za-z0-9_-]{1,80}', stamp): + raise ValueError('Invalid installation attempt identifier') + remote = '/media/fat/.coco3plus/install-' + stamp + backup, stage = remote + '/backup', remote + '/stage' + receipt_path = package / ('install-receipt-' + stamp + '.json') + latest_path = package / 'install-receipt.json' + if receipt_path.exists(): + raise RuntimeError('Preserve existing installation attempt: ' + str(receipt_path)) + previous = None + if latest_path.exists(): + previous = package / ('install-receipt-before-' + stamp + '.json') + with previous.open('xb') as file: + file.write(latest_path.read_bytes()) + plans = { + name: dict(source=stage + '/fat/' + name, + destination='/media/fat/' + name, + backup=backup + '/' + name, + temporary='/media/fat/' + name + '.install-' + stamp + '.tmp', + policy=entry['policy']) + for name, entry in manifest['files'].items() + } + receipt = dict( + host=host, user=user, attempt=stamp, status='planned', phase='planned', + remote=remote, stage=stage, backup=backup, manifest=manifest, + receipt=str(receipt_path), previous_receipt=str(previous) if previous else None, + launcher='/media/fat/' + LAUNCHER, + launcher_temporary='/media/fat/' + LAUNCHER + '.install-' + stamp + '.tmp', + activation_attempted=False, launcher_activated=False, load_requested=load, + loaded=False, planned_files=plans, active_file=None, + completed_files=[], history=['planned'], + recovery='Preserve stage and backup. Before activation the old launcher ' + 'remains, but supporting files may already have changed. Restore ' + 'their backed-up copies if rollback is needed. If activation ' + 'was attempted but not confirmed, inspect the launcher before ' + 'loading; restore the backed-up launcher if needed.') + def save(): + atomic_json(receipt_path, receipt) + atomic_json(latest_path, receipt) + + def phase(name): + receipt['status'] = 'in-progress' + receipt['phase'] = name + receipt['history'].append(name) + save() + + # A durable local plan names all recovery locations before target mutation. + save() + print('Planned receipt: ' + str(receipt_path), flush=True) + print('Remote stage: ' + stage, flush=True) + print('Remote backup: ' + backup, flush=True) + client = target if target is not None else OpenSshTarget(host, user) + try: + phase('preflight') + receipt['preflight_output'] = client.preflight() + phase('staging') + client.prepare(stage, backup) + client.upload(package / 'fat', stage) + phase('staged-verification') + client.verify({stage + '/fat/' + n: v['sha256'] + for n, v in manifest['files'].items()}) + phase('returning-to-menu') + client.enter_menu() + phase('supporting-assets') + installed = {} + for name in sorted(n for n in manifest['files'] if n != LAUNCHER): + entry = manifest['files'][name] + receipt['active_file'] = name + save() + destination = '/media/fat/' + name + result = client.install_file( + stage + '/fat/' + name, destination, backup + '/' + name, entry, + destination + '.install-' + stamp + '.tmp') + receipt['completed_files'].append(dict(path=name, result=result)) + receipt['active_file'] = None + if result == 'installed': + installed[destination] = entry['sha256'] + save() + client.verify(installed) + client.sync() + phase('activating-launcher') + receipt['activation_attempted'] = True + receipt['active_file'] = LAUNCHER + # A transport failure during rename can leave activation uncertain. + receipt['launcher_activated'] = None + save() + client.activate_launcher( + stage + '/fat/' + LAUNCHER, receipt['launcher'], + backup + '/' + LAUNCHER, manifest['files'][LAUNCHER], + receipt['launcher_temporary']) + receipt['launcher_activated'] = True + receipt['active_file'] = None + phase('launcher-activated') + client.sync() + if load: + phase('loading') + receipt['loaded'] = None + save() + client.load() + receipt['loaded'] = True + receipt['status'] = 'complete' + receipt['phase'] = 'complete' + receipt['history'].append('complete') + save() + except BaseException as error: + receipt['status'] = 'failed' + receipt['history'].append('failed') + receipt['error'] = dict(type=type(error).__name__, message=str(error)) + output = getattr(error, 'output', None) + if output: + receipt['error']['output'] = ( + output.decode(errors='replace') if isinstance(output, bytes) else str(output))[-8192:] + try: + save() + except Exception as save_error: + print('Could not update failure receipt; prior plan remains at ' + + str(receipt_path) + ': ' + str(save_error), file=sys.stderr) + raise + print('Installed Coco 3 Plus; backup: ' + backup + '; RBF ' + manifest['rbf_sha256']) + print('Receipt: ' + str(receipt_path)) + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--host', help='IP or DNS hostname; prompted when omitted') + parser.add_argument('--user', default='root') + parser.add_argument('--release', type=Path, help='Existing tools/release.py output; otherwise package defaults') + parser.add_argument('--without-ssc', action='store_true') + parser.add_argument('--load', action='store_true', help='Return to MENU then load the new launcher') + args = parser.parse_args() + args.host = args.host or input('MiSTer IP address or hostname: ').strip() + stamp = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%S%fZ') + package = args.release.resolve() if args.release else ROOT / 'dist' / ('install-' + stamp) + if not args.release: + make_release(package, ROOT / 'build/fpga', ROOT / 'build/roms', + ROOT / 'roms/ssc', args.without_ssc) + install_release(package, args.host, args.user, args.load, stamp=stamp) + + +if __name__ == '__main__': + main() diff --git a/tools/release.py b/tools/release.py index e00b3d2..b342a7e 100644 --- a/tools/release.py +++ b/tools/release.py @@ -1,8 +1,23 @@ #!/usr/bin/env python3 """Create a checked, copyable MiSTer FAT-volume overlay from a standalone build.""" import argparse,datetime,hashlib,json,shutil,subprocess,zipfile -from pathlib import Path +from pathlib import Path,PurePosixPath ROOT=Path(__file__).resolve().parents[1] +# Reviewed source-package content. The repository retains all upstream history; +# a source-review ZIP deliberately omits historical RBFs, Office files and PDFs. +SOURCE_ROOT_FILES={'.gitattributes','.gitignore','AGENTS.md','CHANGELOG.md', + 'COPYRIGHT.md','LICENSING.md','NOTICE.md','README.md','CoCo3.qpf','CoCo3.qsf', + 'CoCo3.sv','CoCo3FPGA.sdc','build_id_num.v','changes.txt','external-io.sdc', + 'files.qip','pll.xml','reset-cpu.sdc','uart-reset.sdc'} +SOURCE_SUFFIXES={ + 'docs':{'.md','.vhd'},'roms':{'.md'}, + 'rtl':{'.sv','.v','.vh','.vhd','.mif','.hex','.qip','.qsys','.ip','.tcl','.sdc', + '.ppf','.sip','.spd'}, + 'sys':{'.sv','.v','.vh','.vhd','.mif','.hex','.qip','.qsys','.ip','.tcl','.sdc'}, + 'scripts':{'.py','.tcl'},'tests':{'.py','.sv'},'tools':{'.py','.json'}, + 'validation':{'.json'},'windows':{'.ps1'}} +SOURCE_REPOSITORY='https://vccplus-git.brianholman.com/VCCPlus/CoCo3_Plus_MiSTer.git' +NON_HASHED_BUILD_FILES={'.gitignore','README.md','changes.txt'} SSC={'pic-7040-510.bin':(4096,'0013cd78482cc5f2360fdf584e9ba51d34861e8ccd9d88247f1a54039c16c365'), 'sp0256-al2.bin':(2048,'98c280fd082169c54081e503dfe7d7d9a6fd46d4ffcc973e199ea0e1d9bc1ac9')} BOOT={'boot0.rom':(32768,'ad358430568250eb8330a37f85361d9ce67e5ae6d5a28c17b31802cd3953a5d2'), @@ -11,6 +26,43 @@ def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() def checked(p,size,digest): if not p.is_file() or p.stat().st_size!=size or sha(p)!=digest:raise RuntimeError('Missing or incorrect input: '+str(p)) return p +def qualify_inputs(record): + inventory_path=ROOT/'tools/project_inputs.json' + if 'project_inventory_sha256' not in record: + raise RuntimeError('Legacy FPGA receipt lacks project_inventory_sha256; preserve it unchanged and run a fresh independently qualified build. Do not backfill fit evidence.') + if record['project_inventory_sha256']!=sha(inventory_path): + raise RuntimeError('FPGA receipt does not qualify the current tools/project_inputs.json') + names=json.loads(inventory_path.read_text())['files'] + if not isinstance(names,list) or len(names)!=len(set(names)): + raise RuntimeError('Invalid or duplicate project input inventory') + for name in names: + path=PurePosixPath(name) + if path.is_absolute() or '..' in path.parts or '\\' in name or ':' in name: + raise RuntimeError('Unsafe project input: '+name) + expected=set(names)-NON_HASHED_BUILD_FILES + actual=record.get('inputs') + if not isinstance(actual,dict) or set(actual)!=expected: + raise RuntimeError('FPGA receipt must qualify the exact current project input set') + for name,digest in actual.items(): + path=ROOT/name + if not path.is_file() or path.is_symlink() or sha(path)!=digest: + raise RuntimeError('Source differs from built input: '+name) +def committed_tree(revision): + tree={} + for entry in subprocess.check_output(['git','ls-tree','-rz','--full-tree',revision],cwd=ROOT).split(b'\0'): + if not entry:continue + metadata,name=entry.split(b'\t',1) + mode,kind,object_id=metadata.split() + if kind==b'blob' and mode in (b'100644',b'100755'): + tree[name.decode('utf-8')]=object_id.decode('ascii') + return tree +def committed_blob(object_id): + return subprocess.check_output(['git','cat-file','blob',object_id],cwd=ROOT) +def source_member(name): + path=PurePosixPath(name) + if name in SOURCE_ROOT_FILES or name=='rtl/ssc/LICENSE':return True + if len(path.parts)==2 and path.parts[0]=='LICENSES':return True + return len(path.parts)>1 and path.suffix.lower() in SOURCE_SUFFIXES.get(path.parts[0],set()) def default_config(ssc): # Main serializes cur_status[16] in byte order, low bits first. # Audio / SDC / ROM / Floppy, startup Slot4, 512K; migrated-layout marker. @@ -21,23 +73,24 @@ def make_release(output,build,roms,ssc,without_ssc=False,public=False): record=json.loads((build/'build.json').read_text()) if not record.get('passed'):raise RuntimeError('Successful standalone FPGA build required') rbf=checked(build/'project/output_files/CoCo3.rbf',record['rbf_bytes'],record['rbf_sha256']) - for n,h in record['inputs'].items(): - if sha(ROOT/n)!=h:raise RuntimeError('Source differs from built input: '+n) + qualify_inputs(record) inputs={} if public else {f'games/coco3plus/{n}':checked(roms/n,*identity) for n,identity in BOOT.items()} if not without_ssc and not public:inputs.update({f'games/coco3plus/{n}':checked(ssc/n,*v) for n,v in SSC.items()}) if subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=ROOT).strip():raise RuntimeError('Commit reviewed changes before creating a distributable source package') source_revision=subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT).decode().strip() + tree=committed_tree(source_revision) + notice_names={'NOTICE.md','LICENSING.md','COPYRIGHT.md','rtl/ssc/LICENSE'} + notice_names.update(name for name in tree if len(PurePosixPath(name).parts)==2 and name.startswith('LICENSES/')) + if not notice_names<=tree.keys():raise RuntimeError('Required committed license notices missing') if output.exists():raise RuntimeError('Preserve previous release: '+str(output)) fat=output/'fat';fat.mkdir(parents=True) core=f'_Computer/.coco3plus/{record["rbf_sha256"][:12]}/coco3plus.rbf';inputs[core]=rbf - for name in ('NOTICE.md','LICENSING.md','COPYRIGHT.md'): - inputs['games/coco3plus/licenses/'+name]=ROOT/name - for folder in (ROOT/'LICENSES',): - for file in folder.glob('*'): - if file.is_file():inputs['games/coco3plus/licenses/'+file.name]=file - inputs['games/coco3plus/licenses/SSC-BSD-3-Clause.txt']=ROOT/'rtl/ssc/LICENSE' for n,p in inputs.items(): target=fat/n;target.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,target) + for name in sorted(notice_names): + leaf='SSC-BSD-3-Clause.txt' if name=='rtl/ssc/LICENSE' else PurePosixPath(name).name + target=fat/'games/coco3plus/licenses'/leaf;target.parent.mkdir(parents=True,exist_ok=True) + target.write_bytes(committed_blob(tree[name])) launcher=fat/'_Computer/Coco 3 Plus.mgl';launcher.parent.mkdir(exist_ok=True) launcher.write_text(f'\n {core[:-4]}\n coco3plus\n\n',newline='\n') config=fat/'config';config.mkdir() @@ -56,11 +109,27 @@ def make_release(output,build,roms,ssc,without_ssc=False,public=False): (output/'README.txt').write_text('Coco 3 Plus FAT-volume overlay. This is not a bootable disk image.\nCopy fat/ CONTENTS to the MiSTer FAT-volume root with MiSTer shut down.\nPreserve existing config/coco3plus_v51.CFG. Back up files before replacing.\nOriginal CoCo3 and other cores are not replaced. Start Computer/Coco 3 Plus.\nSSC ROMs, when included, are separately supplied local inputs; do not redistribute without permission.\n') if public: with (output/'README.txt').open('a') as file:file.write('PUBLIC CORE/SOURCE PACKAGE: no CoCo boot or SSC ROMs included.\nBefore launching, supply boot0.rom and boot1.rom in games/coco3plus and the SSC ROMs if using SSC.\nSee roms/README.md and docs/INSTALL.md in corresponding-source.zip for exact identities and source-build installation.\nThis incomplete-ROM public package is not accepted directly by the SSH installer. Build a complete local package using your own verified ROM inputs.\n') - # Git's reviewed committed tree is the only source-export allowlist. Untracked - # notes, .env files and ignored ROM/cache inputs can never enter this archive. - with (output/'corresponding-source.zip').open('xb') as file: - subprocess.run(['git','archive','--format=zip','--prefix=coco3plus/',source_revision],cwd=ROOT,stdout=file,check=True) + # Use reviewed paths AND committed bytes. This ZIP has no .git metadata; the + # supported build route is a pinned full clone, not an extracted archive. + source_names=sorted(name for name in tree if source_member(name)) + with zipfile.ZipFile(output/'corresponding-source.zip','x',zipfile.ZIP_DEFLATED) as archive: + for name in source_names:archive.writestr('coco3plus/'+name,committed_blob(tree[name])) + checkout=('SOURCE REVIEW ARCHIVE: no Git metadata or historical binary/manual artifacts.\n' + 'Build tools require Git metadata and the full maintained project inventory.\n' + 'Use Git for Windows, Python 3.10+ and the pinned Quartus toolchain.\n' + 'Clone the full repository (do not use --depth), then detach at this exact revision:\n' + f'git clone {SOURCE_REPOSITORY} coco3plus\n' + 'cd coco3plus\n' + f'git checkout --detach {source_revision}\n' + 'git rev-parse HEAD\n' + 'Follow docs/BUILD_WINDOWS.md in that checkout. Repository access is required.\n' + 'The source review ZIP alone is not a standalone build checkout.\n') + (output/'SOURCE-CHECKOUT.txt').write_text(checkout) + with (output/'README.txt').open('a') as file:file.write('For the supported pinned Git build route, see SOURCE-CHECKOUT.txt. The source-review ZIP alone cannot run the build scripts.\n') manifest['source_revision']=source_revision + manifest['project_inventory_sha256']=record['project_inventory_sha256'] + manifest['source_files']=source_names + manifest['source_build_route']='Pinned full Git checkout; see SOURCE-CHECKOUT.txt' manifest['corresponding_source_sha256']=sha(output/'corresponding-source.zip') (output/'manifest.json').write_text(json.dumps(manifest,indent=2)+'\n') archive_path=output.with_name(output.name+'.zip')