Files
CoCo3_Plus_MiSTer/scripts/run_regressions.py
T

88 lines
4.2 KiB
Python

#!/usr/bin/env python3
"""Run independent MPI and inherited regressions against an isolated source copy.
This does not compile Quartus, qualify timing, install a core, or test hardware.
Each test owns a fresh output folder and its source-hashed evidence.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('project', type=Path, nargs='?', default=ROOT)
p.add_argument('--suite', type=Path, required=True)
p.add_argument('--main-reference', type=Path, required=True)
p.add_argument('--output',type=Path,default=ROOT/'build/regressions')
p.add_argument('--jobs', type=int, default=3, choices=range(1, 5))
a = p.parse_args()
project = a.project.resolve()
if project != ROOT and not project.is_relative_to(ROOT / 'build'):
p.error('Use a prepared build/ project')
base = a.output.resolve()
if not base.is_relative_to(ROOT/'build'):p.error('Use an output under build/')
base.mkdir(parents=True)
logs = base / 'regression-logs'
logs.mkdir() # Preserve any earlier run; never overwrite its evidence.
from source_snapshot import original_sources
original=original_sources('coco3-mister')
jobs = [
('test_bh_mpi.py', 'mpi-functional', ['--project', str(project)]),
('test_bh_mpi_core.py', 'mpi-core-functional', [str(project)]),
('test_ssc_mister_system.py', 'ssc-system-functional', [str(project)]),
('test_registered_interfaces.py', 'registered-functional', [str(project)]),
('test_reset_domains.py', 'reset-functional', [str(project)]),
('test_keyboard_control_phase.py', 'keyboard-functional', [str(project)]),
('test_joystick_button_routing.py', 'joystick-buttons-functional', [str(project)]),
('test_peripheral_phase.py', 'peripheral-functional', [str(project)]),
('test_external_sdram_commands.py', 'commands-functional', [str(project)]),
('test_external_sdram.py', 'delayed-functional', [str(project)]),
('test_external_audio.py', 'audio-functional', [str(project)]),
('test_external_pixel.py', 'pixel-functional', [str(project)]),
('test_external_i2c.py', 'i2c-functional', [str(project)]),
('test_coco3_bh_startup.py', 'startup-functional',
[str(project), str(a.main_reference.resolve())]),
('test_bh_menu.py', 'menu-functional', [str(project), str(a.main_reference.resolve())]),
('test_video_sdram.py', 'video-functional', [str(original),str(project)]),
]
def run(job):
script, name, inputs = job
command = [sys.executable, '-X', 'utf8', '-B', str(ROOT / 'scripts' / script),
*inputs, str(base / name)]
if script not in ('test_coco3_bh_startup.py','test_bh_menu.py'):
command += ['--suite', str(a.suite.resolve())]
if script == 'test_external_sdram.py':
command += ['--check-cpu-data']
started = time.monotonic()
print('START ' + name, flush=True)
with (logs / (name + '.log')).open('w', encoding='utf-8') as log:
result = subprocess.run(command, cwd=ROOT, stdout=log,
stderr=subprocess.STDOUT)
receipt = dict(name=name, command=command, exit_code=result.returncode,
seconds=round(time.monotonic() - started, 2))
print(('PASS ' if result.returncode == 0 else 'FAIL ') + name, flush=True)
return receipt
results = []
with ThreadPoolExecutor(max_workers=a.jobs) as pool:
for future in as_completed([pool.submit(run, job) for job in jobs]):
results.append(future.result())
passed = all(item['exit_code'] == 0 for item in results)
(logs / 'result.json').write_text(json.dumps(dict(
passed=passed, scope='Offline functional regressions; no timing/hardware signoff',
results=sorted(results, key=lambda x: x['name'])), indent=2) + '\n')
if not passed:
raise SystemExit('One or more regressions failed; inspect regression-logs')
if __name__ == '__main__':
main()