88 lines
4.2 KiB
Python
88 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Run independent MPI and inherited regressions against an isolated source copy.
|
|
|
|
This does not compile Quartus, qualify timing, install a core, or test hardware.
|
|
Each test owns a fresh output folder and its source-hashed evidence.
|
|
"""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('project', type=Path, nargs='?', default=ROOT)
|
|
p.add_argument('--suite', type=Path, required=True)
|
|
p.add_argument('--main-reference', type=Path, required=True)
|
|
p.add_argument('--output',type=Path,default=ROOT/'build/regressions')
|
|
p.add_argument('--jobs', type=int, default=3, choices=range(1, 5))
|
|
a = p.parse_args()
|
|
project = a.project.resolve()
|
|
if project != ROOT and not project.is_relative_to(ROOT / 'build'):
|
|
p.error('Use a prepared build/ project')
|
|
base = a.output.resolve()
|
|
if not base.is_relative_to(ROOT/'build'):p.error('Use an output under build/')
|
|
base.mkdir(parents=True)
|
|
logs = base / 'regression-logs'
|
|
logs.mkdir() # Preserve any earlier run; never overwrite its evidence.
|
|
from source_snapshot import original_sources
|
|
original=original_sources('coco3-mister')
|
|
jobs = [
|
|
('test_bh_mpi.py', 'mpi-functional', ['--project', str(project)]),
|
|
('test_bh_mpi_core.py', 'mpi-core-functional', [str(project)]),
|
|
('test_ssc_mister_system.py', 'ssc-system-functional', [str(project)]),
|
|
('test_registered_interfaces.py', 'registered-functional', [str(project)]),
|
|
('test_reset_domains.py', 'reset-functional', [str(project)]),
|
|
('test_keyboard_control_phase.py', 'keyboard-functional', [str(project)]),
|
|
('test_joystick_button_routing.py', 'joystick-buttons-functional', [str(project)]),
|
|
('test_peripheral_phase.py', 'peripheral-functional', [str(project)]),
|
|
('test_external_sdram_commands.py', 'commands-functional', [str(project)]),
|
|
('test_external_sdram.py', 'delayed-functional', [str(project)]),
|
|
('test_external_audio.py', 'audio-functional', [str(project)]),
|
|
('test_external_pixel.py', 'pixel-functional', [str(project)]),
|
|
('test_external_i2c.py', 'i2c-functional', [str(project)]),
|
|
('test_coco3_bh_startup.py', 'startup-functional',
|
|
[str(project), str(a.main_reference.resolve())]),
|
|
('test_bh_menu.py', 'menu-functional', [str(project), str(a.main_reference.resolve())]),
|
|
('test_video_sdram.py', 'video-functional', [str(original),str(project)]),
|
|
]
|
|
|
|
def run(job):
|
|
script, name, inputs = job
|
|
command = [sys.executable, '-X', 'utf8', '-B', str(ROOT / 'scripts' / script),
|
|
*inputs, str(base / name)]
|
|
if script not in ('test_coco3_bh_startup.py','test_bh_menu.py'):
|
|
command += ['--suite', str(a.suite.resolve())]
|
|
if script == 'test_external_sdram.py':
|
|
command += ['--check-cpu-data']
|
|
started = time.monotonic()
|
|
print('START ' + name, flush=True)
|
|
with (logs / (name + '.log')).open('w', encoding='utf-8') as log:
|
|
result = subprocess.run(command, cwd=ROOT, stdout=log,
|
|
stderr=subprocess.STDOUT)
|
|
receipt = dict(name=name, command=command, exit_code=result.returncode,
|
|
seconds=round(time.monotonic() - started, 2))
|
|
print(('PASS ' if result.returncode == 0 else 'FAIL ') + name, flush=True)
|
|
return receipt
|
|
|
|
results = []
|
|
with ThreadPoolExecutor(max_workers=a.jobs) as pool:
|
|
for future in as_completed([pool.submit(run, job) for job in jobs]):
|
|
results.append(future.result())
|
|
passed = all(item['exit_code'] == 0 for item in results)
|
|
(logs / 'result.json').write_text(json.dumps(dict(
|
|
passed=passed, scope='Offline functional regressions; no timing/hardware signoff',
|
|
results=sorted(results, key=lambda x: x['name'])), indent=2) + '\n')
|
|
if not passed:
|
|
raise SystemExit('One or more regressions failed; inspect regression-logs')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|