Files
VCCPlusDiagnostics/examples/sound-lab/tests/test_reports.py
T

359 lines
20 KiB
Python

# SPDX-License-Identifier: GPL-3.0-or-later
"""Authored, ROM-free fixtures for Sound Lab report evidence and DECB extraction."""
import hashlib
import importlib.util
import json
from pathlib import Path
import tempfile
import unittest
MODULE = Path(__file__).resolve().parents[1] / "host" / "soundlab_reports.py"
spec = importlib.util.spec_from_file_location("soundlab_reports", MODULE)
reports = importlib.util.module_from_spec(spec)
spec.loader.exec_module(reports)
def make_report(*, auto="OBSERVED", human="HEARD", ending=True, case="TON01-A", crc="7ABC", build="TEST-1", vector_set="1234", note=None):
lines = ["FORMAT=SOUNDLAB-1", f"RUN=0001 TARGET=REALCOCO BUILD_ID={build} VECTOR_SET_CRC16={vector_set}",
"MACHINE=COCO3 VIDEO=NTSC SPEED=NORMAL CART=26-3144A MOD=UNKNOWN",
f"CASE={case} VECTOR_CRC16={crc} AUTO={auto} HUMAN={human}",
"TX_COUNT=11 READY_TIMEOUTS=0 TICK_UNIT=VSYNC TICK_RATE_NUM=60000 TICK_RATE_DEN=1001",
"START_TICK=1234 END_TICK=1260 POLL_RESOLUTION_US=UNKNOWN", "STATUS_BEFORE=E0 STATUS_AFTER=E0"]
if note is not None:
lines.extend([f"NOTE_LENGTH={len(reports.decode_note(note))}", "NOTE=" + note])
lines.append(f"ENDCASE={case}")
if ending:
checksum = reports.crc16(('\r\n'.join(lines) + '\r\n').encode())
lines.append(f"ENDRUN=0001 COMPLETE=YES RECORD_COUNT={len(lines)} CHECKSUM_CRC16={checksum:04X}")
return ('\r\n'.join(lines) + '\r\n').encode()
def make_disk(files):
"""files is (name, data, explicit fragmented granule chain) tuples."""
disk = bytearray([0xFF] * reports.DISK_SIZE)
for index, (name, data, chain) in enumerate(files):
if not 0 < len(data) <= 2304 * len(chain) or len(data) <= 2304 * (len(chain) - 1):
raise ValueError("Fixture chain must exactly cover nonempty data")
base, ext = name.split(".")
entry = bytearray(32)
entry[:8] = base.encode().ljust(8, b' ')
entry[8:11] = ext.encode().ljust(3, b' ')
entry[11:14] = bytes([3, 0xFF, chain[0]])
entry[14:16] = (((len(data) - 1) % 256) + 1).to_bytes(2, 'big')
start = reports.DIRECTORY_OFFSET + index * 32
disk[start:start + 32] = entry
for position, granule in enumerate(chain):
last = position == len(chain) - 1
remaining = len(data) - position * 2304
disk[reports.FAT_OFFSET + granule] = 0xC0 + ((remaining + 255) // 256) if last else chain[position + 1]
chunk = data[position * 2304:(position + 1) * 2304]
offset = reports.granule_offset(granule)
disk[offset:offset + len(chunk)] = chunk
return disk
class DiskTests(unittest.TestCase):
def test_fragmented_chain_skips_directory_track_and_preserves_original(self):
content = b'A' * 2304 + b'B' * 2304 + b'END\r\n'
disk = make_disk([('snd0001.txt', content, [33, 34, 5])])
original = bytes(disk)
selected = reports.extract_reports(disk)
self.assertEqual(selected[0]['data'], content)
self.assertEqual(selected[0]['filename'], 'SND0001.TXT')
self.assertEqual(selected[0]['granules'], [33, 34, 5])
self.assertEqual(bytes(disk), original)
self.assertEqual(selected[0]['sha256'], hashlib.sha256(content).hexdigest())
self.assertEqual(reports.granule_offset(34), 18 * 18 * 256)
def test_full_sector_and_full_granule(self):
for size in (256, 2304, 2305):
with self.subTest(size=size):
data = bytes(i % 256 for i in range(size))
disk = make_disk([('SND0001.TXT', data, [0] if size <= 2304 else [0, 67])])
self.assertEqual(reports.extract_reports(disk)[0]['data'], data)
def test_only_report_names_extracted_and_binary_reports_rejected(self):
disk = make_disk([('NOTES.TXT', b'notes', [0]), ('SND0002.TXT', b'report', [1])])
self.assertEqual([item['filename'] for item in reports.extract_reports(disk)], ['SND0002.TXT'])
disk[reports.DIRECTORY_OFFSET + 32 + 12] = 0
with self.assertRaisesRegex(reports.SoundLabError, 'not marked.*ASCII'):
reports.extract_reports(disk)
def test_wrong_size_rejected_including_truncation(self):
for size in (0, reports.DISK_SIZE - 1, reports.DISK_SIZE + 1):
with self.subTest(size=size), self.assertRaises(reports.SoundLabError):
reports.extract_reports(bytes(size))
def test_invalid_fat_value(self):
disk = make_disk([('SND0001.TXT', b'hello', [0])])
disk[reports.FAT_OFFSET] = 0xCA
with self.assertRaisesRegex(reports.SoundLabError, 'Invalid FAT'):
reports.extract_reports(disk)
def test_fat_loop(self):
disk = make_disk([('SND0001.TXT', b'hello', [0])])
disk[reports.FAT_OFFSET] = 2
disk[reports.FAT_OFFSET + 2] = 0
with self.assertRaisesRegex(reports.SoundLabError, 'loop'):
reports.extract_reports(disk)
def test_free_granule_inside_chain(self):
disk = make_disk([('SND0001.TXT', b'hello', [0])])
disk[reports.FAT_OFFSET] = 0xFF
with self.assertRaisesRegex(reports.SoundLabError, 'free granule'):
reports.extract_reports(disk)
def test_crosslinks_with_unselected_file(self):
disk = make_disk([('SND0001.TXT', b'hello', [0]), ('OTHER.DAT', b'data', [1])])
disk[reports.DIRECTORY_OFFSET + 32 + 13] = 0
with self.assertRaisesRegex(reports.SoundLabError, 'Crosslinked'):
reports.extract_reports(disk)
def test_final_sector_count_and_empty_marker(self):
for byte_count in (257, 65535):
disk = make_disk([('SND0001.TXT', b'hello', [0])])
disk[reports.DIRECTORY_OFFSET + 14:reports.DIRECTORY_OFFSET + 16] = byte_count.to_bytes(2, 'big')
with self.subTest(byte_count=byte_count), self.assertRaisesRegex(reports.SoundLabError, 'byte count'):
reports.extract_reports(disk)
disk = make_disk([('SND0001.TXT', b'hello', [0])])
disk[reports.FAT_OFFSET] = 0xC0
with self.assertRaisesRegex(reports.SoundLabError, 'empty final granule'):
reports.extract_reports(disk)
def test_duplicate_names_and_unsafe_name(self):
disk = make_disk([('SND0001.TXT', b'a', [0]), ('snd0001.txt', b'b', [1])])
with self.assertRaisesRegex(reports.SoundLabError, 'Duplicate'):
reports.extract_reports(disk)
disk = make_disk([('SND0001.TXT', b'a', [0])])
disk[reports.DIRECTORY_OFFSET] = ord('/')
with self.assertRaisesRegex(reports.SoundLabError, 'Unsafe'):
reports.extract_reports(disk)
class ReportTests(unittest.TestCase):
def test_crc_known_vector(self):
self.assertEqual(reports.crc16(b'123456789'), 0x29B1)
def test_complete_report_and_escaped_note(self):
result = reports.parse_report(make_report(note=r'LINE\nTWO\\END\x21'))
self.assertTrue(result['complete'], result['issues'])
self.assertEqual(result['cases'][0]['notes'], ['LINE\nTWO\\END!'])
self.assertEqual(result['integrity'], {'record_count': 'VERIFIED', 'checksum_crc16': 'VERIFIED'})
self.assertEqual(result['cases'][0]['fields']['POLL_RESOLUTION_US'], 'UNKNOWN')
def test_lf_and_crlf_same_canonical_checksum(self):
self.assertTrue(reports.parse_report(make_report().replace(b'\r\n', b'\n'))['complete'])
def test_partial_run_keeps_case_evidence(self):
text = make_report(ending=False).rsplit(b'ENDCASE=', 1)[0]
result = reports.parse_report(text)
self.assertFalse(result['complete'])
self.assertFalse(result['cases'][0]['closed'])
self.assertEqual(result['cases'][0]['human'], 'HEARD')
self.assertTrue(any('partial' in issue for issue in result['issues']))
def test_bad_checksum_and_count(self):
data = make_report().replace(b'TX_COUNT=11', b'TX_COUNT=12').replace(b'RECORD_COUNT=8', b'RECORD_COUNT=7')
result = reports.parse_report(data)
self.assertFalse(result['valid'])
self.assertEqual(result['integrity']['checksum_crc16'], 'FAILED')
self.assertEqual(result['integrity']['record_count'], 'FAILED')
def test_optional_integrity_fields_are_honestly_unverified(self):
data = make_report().split(b' RECORD_COUNT=')[0] + b'\r\n'
result = reports.parse_report(data)
self.assertTrue(result['complete'])
self.assertEqual(result['integrity']['checksum_crc16'], 'NOT_PROVIDED')
def test_missing_human_never_infers_heard(self):
data = make_report().replace(b' HUMAN=HEARD', b'')
result = reports.parse_report(data)
self.assertEqual(result['cases'][0]['human'], 'UNKNOWN')
self.assertFalse(result['valid'])
def test_endcase_identity_and_trailing_records(self):
data = make_report().replace(b'ENDCASE=TON01-A', b'ENDCASE=OTHER') + b'CASE=LATE\r\n'
result = reports.parse_report(data)
self.assertFalse(result['valid'])
self.assertTrue(any('ENDCASE does not match' in value for value in result['issues']))
self.assertTrue(any('after ENDRUN' in value for value in result['issues']))
def test_notes_length_and_unknown_escape(self):
for bad in (b'NOTE_LENGTH=99\r\nNOTE=OK', b'NOTE_LENGTH=2\r\nNOTE=\\q'):
data = make_report(note='OK').replace(b'NOTE_LENGTH=2\r\nNOTE=OK', bad)
result = reports.parse_report(data)
self.assertFalse(result['valid'])
self.assertTrue(any('NOTE_LENGTH mismatch' in value or 'escape' in value for value in result['issues']))
def test_incomplete_note_and_run_identity(self):
data = make_report(ending=False).replace(b'ENDCASE=TON01-A\r\n', b'NOTE_LENGTH=3\r\n')
self.assertTrue(any('Truncated NOTE' in issue for issue in reports.parse_report(data)['issues']))
result = reports.parse_report(make_report().replace(b'ENDRUN=0001', b'ENDRUN=0002'))
self.assertTrue(any('does not match RUN' in issue for issue in result['issues']))
def test_invalid_ascii_and_format(self):
for data in (b'FORMAT=SOUNDLAB-2\n', b'FORMAT=SOUNDLAB-1\rRUN=1', b'FORMAT=SOUNDLAB-1\nNOTE=\xff', b'FORMAT=SOUNDLAB-1\n\x00'):
with self.subTest(data=data), self.assertRaises(reports.SoundLabError):
reports.parse_report(data)
def test_mixed_case_outcomes_compare_separately(self):
left = reports.parse_report(make_report(auto='observed', human='heard', case='ton01-a', crc='7abc'), 'left')
right = reports.parse_report(make_report(), 'right')
result = reports.compare_reports(left, right)
self.assertEqual(result['cases'][0]['comparison'], 'SAME')
other = reports.parse_report(make_report(human='NO_SOUND'), 'other')
result = reports.compare_reports(right, other)
self.assertEqual(result['cases'][0]['left_auto'], result['cases'][0]['right_auto'])
self.assertEqual(result['cases'][0]['comparison'], 'DIFFERENT')
def test_identity_mismatch_makes_not_comparable(self):
baseline = reports.parse_report(make_report())
for changed in (make_report(crc='ABCD'), make_report(build='OTHER'), make_report(vector_set='FFFF'), make_report(case='OTHER')):
result = reports.compare_reports(baseline, reports.parse_report(changed))
self.assertTrue(all(case['comparison'] == 'NOT_COMPARABLE' for case in result['cases']))
def test_authoritative_manifest_identity(self):
report = reports.parse_report(make_report())
manifest = {'build_id': 'TEST-1', 'vector_set_crc16': '1234', 'vectors': [{'case_id': 'TON01-A', 'crc16': '7ABC'}]}
self.assertEqual(reports.validate_manifest(report, manifest), [])
manifest['vectors'][0]['crc16'] = '1234'
compared = reports.compare_reports(report, report, manifest)
self.assertEqual(compared['cases'][0]['comparison'], 'NOT_COMPARABLE')
def test_repeated_attempts_preserved_and_compared_by_repeat(self):
base = make_report(ending=False).decode('ascii')
header, body = base.split('CASE=', 1)
case0 = 'CASE=' + body.replace('ENDCASE=', 'REPEAT=0\r\nENDCASE=')
case1 = case0.replace('REPEAT=0', 'REPEAT=1').replace('HUMAN=HEARD', 'HUMAN=UNSURE')
def finish(text):
raw = text.encode('ascii')
count = len(text.splitlines())
return raw + f'ENDRUN=0001 COMPLETE=YES RECORD_COUNT={count} CHECKSUM_CRC16={reports.crc16(raw):04X}\r\n'.encode('ascii')
left = reports.parse_report(finish(header + case0 + case1))
self.assertTrue(left['valid'], left['issues'])
self.assertEqual([case['repeat'] for case in left['cases']], [0, 1])
right = reports.parse_report(finish(header + case1 + case0))
self.assertTrue(right['valid'], right['issues'])
comparison = reports.compare_reports(left, right)
self.assertEqual([row['repeat'] for row in comparison['cases']], [0, 1])
self.assertEqual([row['comparison'] for row in comparison['cases']], ['SAME', 'SAME'])
changed = reports.parse_report(finish(header + case0 + case1.replace('HUMAN=UNSURE', 'HUMAN=NO_SOUND')))
comparison = reports.compare_reports(left, changed)
self.assertEqual([row['comparison'] for row in comparison['cases']], ['SAME', 'DIFFERENT'])
self.assertIn('case,repeat,auto,human', reports.render_csv(left))
self.assertIn('<th scope="col">repeat</th>', reports.render_html(left))
duplicate = reports.parse_report(finish(header + case0 + case0))
self.assertFalse(duplicate['valid'])
self.assertTrue(any('duplicate REPEAT=0' in issue for issue in duplicate['issues']))
def test_bad_repeat_and_missing_repeat_are_honest(self):
baseline = make_report(ending=False)
for value in ('-1', '65536', 'abc'):
invalid = baseline.replace(b'ENDCASE=', f'REPEAT={value}\r\nENDCASE='.encode()) + b'ENDRUN=0001 COMPLETE=YES\r\n'
parsed = reports.parse_report(invalid)
self.assertFalse(parsed['valid'])
self.assertIsNone(parsed['cases'][0]['repeat'])
self.assertEqual(reports.parse_report(make_report())['cases'][0]['repeat'], 0)
def test_raw_metric_differences_visible_even_with_same_outcomes(self):
a = make_report()
b = a.replace(b'TX_COUNT=11', b'TX_COUNT=12').split(b' RECORD_COUNT=')[0] + b'\r\n'
# Remove optional integrity from both: this fixture isolates metrics comparison.
a = a.split(b' RECORD_COUNT=')[0] + b'\r\n'
result = reports.compare_reports(reports.parse_report(a), reports.parse_report(b))
self.assertEqual(result['cases'][0]['comparison'], 'DIFFERENT')
self.assertEqual(result['cases'][0]['recorded_differences']['TX_COUNT'], {'left': '11', 'right': '12'})
def test_manifest_crc_aliases_and_invalid_schema(self):
report = reports.parse_report(make_report())
manifest = {'build_id': 'TEST-1', 'vector_set_crc16': 0x1234, 'vectors': {'TON01-A': '0x7abc'}}
self.assertEqual(reports.validate_manifest(report, manifest), [])
for bad in ([], {'vectors': 3}, {'vectors': ['not an object']}):
with self.subTest(manifest=bad), self.assertRaises(reports.SoundLabError):
reports.validate_manifest(report, bad)
def test_repeated_header_or_conflicting_controls(self):
for changed in (make_report().replace(b'ENDCASE=TON01-A', b'FORMAT=SOUNDLAB-1\r\nENDCASE=TON01-A'),
make_report().replace(b'CASE=TON01-A VECTOR', b'CASE=TON01-A RUN=0001 VECTOR')):
self.assertFalse(reports.parse_report(changed)['valid'])
def test_html_escapes_notes_and_csv_keeps_timing(self):
result = reports.parse_report(make_report(note='<script>alert(1)</script>'))
rendered = reports.render_html(result)
self.assertNotIn('<script>', rendered)
self.assertIn('&lt;script&gt;', rendered)
csv = reports.render_csv(result)
self.assertIn('TICK_RATE_DEN', csv)
self.assertIn('1001', csv)
class CliTests(unittest.TestCase):
def test_extract_convert_and_no_overwrites(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
disk = root / 'SOUNDLAB.DSK'
disk.write_bytes(make_disk([('SND0001.TXT', make_report(), [0])]))
output = root / 'reports'
self.assertEqual(reports.main(['extract', str(disk), '--output', str(output)]), 0)
self.assertEqual((output / 'SND0001.TXT').read_bytes(), make_report())
self.assertEqual(reports.main(['extract', str(disk), '--output', str(output)]), 1)
self.assertEqual(reports.main(['convert', str(output / 'SND0001.TXT'), '--output', str(root / 'converted')]), 0)
result = json.loads((root / 'converted' / 'SND0001.json').read_text())
self.assertTrue(result['complete'])
self.assertEqual(reports.main(['compare', str(output / 'SND0001.TXT'), str(output / 'SND0001.TXT'), '--output', str(root / 'comparison')]), 0)
def test_partial_conversion_returns_warning_with_evidence(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
report = root / 'partial.txt'
report.write_bytes(make_report(ending=False))
self.assertEqual(reports.main(['convert', str(report), '--output', str(root / 'result')]), 2)
self.assertTrue((root / 'result' / 'partial.html').is_file())
def test_inputs_cannot_be_overwritten_even_explicitly(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
# A source named *.json is still parsed as text, but conversion would target it.
report = root / 'report.json'
data = make_report()
report.write_bytes(data)
self.assertEqual(reports.main(['convert', str(report), '--output', str(root), '--overwrite']), 1)
self.assertEqual(report.read_bytes(), data)
def test_hardlinked_source_cannot_be_overwritten(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
report = root / 'report.txt'
report.write_bytes(make_report())
(root / 'report.json').hardlink_to(report)
self.assertEqual(reports.main(['convert', str(report), '--output', str(root), '--overwrite']), 1)
self.assertEqual(report.read_bytes(), make_report())
def test_explicit_overwrite_and_invalid_manifest(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
report = root / 'report.txt'
report.write_bytes(make_report())
output = root / 'output'
self.assertEqual(reports.main(['convert', str(report), '--output', str(output)]), 0)
self.assertEqual(reports.main(['convert', str(report), '--output', str(output), '--overwrite']), 0)
manifest = root / 'bad.json'
for invalid in ('[]', 'null'):
manifest.write_text(invalid)
self.assertEqual(reports.main(['convert', str(report), '--output', str(root / 'bad'), '--manifest', str(manifest)]), 1)
def test_preflight_rejects_all_outputs_before_writing(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'exists').write_bytes(b'old')
with self.assertRaises(reports.SoundLabError):
reports.write_outputs({root / 'new': b'new', root / 'exists': b'replace'})
self.assertFalse((root / 'new').exists())
(root / 'link').symlink_to(root / 'exists')
with self.assertRaises(reports.SoundLabError):
reports.write_outputs({root / 'link': b'replace'}, overwrite=True)
if __name__ == '__main__':
unittest.main()