#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-or-later """Read-only Sound Lab Disk BASIC extraction and evidence reporting (stdlib only).""" from __future__ import annotations import argparse import csv import hashlib import html import io import json from pathlib import Path import re import sys from typing import Any DISK_SIZE = 35 * 18 * 256 FAT_OFFSET = (17 * 18 + 1) * 256 DIRECTORY_OFFSET = (17 * 18 + 2) * 256 REPORT_NAME = re.compile(r"SND[0-9]{4}\.TXT", re.IGNORECASE) KEY = re.compile(r"[A-Z][A-Z0-9_]*") AUTO = frozenset("PASS FAIL OBSERVED TIMEOUT NOT_RUN NOT_APPLICABLE".split()) HUMAN = frozenset("HEARD WRONG NO_SOUND UNSURE NOT_OBSERVED SKIPPED".split()) class SoundLabError(ValueError): """An invalid disk/report or unsafe output operation.""" def crc16(data: bytes) -> int: """CRC-16/CCITT-FALSE: polynomial 1021, initial FFFF, no reflection/xor.""" result = 0xFFFF for value in data: result ^= value << 8 for _ in range(8): result = ((result << 1) ^ (0x1021 if result & 0x8000 else 0)) & 0xFFFF return result def granule_offset(granule: int) -> int: track = granule // 2 + (1 if granule >= 34 else 0) return (track * 18 + (granule % 2) * 9) * 256 def extract_reports(disk: bytes) -> list[dict[str, Any]]: """Validate active DECB files, return SNDnnnn.TXT ASCII bytes; never mutate disk. All active files are walked to catch crosslinks into a selected report. Unused FAT allocations are tolerated: they can remain after an interrupted write. """ if len(disk) != DISK_SIZE: raise SoundLabError(f"Expected raw 35-track DSK ({DISK_SIZE} bytes), got {len(disk)}") fat = disk[FAT_OFFSET:FAT_OFFSET + 68] for index, value in enumerate(fat): if not (value < 68 or 0xC0 <= value <= 0xC9 or value == 0xFF): raise SoundLabError(f"Invalid FAT value {value:02X} at granule {index}") owned: dict[int, str] = {} names: set[str] = set() reports = [] for index in range(72): entry = disk[DIRECTORY_OFFSET + index * 32:DIRECTORY_OFFSET + (index + 1) * 32] if entry[0] in (0, 0xFF): continue try: base = entry[:8].decode("ascii").rstrip(" ") ext = entry[8:11].decode("ascii").rstrip(" ") except UnicodeDecodeError as error: raise SoundLabError(f"Non-ASCII directory name at entry {index}") from error if not base or any(ord(c) < 33 or c in '/\\:.' for c in base + ext): raise SoundLabError(f"Unsafe directory name at entry {index}") name = f"{base}.{ext}" if ext else base normalized = name.upper() if normalized in names: raise SoundLabError(f"Duplicate directory name: {name}") names.add(normalized) if entry[11] > 3 or entry[12] not in (0, 0xFF): raise SoundLabError(f"Invalid file type/ASCII flag for {name}") last_bytes = int.from_bytes(entry[14:16], "big") if last_bytes > 256: raise SoundLabError(f"Invalid final-sector byte count for {name}") chain = [] current = entry[13] while True: if current >= 68: raise SoundLabError(f"Out-of-range granule {current} for {name}") if current in chain: raise SoundLabError(f"FAT loop for {name}") if current in owned: raise SoundLabError(f"Crosslinked granule {current}: {name} and {owned[current]}") chain.append(current) owned[current] = name value = fat[current] if 0xC0 <= value <= 0xC9: final_sectors = value - 0xC0 if final_sectors == 0 and (len(chain) != 1 or last_bytes != 0): raise SoundLabError(f"Invalid empty final granule for {name}") if final_sectors > 0 and last_bytes == 0: # DOS stores 256, not zero, for a completely used last sector. raise SoundLabError(f"Missing final-sector bytes for {name}") break if value == 0xFF: raise SoundLabError(f"Allocated file {name} points to free granule {current}") current = value length = (len(chain) - 1) * 2304 if final_sectors: length += (final_sectors - 1) * 256 + last_bytes if REPORT_NAME.fullmatch(name): if entry[12] != 0xFF: raise SoundLabError(f"Report {name} is not marked Disk BASIC ASCII") content = b"".join(disk[granule_offset(g):granule_offset(g) + 2304] for g in chain)[:length] reports.append({"filename": normalized, "data": content, "byte_count": len(content), "sha256": hashlib.sha256(content).hexdigest(), "granules": chain, "directory_type": entry[11]}) return sorted(reports, key=lambda item: item["filename"]) def decode_note(value: str) -> str: decoded = bytearray() i = 0 while i < len(value): char = value[i] if char != "\\": if not 32 <= ord(char) <= 126: raise SoundLabError("Notes must use printable ASCII or escaped control bytes") decoded.append(ord(char)) i += 1 continue i += 1 if i >= len(value): raise SoundLabError("Incomplete note escape") escaped = value[i] if escaped in "\\nrt": decoded.append({"\\": 92, "n": 10, "r": 13, "t": 9}[escaped]) i += 1 elif escaped == "x" and i + 2 < len(value) and re.fullmatch(r"[0-9A-Fa-f]{2}", value[i + 1:i + 3]): byte = int(value[i + 1:i + 3], 16) if byte > 127: raise SoundLabError("Escaped notes must remain ASCII") decoded.append(byte) i += 3 else: raise SoundLabError(f"Unknown note escape at position {i}") return decoded.decode("ascii") def parse_tokens(line: str) -> dict[str, str]: fields = {} for token in line.split(): if "=" not in token: raise SoundLabError(f"Expected KEY=VALUE token: {token}") key, value = token.split("=", 1) if not KEY.fullmatch(key) or not value or key in fields: raise SoundLabError(f"Invalid or duplicate token: {token}") fields[key] = value if not fields: raise SoundLabError("Empty record") return fields def parse_report(data: bytes, source: str = "") -> dict[str, Any]: """Parse evidence without turning automatic observations into listening results. Incomplete files are useful evidence, returned with complete=False and issues. Structural and integrity failures likewise remain visible in JSON/HTML; a malformed header/encoding is rejected because it is not a SOUNDLAB-1 report. """ try: text = data.decode("ascii") except UnicodeDecodeError as error: raise SoundLabError("Report is not ASCII") from error if "\x00" in text or re.search(r"[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]", text): raise SoundLabError("Report contains unescaped control bytes") if "\r" in text.replace("\r\n", ""): raise SoundLabError("Report has bare CR line endings") lines = [line for line in text.replace("\r\n", "\n").split("\n") if line.strip()] if not lines or lines[0] != "FORMAT=SOUNDLAB-1": raise SoundLabError("Missing supported FORMAT=SOUNDLAB-1 header") result: dict[str, Any] = {"format": "SOUNDLAB-1", "source": source, "source_sha256": hashlib.sha256(data).hexdigest(), "metadata": {}, "cases": [], "issues": [], "complete": False, "integrity": {"record_count": "NOT_PROVIDED", "checksum_crc16": "NOT_PROVIDED"}} current = None footer = None note_length = None records = [] def issue(message: str) -> None: result["issues"].append(message) def merge(target: dict[str, str], fields: dict[str, str], number: int) -> None: for key, value in fields.items(): if key in target: issue(f"Line {number}: duplicate field {key}") else: target[key] = value for number, line in enumerate(lines, 1): if footer is not None: issue(f"Line {number}: record after ENDRUN") continue if line.startswith("NOTE="): records.append(line) try: note = decode_note(line[5:]) if current is None: issue(f"Line {number}: NOTE outside CASE") elif note_length is None: issue(f"Line {number}: NOTE without NOTE_LENGTH") else: current["notes"].append(note) if len(note.encode("ascii")) != note_length: issue(f"Line {number}: NOTE_LENGTH mismatch") except SoundLabError as error: issue(f"Line {number}: {error}") note_length = None continue if note_length is not None: issue(f"Line {number}: NOTE_LENGTH not followed by NOTE") note_length = None try: fields = parse_tokens(line) except SoundLabError as error: records.append(line) issue(f"Line {number}: {error}") continue controls = set(fields) & {"FORMAT", "RUN", "CASE", "ENDCASE", "ENDRUN", "NOTE_LENGTH"} if len(controls) > 1: issue(f"Line {number}: conflicting control records") if "FORMAT" in fields and number != 1: issue(f"Line {number}: repeated FORMAT header") if "RUN" in fields and (current is not None or result["cases"]): issue(f"Line {number}: RUN header after CASE") if "ENDRUN" in fields: footer = fields result["footer"] = fields if current is not None: issue(f"Line {number}: ENDRUN before ENDCASE") if fields["ENDRUN"] != result["metadata"].get("RUN"): issue(f"Line {number}: ENDRUN does not match RUN") if fields.get("COMPLETE", "").upper() not in ("YES", "NO"): issue(f"Line {number}: COMPLETE must be YES or NO") continue records.append(line) if number == 1: continue if "CASE" in fields: if current is not None: issue(f"Line {number}: CASE before previous ENDCASE") identifier = fields["CASE"].upper() current = {"id": identifier, "fields": {}, "notes": [], "closed": False} result["cases"].append(current) merge(current["fields"], fields, number) elif "ENDCASE" in fields: if current is None: issue(f"Line {number}: ENDCASE outside CASE") else: if fields["ENDCASE"].upper() != current["id"]: issue(f"Line {number}: ENDCASE does not match CASE") else: current["closed"] = True current = None if len(fields) != 1: issue(f"Line {number}: unexpected ENDCASE fields") elif "NOTE_LENGTH" in fields: if len(fields) != 1 or current is None: issue(f"Line {number}: NOTE_LENGTH must be alone inside CASE") try: note_length = int(fields["NOTE_LENGTH"]) if note_length < 0 or note_length > 65535: raise ValueError() except ValueError: note_length = None issue(f"Line {number}: invalid NOTE_LENGTH") else: merge(current["fields"] if current is not None else result["metadata"], fields, number) if note_length is not None: issue("Truncated NOTE after NOTE_LENGTH") if not result["metadata"].get("RUN"): issue("Missing RUN identity") attempts: set[tuple[str, int]] = set() for case in result["cases"]: try: repeat_text = case["fields"].get("REPEAT", "0") if not re.fullmatch(r"[0-9]+", repeat_text): raise ValueError() repeat = int(repeat_text) if repeat > 65535: raise ValueError() case["repeat"] = repeat identity = (case["id"], repeat) if identity in attempts: issue(f"CASE {case['id']}: duplicate REPEAT={repeat}") attempts.add(identity) except ValueError: case["repeat"] = None issue(f"CASE {case['id']}: invalid REPEAT") for key, allowed in (("AUTO", AUTO), ("HUMAN", HUMAN)): value = case["fields"].get(key) if value is None: # UNKNOWN is deliberate: absence is not a positive or negative observation. case[key.lower()] = "UNKNOWN" issue(f"CASE {case['id']}: missing {key}") else: case[key.lower()] = value.upper() if value.upper() not in allowed: issue(f"CASE {case['id']}: unknown {key} value {value}") vector_crc = case["fields"].get("VECTOR_CRC16", "") if not re.fullmatch(r"[0-9A-Fa-f]{4}", vector_crc): issue(f"CASE {case['id']}: missing or invalid VECTOR_CRC16") if not case["closed"]: issue(f"CASE {case['id']}: missing ENDCASE") if footer is None: issue("Missing ENDRUN; partial run") else: if "RECORD_COUNT" in footer: try: valid_count = int(footer["RECORD_COUNT"]) == len(records) except ValueError: valid_count = False result["integrity"]["record_count"] = "VERIFIED" if valid_count else "FAILED" if not valid_count: issue("RECORD_COUNT mismatch") if "CHECKSUM_CRC16" in footer: canonical = ("\r\n".join(records) + "\r\n").encode("ascii") expected = f"{crc16(canonical):04X}" valid_crc = footer["CHECKSUM_CRC16"].upper() == expected result["integrity"]["checksum_crc16"] = "VERIFIED" if valid_crc else "FAILED" if not valid_crc: issue(f"CHECKSUM_CRC16 mismatch (calculated {expected})") if footer.get("COMPLETE", "").upper() != "YES": issue("Run marked incomplete") result["record_count"] = len(records) result["complete"] = footer is not None and footer.get("COMPLETE", "").upper() == "YES" and not result["issues"] result["valid"] = not result["issues"] return result def validate_manifest(report: dict[str, Any], manifest: dict[str, Any]) -> list[str]: """Return exact identity failures; a manifest is evidence, not a trust signature.""" if not isinstance(manifest, dict): raise SoundLabError("Manifest must be a JSON object") problems = [] metadata = report["metadata"] def crc_text(value: Any) -> str: if isinstance(value, int): return f"{value:04X}" return str(value).upper().removeprefix("0X") for key in ("BUILD_ID", "VECTOR_SET_CRC16"): expected = manifest.get(key.lower(), manifest.get(key)) expected_text = crc_text(expected) if key.endswith("CRC16") else str(expected).upper() if expected is None or expected_text != metadata.get(key, "").upper(): problems.append(f"{key} differs from manifest or is absent") vectors = manifest.get("vectors", {}) if isinstance(vectors, list): if not all(isinstance(item, dict) for item in vectors): raise SoundLabError("Manifest vectors must be objects") vectors = {item.get("id", item.get("case_id", "")): item.get("crc16", item.get("vector_crc16", "")) for item in vectors} if not isinstance(vectors, dict): raise SoundLabError("Manifest vectors must be a list or object") vectors = {str(key).upper(): crc_text(value) for key, value in vectors.items()} for case in report["cases"]: expected = vectors.get(case["id"]) if expected is None or expected != case["fields"].get("VECTOR_CRC16", "").upper(): problems.append(f"CASE {case['id']}: vector differs from manifest or is absent") return problems def compare_reports(left: dict[str, Any], right: dict[str, Any], manifest: dict[str, Any] | None = None) -> dict[str, Any]: problems = [] for key in ("BUILD_ID", "VECTOR_SET_CRC16"): a, b = left["metadata"].get(key), right["metadata"].get(key) if not a or not b or a.upper() != b.upper(): problems.append(f"{key} differs or is absent") if manifest is not None: problems.extend("Left: " + value for value in validate_manifest(left, manifest)) problems.extend("Right: " + value for value in validate_manifest(right, manifest)) if not left["valid"] or not right["valid"]: problems.append("One or both reports are partial or invalid; outcomes are provisional") by_id = [{(case["id"], case["repeat"]): case for case in report["cases"]} for report in (left, right)] rows = [] for identity in sorted(set(by_id[0]) | set(by_id[1]), key=lambda item: (item[0], -1 if item[1] is None else item[1])): identifier, repeat = identity a, b = by_id[0].get(identity), by_id[1].get(identity) reason = list(problems) if a is None or b is None: reason.append("CASE missing from one report") elif a["fields"].get("VECTOR_CRC16", "").upper() != b["fields"].get("VECTOR_CRC16", "").upper(): reason.append("VECTOR_CRC16 differs") row = {"case": identifier, "repeat": repeat, "left_auto": a["auto"] if a else "UNKNOWN", "right_auto": b["auto"] if b else "UNKNOWN", "left_human": a["human"] if a else "UNKNOWN", "right_human": b["human"] if b else "UNKNOWN", "reasons": reason} changes = {} if a is not None and b is not None: for key in sorted(set(a["fields"]) | set(b["fields"])): if key in ("CASE", "REPEAT", "VECTOR_CRC16", "AUTO", "HUMAN"): continue av, bv = a["fields"].get(key), b["fields"].get(key) if av != bv: changes[key] = {"left": av, "right": bv} if a["notes"] != b["notes"]: changes["NOTES"] = {"left": a["notes"], "right": b["notes"]} row["recorded_differences"] = changes outcomes_equal = row["left_auto"] == row["right_auto"] and row["left_human"] == row["right_human"] row["comparison"] = "NOT_COMPARABLE" if reason else ("SAME" if outcomes_equal and not changes else "DIFFERENT") rows.append(row) return {"left": left["source"], "right": right["source"], "identity_issues": problems, "manifest_checked": manifest is not None, "identity_basis": "MANIFEST" if manifest is not None else "REPORT_CLAIMS_ONLY", "cases": rows, "notice": "Matching automatic observations do not establish audible equivalence. HUMAN is independent."} def render_html(document: dict[str, Any]) -> str: esc = lambda value: html.escape(str(value)) comparison = "left" in document title = "Sound Lab comparison" if comparison else "Sound Lab report" pieces = [f'
Automatic observations and human listening results are separate evidence. Unknown means unreported; it never means heard.
'] issues = document.get("identity_issues", document.get("issues", [])) if issues: pieces.append('Source: {esc(document["source"])}
SHA-256: {esc(document["source_sha256"])}
Complete: {esc(document["complete"])}
' + esc(json.dumps({"metadata": document["metadata"], "integrity": document["integrity"]}, indent=2)) + '')
keys = ["case", "repeat", "auto", "human", "closed", "notes", "recorded_fields"]
rows = [{"case": case["id"], "repeat": case["repeat"], "auto": case["auto"], "human": case["human"], "closed": case["closed"], "notes": "\n".join(case["notes"]), "recorded_fields": json.dumps(case["fields"], sort_keys=True)} for case in document["cases"]]
pieces.append('| {esc(key)} | ' for key in keys) + '
|---|
{esc(row.get(key, ""))} | ' for key in keys) + '