#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-or-later """Read-only Sound Lab Disk BASIC extraction and evidence reporting (stdlib only).""" from __future__ import annotations import argparse import csv import hashlib import html import io import json from pathlib import Path import re import sys from typing import Any DISK_SIZE = 35 * 18 * 256 FAT_OFFSET = (17 * 18 + 1) * 256 DIRECTORY_OFFSET = (17 * 18 + 2) * 256 REPORT_NAME = re.compile(r"SND[0-9]{4}\.TXT", re.IGNORECASE) KEY = re.compile(r"[A-Z][A-Z0-9_]*") AUTO = frozenset("PASS FAIL OBSERVED TIMEOUT NOT_RUN NOT_APPLICABLE".split()) HUMAN = frozenset("HEARD WRONG NO_SOUND UNSURE NOT_OBSERVED SKIPPED".split()) class SoundLabError(ValueError): """An invalid disk/report or unsafe output operation.""" def crc16(data: bytes) -> int: """CRC-16/CCITT-FALSE: polynomial 1021, initial FFFF, no reflection/xor.""" result = 0xFFFF for value in data: result ^= value << 8 for _ in range(8): result = ((result << 1) ^ (0x1021 if result & 0x8000 else 0)) & 0xFFFF return result def granule_offset(granule: int) -> int: track = granule // 2 + (1 if granule >= 34 else 0) return (track * 18 + (granule % 2) * 9) * 256 def extract_reports(disk: bytes) -> list[dict[str, Any]]: """Validate active DECB files, return SNDnnnn.TXT ASCII bytes; never mutate disk. All active files are walked to catch crosslinks into a selected report. Unused FAT allocations are tolerated: they can remain after an interrupted write. """ if len(disk) != DISK_SIZE: raise SoundLabError(f"Expected raw 35-track DSK ({DISK_SIZE} bytes), got {len(disk)}") fat = disk[FAT_OFFSET:FAT_OFFSET + 68] for index, value in enumerate(fat): if not (value < 68 or 0xC0 <= value <= 0xC9 or value == 0xFF): raise SoundLabError(f"Invalid FAT value {value:02X} at granule {index}") owned: dict[int, str] = {} names: set[str] = set() reports = [] for index in range(72): entry = disk[DIRECTORY_OFFSET + index * 32:DIRECTORY_OFFSET + (index + 1) * 32] if entry[0] in (0, 0xFF): continue try: base = entry[:8].decode("ascii").rstrip(" ") ext = entry[8:11].decode("ascii").rstrip(" ") except UnicodeDecodeError as error: raise SoundLabError(f"Non-ASCII directory name at entry {index}") from error if not base or any(ord(c) < 33 or c in '/\\:.' for c in base + ext): raise SoundLabError(f"Unsafe directory name at entry {index}") name = f"{base}.{ext}" if ext else base normalized = name.upper() if normalized in names: raise SoundLabError(f"Duplicate directory name: {name}") names.add(normalized) if entry[11] > 3 or entry[12] not in (0, 0xFF): raise SoundLabError(f"Invalid file type/ASCII flag for {name}") last_bytes = int.from_bytes(entry[14:16], "big") if last_bytes > 256: raise SoundLabError(f"Invalid final-sector byte count for {name}") chain = [] current = entry[13] while True: if current >= 68: raise SoundLabError(f"Out-of-range granule {current} for {name}") if current in chain: raise SoundLabError(f"FAT loop for {name}") if current in owned: raise SoundLabError(f"Crosslinked granule {current}: {name} and {owned[current]}") chain.append(current) owned[current] = name value = fat[current] if 0xC0 <= value <= 0xC9: final_sectors = value - 0xC0 if final_sectors == 0 and (len(chain) != 1 or last_bytes != 0): raise SoundLabError(f"Invalid empty final granule for {name}") if final_sectors > 0 and last_bytes == 0: # DOS stores 256, not zero, for a completely used last sector. raise SoundLabError(f"Missing final-sector bytes for {name}") break if value == 0xFF: raise SoundLabError(f"Allocated file {name} points to free granule {current}") current = value length = (len(chain) - 1) * 2304 if final_sectors: length += (final_sectors - 1) * 256 + last_bytes if REPORT_NAME.fullmatch(name): if entry[12] != 0xFF: raise SoundLabError(f"Report {name} is not marked Disk BASIC ASCII") content = b"".join(disk[granule_offset(g):granule_offset(g) + 2304] for g in chain)[:length] reports.append({"filename": normalized, "data": content, "byte_count": len(content), "sha256": hashlib.sha256(content).hexdigest(), "granules": chain, "directory_type": entry[11]}) return sorted(reports, key=lambda item: item["filename"]) def decode_note(value: str) -> str: decoded = bytearray() i = 0 while i < len(value): char = value[i] if char != "\\": if not 32 <= ord(char) <= 126: raise SoundLabError("Notes must use printable ASCII or escaped control bytes") decoded.append(ord(char)) i += 1 continue i += 1 if i >= len(value): raise SoundLabError("Incomplete note escape") escaped = value[i] if escaped in "\\nrt": decoded.append({"\\": 92, "n": 10, "r": 13, "t": 9}[escaped]) i += 1 elif escaped == "x" and i + 2 < len(value) and re.fullmatch(r"[0-9A-Fa-f]{2}", value[i + 1:i + 3]): byte = int(value[i + 1:i + 3], 16) if byte > 127: raise SoundLabError("Escaped notes must remain ASCII") decoded.append(byte) i += 3 else: raise SoundLabError(f"Unknown note escape at position {i}") return decoded.decode("ascii") def parse_tokens(line: str) -> dict[str, str]: fields = {} for token in line.split(): if "=" not in token: raise SoundLabError(f"Expected KEY=VALUE token: {token}") key, value = token.split("=", 1) if not KEY.fullmatch(key) or not value or key in fields: raise SoundLabError(f"Invalid or duplicate token: {token}") fields[key] = value if not fields: raise SoundLabError("Empty record") return fields def parse_report(data: bytes, source: str = "") -> dict[str, Any]: """Parse evidence without turning automatic observations into listening results. Incomplete files are useful evidence, returned with complete=False and issues. Structural and integrity failures likewise remain visible in JSON/HTML; a malformed header/encoding is rejected because it is not a SOUNDLAB-1 report. """ try: text = data.decode("ascii") except UnicodeDecodeError as error: raise SoundLabError("Report is not ASCII") from error if "\x00" in text or re.search(r"[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]", text): raise SoundLabError("Report contains unescaped control bytes") if "\r" in text.replace("\r\n", ""): raise SoundLabError("Report has bare CR line endings") lines = [line for line in text.replace("\r\n", "\n").split("\n") if line.strip()] if not lines or lines[0] != "FORMAT=SOUNDLAB-1": raise SoundLabError("Missing supported FORMAT=SOUNDLAB-1 header") result: dict[str, Any] = {"format": "SOUNDLAB-1", "source": source, "source_sha256": hashlib.sha256(data).hexdigest(), "metadata": {}, "cases": [], "issues": [], "complete": False, "integrity": {"record_count": "NOT_PROVIDED", "checksum_crc16": "NOT_PROVIDED"}} current = None footer = None note_length = None records = [] def issue(message: str) -> None: result["issues"].append(message) def merge(target: dict[str, str], fields: dict[str, str], number: int) -> None: for key, value in fields.items(): if key in target: issue(f"Line {number}: duplicate field {key}") else: target[key] = value for number, line in enumerate(lines, 1): if footer is not None: issue(f"Line {number}: record after ENDRUN") continue if line.startswith("NOTE="): records.append(line) try: note = decode_note(line[5:]) if current is None: issue(f"Line {number}: NOTE outside CASE") elif note_length is None: issue(f"Line {number}: NOTE without NOTE_LENGTH") else: current["notes"].append(note) if len(note.encode("ascii")) != note_length: issue(f"Line {number}: NOTE_LENGTH mismatch") except SoundLabError as error: issue(f"Line {number}: {error}") note_length = None continue if note_length is not None: issue(f"Line {number}: NOTE_LENGTH not followed by NOTE") note_length = None try: fields = parse_tokens(line) except SoundLabError as error: records.append(line) issue(f"Line {number}: {error}") continue controls = set(fields) & {"FORMAT", "RUN", "CASE", "ENDCASE", "ENDRUN", "NOTE_LENGTH"} if len(controls) > 1: issue(f"Line {number}: conflicting control records") if "FORMAT" in fields and number != 1: issue(f"Line {number}: repeated FORMAT header") if "RUN" in fields and (current is not None or result["cases"]): issue(f"Line {number}: RUN header after CASE") if "ENDRUN" in fields: footer = fields result["footer"] = fields if current is not None: issue(f"Line {number}: ENDRUN before ENDCASE") if fields["ENDRUN"] != result["metadata"].get("RUN"): issue(f"Line {number}: ENDRUN does not match RUN") if fields.get("COMPLETE", "").upper() not in ("YES", "NO"): issue(f"Line {number}: COMPLETE must be YES or NO") continue records.append(line) if number == 1: continue if "CASE" in fields: if current is not None: issue(f"Line {number}: CASE before previous ENDCASE") identifier = fields["CASE"].upper() current = {"id": identifier, "fields": {}, "notes": [], "closed": False} result["cases"].append(current) merge(current["fields"], fields, number) elif "ENDCASE" in fields: if current is None: issue(f"Line {number}: ENDCASE outside CASE") else: if fields["ENDCASE"].upper() != current["id"]: issue(f"Line {number}: ENDCASE does not match CASE") else: current["closed"] = True current = None if len(fields) != 1: issue(f"Line {number}: unexpected ENDCASE fields") elif "NOTE_LENGTH" in fields: if len(fields) != 1 or current is None: issue(f"Line {number}: NOTE_LENGTH must be alone inside CASE") try: note_length = int(fields["NOTE_LENGTH"]) if note_length < 0 or note_length > 65535: raise ValueError() except ValueError: note_length = None issue(f"Line {number}: invalid NOTE_LENGTH") else: merge(current["fields"] if current is not None else result["metadata"], fields, number) if note_length is not None: issue("Truncated NOTE after NOTE_LENGTH") if not result["metadata"].get("RUN"): issue("Missing RUN identity") attempts: set[tuple[str, int]] = set() for case in result["cases"]: try: repeat_text = case["fields"].get("REPEAT", "0") if not re.fullmatch(r"[0-9]+", repeat_text): raise ValueError() repeat = int(repeat_text) if repeat > 65535: raise ValueError() case["repeat"] = repeat identity = (case["id"], repeat) if identity in attempts: issue(f"CASE {case['id']}: duplicate REPEAT={repeat}") attempts.add(identity) except ValueError: case["repeat"] = None issue(f"CASE {case['id']}: invalid REPEAT") for key, allowed in (("AUTO", AUTO), ("HUMAN", HUMAN)): value = case["fields"].get(key) if value is None: # UNKNOWN is deliberate: absence is not a positive or negative observation. case[key.lower()] = "UNKNOWN" issue(f"CASE {case['id']}: missing {key}") else: case[key.lower()] = value.upper() if value.upper() not in allowed: issue(f"CASE {case['id']}: unknown {key} value {value}") vector_crc = case["fields"].get("VECTOR_CRC16", "") if not re.fullmatch(r"[0-9A-Fa-f]{4}", vector_crc): issue(f"CASE {case['id']}: missing or invalid VECTOR_CRC16") if not case["closed"]: issue(f"CASE {case['id']}: missing ENDCASE") if footer is None: issue("Missing ENDRUN; partial run") else: if "RECORD_COUNT" in footer: try: valid_count = int(footer["RECORD_COUNT"]) == len(records) except ValueError: valid_count = False result["integrity"]["record_count"] = "VERIFIED" if valid_count else "FAILED" if not valid_count: issue("RECORD_COUNT mismatch") if "CHECKSUM_CRC16" in footer: canonical = ("\r\n".join(records) + "\r\n").encode("ascii") expected = f"{crc16(canonical):04X}" valid_crc = footer["CHECKSUM_CRC16"].upper() == expected result["integrity"]["checksum_crc16"] = "VERIFIED" if valid_crc else "FAILED" if not valid_crc: issue(f"CHECKSUM_CRC16 mismatch (calculated {expected})") if footer.get("COMPLETE", "").upper() != "YES": issue("Run marked incomplete") result["record_count"] = len(records) result["complete"] = footer is not None and footer.get("COMPLETE", "").upper() == "YES" and not result["issues"] result["valid"] = not result["issues"] return result def validate_manifest(report: dict[str, Any], manifest: dict[str, Any]) -> list[str]: """Return exact identity failures; a manifest is evidence, not a trust signature.""" if not isinstance(manifest, dict): raise SoundLabError("Manifest must be a JSON object") problems = [] metadata = report["metadata"] def crc_text(value: Any) -> str: if isinstance(value, int): return f"{value:04X}" return str(value).upper().removeprefix("0X") for key in ("BUILD_ID", "VECTOR_SET_CRC16"): expected = manifest.get(key.lower(), manifest.get(key)) expected_text = crc_text(expected) if key.endswith("CRC16") else str(expected).upper() if expected is None or expected_text != metadata.get(key, "").upper(): problems.append(f"{key} differs from manifest or is absent") vectors = manifest.get("vectors", {}) if isinstance(vectors, list): if not all(isinstance(item, dict) for item in vectors): raise SoundLabError("Manifest vectors must be objects") vectors = {item.get("id", item.get("case_id", "")): item.get("crc16", item.get("vector_crc16", "")) for item in vectors} if not isinstance(vectors, dict): raise SoundLabError("Manifest vectors must be a list or object") vectors = {str(key).upper(): crc_text(value) for key, value in vectors.items()} for case in report["cases"]: expected = vectors.get(case["id"]) if expected is None or expected != case["fields"].get("VECTOR_CRC16", "").upper(): problems.append(f"CASE {case['id']}: vector differs from manifest or is absent") return problems def compare_reports(left: dict[str, Any], right: dict[str, Any], manifest: dict[str, Any] | None = None) -> dict[str, Any]: problems = [] for key in ("BUILD_ID", "VECTOR_SET_CRC16"): a, b = left["metadata"].get(key), right["metadata"].get(key) if not a or not b or a.upper() != b.upper(): problems.append(f"{key} differs or is absent") if manifest is not None: problems.extend("Left: " + value for value in validate_manifest(left, manifest)) problems.extend("Right: " + value for value in validate_manifest(right, manifest)) if not left["valid"] or not right["valid"]: problems.append("One or both reports are partial or invalid; outcomes are provisional") by_id = [{(case["id"], case["repeat"]): case for case in report["cases"]} for report in (left, right)] rows = [] for identity in sorted(set(by_id[0]) | set(by_id[1]), key=lambda item: (item[0], -1 if item[1] is None else item[1])): identifier, repeat = identity a, b = by_id[0].get(identity), by_id[1].get(identity) reason = list(problems) if a is None or b is None: reason.append("CASE missing from one report") elif a["fields"].get("VECTOR_CRC16", "").upper() != b["fields"].get("VECTOR_CRC16", "").upper(): reason.append("VECTOR_CRC16 differs") row = {"case": identifier, "repeat": repeat, "left_auto": a["auto"] if a else "UNKNOWN", "right_auto": b["auto"] if b else "UNKNOWN", "left_human": a["human"] if a else "UNKNOWN", "right_human": b["human"] if b else "UNKNOWN", "reasons": reason} changes = {} if a is not None and b is not None: for key in sorted(set(a["fields"]) | set(b["fields"])): if key in ("CASE", "REPEAT", "VECTOR_CRC16", "AUTO", "HUMAN"): continue av, bv = a["fields"].get(key), b["fields"].get(key) if av != bv: changes[key] = {"left": av, "right": bv} if a["notes"] != b["notes"]: changes["NOTES"] = {"left": a["notes"], "right": b["notes"]} row["recorded_differences"] = changes outcomes_equal = row["left_auto"] == row["right_auto"] and row["left_human"] == row["right_human"] row["comparison"] = "NOT_COMPARABLE" if reason else ("SAME" if outcomes_equal and not changes else "DIFFERENT") rows.append(row) return {"left": left["source"], "right": right["source"], "identity_issues": problems, "manifest_checked": manifest is not None, "identity_basis": "MANIFEST" if manifest is not None else "REPORT_CLAIMS_ONLY", "cases": rows, "notice": "Matching automatic observations do not establish audible equivalence. HUMAN is independent."} def render_html(document: dict[str, Any]) -> str: esc = lambda value: html.escape(str(value)) comparison = "left" in document title = "Sound Lab comparison" if comparison else "Sound Lab report" pieces = [f'{title}', '', f'

{title}

', '

Automatic observations and human listening results are separate evidence. Unknown means unreported; it never means heard.

'] issues = document.get("identity_issues", document.get("issues", [])) if issues: pieces.append('
Incomplete or unverified evidence
') if comparison: keys = ["case", "repeat", "left_auto", "right_auto", "left_human", "right_human", "comparison", "recorded_differences", "reasons"] rows = document["cases"] else: pieces.append(f'

Source: {esc(document["source"])}
SHA-256: {esc(document["source_sha256"])}
Complete: {esc(document["complete"])}

') pieces.append('

Run metadata and integrity

' + esc(json.dumps({"metadata": document["metadata"], "integrity": document["integrity"]}, indent=2)) + '
') keys = ["case", "repeat", "auto", "human", "closed", "notes", "recorded_fields"] rows = [{"case": case["id"], "repeat": case["repeat"], "auto": case["auto"], "human": case["human"], "closed": case["closed"], "notes": "\n".join(case["notes"]), "recorded_fields": json.dumps(case["fields"], sort_keys=True)} for case in document["cases"]] pieces.append('

Cases

' + ''.join(f'' for key in keys) + '') for row in rows: pieces.append('' + ''.join(f'' for key in keys) + '') pieces.append('
{esc(key)}
{esc(row.get(key, ""))}
') return '\n'.join(pieces) def render_csv(document: dict[str, Any]) -> str: stream = io.StringIO(newline="") if "left" in document: fields = ["case", "repeat", "left_auto", "right_auto", "left_human", "right_human", "comparison", "recorded_differences", "reasons"] rows = document["cases"] else: # Union preserves timing/status fields, rather than reducing reports to outcomes. extra = sorted({key for case in document["cases"] for key in case["fields"]} - {"CASE", "REPEAT", "AUTO", "HUMAN"}) fields = ["case", "repeat", "auto", "human", "closed", "notes"] + extra rows = [{"case": case["id"], "repeat": case["repeat"], "auto": case["auto"], "human": case["human"], "closed": case["closed"], "notes": '\n'.join(case["notes"]), **{key: case["fields"].get(key, "") for key in extra}} for case in document["cases"]] writer = csv.DictWriter(stream, fieldnames=fields) writer.writeheader() writer.writerows(rows) return stream.getvalue() def write_outputs(outputs: dict[Path, bytes], overwrite: bool = False) -> None: """Preflight every target before any writes; originals are never implicitly replaced.""" for path in outputs: if path.is_symlink() or (path.exists() and (not overwrite or not path.is_file())): raise SoundLabError(f"Refusing to replace {path}; select a fresh folder or use --overwrite for regular files") for path, data in outputs.items(): path.parent.mkdir(parents=True, exist_ok=True) with path.open("wb" if overwrite else "xb") as handle: handle.write(data) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="operation", required=True) for operation in ("extract", "convert", "compare"): command = sub.add_parser(operation) command.add_argument("source", type=Path) if operation == "compare": command.add_argument("other", type=Path) command.add_argument("--output", required=True, type=Path) command.add_argument("--overwrite", action="store_true") if operation != "extract": command.add_argument("--manifest", type=Path, help="Authoritative build/vector JSON manifest") args = parser.parse_args(argv) try: outputs = {} if args.operation == "extract": disk = args.source.read_bytes() reports = extract_reports(disk) if not reports: raise SoundLabError("No SNDnnnn.TXT ASCII reports found") receipt = {"disk": args.source.name, "disk_byte_count": len(disk), "disk_sha256": hashlib.sha256(disk).hexdigest(), "reports": []} for report in reports: outputs[args.output / report["filename"]] = report["data"] receipt["reports"].append({key: value for key, value in report.items() if key != "data"}) outputs[args.output / "extraction.json"] = (json.dumps(receipt, indent=2) + '\n').encode() else: left = parse_report(args.source.read_bytes(), args.source.name) manifest = json.loads(args.manifest.read_text()) if args.manifest else None if args.manifest and not isinstance(manifest, dict): raise SoundLabError("Manifest must be a JSON object") if args.operation == "compare": document = compare_reports(left, parse_report(args.other.read_bytes(), args.other.name), manifest) stem = "comparison" else: document = left if manifest is not None: document["manifest_issues"] = validate_manifest(document, manifest) document["issues"].extend(document["manifest_issues"]) document["valid"] = document["valid"] and not document["manifest_issues"] document["complete"] = document["complete"] and document["valid"] stem = args.source.stem outputs = {args.output / f"{stem}.json": (json.dumps(document, indent=2) + '\n').encode(), args.output / f"{stem}.csv": render_csv(document).encode(), args.output / f"{stem}.html": render_html(document).encode()} # Never allow an output to alias any supplied source, even with --overwrite. sources = [args.source] + ([args.other] if args.operation == "compare" else []) if getattr(args, "manifest", None): sources.append(args.manifest) source_paths = {path.resolve() for path in sources} if any(path.resolve() in source_paths or any(path.exists() and path.samefile(source) for source in sources) for path in outputs): raise SoundLabError("Output would replace an input; choose another folder") write_outputs(outputs, args.overwrite) print(f"Wrote {len(outputs)} files to {args.output}") if args.operation != "extract" and (document.get("issues") or document.get("identity_issues") or any(row.get("comparison") == "NOT_COMPARABLE" for row in document["cases"])): print("Warning: evidence is incomplete, invalid, or not comparable; inspect JSON/HTML", file=sys.stderr) return 2 return 0 except (OSError, SoundLabError, json.JSONDecodeError) as error: print(f"soundlab: {error}", file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())