Files

526 lines
26 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Read-only Sound Lab Disk BASIC extraction and evidence reporting (stdlib only)."""
from __future__ import annotations
import argparse
import csv
import hashlib
import html
import io
import json
from pathlib import Path
import re
import sys
from typing import Any
DISK_SIZE = 35 * 18 * 256
FAT_OFFSET = (17 * 18 + 1) * 256
DIRECTORY_OFFSET = (17 * 18 + 2) * 256
REPORT_NAME = re.compile(r"SND[0-9]{4}\.TXT", re.IGNORECASE)
KEY = re.compile(r"[A-Z][A-Z0-9_]*")
AUTO = frozenset("PASS FAIL OBSERVED TIMEOUT NOT_RUN NOT_APPLICABLE".split())
HUMAN = frozenset("HEARD WRONG NO_SOUND UNSURE NOT_OBSERVED SKIPPED".split())
class SoundLabError(ValueError):
"""An invalid disk/report or unsafe output operation."""
def crc16(data: bytes) -> int:
"""CRC-16/CCITT-FALSE: polynomial 1021, initial FFFF, no reflection/xor."""
result = 0xFFFF
for value in data:
result ^= value << 8
for _ in range(8):
result = ((result << 1) ^ (0x1021 if result & 0x8000 else 0)) & 0xFFFF
return result
def granule_offset(granule: int) -> int:
track = granule // 2 + (1 if granule >= 34 else 0)
return (track * 18 + (granule % 2) * 9) * 256
def extract_reports(disk: bytes) -> list[dict[str, Any]]:
"""Validate active DECB files, return SNDnnnn.TXT ASCII bytes; never mutate disk.
All active files are walked to catch crosslinks into a selected report. Unused
FAT allocations are tolerated: they can remain after an interrupted write.
"""
if len(disk) != DISK_SIZE:
raise SoundLabError(f"Expected raw 35-track DSK ({DISK_SIZE} bytes), got {len(disk)}")
fat = disk[FAT_OFFSET:FAT_OFFSET + 68]
for index, value in enumerate(fat):
if not (value < 68 or 0xC0 <= value <= 0xC9 or value == 0xFF):
raise SoundLabError(f"Invalid FAT value {value:02X} at granule {index}")
owned: dict[int, str] = {}
names: set[str] = set()
reports = []
for index in range(72):
entry = disk[DIRECTORY_OFFSET + index * 32:DIRECTORY_OFFSET + (index + 1) * 32]
if entry[0] in (0, 0xFF):
continue
try:
base = entry[:8].decode("ascii").rstrip(" ")
ext = entry[8:11].decode("ascii").rstrip(" ")
except UnicodeDecodeError as error:
raise SoundLabError(f"Non-ASCII directory name at entry {index}") from error
if not base or any(ord(c) < 33 or c in '/\\:.' for c in base + ext):
raise SoundLabError(f"Unsafe directory name at entry {index}")
name = f"{base}.{ext}" if ext else base
normalized = name.upper()
if normalized in names:
raise SoundLabError(f"Duplicate directory name: {name}")
names.add(normalized)
if entry[11] > 3 or entry[12] not in (0, 0xFF):
raise SoundLabError(f"Invalid file type/ASCII flag for {name}")
last_bytes = int.from_bytes(entry[14:16], "big")
if last_bytes > 256:
raise SoundLabError(f"Invalid final-sector byte count for {name}")
chain = []
current = entry[13]
while True:
if current >= 68:
raise SoundLabError(f"Out-of-range granule {current} for {name}")
if current in chain:
raise SoundLabError(f"FAT loop for {name}")
if current in owned:
raise SoundLabError(f"Crosslinked granule {current}: {name} and {owned[current]}")
chain.append(current)
owned[current] = name
value = fat[current]
if 0xC0 <= value <= 0xC9:
final_sectors = value - 0xC0
if final_sectors == 0 and (len(chain) != 1 or last_bytes != 0):
raise SoundLabError(f"Invalid empty final granule for {name}")
if final_sectors > 0 and last_bytes == 0:
# DOS stores 256, not zero, for a completely used last sector.
raise SoundLabError(f"Missing final-sector bytes for {name}")
break
if value == 0xFF:
raise SoundLabError(f"Allocated file {name} points to free granule {current}")
current = value
length = (len(chain) - 1) * 2304
if final_sectors:
length += (final_sectors - 1) * 256 + last_bytes
if REPORT_NAME.fullmatch(name):
if entry[12] != 0xFF:
raise SoundLabError(f"Report {name} is not marked Disk BASIC ASCII")
content = b"".join(disk[granule_offset(g):granule_offset(g) + 2304] for g in chain)[:length]
reports.append({"filename": normalized, "data": content,
"byte_count": len(content), "sha256": hashlib.sha256(content).hexdigest(),
"granules": chain, "directory_type": entry[11]})
return sorted(reports, key=lambda item: item["filename"])
def decode_note(value: str) -> str:
decoded = bytearray()
i = 0
while i < len(value):
char = value[i]
if char != "\\":
if not 32 <= ord(char) <= 126:
raise SoundLabError("Notes must use printable ASCII or escaped control bytes")
decoded.append(ord(char))
i += 1
continue
i += 1
if i >= len(value):
raise SoundLabError("Incomplete note escape")
escaped = value[i]
if escaped in "\\nrt":
decoded.append({"\\": 92, "n": 10, "r": 13, "t": 9}[escaped])
i += 1
elif escaped == "x" and i + 2 < len(value) and re.fullmatch(r"[0-9A-Fa-f]{2}", value[i + 1:i + 3]):
byte = int(value[i + 1:i + 3], 16)
if byte > 127:
raise SoundLabError("Escaped notes must remain ASCII")
decoded.append(byte)
i += 3
else:
raise SoundLabError(f"Unknown note escape at position {i}")
return decoded.decode("ascii")
def parse_tokens(line: str) -> dict[str, str]:
fields = {}
for token in line.split():
if "=" not in token:
raise SoundLabError(f"Expected KEY=VALUE token: {token}")
key, value = token.split("=", 1)
if not KEY.fullmatch(key) or not value or key in fields:
raise SoundLabError(f"Invalid or duplicate token: {token}")
fields[key] = value
if not fields:
raise SoundLabError("Empty record")
return fields
def parse_report(data: bytes, source: str = "") -> dict[str, Any]:
"""Parse evidence without turning automatic observations into listening results.
Incomplete files are useful evidence, returned with complete=False and issues.
Structural and integrity failures likewise remain visible in JSON/HTML; a
malformed header/encoding is rejected because it is not a SOUNDLAB-1 report.
"""
try:
text = data.decode("ascii")
except UnicodeDecodeError as error:
raise SoundLabError("Report is not ASCII") from error
if "\x00" in text or re.search(r"[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]", text):
raise SoundLabError("Report contains unescaped control bytes")
if "\r" in text.replace("\r\n", ""):
raise SoundLabError("Report has bare CR line endings")
lines = [line for line in text.replace("\r\n", "\n").split("\n") if line.strip()]
if not lines or lines[0] != "FORMAT=SOUNDLAB-1":
raise SoundLabError("Missing supported FORMAT=SOUNDLAB-1 header")
result: dict[str, Any] = {"format": "SOUNDLAB-1", "source": source,
"source_sha256": hashlib.sha256(data).hexdigest(),
"metadata": {}, "cases": [], "issues": [], "complete": False,
"integrity": {"record_count": "NOT_PROVIDED", "checksum_crc16": "NOT_PROVIDED"}}
current = None
footer = None
note_length = None
records = []
def issue(message: str) -> None:
result["issues"].append(message)
def merge(target: dict[str, str], fields: dict[str, str], number: int) -> None:
for key, value in fields.items():
if key in target:
issue(f"Line {number}: duplicate field {key}")
else:
target[key] = value
for number, line in enumerate(lines, 1):
if footer is not None:
issue(f"Line {number}: record after ENDRUN")
continue
if line.startswith("NOTE="):
records.append(line)
try:
note = decode_note(line[5:])
if current is None:
issue(f"Line {number}: NOTE outside CASE")
elif note_length is None:
issue(f"Line {number}: NOTE without NOTE_LENGTH")
else:
current["notes"].append(note)
if len(note.encode("ascii")) != note_length:
issue(f"Line {number}: NOTE_LENGTH mismatch")
except SoundLabError as error:
issue(f"Line {number}: {error}")
note_length = None
continue
if note_length is not None:
issue(f"Line {number}: NOTE_LENGTH not followed by NOTE")
note_length = None
try:
fields = parse_tokens(line)
except SoundLabError as error:
records.append(line)
issue(f"Line {number}: {error}")
continue
controls = set(fields) & {"FORMAT", "RUN", "CASE", "ENDCASE", "ENDRUN", "NOTE_LENGTH"}
if len(controls) > 1:
issue(f"Line {number}: conflicting control records")
if "FORMAT" in fields and number != 1:
issue(f"Line {number}: repeated FORMAT header")
if "RUN" in fields and (current is not None or result["cases"]):
issue(f"Line {number}: RUN header after CASE")
if "ENDRUN" in fields:
footer = fields
result["footer"] = fields
if current is not None:
issue(f"Line {number}: ENDRUN before ENDCASE")
if fields["ENDRUN"] != result["metadata"].get("RUN"):
issue(f"Line {number}: ENDRUN does not match RUN")
if fields.get("COMPLETE", "").upper() not in ("YES", "NO"):
issue(f"Line {number}: COMPLETE must be YES or NO")
continue
records.append(line)
if number == 1:
continue
if "CASE" in fields:
if current is not None:
issue(f"Line {number}: CASE before previous ENDCASE")
identifier = fields["CASE"].upper()
current = {"id": identifier, "fields": {}, "notes": [], "closed": False}
result["cases"].append(current)
merge(current["fields"], fields, number)
elif "ENDCASE" in fields:
if current is None:
issue(f"Line {number}: ENDCASE outside CASE")
else:
if fields["ENDCASE"].upper() != current["id"]:
issue(f"Line {number}: ENDCASE does not match CASE")
else:
current["closed"] = True
current = None
if len(fields) != 1:
issue(f"Line {number}: unexpected ENDCASE fields")
elif "NOTE_LENGTH" in fields:
if len(fields) != 1 or current is None:
issue(f"Line {number}: NOTE_LENGTH must be alone inside CASE")
try:
note_length = int(fields["NOTE_LENGTH"])
if note_length < 0 or note_length > 65535:
raise ValueError()
except ValueError:
note_length = None
issue(f"Line {number}: invalid NOTE_LENGTH")
else:
merge(current["fields"] if current is not None else result["metadata"], fields, number)
if note_length is not None:
issue("Truncated NOTE after NOTE_LENGTH")
if not result["metadata"].get("RUN"):
issue("Missing RUN identity")
attempts: set[tuple[str, int]] = set()
for case in result["cases"]:
try:
repeat_text = case["fields"].get("REPEAT", "0")
if not re.fullmatch(r"[0-9]+", repeat_text):
raise ValueError()
repeat = int(repeat_text)
if repeat > 65535:
raise ValueError()
case["repeat"] = repeat
identity = (case["id"], repeat)
if identity in attempts:
issue(f"CASE {case['id']}: duplicate REPEAT={repeat}")
attempts.add(identity)
except ValueError:
case["repeat"] = None
issue(f"CASE {case['id']}: invalid REPEAT")
for key, allowed in (("AUTO", AUTO), ("HUMAN", HUMAN)):
value = case["fields"].get(key)
if value is None:
# UNKNOWN is deliberate: absence is not a positive or negative observation.
case[key.lower()] = "UNKNOWN"
issue(f"CASE {case['id']}: missing {key}")
else:
case[key.lower()] = value.upper()
if value.upper() not in allowed:
issue(f"CASE {case['id']}: unknown {key} value {value}")
vector_crc = case["fields"].get("VECTOR_CRC16", "")
if not re.fullmatch(r"[0-9A-Fa-f]{4}", vector_crc):
issue(f"CASE {case['id']}: missing or invalid VECTOR_CRC16")
if not case["closed"]:
issue(f"CASE {case['id']}: missing ENDCASE")
if footer is None:
issue("Missing ENDRUN; partial run")
else:
if "RECORD_COUNT" in footer:
try:
valid_count = int(footer["RECORD_COUNT"]) == len(records)
except ValueError:
valid_count = False
result["integrity"]["record_count"] = "VERIFIED" if valid_count else "FAILED"
if not valid_count:
issue("RECORD_COUNT mismatch")
if "CHECKSUM_CRC16" in footer:
canonical = ("\r\n".join(records) + "\r\n").encode("ascii")
expected = f"{crc16(canonical):04X}"
valid_crc = footer["CHECKSUM_CRC16"].upper() == expected
result["integrity"]["checksum_crc16"] = "VERIFIED" if valid_crc else "FAILED"
if not valid_crc:
issue(f"CHECKSUM_CRC16 mismatch (calculated {expected})")
if footer.get("COMPLETE", "").upper() != "YES":
issue("Run marked incomplete")
result["record_count"] = len(records)
result["complete"] = footer is not None and footer.get("COMPLETE", "").upper() == "YES" and not result["issues"]
result["valid"] = not result["issues"]
return result
def validate_manifest(report: dict[str, Any], manifest: dict[str, Any]) -> list[str]:
"""Return exact identity failures; a manifest is evidence, not a trust signature."""
if not isinstance(manifest, dict):
raise SoundLabError("Manifest must be a JSON object")
problems = []
metadata = report["metadata"]
def crc_text(value: Any) -> str:
if isinstance(value, int):
return f"{value:04X}"
return str(value).upper().removeprefix("0X")
for key in ("BUILD_ID", "VECTOR_SET_CRC16"):
expected = manifest.get(key.lower(), manifest.get(key))
expected_text = crc_text(expected) if key.endswith("CRC16") else str(expected).upper()
if expected is None or expected_text != metadata.get(key, "").upper():
problems.append(f"{key} differs from manifest or is absent")
vectors = manifest.get("vectors", {})
if isinstance(vectors, list):
if not all(isinstance(item, dict) for item in vectors):
raise SoundLabError("Manifest vectors must be objects")
vectors = {item.get("id", item.get("case_id", "")): item.get("crc16", item.get("vector_crc16", "")) for item in vectors}
if not isinstance(vectors, dict):
raise SoundLabError("Manifest vectors must be a list or object")
vectors = {str(key).upper(): crc_text(value) for key, value in vectors.items()}
for case in report["cases"]:
expected = vectors.get(case["id"])
if expected is None or expected != case["fields"].get("VECTOR_CRC16", "").upper():
problems.append(f"CASE {case['id']}: vector differs from manifest or is absent")
return problems
def compare_reports(left: dict[str, Any], right: dict[str, Any], manifest: dict[str, Any] | None = None) -> dict[str, Any]:
problems = []
for key in ("BUILD_ID", "VECTOR_SET_CRC16"):
a, b = left["metadata"].get(key), right["metadata"].get(key)
if not a or not b or a.upper() != b.upper():
problems.append(f"{key} differs or is absent")
if manifest is not None:
problems.extend("Left: " + value for value in validate_manifest(left, manifest))
problems.extend("Right: " + value for value in validate_manifest(right, manifest))
if not left["valid"] or not right["valid"]:
problems.append("One or both reports are partial or invalid; outcomes are provisional")
by_id = [{(case["id"], case["repeat"]): case for case in report["cases"]} for report in (left, right)]
rows = []
for identity in sorted(set(by_id[0]) | set(by_id[1]), key=lambda item: (item[0], -1 if item[1] is None else item[1])):
identifier, repeat = identity
a, b = by_id[0].get(identity), by_id[1].get(identity)
reason = list(problems)
if a is None or b is None:
reason.append("CASE missing from one report")
elif a["fields"].get("VECTOR_CRC16", "").upper() != b["fields"].get("VECTOR_CRC16", "").upper():
reason.append("VECTOR_CRC16 differs")
row = {"case": identifier, "repeat": repeat, "left_auto": a["auto"] if a else "UNKNOWN",
"right_auto": b["auto"] if b else "UNKNOWN", "left_human": a["human"] if a else "UNKNOWN",
"right_human": b["human"] if b else "UNKNOWN", "reasons": reason}
changes = {}
if a is not None and b is not None:
for key in sorted(set(a["fields"]) | set(b["fields"])):
if key in ("CASE", "REPEAT", "VECTOR_CRC16", "AUTO", "HUMAN"):
continue
av, bv = a["fields"].get(key), b["fields"].get(key)
if av != bv:
changes[key] = {"left": av, "right": bv}
if a["notes"] != b["notes"]:
changes["NOTES"] = {"left": a["notes"], "right": b["notes"]}
row["recorded_differences"] = changes
outcomes_equal = row["left_auto"] == row["right_auto"] and row["left_human"] == row["right_human"]
row["comparison"] = "NOT_COMPARABLE" if reason else ("SAME" if outcomes_equal and not changes else "DIFFERENT")
rows.append(row)
return {"left": left["source"], "right": right["source"], "identity_issues": problems,
"manifest_checked": manifest is not None, "identity_basis": "MANIFEST" if manifest is not None else "REPORT_CLAIMS_ONLY", "cases": rows,
"notice": "Matching automatic observations do not establish audible equivalence. HUMAN is independent."}
def render_html(document: dict[str, Any]) -> str:
esc = lambda value: html.escape(str(value))
comparison = "left" in document
title = "Sound Lab comparison" if comparison else "Sound Lab report"
pieces = [f'<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>{title}</title>',
'<style>body{font:16px system-ui;max-width:75rem;margin:2rem auto;padding:0 1rem;line-height:1.5;color:#20252b;background:#fff}table{border-collapse:collapse;width:100%}td,th{padding:.5rem;border:1px solid #999;text-align:left;vertical-align:top}pre{white-space:pre-wrap;overflow-wrap:anywhere}.warning{background:#fff0c2;padding:1rem}code{overflow-wrap:anywhere}</style>', f'<h1>{title}</h1>', '<p>Automatic observations and human listening results are separate evidence. Unknown means unreported; it never means heard.</p>']
issues = document.get("identity_issues", document.get("issues", []))
if issues:
pieces.append('<div class="warning"><strong>Incomplete or unverified evidence</strong><ul>' + ''.join(f'<li>{esc(value)}</li>' for value in issues) + '</ul></div>')
if comparison:
keys = ["case", "repeat", "left_auto", "right_auto", "left_human", "right_human", "comparison", "recorded_differences", "reasons"]
rows = document["cases"]
else:
pieces.append(f'<p>Source: <code>{esc(document["source"])}</code><br>SHA-256: <code>{esc(document["source_sha256"])}</code><br>Complete: {esc(document["complete"])}</p>')
pieces.append('<h2>Run metadata and integrity</h2><pre>' + esc(json.dumps({"metadata": document["metadata"], "integrity": document["integrity"]}, indent=2)) + '</pre>')
keys = ["case", "repeat", "auto", "human", "closed", "notes", "recorded_fields"]
rows = [{"case": case["id"], "repeat": case["repeat"], "auto": case["auto"], "human": case["human"], "closed": case["closed"], "notes": "\n".join(case["notes"]), "recorded_fields": json.dumps(case["fields"], sort_keys=True)} for case in document["cases"]]
pieces.append('<h2>Cases</h2><table><thead><tr>' + ''.join(f'<th scope="col">{esc(key)}</th>' for key in keys) + '</tr></thead><tbody>')
for row in rows:
pieces.append('<tr>' + ''.join(f'<td><pre>{esc(row.get(key, ""))}</pre></td>' for key in keys) + '</tr>')
pieces.append('</tbody></table></html>')
return '\n'.join(pieces)
def render_csv(document: dict[str, Any]) -> str:
stream = io.StringIO(newline="")
if "left" in document:
fields = ["case", "repeat", "left_auto", "right_auto", "left_human", "right_human", "comparison", "recorded_differences", "reasons"]
rows = document["cases"]
else:
# Union preserves timing/status fields, rather than reducing reports to outcomes.
extra = sorted({key for case in document["cases"] for key in case["fields"]} - {"CASE", "REPEAT", "AUTO", "HUMAN"})
fields = ["case", "repeat", "auto", "human", "closed", "notes"] + extra
rows = [{"case": case["id"], "repeat": case["repeat"], "auto": case["auto"], "human": case["human"], "closed": case["closed"], "notes": '\n'.join(case["notes"]), **{key: case["fields"].get(key, "") for key in extra}} for case in document["cases"]]
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
writer.writerows(rows)
return stream.getvalue()
def write_outputs(outputs: dict[Path, bytes], overwrite: bool = False) -> None:
"""Preflight every target before any writes; originals are never implicitly replaced."""
for path in outputs:
if path.is_symlink() or (path.exists() and (not overwrite or not path.is_file())):
raise SoundLabError(f"Refusing to replace {path}; select a fresh folder or use --overwrite for regular files")
for path, data in outputs.items():
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("wb" if overwrite else "xb") as handle:
handle.write(data)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="operation", required=True)
for operation in ("extract", "convert", "compare"):
command = sub.add_parser(operation)
command.add_argument("source", type=Path)
if operation == "compare":
command.add_argument("other", type=Path)
command.add_argument("--output", required=True, type=Path)
command.add_argument("--overwrite", action="store_true")
if operation != "extract":
command.add_argument("--manifest", type=Path, help="Authoritative build/vector JSON manifest")
args = parser.parse_args(argv)
try:
outputs = {}
if args.operation == "extract":
disk = args.source.read_bytes()
reports = extract_reports(disk)
if not reports:
raise SoundLabError("No SNDnnnn.TXT ASCII reports found")
receipt = {"disk": args.source.name, "disk_byte_count": len(disk), "disk_sha256": hashlib.sha256(disk).hexdigest(), "reports": []}
for report in reports:
outputs[args.output / report["filename"]] = report["data"]
receipt["reports"].append({key: value for key, value in report.items() if key != "data"})
outputs[args.output / "extraction.json"] = (json.dumps(receipt, indent=2) + '\n').encode()
else:
left = parse_report(args.source.read_bytes(), args.source.name)
manifest = json.loads(args.manifest.read_text()) if args.manifest else None
if args.manifest and not isinstance(manifest, dict):
raise SoundLabError("Manifest must be a JSON object")
if args.operation == "compare":
document = compare_reports(left, parse_report(args.other.read_bytes(), args.other.name), manifest)
stem = "comparison"
else:
document = left
if manifest is not None:
document["manifest_issues"] = validate_manifest(document, manifest)
document["issues"].extend(document["manifest_issues"])
document["valid"] = document["valid"] and not document["manifest_issues"]
document["complete"] = document["complete"] and document["valid"]
stem = args.source.stem
outputs = {args.output / f"{stem}.json": (json.dumps(document, indent=2) + '\n').encode(), args.output / f"{stem}.csv": render_csv(document).encode(), args.output / f"{stem}.html": render_html(document).encode()}
# Never allow an output to alias any supplied source, even with --overwrite.
sources = [args.source] + ([args.other] if args.operation == "compare" else [])
if getattr(args, "manifest", None):
sources.append(args.manifest)
source_paths = {path.resolve() for path in sources}
if any(path.resolve() in source_paths or any(path.exists() and path.samefile(source) for source in sources) for path in outputs):
raise SoundLabError("Output would replace an input; choose another folder")
write_outputs(outputs, args.overwrite)
print(f"Wrote {len(outputs)} files to {args.output}")
if args.operation != "extract" and (document.get("issues") or document.get("identity_issues") or any(row.get("comparison") == "NOT_COMPARABLE" for row in document["cases"])):
print("Warning: evidence is incomplete, invalid, or not comparable; inspect JSON/HTML", file=sys.stderr)
return 2
return 0
except (OSError, SoundLabError, json.JSONDecodeError) as error:
print(f"soundlab: {error}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())