#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-or-later """Apply reviewed software-to-Archive links without modifying source databases. The curation file is maintainer input. Filename matches are candidates only; accepted links bind an existing software identity and exact indexed document. """ import argparse import hashlib import json from pathlib import Path import re import os import tempfile import sqlite3 from contextlib import closing CURATION = Path(__file__).with_name('curation') / 'related-archive.json' ROLES = ('Guide', 'Tutorial', 'Reference', 'Hints', 'Walkthrough', 'Solution', 'Map') SHA = re.compile(r'^[0-9a-f]{64}$') def load_curation(path=CURATION): data = json.loads(Path(path).read_text()) if data.get('schema_version') != 1 or not isinstance(data.get('relationships'), list): raise ValueError('Unsupported related Archive curation contract') ids = set() for row in data['relationships']: if row.get('id') in ids or not isinstance(row.get('id'), str) or not row['id']: raise ValueError('Missing or duplicate relationship identity') ids.add(row['id']) if row.get('disposition') not in ('accepted', 'rejected', 'ambiguous'): raise ValueError('Relationship needs an explicit review disposition') if row['disposition'] != 'accepted': continue if not row.get('title_id') or not row.get('evidence') or not row.get('title'): raise ValueError('Accepted relationship needs a title and review evidence') if not SHA.fullmatch(row.get('document_id', '')) or row.get('sha256') != row['document_id']: raise ValueError('Archive identity must bind exact indexed bytes') if row.get('role') not in ROLES: raise ValueError('Unknown related-document role') page = row.get('page') if page is not None and (isinstance(page, bool) or not isinstance(page, int) or page < 1): raise ValueError('Page/section location must be a positive physical location') if not isinstance(row.get('spoiler', False), bool) or not isinstance(row.get('order', 999), int): raise ValueError('Invalid relationship presentation metadata') return data def apply_curated(db, curation=CURATION, archive=None, strict=False): """Apply accepted links in caller transaction; never execute/download content.""" data = load_curation(curation) titles = {row[0] for row in db.execute('SELECT id FROM title')} releases = {row[0]: row[1] for row in db.execute('SELECT id,title_id FROM release')} groups = {} for row in data['relationships']: if row['disposition'] != 'accepted': continue title_id = row['title_id'] if title_id not in titles: if strict: raise ValueError('Related reference has an unknown software title: ' + title_id) continue # Empty/source or scoped catalog omits titles deliberately. edition_ids = row.get('edition_ids', []) if not isinstance(edition_ids, list) or any(releases.get(i) != title_id for i in edition_ids): raise ValueError('Related reference has an unknown or conflicting edition') if archive is not None: record = archive.execute('SELECT page_count FROM document WHERE sha256=?', (row['document_id'],)).fetchone() if record is None or (row.get('page') or 1) > record[0]: raise ValueError('Related reference is missing or its location no longer exists') selected = edition_ids or [i for i, title in releases.items() if title == title_id] for release in selected: projected = {key: row[key] for key in ('id', 'title', 'document_id', 'sha256', 'role')} projected.update(page=row.get('page'), spoiler=row.get('spoiler', False), order=row.get('order', 999)) group = groups.setdefault(release, []) if any((r['document_id'], r['page']) == (projected['document_id'], projected['page']) for r in group): raise ValueError('Duplicate accepted related-document target') group.append(projected) for release, in db.execute('SELECT id FROM release').fetchall(): original = db.execute('SELECT metadata_json FROM release WHERE id=?', (release,)).fetchone()[0] metadata = json.loads(original) previous = dict(metadata) # Reviewed curation is authoritative for this capability. Later guesses # cannot promote rejected or removed relationships by reusing old output. metadata.pop('related_archive', None) metadata.pop('related_archive_contract', None) if release in groups: metadata['related_archive_contract'] = 1 metadata['related_archive'] = groups[release] if metadata != previous: db.execute('UPDATE release SET metadata_json=? WHERE id=?', (json.dumps(metadata, ensure_ascii=False, sort_keys=True), release)) if groups and db.execute("SELECT value FROM metadata WHERE key='related_archive_contract'").fetchone() != ('1',): db.execute("INSERT OR REPLACE INTO metadata VALUES('related_archive_contract','1')") return {'relationships': sum(map(len, groups.values())), 'editions': len(groups), 'titles': len({releases[i] for i in groups})} def candidates(catalog, archive): def normal(text): return re.sub(r'[^a-z0-9]', '', re.sub(r'^the\s+', '', text.lower())) titles = catalog.execute('SELECT id,name FROM title').fetchall() rows = [] for doc, title, collection, source in archive.execute("SELECT sha256,title,collection,source_url FROM document WHERE category IN ('Walkthroughs','Maps') ORDER BY title"): prefix = re.split(r'\b(?:walktrough|walkthrough|hints|map|solution|help|spell book)\b|\(', title, maxsplit=1, flags=re.I)[0] key = normal(prefix) matches = [{'id': i, 'title': name} for i, name in titles if normal(name) == key] rows.append({'document_id': doc, 'title': title, 'collection': collection, 'source_url': source, 'candidate_titles': matches, 'status': 'candidate-only' if matches else 'unmatched'}) return rows def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('catalog', type=Path) parser.add_argument('--archive', required=True, type=Path) parser.add_argument('--curation', default=CURATION, type=Path) parser.add_argument('--output', type=Path) parser.add_argument('--candidate-report', type=Path) args = parser.parse_args() with closing(sqlite3.connect(args.catalog.resolve().as_uri() + '?mode=ro', uri=True)) as source, closing(sqlite3.connect(args.archive.resolve().as_uri() + '?mode=ro', uri=True)) as archive: if args.candidate_report: args.candidate_report.parent.mkdir(parents=True, exist_ok=True) rows = candidates(source, archive) args.candidate_report.write_text(json.dumps({'schema_version': 1, 'status': 'candidates-not-approved', 'documents': rows}, indent=2) + '\n') if args.output: if args.output.exists() or args.output.is_symlink() or args.output.resolve() == args.catalog.resolve(): raise ValueError('Choose a new candidate catalog output') args.output.parent.mkdir(parents=True, exist_ok=True) descriptor, name = tempfile.mkstemp(prefix='.related-candidate-', suffix='.sqlite', dir=args.output.parent) os.close(descriptor) temporary = Path(name) try: with closing(sqlite3.connect(temporary)) as output: source.backup(output) report = apply_curated(output, args.curation, archive, strict=True) output.commit() # Exclusive promotion: a concurrent process cannot be overwritten. os.link(temporary, args.output) print(json.dumps(report)) finally: temporary.unlink(missing_ok=True) if __name__ == '__main__': main()