Sync source edition with VCC Plus 3.7.0 build 63
This commit is contained in:
1 parent
c652a5276d
commit
b2ecec4d87
86 files changed
+10519
-472
No files matched your search
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Apply reviewed software-to-Archive links without modifying source databases.
|
||||
|
||||
The curation file is maintainer input. Filename matches are candidates only;
|
||||
accepted links bind an existing software identity and exact indexed document.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import os
|
||||
import tempfile
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
|
||||
CURATION = Path(__file__).with_name('curation') / 'related-archive.json'
|
||||
ROLES = ('Guide', 'Tutorial', 'Reference', 'Hints', 'Walkthrough', 'Solution', 'Map')
|
||||
SHA = re.compile(r'^[0-9a-f]{64}$')
|
||||
|
||||
|
||||
def load_curation(path=CURATION):
|
||||
data = json.loads(Path(path).read_text())
|
||||
if data.get('schema_version') != 1 or not isinstance(data.get('relationships'), list):
|
||||
raise ValueError('Unsupported related Archive curation contract')
|
||||
ids = set()
|
||||
for row in data['relationships']:
|
||||
if row.get('id') in ids or not isinstance(row.get('id'), str) or not row['id']:
|
||||
raise ValueError('Missing or duplicate relationship identity')
|
||||
ids.add(row['id'])
|
||||
if row.get('disposition') not in ('accepted', 'rejected', 'ambiguous'):
|
||||
raise ValueError('Relationship needs an explicit review disposition')
|
||||
if row['disposition'] != 'accepted':
|
||||
continue
|
||||
if not row.get('title_id') or not row.get('evidence') or not row.get('title'):
|
||||
raise ValueError('Accepted relationship needs a title and review evidence')
|
||||
if not SHA.fullmatch(row.get('document_id', '')) or row.get('sha256') != row['document_id']:
|
||||
raise ValueError('Archive identity must bind exact indexed bytes')
|
||||
if row.get('role') not in ROLES:
|
||||
raise ValueError('Unknown related-document role')
|
||||
page = row.get('page')
|
||||
if page is not None and (isinstance(page, bool) or not isinstance(page, int) or page < 1):
|
||||
raise ValueError('Page/section location must be a positive physical location')
|
||||
if not isinstance(row.get('spoiler', False), bool) or not isinstance(row.get('order', 999), int):
|
||||
raise ValueError('Invalid relationship presentation metadata')
|
||||
return data
|
||||
|
||||
|
||||
def apply_curated(db, curation=CURATION, archive=None, strict=False):
|
||||
"""Apply accepted links in caller transaction; never execute/download content."""
|
||||
data = load_curation(curation)
|
||||
titles = {row[0] for row in db.execute('SELECT id FROM title')}
|
||||
releases = {row[0]: row[1] for row in db.execute('SELECT id,title_id FROM release')}
|
||||
groups = {}
|
||||
for row in data['relationships']:
|
||||
if row['disposition'] != 'accepted':
|
||||
continue
|
||||
title_id = row['title_id']
|
||||
if title_id not in titles:
|
||||
if strict:
|
||||
raise ValueError('Related reference has an unknown software title: ' + title_id)
|
||||
continue # Empty/source or scoped catalog omits titles deliberately.
|
||||
edition_ids = row.get('edition_ids', [])
|
||||
if not isinstance(edition_ids, list) or any(releases.get(i) != title_id for i in edition_ids):
|
||||
raise ValueError('Related reference has an unknown or conflicting edition')
|
||||
if archive is not None:
|
||||
record = archive.execute('SELECT page_count FROM document WHERE sha256=?', (row['document_id'],)).fetchone()
|
||||
if record is None or (row.get('page') or 1) > record[0]:
|
||||
raise ValueError('Related reference is missing or its location no longer exists')
|
||||
selected = edition_ids or [i for i, title in releases.items() if title == title_id]
|
||||
for release in selected:
|
||||
projected = {key: row[key] for key in ('id', 'title', 'document_id', 'sha256', 'role')}
|
||||
projected.update(page=row.get('page'), spoiler=row.get('spoiler', False), order=row.get('order', 999))
|
||||
group = groups.setdefault(release, [])
|
||||
if any((r['document_id'], r['page']) == (projected['document_id'], projected['page']) for r in group):
|
||||
raise ValueError('Duplicate accepted related-document target')
|
||||
group.append(projected)
|
||||
for release, in db.execute('SELECT id FROM release').fetchall():
|
||||
original = db.execute('SELECT metadata_json FROM release WHERE id=?', (release,)).fetchone()[0]
|
||||
metadata = json.loads(original)
|
||||
previous = dict(metadata)
|
||||
# Reviewed curation is authoritative for this capability. Later guesses
|
||||
# cannot promote rejected or removed relationships by reusing old output.
|
||||
metadata.pop('related_archive', None)
|
||||
metadata.pop('related_archive_contract', None)
|
||||
if release in groups:
|
||||
metadata['related_archive_contract'] = 1
|
||||
metadata['related_archive'] = groups[release]
|
||||
if metadata != previous:
|
||||
db.execute('UPDATE release SET metadata_json=? WHERE id=?', (json.dumps(metadata, ensure_ascii=False, sort_keys=True), release))
|
||||
if groups and db.execute("SELECT value FROM metadata WHERE key='related_archive_contract'").fetchone() != ('1',):
|
||||
db.execute("INSERT OR REPLACE INTO metadata VALUES('related_archive_contract','1')")
|
||||
return {'relationships': sum(map(len, groups.values())), 'editions': len(groups),
|
||||
'titles': len({releases[i] for i in groups})}
|
||||
|
||||
|
||||
def candidates(catalog, archive):
|
||||
def normal(text):
|
||||
return re.sub(r'[^a-z0-9]', '', re.sub(r'^the\s+', '', text.lower()))
|
||||
titles = catalog.execute('SELECT id,name FROM title').fetchall()
|
||||
rows = []
|
||||
for doc, title, collection, source in archive.execute("SELECT sha256,title,collection,source_url FROM document WHERE category IN ('Walkthroughs','Maps') ORDER BY title"):
|
||||
prefix = re.split(r'\b(?:walktrough|walkthrough|hints|map|solution|help|spell book)\b|\(', title, maxsplit=1, flags=re.I)[0]
|
||||
key = normal(prefix)
|
||||
matches = [{'id': i, 'title': name} for i, name in titles if normal(name) == key]
|
||||
rows.append({'document_id': doc, 'title': title, 'collection': collection, 'source_url': source,
|
||||
'candidate_titles': matches, 'status': 'candidate-only' if matches else 'unmatched'})
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('catalog', type=Path)
|
||||
parser.add_argument('--archive', required=True, type=Path)
|
||||
parser.add_argument('--curation', default=CURATION, type=Path)
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--candidate-report', type=Path)
|
||||
args = parser.parse_args()
|
||||
with closing(sqlite3.connect(args.catalog.resolve().as_uri() + '?mode=ro', uri=True)) as source, closing(sqlite3.connect(args.archive.resolve().as_uri() + '?mode=ro', uri=True)) as archive:
|
||||
if args.candidate_report:
|
||||
args.candidate_report.parent.mkdir(parents=True, exist_ok=True)
|
||||
rows = candidates(source, archive)
|
||||
args.candidate_report.write_text(json.dumps({'schema_version': 1, 'status': 'candidates-not-approved', 'documents': rows}, indent=2) + '\n')
|
||||
if args.output:
|
||||
if args.output.exists() or args.output.is_symlink() or args.output.resolve() == args.catalog.resolve():
|
||||
raise ValueError('Choose a new candidate catalog output')
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, name = tempfile.mkstemp(prefix='.related-candidate-', suffix='.sqlite', dir=args.output.parent)
|
||||
os.close(descriptor)
|
||||
temporary = Path(name)
|
||||
try:
|
||||
with closing(sqlite3.connect(temporary)) as output:
|
||||
source.backup(output)
|
||||
report = apply_curated(output, args.curation, archive, strict=True)
|
||||
output.commit()
|
||||
# Exclusive promotion: a concurrent process cannot be overwritten.
|
||||
os.link(temporary, args.output)
|
||||
print(json.dumps(report))
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user