Sync source edition with VCC Plus 3.7.0 build 63
This commit is contained in:
1 parent
c652a5276d
commit
b2ecec4d87
86 files changed
+10519
-472
No files matched your search
@@ -0,0 +1,165 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"relationships": [
|
||||
{
|
||||
"id": "02587e81cceaec9ac0c11fe9",
|
||||
"title_id": "486dda54b9351ac926c382f70f328aa5",
|
||||
"software_title": "Raaka-Tu",
|
||||
"title": "Raaka-Tu Walkthrough (Tandy)",
|
||||
"document_id": "e55b30ce36985c0ff35ade23152828e090014bae1b8ddcadd50b02b90f72c7dc",
|
||||
"sha256": "e55b30ce36985c0ff35ade23152828e090014bae1b8ddcadd50b02b90f72c7dc",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Raaka-Tu%20Walkthrough%20(Tandy).txt",
|
||||
"role": "Walkthrough",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed the indexed command-by-command solution, including item acquisition and movement; separate from the game manual."
|
||||
},
|
||||
{
|
||||
"id": "66aa0cf28072b0be9a60f72b",
|
||||
"title_id": "2a384f275ebcae78dca310d3312e9cdf",
|
||||
"software_title": "Bedlam",
|
||||
"title": "Bedlam Walkthrough (Tandy)",
|
||||
"document_id": "a10ac100f61319832afb5ff9ca2f2114977e5c82a85b7bdd69229b21dd22ea71",
|
||||
"sha256": "a10ac100f61319832afb5ff9ca2f2114977e5c82a85b7bdd69229b21dd22ea71",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Bedlam%20Walkthrough%20(Tandy).txt",
|
||||
"role": "Walkthrough",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed the indexed solution sequence and conditional dog/injection advice; useful game-specific help."
|
||||
},
|
||||
{
|
||||
"id": "7e3c07f0d050240f97445e03",
|
||||
"title_id": "bced2c52bb9d1f1e864cb8cc7d2ce9e5",
|
||||
"software_title": "Pyramid 2000",
|
||||
"title": "Pyramid 2000 Walkthrough (Tandy)",
|
||||
"document_id": "9e2d4b3de4547f809920acafa48a57ed0272e5cf458c5bc313f73d3c9edef3c6",
|
||||
"sha256": "9e2d4b3de4547f809920acafa48a57ed0272e5cf458c5bc313f73d3c9edef3c6",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Pyramid%202000%20Walkthrough%20(Tandy).txt",
|
||||
"role": "Solution",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed the indexed numbered puzzle/movement solution and score completion; separate spoiler material."
|
||||
},
|
||||
{
|
||||
"id": "91bc669a5429e19f33e23496",
|
||||
"title_id": "e0af7e3113d4a0666c4d735df8b70e59",
|
||||
"software_title": "Calixto Island",
|
||||
"title": "Calixto Island Walkthrough (Mark Data Products)",
|
||||
"document_id": "3c0e7e7a94357a6fe97b6d59ded462d4988c3cc8f6edca512bc332701ae06acb",
|
||||
"sha256": "3c0e7e7a94357a6fe97b6d59ded462d4988c3cc8f6edca512bc332701ae06acb",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Calixto%20Island%20Walkthrough%20(Mark%20Data%20Products).txt",
|
||||
"role": "Walkthrough",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed the indexed command sequence covering island equipment, boat and temple progression."
|
||||
},
|
||||
{
|
||||
"id": "1f432b0ab98df3b00911d53f",
|
||||
"title_id": "e0af7e3113d4a0666c4d735df8b70e59",
|
||||
"software_title": "Calixto Island",
|
||||
"title": "Calixto Island Map (Mark Data Products)",
|
||||
"document_id": "afd0c53c3a8ae5dd685258c754a5b161863a7f1d824870c12d660c6410c1e18f",
|
||||
"sha256": "afd0c53c3a8ae5dd685258c754a5b161863a7f1d824870c12d660c6410c1e18f",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Calixto%20Island%20Map%20(Mark%20Data%20Products).jpg",
|
||||
"role": "Map",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Visually inspected the retained map: named Calixto Island rooms, island/boat links and item annotations."
|
||||
},
|
||||
{
|
||||
"id": "53bbd42bf0b5e6edb01c7085",
|
||||
"title_id": "dbd7d8cd515b8f6f4bc58fa25482ab7c",
|
||||
"software_title": "Madness and the Minotaur",
|
||||
"title": "Madness and the Minotaur Hints (Tandy)",
|
||||
"document_id": "11925450e2d10999099cc4e589fb96e8ef18a66ab70bc221c92445599c709702",
|
||||
"sha256": "11925450e2d10999099cc4e589fb96e8ef18a66ab70bc221c92445599c709702",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Madness%20and%20the%20Minotaur%20Hints%20(Tandy).txt",
|
||||
"role": "Hints",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed indexed room-coded hints about the lamp, spells, creatures and randomized room details."
|
||||
},
|
||||
{
|
||||
"id": "c8daed0e01578988d888a866",
|
||||
"title_id": "255cf6ae5ff4eb045362b0afa2c68de9",
|
||||
"software_title": "Sands of Egypt",
|
||||
"title": "Sands of Egypt - Help & Maps (Tandy)",
|
||||
"document_id": "76ed8a92c8ff6c6d6903977d16d8c9e4ecdc2a66f443abf6a413e3e62a48e07d",
|
||||
"sha256": "76ed8a92c8ff6c6d6903977d16d8c9e4ecdc2a66f443abf6a413e3e62a48e07d",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Sands%20of%20Egypt%20-%20Help%20%26%20Maps%20(Tandy).txt",
|
||||
"role": "Guide",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed indexed introduction and preposition/input guidance; the article explicitly supplements the adventure with syntax help and maps."
|
||||
},
|
||||
{
|
||||
"id": "820e2bff9f157f837edf18b2",
|
||||
"title_id": "d3167ac84f010c802cb8a5f1ec35c7b5",
|
||||
"software_title": "Syzygy",
|
||||
"title": "Syzygy Hints (Spectral Associates)",
|
||||
"document_id": "5f9f440af190cf1e32750a35ba6db14d166afe82f06a58fb8cdf856f68165b5c",
|
||||
"sha256": "5f9f440af190cf1e32750a35ba6db14d166afe82f06a58fb8cdf856f68165b5c",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Syzygy%20Hints%20(Spectral%20Associates).txt",
|
||||
"role": "Hints",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed indexed coordinate-entry guidance and destinations; software-specific puzzle assistance."
|
||||
},
|
||||
{
|
||||
"id": "9640d9703099d24f76ad781c",
|
||||
"title_id": "3d23047f748cdb3eb7cced8f29ad7182",
|
||||
"software_title": "Trekboer",
|
||||
"title": "Trekboer Walkthrough (Mark Data Products)",
|
||||
"document_id": "e5b50515cd52d95d5f5ed660bb9684658bf9dafb1ae6e6484a750ba994c2e10f",
|
||||
"sha256": "e5b50515cd52d95d5f5ed660bb9684658bf9dafb1ae6e6484a750ba994c2e10f",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Walkthroughs/Trekboer%20Walkthrough%20(Mark%20Data%20Products).TXT",
|
||||
"role": "Solution",
|
||||
"spoiler": true,
|
||||
"order": 999,
|
||||
"page": null,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed indexed complete Mark-Data adventure solution and command sequence."
|
||||
},
|
||||
{
|
||||
"id": "2be433ec8a3a9c61002d22ab",
|
||||
"title_id": "b7debb76ab93b3d0a2bf9c3abfdbb284",
|
||||
"software_title": "Deskmate 3",
|
||||
"title": "DeskMate Quick Reference Quide (Tandy)",
|
||||
"document_id": "8856b2abd5e76971e3c87bb2ea28a2a34c127a82c36937fa0af7022a01d5665e",
|
||||
"sha256": "8856b2abd5e76971e3c87bb2ea28a2a34c127a82c36937fa0af7022a01d5665e",
|
||||
"source_url": "https://colorcomputerarchive.com/repo/Documents/Manuals/Applications/DeskMate%20Quick%20Reference%20Quide%20(Tandy).pdf",
|
||||
"role": "Reference",
|
||||
"spoiler": false,
|
||||
"order": 999,
|
||||
"page": 1,
|
||||
"disposition": "accepted",
|
||||
"reviewed_at": "2026-11-13",
|
||||
"evidence": "Reviewed indexed first page: Color Computer 3 DeskMate title, general keys, menus and mouse/joystick techniques; selected base DeskMate 3 title only."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -105,8 +105,10 @@ def source_catalog(output):
|
||||
releases=0, assets=0, thumbnails=0, empty=True)
|
||||
|
||||
|
||||
def package(source, output, require_full=False, *, content_profile='normal'):
|
||||
def package(source, output, require_full=False, *, content_profile='normal', suppression_manifest=None, rights_cache=None):
|
||||
if resolve_content_profile(content_profile) == 'source':
|
||||
if suppression_manifest is not None:
|
||||
raise ValueError('A rights manifest applies to a full candidate, not the empty source-content profile')
|
||||
if require_full:
|
||||
raise ValueError('--require-full cannot be combined with the source content profile')
|
||||
return source_catalog(output)
|
||||
@@ -128,6 +130,16 @@ def package(source, output, require_full=False, *, content_profile='normal'):
|
||||
else:
|
||||
print('warning: Full Software Library catalog is absent; bundling an empty development catalog. See tools/catalog/README.md.')
|
||||
empty_catalog(temporary)
|
||||
# Derived metadata is prepared from tracked reviewed associations; the
|
||||
# source catalog remains unchanged. Retained rights exclusions never
|
||||
# expire during offline builds; a fresh LIVE gate governs publication.
|
||||
from related_archive import apply_curated
|
||||
from rights import apply_cached, read_json
|
||||
with closing(sqlite3.connect(temporary)) as candidate:
|
||||
apply_curated(candidate)
|
||||
manifest = read_json(suppression_manifest) if isinstance(suppression_manifest, (str, Path)) else suppression_manifest
|
||||
apply_cached(candidate, cache_path=rights_cache, extra=manifest)
|
||||
candidate.commit()
|
||||
counts = validate(temporary, require_full=require_full)
|
||||
digest = hashlib.sha256(temporary.read_bytes()).hexdigest()
|
||||
os.chmod(temporary, 0o444)
|
||||
@@ -250,12 +262,16 @@ def main():
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--check-only', action='store_true', help='Validate input without staging a resource')
|
||||
parser.add_argument('--require-full', action='store_true')
|
||||
parser.add_argument('--suppression-manifest', type=Path, help='Explicit fresh reviewed rights-withdrawal manifest for candidate preparation')
|
||||
parser.add_argument('--rights-cache', type=Path, help='Retained exclusion cache override; ordinary full preparation uses the persistent maintainer cache')
|
||||
args = parser.parse_args()
|
||||
if not args.check_only and args.output is None:
|
||||
parser.error('--output is required unless --check-only is used')
|
||||
try:
|
||||
profile = resolve_content_profile(args.content_profile)
|
||||
if profile == 'source':
|
||||
if args.suppression_manifest is not None:
|
||||
raise ValueError('A rights manifest applies to a full candidate, not the empty source-content profile')
|
||||
if args.require_full:
|
||||
raise ValueError('--require-full cannot be combined with the source content profile')
|
||||
if args.check_only:
|
||||
@@ -265,7 +281,14 @@ def main():
|
||||
result = source_catalog(args.output)
|
||||
else:
|
||||
source = args.source if args.source is not None else DEFAULT_SOURCE
|
||||
result = validate(source, args.require_full) if args.check_only else package(source, args.output, args.require_full)
|
||||
if args.check_only:
|
||||
validate(source, args.require_full)
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-suppression-check-') as temporary:
|
||||
result = package(source, Path(temporary) / 'catalog.sqlite', args.require_full,
|
||||
suppression_manifest=args.suppression_manifest, rights_cache=args.rights_cache)
|
||||
result['suppression_checked'] = True
|
||||
else:
|
||||
result = package(source, args.output, args.require_full, suppression_manifest=args.suppression_manifest, rights_cache=args.rights_cache)
|
||||
if not args.check_only:
|
||||
artwork = args.output.parent / 'Artwork'
|
||||
if artwork.is_symlink(): raise ValueError('Artwork destination must not be a symlink')
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Apply reviewed software-to-Archive links without modifying source databases.
|
||||
|
||||
The curation file is maintainer input. Filename matches are candidates only;
|
||||
accepted links bind an existing software identity and exact indexed document.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import os
|
||||
import tempfile
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
|
||||
CURATION = Path(__file__).with_name('curation') / 'related-archive.json'
|
||||
ROLES = ('Guide', 'Tutorial', 'Reference', 'Hints', 'Walkthrough', 'Solution', 'Map')
|
||||
SHA = re.compile(r'^[0-9a-f]{64}$')
|
||||
|
||||
|
||||
def load_curation(path=CURATION):
|
||||
data = json.loads(Path(path).read_text())
|
||||
if data.get('schema_version') != 1 or not isinstance(data.get('relationships'), list):
|
||||
raise ValueError('Unsupported related Archive curation contract')
|
||||
ids = set()
|
||||
for row in data['relationships']:
|
||||
if row.get('id') in ids or not isinstance(row.get('id'), str) or not row['id']:
|
||||
raise ValueError('Missing or duplicate relationship identity')
|
||||
ids.add(row['id'])
|
||||
if row.get('disposition') not in ('accepted', 'rejected', 'ambiguous'):
|
||||
raise ValueError('Relationship needs an explicit review disposition')
|
||||
if row['disposition'] != 'accepted':
|
||||
continue
|
||||
if not row.get('title_id') or not row.get('evidence') or not row.get('title'):
|
||||
raise ValueError('Accepted relationship needs a title and review evidence')
|
||||
if not SHA.fullmatch(row.get('document_id', '')) or row.get('sha256') != row['document_id']:
|
||||
raise ValueError('Archive identity must bind exact indexed bytes')
|
||||
if row.get('role') not in ROLES:
|
||||
raise ValueError('Unknown related-document role')
|
||||
page = row.get('page')
|
||||
if page is not None and (isinstance(page, bool) or not isinstance(page, int) or page < 1):
|
||||
raise ValueError('Page/section location must be a positive physical location')
|
||||
if not isinstance(row.get('spoiler', False), bool) or not isinstance(row.get('order', 999), int):
|
||||
raise ValueError('Invalid relationship presentation metadata')
|
||||
return data
|
||||
|
||||
|
||||
def apply_curated(db, curation=CURATION, archive=None, strict=False):
|
||||
"""Apply accepted links in caller transaction; never execute/download content."""
|
||||
data = load_curation(curation)
|
||||
titles = {row[0] for row in db.execute('SELECT id FROM title')}
|
||||
releases = {row[0]: row[1] for row in db.execute('SELECT id,title_id FROM release')}
|
||||
groups = {}
|
||||
for row in data['relationships']:
|
||||
if row['disposition'] != 'accepted':
|
||||
continue
|
||||
title_id = row['title_id']
|
||||
if title_id not in titles:
|
||||
if strict:
|
||||
raise ValueError('Related reference has an unknown software title: ' + title_id)
|
||||
continue # Empty/source or scoped catalog omits titles deliberately.
|
||||
edition_ids = row.get('edition_ids', [])
|
||||
if not isinstance(edition_ids, list) or any(releases.get(i) != title_id for i in edition_ids):
|
||||
raise ValueError('Related reference has an unknown or conflicting edition')
|
||||
if archive is not None:
|
||||
record = archive.execute('SELECT page_count FROM document WHERE sha256=?', (row['document_id'],)).fetchone()
|
||||
if record is None or (row.get('page') or 1) > record[0]:
|
||||
raise ValueError('Related reference is missing or its location no longer exists')
|
||||
selected = edition_ids or [i for i, title in releases.items() if title == title_id]
|
||||
for release in selected:
|
||||
projected = {key: row[key] for key in ('id', 'title', 'document_id', 'sha256', 'role')}
|
||||
projected.update(page=row.get('page'), spoiler=row.get('spoiler', False), order=row.get('order', 999))
|
||||
group = groups.setdefault(release, [])
|
||||
if any((r['document_id'], r['page']) == (projected['document_id'], projected['page']) for r in group):
|
||||
raise ValueError('Duplicate accepted related-document target')
|
||||
group.append(projected)
|
||||
for release, in db.execute('SELECT id FROM release').fetchall():
|
||||
original = db.execute('SELECT metadata_json FROM release WHERE id=?', (release,)).fetchone()[0]
|
||||
metadata = json.loads(original)
|
||||
previous = dict(metadata)
|
||||
# Reviewed curation is authoritative for this capability. Later guesses
|
||||
# cannot promote rejected or removed relationships by reusing old output.
|
||||
metadata.pop('related_archive', None)
|
||||
metadata.pop('related_archive_contract', None)
|
||||
if release in groups:
|
||||
metadata['related_archive_contract'] = 1
|
||||
metadata['related_archive'] = groups[release]
|
||||
if metadata != previous:
|
||||
db.execute('UPDATE release SET metadata_json=? WHERE id=?', (json.dumps(metadata, ensure_ascii=False, sort_keys=True), release))
|
||||
if groups and db.execute("SELECT value FROM metadata WHERE key='related_archive_contract'").fetchone() != ('1',):
|
||||
db.execute("INSERT OR REPLACE INTO metadata VALUES('related_archive_contract','1')")
|
||||
return {'relationships': sum(map(len, groups.values())), 'editions': len(groups),
|
||||
'titles': len({releases[i] for i in groups})}
|
||||
|
||||
|
||||
def candidates(catalog, archive):
|
||||
def normal(text):
|
||||
return re.sub(r'[^a-z0-9]', '', re.sub(r'^the\s+', '', text.lower()))
|
||||
titles = catalog.execute('SELECT id,name FROM title').fetchall()
|
||||
rows = []
|
||||
for doc, title, collection, source in archive.execute("SELECT sha256,title,collection,source_url FROM document WHERE category IN ('Walkthroughs','Maps') ORDER BY title"):
|
||||
prefix = re.split(r'\b(?:walktrough|walkthrough|hints|map|solution|help|spell book)\b|\(', title, maxsplit=1, flags=re.I)[0]
|
||||
key = normal(prefix)
|
||||
matches = [{'id': i, 'title': name} for i, name in titles if normal(name) == key]
|
||||
rows.append({'document_id': doc, 'title': title, 'collection': collection, 'source_url': source,
|
||||
'candidate_titles': matches, 'status': 'candidate-only' if matches else 'unmatched'})
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('catalog', type=Path)
|
||||
parser.add_argument('--archive', required=True, type=Path)
|
||||
parser.add_argument('--curation', default=CURATION, type=Path)
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--candidate-report', type=Path)
|
||||
args = parser.parse_args()
|
||||
with closing(sqlite3.connect(args.catalog.resolve().as_uri() + '?mode=ro', uri=True)) as source, closing(sqlite3.connect(args.archive.resolve().as_uri() + '?mode=ro', uri=True)) as archive:
|
||||
if args.candidate_report:
|
||||
args.candidate_report.parent.mkdir(parents=True, exist_ok=True)
|
||||
rows = candidates(source, archive)
|
||||
args.candidate_report.write_text(json.dumps({'schema_version': 1, 'status': 'candidates-not-approved', 'documents': rows}, indent=2) + '\n')
|
||||
if args.output:
|
||||
if args.output.exists() or args.output.is_symlink() or args.output.resolve() == args.catalog.resolve():
|
||||
raise ValueError('Choose a new candidate catalog output')
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, name = tempfile.mkstemp(prefix='.related-candidate-', suffix='.sqlite', dir=args.output.parent)
|
||||
os.close(descriptor)
|
||||
temporary = Path(name)
|
||||
try:
|
||||
with closing(sqlite3.connect(temporary)) as output:
|
||||
source.backup(output)
|
||||
report = apply_curated(output, args.curation, archive, strict=True)
|
||||
output.commit()
|
||||
# Exclusive promotion: a concurrent process cannot be overwritten.
|
||||
os.link(temporary, args.output)
|
||||
print(json.dumps(report))
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Retain rights exclusions offline and inventory an actual portable app catalog.
|
||||
|
||||
Refresh consumes an explicitly obtained authenticated snapshot; this tool does
|
||||
not authenticate, contact servers, publish content or infer rights authority.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timezone
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_CACHE = ROOT / 'media/software-library-full/curation/rights-suppressions.json'
|
||||
DEFAULT_CATALOG = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite'
|
||||
MAX_CACHE_BYTES = 8 * 1024 * 1024
|
||||
MAX_INVENTORY_BYTES = 16 * 1024 * 1024
|
||||
MAX_CASES = 10000
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def read_json(path, maximum=MAX_CACHE_BYTES):
|
||||
path = Path(path)
|
||||
require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum,
|
||||
'Rights input must be a bounded regular nonsymlink file')
|
||||
with path.open('rb') as stream:
|
||||
data = stream.read(maximum + 1)
|
||||
require(len(data) <= maximum, 'Rights input exceeds its byte limit')
|
||||
return json.loads(data)
|
||||
|
||||
|
||||
def normalize(value, *, as_of=None, fresh=False):
|
||||
from submission_intake import suppressions, time_value
|
||||
require(isinstance(value, dict), 'Invalid rights snapshot')
|
||||
records = value.get('suppressions')
|
||||
require(isinstance(records, list) and len(records) <= MAX_CASES, 'Rights case count exceeds bounds')
|
||||
require(all(isinstance(item, dict) for item in records), 'Invalid rights case')
|
||||
suppressions(value, as_of, require_fresh=fresh)
|
||||
result = []
|
||||
for item in records:
|
||||
scope = {}
|
||||
for key, values in item['scope'].items():
|
||||
require(all(len(x.encode('utf-8')) <= 4096 and '\0' not in x for x in values), 'Rights scope identity exceeds bounds')
|
||||
if values:
|
||||
scope[key] = sorted(set(values))
|
||||
result.append({'case_reference': item['case_reference'],
|
||||
'effective_at': time_value(item['effective_at']).isoformat(), 'scope': scope})
|
||||
unique = {json.dumps(item, sort_keys=True, separators=(',', ':')): item for item in result}
|
||||
document = {'schema_version': 1, 'suppressions': [unique[key] for key in sorted(unique)]}
|
||||
if 'generated_at' in value:
|
||||
document['generated_at'] = time_value(value['generated_at']).isoformat()
|
||||
require(len(json.dumps(document, ensure_ascii=False).encode()) <= MAX_CACHE_BYTES, 'Rights cache exceeds byte limit')
|
||||
return document
|
||||
|
||||
|
||||
def cached(path=None):
|
||||
path = Path(path) if path is not None else DEFAULT_CACHE
|
||||
if not path.exists() and not path.is_symlink():
|
||||
return {'schema_version': 1, 'suppressions': []}
|
||||
return normalize(read_json(path))
|
||||
|
||||
|
||||
def combine(*documents):
|
||||
records = {}
|
||||
for document in documents:
|
||||
for item in normalize(document)['suppressions']:
|
||||
records[json.dumps(item, sort_keys=True, separators=(',', ':'))] = item
|
||||
# No omission, changed timestamp or narrowed replacement scope revokes an
|
||||
# earlier exclusion. Future cases remain retained until they take effect.
|
||||
return normalize({'schema_version': 1, 'suppressions': list(records.values())})
|
||||
|
||||
|
||||
def apply_cached(db, *, cache_path=None, extra=None, as_of=None):
|
||||
from submission_intake import apply_suppressions
|
||||
document = combine(cached(cache_path), *([extra] if extra is not None else []))
|
||||
return apply_suppressions(db, document, as_of)
|
||||
|
||||
|
||||
def _atomic_json(path, value, maximum, *, replace=False):
|
||||
path = Path(path)
|
||||
require(not path.is_symlink() and (replace or not path.exists()), 'Rights output already exists or is a symlink')
|
||||
data = (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + '\n').encode()
|
||||
require(len(data) <= maximum, 'Rights output exceeds byte limit')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix='.rights-', dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(data); stream.flush(); os.fsync(stream.fileno())
|
||||
if replace:
|
||||
require(not path.is_symlink(), 'Rights output became a symlink')
|
||||
os.replace(temporary, path)
|
||||
else:
|
||||
os.link(temporary, path) # Exclusive promotion, including racing writers.
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _hash(value, *, optional=False):
|
||||
import re
|
||||
require((optional and value is None) or isinstance(value, str) and re.fullmatch('[a-f0-9]{64}', value), 'Invalid catalog inventory hash')
|
||||
return value
|
||||
|
||||
|
||||
def _identity(value):
|
||||
require(isinstance(value, str) and 0 < len(value.encode('utf-8')) <= 4096 and '\0' not in value,
|
||||
'Invalid catalog inventory identity')
|
||||
return value
|
||||
|
||||
|
||||
def _urls(values):
|
||||
result = sorted(set(value for value in values if value))
|
||||
require(len(result) <= 64, 'Catalog inventory URL count exceeds bounds')
|
||||
for url in result:
|
||||
_identity(url)
|
||||
parsed = urlsplit(url)
|
||||
require(parsed.scheme in ('http', 'https') and parsed.hostname and parsed.username is None and parsed.password is None
|
||||
and not any(ord(c) < 32 for c in url), 'Catalog inventory requires public HTTP references without credentials')
|
||||
return result
|
||||
|
||||
|
||||
def inventory(catalog):
|
||||
"""Read only the actual app export, never a maintainer DB or media corpus."""
|
||||
from package import validate
|
||||
from submission_intake import digest
|
||||
catalog = Path(catalog)
|
||||
validate(catalog)
|
||||
before = digest(catalog)
|
||||
result = {'schema_version': 1, 'catalog_sha256': before, 'releases': [], 'assets': []}
|
||||
with closing(sqlite3.connect(catalog.resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
||||
db.execute('PRAGMA query_only=ON')
|
||||
require(db.execute('SELECT count(*) FROM release').fetchone()[0] <= 50000
|
||||
and db.execute('SELECT count(*) FROM asset').fetchone()[0] <= 100000, 'Catalog inventory row count exceeds bounds')
|
||||
for title, identity, sha, url, metadata in db.execute('SELECT title_id,id,package_sha256,source_url,metadata_json FROM release ORDER BY id'):
|
||||
reference = json.loads(metadata).get('submission_reference')
|
||||
if reference is not None:
|
||||
from submission_intake import REFERENCE
|
||||
require(isinstance(reference, str) and REFERENCE.fullmatch(reference), 'Invalid inventory submission reference')
|
||||
urls = _urls([url] + [r[0] for r in db.execute('SELECT url FROM download_location WHERE package_sha256=? ORDER BY url', (sha,))])
|
||||
result['releases'].append({'title_id': _identity(title), 'id': _identity(identity), 'package_sha256': _hash(sha),
|
||||
'source_url': url, 'source_urls': urls, 'submission_reference': reference})
|
||||
for identity, sha, source, manual, thumbnail, thumbnail_sha in db.execute(
|
||||
'SELECT a.id,a.sha256,a.source_url,d.manual_url,d.thumbnail_url,d.thumbnail_sha256 FROM asset a LEFT JOIN asset_delivery d ON d.asset_id=a.id ORDER BY a.id'):
|
||||
row = {'id': _identity(identity), 'sha256': _hash(sha, optional=True), 'source_urls': _urls([source, manual, thumbnail])}
|
||||
if thumbnail_sha is not None:
|
||||
row['thumbnail_sha256'] = _hash(thumbnail_sha)
|
||||
result['assets'].append(row)
|
||||
require(digest(catalog) == before, 'Catalog changed while exporting rights inventory')
|
||||
require(len(json.dumps(result, ensure_ascii=False).encode()) <= MAX_INVENTORY_BYTES, 'Catalog inventory exceeds byte limit')
|
||||
return result
|
||||
|
||||
|
||||
def _bind_observed_bytes(document, catalog):
|
||||
"""Preserve exact matching bytes/aliases through later title or URL renames.
|
||||
|
||||
Never promote a package-only scope to an entire title/submission. No fuzzy
|
||||
name matching or new rights authority is inferred from a catalog.
|
||||
"""
|
||||
from submission_intake import applies
|
||||
observed = inventory(catalog)
|
||||
rows = []
|
||||
for item in document['suppressions']:
|
||||
scope = item['scope']
|
||||
hashes, urls = set(scope.get('package_sha256', [])), set(scope.get('source_urls', []))
|
||||
for row in observed['releases']:
|
||||
if applies(scope, row):
|
||||
hashes.add(row['package_sha256']); urls.update(row['source_urls'])
|
||||
for row in observed['assets']:
|
||||
if set(row['source_urls']) & set(scope.get('source_urls', [])) or {row.get('sha256'), row.get('thumbnail_sha256')} & set(scope.get('package_sha256', [])):
|
||||
hashes.update(h for h in (row.get('sha256'), row.get('thumbnail_sha256')) if h); urls.update(row['source_urls'])
|
||||
# Scope arrays have a 100-value limit. Keep the original exact scope and
|
||||
# retain bounded additional records rather than dropping observed aliases.
|
||||
rows.append(item)
|
||||
for key, values in [('package_sha256', sorted(hashes)), ('source_urls', sorted(urls))]:
|
||||
for start in range(0, len(values), 100):
|
||||
rows.append({**item, 'scope': {key: values[start:start+100]}})
|
||||
return combine({'schema_version': 1, 'suppressions': rows})
|
||||
|
||||
|
||||
def refresh(snapshot, cache_path=None, *, catalog=None, as_of=None):
|
||||
document = normalize(snapshot, as_of=as_of, fresh=True)
|
||||
path = Path(cache_path) if cache_path is not None else DEFAULT_CACHE
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock = path.with_name('.' + path.name + '.lock')
|
||||
with os.fdopen(os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600), 'a+b') as stream:
|
||||
fcntl.flock(stream, fcntl.LOCK_EX)
|
||||
retained = combine(cached(path), document)
|
||||
if catalog is not None:
|
||||
retained = _bind_observed_bytes(retained, catalog)
|
||||
retained['generated_at'] = document['generated_at']
|
||||
_atomic_json(path, retained, MAX_CACHE_BYTES, replace=True)
|
||||
return retained
|
||||
|
||||
|
||||
def export_inventory(catalog, output):
|
||||
result = inventory(catalog)
|
||||
_atomic_json(output, result, MAX_INVENTORY_BYTES)
|
||||
return {'catalog_sha256': result['catalog_sha256'], 'inventory_sha256': hashlib.sha256(Path(output).read_bytes()).hexdigest(),
|
||||
'releases': len(result['releases']), 'assets': len(result['assets'])}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest='command', required=True)
|
||||
update = sub.add_parser('refresh', help='Monotonically retain a fresh authenticated suppression snapshot')
|
||||
update.add_argument('--snapshot', required=True, type=Path)
|
||||
update.add_argument('--cache', type=Path, default=DEFAULT_CACHE)
|
||||
update.add_argument('--catalog', type=Path, help='Current app catalog for binding matching bytes and aliases; defaults to the prepared catalog when present')
|
||||
export = sub.add_parser('inventory', help='Export a SHA-bound rights inventory from an actual app catalog')
|
||||
export.add_argument('catalog', type=Path); export.add_argument('--output', required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == 'refresh':
|
||||
catalog = args.catalog if args.catalog is not None else DEFAULT_CATALOG if DEFAULT_CATALOG.exists() else None
|
||||
result = refresh(read_json(args.snapshot), args.cache, catalog=catalog)
|
||||
print(json.dumps({'retained_suppressions': len(result['suppressions'])}))
|
||||
else:
|
||||
print(json.dumps(export_inventory(args.catalog, args.output)))
|
||||
except (ValueError, OSError, sqlite3.DatabaseError) as error:
|
||||
parser.exit(1, 'error: Rights catalog: ' + str(error) + '\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Explicit, offline intake of exact approved software into a NEW app catalog.
|
||||
|
||||
No download, execution, extraction into a project, publication or installed-media
|
||||
mutation. Fresh approval/suppression snapshots come from the authenticated website
|
||||
maintenance channel. Originals and permission records remain in ignored storage.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import stat
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit, unquote
|
||||
import zipfile
|
||||
|
||||
HASH = re.compile(r'[a-f0-9]{64}')
|
||||
REFERENCE = re.compile(r'SW-[0-9]{4}-[0-9]+')
|
||||
MAX_PACKAGE = 256 * 1024 * 1024
|
||||
MAX_EXPANDED = 256 * 1024 * 1024
|
||||
MAX_MEMBERS = 4096
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def encoded(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), sort_keys=True)
|
||||
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def time_value(value):
|
||||
require(isinstance(value, str), 'A timezone-aware timestamp is required')
|
||||
result = datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
require(result.tzinfo is not None, 'Timestamp requires a timezone')
|
||||
return result
|
||||
|
||||
|
||||
def load(path, maximum=8 * 1024 * 1024):
|
||||
path = Path(path)
|
||||
require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum,
|
||||
'Use a bounded regular JSON file')
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def approval(manifest, current, as_of=None):
|
||||
now = as_of or datetime.now(timezone.utc)
|
||||
require(manifest.get('schema_version') == 1 and REFERENCE.fullmatch(manifest.get('submission_reference', '')),
|
||||
'Unsupported approved submission manifest')
|
||||
snapshot = manifest.get('approval_snapshot_json')
|
||||
require(isinstance(snapshot, str) and HASH.fullmatch(manifest.get('approval_sha256', '')),
|
||||
'Exact immutable approval snapshot is required')
|
||||
require(hashlib.sha256(snapshot.encode()).hexdigest() == manifest['approval_sha256'], 'Approval snapshot changed')
|
||||
original = json.loads(snapshot)
|
||||
require({k: v for k, v in manifest.items() if k not in ('approval_snapshot_json', 'approval_sha256')} == original,
|
||||
'Manifest disagrees with its approved snapshot')
|
||||
require(current.get('state') in ('claimed', 'ready') and current.get('manifest', {}).get('approval_sha256') == manifest['approval_sha256']
|
||||
and current['manifest'].get('submission_reference') == manifest['submission_reference']
|
||||
and current['manifest'].get('approval_revision') == manifest.get('approval_revision'),
|
||||
'Current claimed approval must match; withdrawn/stale approvals are refused')
|
||||
checked = time_value(current.get('checked_at'))
|
||||
require(0 <= (now - checked).total_seconds() <= 900, 'Approval check expired; refresh the authenticated claim')
|
||||
permission = manifest.get('permission', {})
|
||||
require(permission.get('operator') == 'Dezota LLC' and permission.get('record_reference')
|
||||
and HASH.fullmatch(permission.get('terms_sha256', '')) and isinstance(permission.get('terms_text'), str)
|
||||
and hashlib.sha256(permission['terms_text'].encode()).hexdigest() == permission['terms_sha256'],
|
||||
'Missing or changed permission record')
|
||||
assets = manifest.get('assets')
|
||||
require(isinstance(assets, list) and 1 <= len(assets) <= 4, 'Approved asset descriptors required')
|
||||
require(sum(a.get('role') == 'package' for a in assets) == 1, 'Exactly one approved package is required')
|
||||
for asset in assets:
|
||||
require(asset.get('role') in ('package', 'manual', 'image') and HASH.fullmatch(asset.get('sha256', ''))
|
||||
and type(asset.get('byte_size')) is int and asset['byte_size'] > 0, 'Invalid approved asset')
|
||||
return manifest
|
||||
|
||||
|
||||
def suppressions(value, as_of=None, require_fresh=False):
|
||||
now = as_of or datetime.now(timezone.utc)
|
||||
require(value.get('schema_version') == 1 and isinstance(value.get('suppressions'), list), 'Unsupported suppression manifest')
|
||||
if require_fresh:
|
||||
age = (now - time_value(value.get('generated_at'))).total_seconds()
|
||||
require(0 <= age <= 900, 'Suppression check expired; refresh it before intake')
|
||||
result = []
|
||||
for item in value['suppressions']:
|
||||
require(isinstance(item.get('case_reference'), str) and re.fullmatch(r'FB-[0-9]{4}-[0-9]+',item['case_reference']), 'Rights case reference required')
|
||||
scope = item.get('scope')
|
||||
require(isinstance(scope, dict) and set(scope) <= {'title_ids', 'release_ids', 'package_sha256', 'source_urls', 'submission_references'}, 'Invalid withdrawal scope')
|
||||
require(any(scope.values()) and all(isinstance(v, list) and len(v) <= 100 and all(isinstance(x, str) and x for x in v) for v in scope.values()), 'Empty or invalid withdrawal scope')
|
||||
require(all(HASH.fullmatch(v) for v in scope.get('package_sha256', [])), 'Invalid withdrawal hash')
|
||||
if time_value(item.get('effective_at')) <= now:
|
||||
result.append(scope)
|
||||
return result
|
||||
|
||||
|
||||
def applies(scope, row):
|
||||
return (row.get('title_id') in scope.get('title_ids', [])
|
||||
or row.get('id') in scope.get('release_ids', [])
|
||||
or row.get('package_sha256') in scope.get('package_sha256', [])
|
||||
or row.get('source_url') in scope.get('source_urls', [])
|
||||
or bool(set(row.get('source_urls', [])) & set(scope.get('source_urls', [])))
|
||||
or row.get('submission_reference') in scope.get('submission_references', []))
|
||||
|
||||
|
||||
def apply_suppressions(db, manifest, as_of=None):
|
||||
"""Remove only scoped catalog rows in caller transaction; never touch files.
|
||||
|
||||
Requires explicit suppression data. Feed this at each candidate preparation;
|
||||
publication additionally requires a freshly authenticated suppression check.
|
||||
Future-dated exclusions remain in the manifest and become effective by time.
|
||||
"""
|
||||
scopes = suppressions(manifest, as_of)
|
||||
removed = []
|
||||
affected_titles = set()
|
||||
affected_assets = set()
|
||||
columns = [row[1] for row in db.execute('PRAGMA table_info(release)')]
|
||||
for raw in db.execute('SELECT * FROM release').fetchall():
|
||||
row = dict(zip(columns, raw))
|
||||
meta = json.loads(row.get('metadata_json') or '{}')
|
||||
row['submission_reference'] = meta.get('submission_reference')
|
||||
row['source_urls'] = [u[0] for u in db.execute('SELECT url FROM download_location WHERE package_sha256=?', (row['package_sha256'],))]
|
||||
if any(applies(scope, row) for scope in scopes):
|
||||
release_id = row['id']
|
||||
affected_titles.add(row['title_id'])
|
||||
affected_assets.update(a[0] for a in db.execute('SELECT asset_id FROM release_asset WHERE release_id=?', (release_id,)))
|
||||
for table in ('release_asset', 'setup', 'member', 'source_hint'):
|
||||
if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone():
|
||||
db.execute(f'DELETE FROM {table} WHERE release_id=?', (release_id,))
|
||||
db.execute('DELETE FROM release WHERE id=?', (release_id,))
|
||||
removed.append(release_id)
|
||||
empty = [title_id for title_id in sorted(affected_titles) if not db.execute('SELECT 1 FROM release WHERE title_id=?', (title_id,)).fetchone()]
|
||||
for title_id in empty:
|
||||
db.execute('DELETE FROM title WHERE id=?', (title_id,))
|
||||
# Asset-only exclusions remove only supporting assets, retaining unrelated software.
|
||||
# Match both upstream/original and hosted derivative URLs, including old catalogs.
|
||||
asset_columns = [r[1] for r in db.execute('PRAGMA table_info(asset)')]
|
||||
for raw in db.execute('SELECT * FROM asset').fetchall():
|
||||
asset = dict(zip(asset_columns, raw))
|
||||
delivery = db.execute('SELECT manual_url,thumbnail_url,thumbnail_sha256 FROM asset_delivery WHERE asset_id=?', (asset['id'],)).fetchone()
|
||||
urls = {asset['source_url'], *(delivery[:2] if delivery else [])}
|
||||
hashes = {asset.get('sha256'), delivery[2] if delivery else None}
|
||||
if any(urls & set(scope.get('source_urls', [])) or hashes & set(scope.get('package_sha256', [])) for scope in scopes):
|
||||
db.execute('DELETE FROM release_asset WHERE asset_id=?', (asset['id'],))
|
||||
affected_assets.add(asset['id'])
|
||||
removed_assets = []
|
||||
for asset_id in sorted(affected_assets):
|
||||
if not db.execute('SELECT 1 FROM release_asset WHERE asset_id=?', (asset_id,)).fetchone():
|
||||
db.execute('DELETE FROM asset_delivery WHERE asset_id=?', (asset_id,))
|
||||
db.execute('DELETE FROM asset WHERE id=?', (asset_id,))
|
||||
removed_assets.append(asset_id)
|
||||
return {'removed_release_ids': removed, 'removed_title_ids': empty, 'removed_asset_ids': removed_assets}
|
||||
|
||||
|
||||
def inspect_zip(path):
|
||||
require(path.stat().st_size <= MAX_PACKAGE, 'Package exceeds size limit')
|
||||
members = []
|
||||
seen = set()
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
infos = archive.infolist()
|
||||
require(1 <= len(infos) <= MAX_MEMBERS, 'ZIP member count is outside bounds')
|
||||
require(sum(i.file_size for i in infos) <= MAX_EXPANDED, 'ZIP expansion exceeds bounds')
|
||||
for info in infos:
|
||||
name = PurePosixPath(info.filename)
|
||||
require(not name.is_absolute() and '\\' not in info.filename and not any(p in ('..', '.', '') for p in name.parts)
|
||||
and not any(p in ('', '.', '..') for p in info.filename.rstrip('/').split('/')) and '\x00' not in info.filename and ':' not in info.filename and not info.flag_bits & 1, 'Unsafe ZIP member path or encrypted entry')
|
||||
require(not stat.S_ISLNK(info.external_attr >> 16), 'ZIP links are refused')
|
||||
key = info.filename.casefold()
|
||||
require(key not in seen, 'ZIP contains colliding member names')
|
||||
seen.add(key)
|
||||
if info.is_dir():
|
||||
continue
|
||||
require(info.file_size <= MAX_EXPANDED and (not info.file_size or info.file_size <= max(1, info.compress_size) * 4096), 'ZIP expansion ratio exceeds bounds')
|
||||
h = hashlib.sha256()
|
||||
count = 0
|
||||
with archive.open(info) as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
count += len(chunk)
|
||||
require(count <= info.file_size, 'ZIP expanded beyond declared member size')
|
||||
h.update(chunk)
|
||||
require(count == info.file_size, 'ZIP member is truncated')
|
||||
kind = {'.dsk': 'disk', '.os9': 'disk', '.vhd': 'hard_disk', '.img': 'hard_disk', '.ccc': 'cartridge', '.cas': 'cassette', '.wav': 'cassette', '.bin': 'binary'}.get(name.suffix.lower(), 'supporting')
|
||||
members.append({'path': info.filename, 'sha256': h.hexdigest(), 'byte_size': count, 'kind': kind})
|
||||
require(any(m['kind'] != 'supporting' for m in members), 'Package contains no recognized CoCo media')
|
||||
return members
|
||||
|
||||
|
||||
def hosted_url(url, role):
|
||||
parsed = urlsplit(url)
|
||||
path = unquote(parsed.path)
|
||||
require(parsed.scheme == 'https' and parsed.netloc == 'vccplus.brianholman.com' and not parsed.query and not parsed.fragment and '\\' not in path and '\x00' not in path and all(p not in ('.', '..') for p in path.split('/')), 'Invalid hosted software location')
|
||||
base = '/dl/software/CoCoLibrary/'
|
||||
prefix, extensions = {'package': (base, ('.zip',)), 'manual': (base + 'Documents/Manuals/', ('.pdf',)), 'image': (base + 'Artwork/Originals/', ('.png', '.jpg', '.jpeg')), 'thumbnail': (base + 'Artwork/Thumbnails/', ('.png',))}[role]
|
||||
require(path.startswith(prefix) and path.lower().endswith(extensions), 'Invalid hosted software location for ' + role)
|
||||
return url
|
||||
|
||||
|
||||
def verified_locations(receipt, assets):
|
||||
require(receipt.get('state') == 'verified' and isinstance(receipt.get('artifacts'), list), 'Verified hosting receipt required')
|
||||
result = {}
|
||||
for asset in assets:
|
||||
matches = [r for r in receipt['artifacts'] if r.get('sha256') == asset['sha256'] and r.get('byte_size') == asset['byte_size'] and r.get('status') == 200]
|
||||
require(len(matches) == 1, 'Missing or ambiguous verified publication for an approved asset')
|
||||
result[asset['sha256']] = {'url': hosted_url(matches[0].get('url', ''), asset['role'])}
|
||||
derived = receipt.get('derived_artifacts', [])
|
||||
require(isinstance(derived, list), 'Invalid derived artwork receipt')
|
||||
for artifact in derived:
|
||||
source = artifact.get('source_sha256')
|
||||
require(any(a['role'] == 'image' and a['sha256'] == source for a in assets), 'Derived thumbnail must identify approved original artwork')
|
||||
require(artifact.get('role') == 'thumbnail' and artifact.get('status') == 200 and isinstance(artifact.get('sha256'), str) and HASH.fullmatch(artifact['sha256']) and type(artifact.get('byte_size')) is int and 0 < artifact['byte_size'] <= 2 * 1024 * 1024, 'Invalid prepared PNG thumbnail receipt')
|
||||
require('thumbnail' not in result[source], 'Ambiguous prepared thumbnail receipt')
|
||||
result[source]['thumbnail'] = {**artifact, 'url': hosted_url(artifact.get('url', ''), 'thumbnail')}
|
||||
return result
|
||||
|
||||
|
||||
def prepare_catalog(source, destination, manifest, current, suppression_manifest, files, hosting_receipt, as_of=None, *, rights_cache=None):
|
||||
manifest = approval(manifest, current, as_of)
|
||||
suppressions(suppression_manifest, as_of, require_fresh=True)
|
||||
from rights import cached, combine
|
||||
retained_suppressions = combine(cached(rights_cache), suppression_manifest)
|
||||
require(Path(source).is_file() and not Path(source).is_symlink(), 'Use an existing regular prepared app catalog')
|
||||
destination = Path(destination)
|
||||
require(destination.resolve() != Path(source).resolve() and not destination.exists(), 'Create a NEW catalog; existing source/output is never replaced')
|
||||
assets = manifest['assets']
|
||||
paths = {}
|
||||
for asset in assets:
|
||||
path = Path(files.get(asset['sha256'], ''))
|
||||
require(path.is_file() and not path.is_symlink() and path.stat().st_size == asset['byte_size'] and digest(path) == asset['sha256'], 'Approved original bytes are missing or changed')
|
||||
paths[asset['sha256']] = path
|
||||
package = next(a for a in assets if a['role'] == 'package')
|
||||
members = inspect_zip(paths[package['sha256']])
|
||||
locations = verified_locations(hosting_receipt, assets)
|
||||
ref = manifest['submission_reference']
|
||||
title_id = 'direct-' + ref.lower()
|
||||
release_id = title_id + '-r' + str(manifest['approval_revision'])
|
||||
row = {'id': release_id, 'title_id': title_id, 'package_sha256': package['sha256'], 'source_url': locations[package['sha256']]['url'], 'submission_reference': ref}
|
||||
require(not any(applies(s, row) for s in suppressions(retained_suppressions, as_of)), 'A rights withdrawal blocks this submission')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix='.submission-', suffix='.sqlite', dir=destination.parent)
|
||||
os.close(fd)
|
||||
try:
|
||||
with closing(sqlite3.connect(f'file:{Path(source).resolve()}?mode=ro', uri=True)) as original, closing(sqlite3.connect(temporary)) as db:
|
||||
original.backup(db)
|
||||
db.execute('PRAGMA foreign_keys=ON')
|
||||
require(db.execute('PRAGMA user_version').fetchone()[0] == 1, 'Unsupported app catalog version')
|
||||
with db:
|
||||
prior = db.execute('SELECT package_sha256 FROM release WHERE id=?', (release_id,)).fetchone()
|
||||
require(prior is None or prior[0] == package['sha256'], 'Existing submission identity conflicts')
|
||||
if prior is None:
|
||||
primary = next(m for m in members if m['kind'] != 'supporting')
|
||||
media = {'hard_disk': 'hard-disk'}.get(primary['kind'], primary['kind'])
|
||||
metadata = {'origin': 'direct_submission', 'submission_reference': ref, 'approval_revision': manifest['approval_revision'], 'author': manifest['author'], 'publisher': manifest['author'], 'software_version': manifest['software_version'], 'license': manifest['metadata']['end_user_terms'], 'project_url': manifest['metadata'].get('project_url', ''), 'description': manifest['description'], 'permission_operator': 'Dezota LLC'}
|
||||
db.execute('INSERT OR IGNORE INTO title VALUES(?,?,?)', (title_id, manifest['title'], 'reviewed-direct-submission'))
|
||||
db.execute('INSERT OR IGNORE INTO package VALUES(?,?)', (package['sha256'], package['byte_size']))
|
||||
db.execute('INSERT INTO release VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)', (release_id,title_id,manifest['software_version'],'Uncategorized',media,'software',locations[package['sha256']]['url'],None,None,None,package['sha256'],'unknown',encoded(metadata)))
|
||||
db.execute('INSERT OR IGNORE INTO download_location VALUES(?,?,?)', (package['sha256'],locations[package['sha256']]['url'],0))
|
||||
for member in members:
|
||||
db.execute('INSERT INTO member VALUES(?,?,?,?,?,?)', (release_id,member['path'],member['sha256'],member['byte_size'],member['kind'],'software' if member['kind'] != 'supporting' else 'supporting'))
|
||||
db.execute('INSERT INTO setup VALUES(?,?,?,?,?,?,?,?)', (release_id,'manual-setup','{}','[]',None,manifest['metadata'].get('setup') or 'Review media and configure the CoCo manually.',encoded(['Direct submission: automatic setup is not runtime-qualified.']),encoded({'scope':'manual-setup','approval_reference':ref})))
|
||||
for number, asset in enumerate(a for a in assets if a['role'] != 'package'):
|
||||
asset_id = release_id + '-asset-' + str(number)
|
||||
kind = 'manual' if asset['role'] == 'manual' else 'cover'
|
||||
db.execute('INSERT INTO asset VALUES(?,?,?,?,?,?,?)',(asset_id,asset['name'],kind,locations[asset['sha256']]['url'],asset['sha256'],asset['byte_size'],manifest['author']))
|
||||
db.execute('INSERT INTO release_asset VALUES(?,?,?,?)',(release_id,asset_id,'reviewed','Approved direct-submission supporting asset'))
|
||||
thumb = locations[asset['sha256']].get('thumbnail', {})
|
||||
db.execute('INSERT INTO asset_delivery VALUES(?,?,?,?,?,?)',(asset_id,locations[asset['sha256']]['url'] if kind=='manual' else None,thumb.get('url'),thumb.get('sha256'),thumb.get('byte_size'),0))
|
||||
exclusions = apply_suppressions(db,retained_suppressions,as_of)
|
||||
db.execute('INSERT OR REPLACE INTO metadata VALUES(?,?)',('submission_approval:'+ref,manifest['approval_sha256']))
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'Candidate catalog integrity failed')
|
||||
# Exclusive hardlink promotion never replaces an output created by a racing operation.
|
||||
os.link(temporary,destination)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
return {'schema_version':1,'submission_reference':ref,'approval_revision':manifest['approval_revision'],'title_id':title_id,'release_id':release_id,'catalog_sha256':digest(destination),'hosting_receipt':hosting_receipt,'target_release':manifest.get('target_release',''),'qualification_record':'Prepared with manual setup; runtime qualification pending.','suppression_result':exclusions}
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('source',type=Path)
|
||||
parser.add_argument('--output',required=True,type=Path)
|
||||
parser.add_argument('--manifest',required=True,type=Path)
|
||||
parser.add_argument('--approval-check',required=True,type=Path)
|
||||
parser.add_argument('--suppressions',required=True,type=Path)
|
||||
parser.add_argument('--files',required=True,type=Path,help='Private JSON mapping approved SHA-256 to local retained file path')
|
||||
parser.add_argument('--hosting-receipt',required=True,type=Path)
|
||||
parser.add_argument('--result',required=True,type=Path)
|
||||
parser.add_argument('--rights-cache',type=Path,help='Persistent exclusion cache override; refresh merges instead of replacing previous exclusions')
|
||||
args=parser.parse_args()
|
||||
from rights import refresh
|
||||
snapshot = load(args.suppressions)
|
||||
# This explicit file/CLI boundary retains authenticated exclusions. The pure
|
||||
# preparation helper above reads the cache but never writes global state.
|
||||
refresh(snapshot,args.rights_cache,catalog=args.source)
|
||||
result=prepare_catalog(args.source,args.output,load(args.manifest),load(args.approval_check),snapshot,load(args.files),load(args.hosting_receipt),rights_cache=args.rights_cache)
|
||||
args.result.parent.mkdir(parents=True,exist_ok=True)
|
||||
with os.fdopen(os.open(args.result, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as stream:stream.write(json.dumps(result,indent=2)+'\n')
|
||||
print('New catalog candidate prepared; runtime qualification/publication remain separate.')
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -17,6 +17,12 @@ RESOURCES = (
|
||||
('tools/catalog/package.py', 'tools/catalog/app-schema.sql', 'SoftwareLibrary/catalog.sqlite'),
|
||||
('tools/knowledge/package_archive.py', 'tools/knowledge/archive-library.sql', 'ArchiveLibrary/ArchiveLibrary.sqlite'),
|
||||
)
|
||||
CATALOG_PACKAGE_INPUTS = (
|
||||
'tools/catalog/related_archive.py',
|
||||
'tools/catalog/curation/related-archive.json',
|
||||
'tools/catalog/rights.py',
|
||||
'tools/catalog/submission_intake.py',
|
||||
)
|
||||
|
||||
|
||||
class BuildContentTests(unittest.TestCase):
|
||||
@@ -26,7 +32,8 @@ class BuildContentTests(unittest.TestCase):
|
||||
self.root = Path(temporary.name)
|
||||
self.source = self.root / 'Source With Spaces'
|
||||
self.build = self.root / 'Build With Spaces'
|
||||
for relative in ['tools/build_content.py', *(p for r in RESOURCES for p in r[:2])]:
|
||||
for relative in ['tools/build_content.py', *(p for r in RESOURCES for p in r[:2]),
|
||||
*CATALOG_PACKAGE_INPUTS]:
|
||||
destination = self.source / relative
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(ROOT / relative, destination)
|
||||
|
||||
Reference in new issue
Block a user