#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-or-later """Explicit, offline intake of exact approved software into a NEW app catalog. No download, execution, extraction into a project, publication or installed-media mutation. Fresh approval/suppression snapshots come from the authenticated website maintenance channel. Originals and permission records remain in ignored storage. """ import argparse from contextlib import closing from datetime import datetime, timezone import hashlib import json import os from pathlib import Path, PurePosixPath import re import shutil import sqlite3 import stat import tempfile from urllib.parse import urlsplit, unquote import zipfile HASH = re.compile(r'[a-f0-9]{64}') REFERENCE = re.compile(r'SW-[0-9]{4}-[0-9]+') MAX_PACKAGE = 256 * 1024 * 1024 MAX_EXPANDED = 256 * 1024 * 1024 MAX_MEMBERS = 4096 def require(condition, message): if not condition: raise ValueError(message) def encoded(value): return json.dumps(value, ensure_ascii=False, separators=(',', ':'), sort_keys=True) def digest(path): h = hashlib.sha256() with Path(path).open('rb') as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b''): h.update(chunk) return h.hexdigest() def time_value(value): require(isinstance(value, str), 'A timezone-aware timestamp is required') result = datetime.fromisoformat(value.replace('Z', '+00:00')) require(result.tzinfo is not None, 'Timestamp requires a timezone') return result def load(path, maximum=8 * 1024 * 1024): path = Path(path) require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum, 'Use a bounded regular JSON file') return json.loads(path.read_text()) def approval(manifest, current, as_of=None): now = as_of or datetime.now(timezone.utc) require(manifest.get('schema_version') == 1 and REFERENCE.fullmatch(manifest.get('submission_reference', '')), 'Unsupported approved submission manifest') snapshot = manifest.get('approval_snapshot_json') require(isinstance(snapshot, str) and HASH.fullmatch(manifest.get('approval_sha256', '')), 'Exact immutable approval snapshot is required') require(hashlib.sha256(snapshot.encode()).hexdigest() == manifest['approval_sha256'], 'Approval snapshot changed') original = json.loads(snapshot) require({k: v for k, v in manifest.items() if k not in ('approval_snapshot_json', 'approval_sha256')} == original, 'Manifest disagrees with its approved snapshot') require(current.get('state') in ('claimed', 'ready') and current.get('manifest', {}).get('approval_sha256') == manifest['approval_sha256'] and current['manifest'].get('submission_reference') == manifest['submission_reference'] and current['manifest'].get('approval_revision') == manifest.get('approval_revision'), 'Current claimed approval must match; withdrawn/stale approvals are refused') checked = time_value(current.get('checked_at')) require(0 <= (now - checked).total_seconds() <= 900, 'Approval check expired; refresh the authenticated claim') permission = manifest.get('permission', {}) require(permission.get('operator') == 'Dezota LLC' and permission.get('record_reference') and HASH.fullmatch(permission.get('terms_sha256', '')) and isinstance(permission.get('terms_text'), str) and hashlib.sha256(permission['terms_text'].encode()).hexdigest() == permission['terms_sha256'], 'Missing or changed permission record') assets = manifest.get('assets') require(isinstance(assets, list) and 1 <= len(assets) <= 4, 'Approved asset descriptors required') require(sum(a.get('role') == 'package' for a in assets) == 1, 'Exactly one approved package is required') for asset in assets: require(asset.get('role') in ('package', 'manual', 'image') and HASH.fullmatch(asset.get('sha256', '')) and type(asset.get('byte_size')) is int and asset['byte_size'] > 0, 'Invalid approved asset') return manifest def suppressions(value, as_of=None, require_fresh=False): now = as_of or datetime.now(timezone.utc) require(value.get('schema_version') == 1 and isinstance(value.get('suppressions'), list), 'Unsupported suppression manifest') if require_fresh: age = (now - time_value(value.get('generated_at'))).total_seconds() require(0 <= age <= 900, 'Suppression check expired; refresh it before intake') result = [] for item in value['suppressions']: require(isinstance(item.get('case_reference'), str) and re.fullmatch(r'FB-[0-9]{4}-[0-9]+',item['case_reference']), 'Rights case reference required') scope = item.get('scope') require(isinstance(scope, dict) and set(scope) <= {'title_ids', 'release_ids', 'package_sha256', 'source_urls', 'submission_references'}, 'Invalid withdrawal scope') require(any(scope.values()) and all(isinstance(v, list) and len(v) <= 100 and all(isinstance(x, str) and x for x in v) for v in scope.values()), 'Empty or invalid withdrawal scope') require(all(HASH.fullmatch(v) for v in scope.get('package_sha256', [])), 'Invalid withdrawal hash') if time_value(item.get('effective_at')) <= now: result.append(scope) return result def applies(scope, row): return (row.get('title_id') in scope.get('title_ids', []) or row.get('id') in scope.get('release_ids', []) or row.get('package_sha256') in scope.get('package_sha256', []) or row.get('source_url') in scope.get('source_urls', []) or bool(set(row.get('source_urls', [])) & set(scope.get('source_urls', []))) or row.get('submission_reference') in scope.get('submission_references', [])) def apply_suppressions(db, manifest, as_of=None): """Remove only scoped catalog rows in caller transaction; never touch files. Requires explicit suppression data. Feed this at each candidate preparation; publication additionally requires a freshly authenticated suppression check. Future-dated exclusions remain in the manifest and become effective by time. """ scopes = suppressions(manifest, as_of) removed = [] affected_titles = set() affected_assets = set() columns = [row[1] for row in db.execute('PRAGMA table_info(release)')] for raw in db.execute('SELECT * FROM release').fetchall(): row = dict(zip(columns, raw)) meta = json.loads(row.get('metadata_json') or '{}') row['submission_reference'] = meta.get('submission_reference') row['source_urls'] = [u[0] for u in db.execute('SELECT url FROM download_location WHERE package_sha256=?', (row['package_sha256'],))] if any(applies(scope, row) for scope in scopes): release_id = row['id'] affected_titles.add(row['title_id']) affected_assets.update(a[0] for a in db.execute('SELECT asset_id FROM release_asset WHERE release_id=?', (release_id,))) for table in ('release_asset', 'setup', 'member', 'source_hint'): if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone(): db.execute(f'DELETE FROM {table} WHERE release_id=?', (release_id,)) db.execute('DELETE FROM release WHERE id=?', (release_id,)) removed.append(release_id) empty = [title_id for title_id in sorted(affected_titles) if not db.execute('SELECT 1 FROM release WHERE title_id=?', (title_id,)).fetchone()] for title_id in empty: db.execute('DELETE FROM title WHERE id=?', (title_id,)) # Asset-only exclusions remove only supporting assets, retaining unrelated software. # Match both upstream/original and hosted derivative URLs, including old catalogs. asset_columns = [r[1] for r in db.execute('PRAGMA table_info(asset)')] for raw in db.execute('SELECT * FROM asset').fetchall(): asset = dict(zip(asset_columns, raw)) delivery = db.execute('SELECT manual_url,thumbnail_url,thumbnail_sha256 FROM asset_delivery WHERE asset_id=?', (asset['id'],)).fetchone() urls = {asset['source_url'], *(delivery[:2] if delivery else [])} hashes = {asset.get('sha256'), delivery[2] if delivery else None} if any(urls & set(scope.get('source_urls', [])) or hashes & set(scope.get('package_sha256', [])) for scope in scopes): db.execute('DELETE FROM release_asset WHERE asset_id=?', (asset['id'],)) affected_assets.add(asset['id']) removed_assets = [] for asset_id in sorted(affected_assets): if not db.execute('SELECT 1 FROM release_asset WHERE asset_id=?', (asset_id,)).fetchone(): db.execute('DELETE FROM asset_delivery WHERE asset_id=?', (asset_id,)) db.execute('DELETE FROM asset WHERE id=?', (asset_id,)) removed_assets.append(asset_id) return {'removed_release_ids': removed, 'removed_title_ids': empty, 'removed_asset_ids': removed_assets} def inspect_zip(path): require(path.stat().st_size <= MAX_PACKAGE, 'Package exceeds size limit') members = [] seen = set() with zipfile.ZipFile(path) as archive: infos = archive.infolist() require(1 <= len(infos) <= MAX_MEMBERS, 'ZIP member count is outside bounds') require(sum(i.file_size for i in infos) <= MAX_EXPANDED, 'ZIP expansion exceeds bounds') for info in infos: name = PurePosixPath(info.filename) require(not name.is_absolute() and '\\' not in info.filename and not any(p in ('..', '.', '') for p in name.parts) and not any(p in ('', '.', '..') for p in info.filename.rstrip('/').split('/')) and '\x00' not in info.filename and ':' not in info.filename and not info.flag_bits & 1, 'Unsafe ZIP member path or encrypted entry') require(not stat.S_ISLNK(info.external_attr >> 16), 'ZIP links are refused') key = info.filename.casefold() require(key not in seen, 'ZIP contains colliding member names') seen.add(key) if info.is_dir(): continue require(info.file_size <= MAX_EXPANDED and (not info.file_size or info.file_size <= max(1, info.compress_size) * 4096), 'ZIP expansion ratio exceeds bounds') h = hashlib.sha256() count = 0 with archive.open(info) as stream: while chunk := stream.read(1024 * 1024): count += len(chunk) require(count <= info.file_size, 'ZIP expanded beyond declared member size') h.update(chunk) require(count == info.file_size, 'ZIP member is truncated') kind = {'.dsk': 'disk', '.os9': 'disk', '.vhd': 'hard_disk', '.img': 'hard_disk', '.ccc': 'cartridge', '.cas': 'cassette', '.wav': 'cassette', '.bin': 'binary'}.get(name.suffix.lower(), 'supporting') members.append({'path': info.filename, 'sha256': h.hexdigest(), 'byte_size': count, 'kind': kind}) require(any(m['kind'] != 'supporting' for m in members), 'Package contains no recognized CoCo media') return members def hosted_url(url, role): parsed = urlsplit(url) path = unquote(parsed.path) require(parsed.scheme == 'https' and parsed.netloc == 'vccplus.brianholman.com' and not parsed.query and not parsed.fragment and '\\' not in path and '\x00' not in path and all(p not in ('.', '..') for p in path.split('/')), 'Invalid hosted software location') base = '/dl/software/CoCoLibrary/' prefix, extensions = {'package': (base, ('.zip',)), 'manual': (base + 'Documents/Manuals/', ('.pdf',)), 'image': (base + 'Artwork/Originals/', ('.png', '.jpg', '.jpeg')), 'thumbnail': (base + 'Artwork/Thumbnails/', ('.png',))}[role] require(path.startswith(prefix) and path.lower().endswith(extensions), 'Invalid hosted software location for ' + role) return url def verified_locations(receipt, assets): require(receipt.get('state') == 'verified' and isinstance(receipt.get('artifacts'), list), 'Verified hosting receipt required') result = {} for asset in assets: matches = [r for r in receipt['artifacts'] if r.get('sha256') == asset['sha256'] and r.get('byte_size') == asset['byte_size'] and r.get('status') == 200] require(len(matches) == 1, 'Missing or ambiguous verified publication for an approved asset') result[asset['sha256']] = {'url': hosted_url(matches[0].get('url', ''), asset['role'])} derived = receipt.get('derived_artifacts', []) require(isinstance(derived, list), 'Invalid derived artwork receipt') for artifact in derived: source = artifact.get('source_sha256') require(any(a['role'] == 'image' and a['sha256'] == source for a in assets), 'Derived thumbnail must identify approved original artwork') require(artifact.get('role') == 'thumbnail' and artifact.get('status') == 200 and isinstance(artifact.get('sha256'), str) and HASH.fullmatch(artifact['sha256']) and type(artifact.get('byte_size')) is int and 0 < artifact['byte_size'] <= 2 * 1024 * 1024, 'Invalid prepared PNG thumbnail receipt') require('thumbnail' not in result[source], 'Ambiguous prepared thumbnail receipt') result[source]['thumbnail'] = {**artifact, 'url': hosted_url(artifact.get('url', ''), 'thumbnail')} return result def prepare_catalog(source, destination, manifest, current, suppression_manifest, files, hosting_receipt, as_of=None, *, rights_cache=None): manifest = approval(manifest, current, as_of) suppressions(suppression_manifest, as_of, require_fresh=True) from rights import cached, combine retained_suppressions = combine(cached(rights_cache), suppression_manifest) require(Path(source).is_file() and not Path(source).is_symlink(), 'Use an existing regular prepared app catalog') destination = Path(destination) require(destination.resolve() != Path(source).resolve() and not destination.exists(), 'Create a NEW catalog; existing source/output is never replaced') assets = manifest['assets'] paths = {} for asset in assets: path = Path(files.get(asset['sha256'], '')) require(path.is_file() and not path.is_symlink() and path.stat().st_size == asset['byte_size'] and digest(path) == asset['sha256'], 'Approved original bytes are missing or changed') paths[asset['sha256']] = path package = next(a for a in assets if a['role'] == 'package') members = inspect_zip(paths[package['sha256']]) locations = verified_locations(hosting_receipt, assets) ref = manifest['submission_reference'] title_id = 'direct-' + ref.lower() release_id = title_id + '-r' + str(manifest['approval_revision']) row = {'id': release_id, 'title_id': title_id, 'package_sha256': package['sha256'], 'source_url': locations[package['sha256']]['url'], 'submission_reference': ref} require(not any(applies(s, row) for s in suppressions(retained_suppressions, as_of)), 'A rights withdrawal blocks this submission') destination.parent.mkdir(parents=True, exist_ok=True) fd, temporary = tempfile.mkstemp(prefix='.submission-', suffix='.sqlite', dir=destination.parent) os.close(fd) try: with closing(sqlite3.connect(f'file:{Path(source).resolve()}?mode=ro', uri=True)) as original, closing(sqlite3.connect(temporary)) as db: original.backup(db) db.execute('PRAGMA foreign_keys=ON') require(db.execute('PRAGMA user_version').fetchone()[0] == 1, 'Unsupported app catalog version') with db: prior = db.execute('SELECT package_sha256 FROM release WHERE id=?', (release_id,)).fetchone() require(prior is None or prior[0] == package['sha256'], 'Existing submission identity conflicts') if prior is None: primary = next(m for m in members if m['kind'] != 'supporting') media = {'hard_disk': 'hard-disk'}.get(primary['kind'], primary['kind']) metadata = {'origin': 'direct_submission', 'submission_reference': ref, 'approval_revision': manifest['approval_revision'], 'author': manifest['author'], 'publisher': manifest['author'], 'software_version': manifest['software_version'], 'license': manifest['metadata']['end_user_terms'], 'project_url': manifest['metadata'].get('project_url', ''), 'description': manifest['description'], 'permission_operator': 'Dezota LLC'} db.execute('INSERT OR IGNORE INTO title VALUES(?,?,?)', (title_id, manifest['title'], 'reviewed-direct-submission')) db.execute('INSERT OR IGNORE INTO package VALUES(?,?)', (package['sha256'], package['byte_size'])) db.execute('INSERT INTO release VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)', (release_id,title_id,manifest['software_version'],'Uncategorized',media,'software',locations[package['sha256']]['url'],None,None,None,package['sha256'],'unknown',encoded(metadata))) db.execute('INSERT OR IGNORE INTO download_location VALUES(?,?,?)', (package['sha256'],locations[package['sha256']]['url'],0)) for member in members: db.execute('INSERT INTO member VALUES(?,?,?,?,?,?)', (release_id,member['path'],member['sha256'],member['byte_size'],member['kind'],'software' if member['kind'] != 'supporting' else 'supporting')) db.execute('INSERT INTO setup VALUES(?,?,?,?,?,?,?,?)', (release_id,'manual-setup','{}','[]',None,manifest['metadata'].get('setup') or 'Review media and configure the CoCo manually.',encoded(['Direct submission: automatic setup is not runtime-qualified.']),encoded({'scope':'manual-setup','approval_reference':ref}))) for number, asset in enumerate(a for a in assets if a['role'] != 'package'): asset_id = release_id + '-asset-' + str(number) kind = 'manual' if asset['role'] == 'manual' else 'cover' db.execute('INSERT INTO asset VALUES(?,?,?,?,?,?,?)',(asset_id,asset['name'],kind,locations[asset['sha256']]['url'],asset['sha256'],asset['byte_size'],manifest['author'])) db.execute('INSERT INTO release_asset VALUES(?,?,?,?)',(release_id,asset_id,'reviewed','Approved direct-submission supporting asset')) thumb = locations[asset['sha256']].get('thumbnail', {}) db.execute('INSERT INTO asset_delivery VALUES(?,?,?,?,?,?)',(asset_id,locations[asset['sha256']]['url'] if kind=='manual' else None,thumb.get('url'),thumb.get('sha256'),thumb.get('byte_size'),0)) exclusions = apply_suppressions(db,retained_suppressions,as_of) db.execute('INSERT OR REPLACE INTO metadata VALUES(?,?)',('submission_approval:'+ref,manifest['approval_sha256'])) require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'Candidate catalog integrity failed') # Exclusive hardlink promotion never replaces an output created by a racing operation. os.link(temporary,destination) finally: Path(temporary).unlink(missing_ok=True) return {'schema_version':1,'submission_reference':ref,'approval_revision':manifest['approval_revision'],'title_id':title_id,'release_id':release_id,'catalog_sha256':digest(destination),'hosting_receipt':hosting_receipt,'target_release':manifest.get('target_release',''),'qualification_record':'Prepared with manual setup; runtime qualification pending.','suppression_result':exclusions} def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('source',type=Path) parser.add_argument('--output',required=True,type=Path) parser.add_argument('--manifest',required=True,type=Path) parser.add_argument('--approval-check',required=True,type=Path) parser.add_argument('--suppressions',required=True,type=Path) parser.add_argument('--files',required=True,type=Path,help='Private JSON mapping approved SHA-256 to local retained file path') parser.add_argument('--hosting-receipt',required=True,type=Path) parser.add_argument('--result',required=True,type=Path) parser.add_argument('--rights-cache',type=Path,help='Persistent exclusion cache override; refresh merges instead of replacing previous exclusions') args=parser.parse_args() from rights import refresh snapshot = load(args.suppressions) # This explicit file/CLI boundary retains authenticated exclusions. The pure # preparation helper above reads the cache but never writes global state. refresh(snapshot,args.rights_cache,catalog=args.source) result=prepare_catalog(args.source,args.output,load(args.manifest),load(args.approval_check),snapshot,load(args.files),load(args.hosting_receipt),rights_cache=args.rights_cache) args.result.parent.mkdir(parents=True,exist_ok=True) with os.fdopen(os.open(args.result, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as stream:stream.write(json.dumps(result,indent=2)+'\n') print('New catalog candidate prepared; runtime qualification/publication remain separate.') if __name__=='__main__':main()