#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-or-later """Retain rights exclusions offline and inventory an actual portable app catalog. Refresh consumes an explicitly obtained authenticated snapshot; this tool does not authenticate, contact servers, publish content or infer rights authority. """ import argparse from contextlib import closing from datetime import datetime, timezone import fcntl import hashlib import json import os from pathlib import Path import sqlite3 import tempfile from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parents[2] DEFAULT_CACHE = ROOT / 'media/software-library-full/curation/rights-suppressions.json' DEFAULT_CATALOG = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite' MAX_CACHE_BYTES = 8 * 1024 * 1024 MAX_INVENTORY_BYTES = 16 * 1024 * 1024 MAX_CASES = 10000 def require(condition, message): if not condition: raise ValueError(message) def read_json(path, maximum=MAX_CACHE_BYTES): path = Path(path) require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum, 'Rights input must be a bounded regular nonsymlink file') with path.open('rb') as stream: data = stream.read(maximum + 1) require(len(data) <= maximum, 'Rights input exceeds its byte limit') return json.loads(data) def normalize(value, *, as_of=None, fresh=False): from submission_intake import suppressions, time_value require(isinstance(value, dict), 'Invalid rights snapshot') records = value.get('suppressions') require(isinstance(records, list) and len(records) <= MAX_CASES, 'Rights case count exceeds bounds') require(all(isinstance(item, dict) for item in records), 'Invalid rights case') suppressions(value, as_of, require_fresh=fresh) result = [] for item in records: scope = {} for key, values in item['scope'].items(): require(all(len(x.encode('utf-8')) <= 4096 and '\0' not in x for x in values), 'Rights scope identity exceeds bounds') if values: scope[key] = sorted(set(values)) result.append({'case_reference': item['case_reference'], 'effective_at': time_value(item['effective_at']).isoformat(), 'scope': scope}) unique = {json.dumps(item, sort_keys=True, separators=(',', ':')): item for item in result} document = {'schema_version': 1, 'suppressions': [unique[key] for key in sorted(unique)]} if 'generated_at' in value: document['generated_at'] = time_value(value['generated_at']).isoformat() require(len(json.dumps(document, ensure_ascii=False).encode()) <= MAX_CACHE_BYTES, 'Rights cache exceeds byte limit') return document def cached(path=None): path = Path(path) if path is not None else DEFAULT_CACHE if not path.exists() and not path.is_symlink(): return {'schema_version': 1, 'suppressions': []} return normalize(read_json(path)) def combine(*documents): records = {} for document in documents: for item in normalize(document)['suppressions']: records[json.dumps(item, sort_keys=True, separators=(',', ':'))] = item # No omission, changed timestamp or narrowed replacement scope revokes an # earlier exclusion. Future cases remain retained until they take effect. return normalize({'schema_version': 1, 'suppressions': list(records.values())}) def apply_cached(db, *, cache_path=None, extra=None, as_of=None): from submission_intake import apply_suppressions document = combine(cached(cache_path), *([extra] if extra is not None else [])) return apply_suppressions(db, document, as_of) def _atomic_json(path, value, maximum, *, replace=False): path = Path(path) require(not path.is_symlink() and (replace or not path.exists()), 'Rights output already exists or is a symlink') data = (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + '\n').encode() require(len(data) <= maximum, 'Rights output exceeds byte limit') path.parent.mkdir(parents=True, exist_ok=True) fd, temporary = tempfile.mkstemp(prefix='.rights-', dir=path.parent) try: with os.fdopen(fd, 'wb') as stream: stream.write(data); stream.flush(); os.fsync(stream.fileno()) if replace: require(not path.is_symlink(), 'Rights output became a symlink') os.replace(temporary, path) else: os.link(temporary, path) # Exclusive promotion, including racing writers. finally: Path(temporary).unlink(missing_ok=True) def _hash(value, *, optional=False): import re require((optional and value is None) or isinstance(value, str) and re.fullmatch('[a-f0-9]{64}', value), 'Invalid catalog inventory hash') return value def _identity(value): require(isinstance(value, str) and 0 < len(value.encode('utf-8')) <= 4096 and '\0' not in value, 'Invalid catalog inventory identity') return value def _urls(values): result = sorted(set(value for value in values if value)) require(len(result) <= 64, 'Catalog inventory URL count exceeds bounds') for url in result: _identity(url) parsed = urlsplit(url) require(parsed.scheme in ('http', 'https') and parsed.hostname and parsed.username is None and parsed.password is None and not any(ord(c) < 32 for c in url), 'Catalog inventory requires public HTTP references without credentials') return result def inventory(catalog): """Read only the actual app export, never a maintainer DB or media corpus.""" from package import validate from submission_intake import digest catalog = Path(catalog) validate(catalog) before = digest(catalog) result = {'schema_version': 1, 'catalog_sha256': before, 'releases': [], 'assets': []} with closing(sqlite3.connect(catalog.resolve().as_uri() + '?mode=ro', uri=True)) as db: db.execute('PRAGMA query_only=ON') require(db.execute('SELECT count(*) FROM release').fetchone()[0] <= 50000 and db.execute('SELECT count(*) FROM asset').fetchone()[0] <= 100000, 'Catalog inventory row count exceeds bounds') for title, identity, sha, url, metadata in db.execute('SELECT title_id,id,package_sha256,source_url,metadata_json FROM release ORDER BY id'): reference = json.loads(metadata).get('submission_reference') if reference is not None: from submission_intake import REFERENCE require(isinstance(reference, str) and REFERENCE.fullmatch(reference), 'Invalid inventory submission reference') urls = _urls([url] + [r[0] for r in db.execute('SELECT url FROM download_location WHERE package_sha256=? ORDER BY url', (sha,))]) result['releases'].append({'title_id': _identity(title), 'id': _identity(identity), 'package_sha256': _hash(sha), 'source_url': url, 'source_urls': urls, 'submission_reference': reference}) for identity, sha, source, manual, thumbnail, thumbnail_sha in db.execute( 'SELECT a.id,a.sha256,a.source_url,d.manual_url,d.thumbnail_url,d.thumbnail_sha256 FROM asset a LEFT JOIN asset_delivery d ON d.asset_id=a.id ORDER BY a.id'): row = {'id': _identity(identity), 'sha256': _hash(sha, optional=True), 'source_urls': _urls([source, manual, thumbnail])} if thumbnail_sha is not None: row['thumbnail_sha256'] = _hash(thumbnail_sha) result['assets'].append(row) require(digest(catalog) == before, 'Catalog changed while exporting rights inventory') require(len(json.dumps(result, ensure_ascii=False).encode()) <= MAX_INVENTORY_BYTES, 'Catalog inventory exceeds byte limit') return result def _bind_observed_bytes(document, catalog): """Preserve exact matching bytes/aliases through later title or URL renames. Never promote a package-only scope to an entire title/submission. No fuzzy name matching or new rights authority is inferred from a catalog. """ from submission_intake import applies observed = inventory(catalog) rows = [] for item in document['suppressions']: scope = item['scope'] hashes, urls = set(scope.get('package_sha256', [])), set(scope.get('source_urls', [])) for row in observed['releases']: if applies(scope, row): hashes.add(row['package_sha256']); urls.update(row['source_urls']) for row in observed['assets']: if set(row['source_urls']) & set(scope.get('source_urls', [])) or {row.get('sha256'), row.get('thumbnail_sha256')} & set(scope.get('package_sha256', [])): hashes.update(h for h in (row.get('sha256'), row.get('thumbnail_sha256')) if h); urls.update(row['source_urls']) # Scope arrays have a 100-value limit. Keep the original exact scope and # retain bounded additional records rather than dropping observed aliases. rows.append(item) for key, values in [('package_sha256', sorted(hashes)), ('source_urls', sorted(urls))]: for start in range(0, len(values), 100): rows.append({**item, 'scope': {key: values[start:start+100]}}) return combine({'schema_version': 1, 'suppressions': rows}) def refresh(snapshot, cache_path=None, *, catalog=None, as_of=None): document = normalize(snapshot, as_of=as_of, fresh=True) path = Path(cache_path) if cache_path is not None else DEFAULT_CACHE path.parent.mkdir(parents=True, exist_ok=True) lock = path.with_name('.' + path.name + '.lock') with os.fdopen(os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600), 'a+b') as stream: fcntl.flock(stream, fcntl.LOCK_EX) retained = combine(cached(path), document) if catalog is not None: retained = _bind_observed_bytes(retained, catalog) retained['generated_at'] = document['generated_at'] _atomic_json(path, retained, MAX_CACHE_BYTES, replace=True) return retained def export_inventory(catalog, output): result = inventory(catalog) _atomic_json(output, result, MAX_INVENTORY_BYTES) return {'catalog_sha256': result['catalog_sha256'], 'inventory_sha256': hashlib.sha256(Path(output).read_bytes()).hexdigest(), 'releases': len(result['releases']), 'assets': len(result['assets'])} def main(): parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest='command', required=True) update = sub.add_parser('refresh', help='Monotonically retain a fresh authenticated suppression snapshot') update.add_argument('--snapshot', required=True, type=Path) update.add_argument('--cache', type=Path, default=DEFAULT_CACHE) update.add_argument('--catalog', type=Path, help='Current app catalog for binding matching bytes and aliases; defaults to the prepared catalog when present') export = sub.add_parser('inventory', help='Export a SHA-bound rights inventory from an actual app catalog') export.add_argument('catalog', type=Path); export.add_argument('--output', required=True, type=Path) args = parser.parse_args() try: if args.command == 'refresh': catalog = args.catalog if args.catalog is not None else DEFAULT_CATALOG if DEFAULT_CATALOG.exists() else None result = refresh(read_json(args.snapshot), args.cache, catalog=catalog) print(json.dumps({'retained_suppressions': len(result['suppressions'])})) else: print(json.dumps(export_inventory(args.catalog, args.output))) except (ValueError, OSError, sqlite3.DatabaseError) as error: parser.exit(1, 'error: Rights catalog: ' + str(error) + '\n') if __name__ == '__main__': main()