235 lines
12 KiB
Python
235 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Retain rights exclusions offline and inventory an actual portable app catalog.
|
|
|
|
Refresh consumes an explicitly obtained authenticated snapshot; this tool does
|
|
not authenticate, contact servers, publish content or infer rights authority.
|
|
"""
|
|
import argparse
|
|
from contextlib import closing
|
|
from datetime import datetime, timezone
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
DEFAULT_CACHE = ROOT / 'media/software-library-full/curation/rights-suppressions.json'
|
|
DEFAULT_CATALOG = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite'
|
|
MAX_CACHE_BYTES = 8 * 1024 * 1024
|
|
MAX_INVENTORY_BYTES = 16 * 1024 * 1024
|
|
MAX_CASES = 10000
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def read_json(path, maximum=MAX_CACHE_BYTES):
|
|
path = Path(path)
|
|
require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum,
|
|
'Rights input must be a bounded regular nonsymlink file')
|
|
with path.open('rb') as stream:
|
|
data = stream.read(maximum + 1)
|
|
require(len(data) <= maximum, 'Rights input exceeds its byte limit')
|
|
return json.loads(data)
|
|
|
|
|
|
def normalize(value, *, as_of=None, fresh=False):
|
|
from submission_intake import suppressions, time_value
|
|
require(isinstance(value, dict), 'Invalid rights snapshot')
|
|
records = value.get('suppressions')
|
|
require(isinstance(records, list) and len(records) <= MAX_CASES, 'Rights case count exceeds bounds')
|
|
require(all(isinstance(item, dict) for item in records), 'Invalid rights case')
|
|
suppressions(value, as_of, require_fresh=fresh)
|
|
result = []
|
|
for item in records:
|
|
scope = {}
|
|
for key, values in item['scope'].items():
|
|
require(all(len(x.encode('utf-8')) <= 4096 and '\0' not in x for x in values), 'Rights scope identity exceeds bounds')
|
|
if values:
|
|
scope[key] = sorted(set(values))
|
|
result.append({'case_reference': item['case_reference'],
|
|
'effective_at': time_value(item['effective_at']).isoformat(), 'scope': scope})
|
|
unique = {json.dumps(item, sort_keys=True, separators=(',', ':')): item for item in result}
|
|
document = {'schema_version': 1, 'suppressions': [unique[key] for key in sorted(unique)]}
|
|
if 'generated_at' in value:
|
|
document['generated_at'] = time_value(value['generated_at']).isoformat()
|
|
require(len(json.dumps(document, ensure_ascii=False).encode()) <= MAX_CACHE_BYTES, 'Rights cache exceeds byte limit')
|
|
return document
|
|
|
|
|
|
def cached(path=None):
|
|
path = Path(path) if path is not None else DEFAULT_CACHE
|
|
if not path.exists() and not path.is_symlink():
|
|
return {'schema_version': 1, 'suppressions': []}
|
|
return normalize(read_json(path))
|
|
|
|
|
|
def combine(*documents):
|
|
records = {}
|
|
for document in documents:
|
|
for item in normalize(document)['suppressions']:
|
|
records[json.dumps(item, sort_keys=True, separators=(',', ':'))] = item
|
|
# No omission, changed timestamp or narrowed replacement scope revokes an
|
|
# earlier exclusion. Future cases remain retained until they take effect.
|
|
return normalize({'schema_version': 1, 'suppressions': list(records.values())})
|
|
|
|
|
|
def apply_cached(db, *, cache_path=None, extra=None, as_of=None):
|
|
from submission_intake import apply_suppressions
|
|
document = combine(cached(cache_path), *([extra] if extra is not None else []))
|
|
return apply_suppressions(db, document, as_of)
|
|
|
|
|
|
def _atomic_json(path, value, maximum, *, replace=False):
|
|
path = Path(path)
|
|
require(not path.is_symlink() and (replace or not path.exists()), 'Rights output already exists or is a symlink')
|
|
data = (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + '\n').encode()
|
|
require(len(data) <= maximum, 'Rights output exceeds byte limit')
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, temporary = tempfile.mkstemp(prefix='.rights-', dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, 'wb') as stream:
|
|
stream.write(data); stream.flush(); os.fsync(stream.fileno())
|
|
if replace:
|
|
require(not path.is_symlink(), 'Rights output became a symlink')
|
|
os.replace(temporary, path)
|
|
else:
|
|
os.link(temporary, path) # Exclusive promotion, including racing writers.
|
|
finally:
|
|
Path(temporary).unlink(missing_ok=True)
|
|
|
|
|
|
def _hash(value, *, optional=False):
|
|
import re
|
|
require((optional and value is None) or isinstance(value, str) and re.fullmatch('[a-f0-9]{64}', value), 'Invalid catalog inventory hash')
|
|
return value
|
|
|
|
|
|
def _identity(value):
|
|
require(isinstance(value, str) and 0 < len(value.encode('utf-8')) <= 4096 and '\0' not in value,
|
|
'Invalid catalog inventory identity')
|
|
return value
|
|
|
|
|
|
def _urls(values):
|
|
result = sorted(set(value for value in values if value))
|
|
require(len(result) <= 64, 'Catalog inventory URL count exceeds bounds')
|
|
for url in result:
|
|
_identity(url)
|
|
parsed = urlsplit(url)
|
|
require(parsed.scheme in ('http', 'https') and parsed.hostname and parsed.username is None and parsed.password is None
|
|
and not any(ord(c) < 32 for c in url), 'Catalog inventory requires public HTTP references without credentials')
|
|
return result
|
|
|
|
|
|
def inventory(catalog):
|
|
"""Read only the actual app export, never a maintainer DB or media corpus."""
|
|
from package import validate
|
|
from submission_intake import digest
|
|
catalog = Path(catalog)
|
|
validate(catalog)
|
|
before = digest(catalog)
|
|
result = {'schema_version': 1, 'catalog_sha256': before, 'releases': [], 'assets': []}
|
|
with closing(sqlite3.connect(catalog.resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
|
db.execute('PRAGMA query_only=ON')
|
|
require(db.execute('SELECT count(*) FROM release').fetchone()[0] <= 50000
|
|
and db.execute('SELECT count(*) FROM asset').fetchone()[0] <= 100000, 'Catalog inventory row count exceeds bounds')
|
|
for title, identity, sha, url, metadata in db.execute('SELECT title_id,id,package_sha256,source_url,metadata_json FROM release ORDER BY id'):
|
|
reference = json.loads(metadata).get('submission_reference')
|
|
if reference is not None:
|
|
from submission_intake import REFERENCE
|
|
require(isinstance(reference, str) and REFERENCE.fullmatch(reference), 'Invalid inventory submission reference')
|
|
urls = _urls([url] + [r[0] for r in db.execute('SELECT url FROM download_location WHERE package_sha256=? ORDER BY url', (sha,))])
|
|
result['releases'].append({'title_id': _identity(title), 'id': _identity(identity), 'package_sha256': _hash(sha),
|
|
'source_url': url, 'source_urls': urls, 'submission_reference': reference})
|
|
for identity, sha, source, manual, thumbnail, thumbnail_sha in db.execute(
|
|
'SELECT a.id,a.sha256,a.source_url,d.manual_url,d.thumbnail_url,d.thumbnail_sha256 FROM asset a LEFT JOIN asset_delivery d ON d.asset_id=a.id ORDER BY a.id'):
|
|
row = {'id': _identity(identity), 'sha256': _hash(sha, optional=True), 'source_urls': _urls([source, manual, thumbnail])}
|
|
if thumbnail_sha is not None:
|
|
row['thumbnail_sha256'] = _hash(thumbnail_sha)
|
|
result['assets'].append(row)
|
|
require(digest(catalog) == before, 'Catalog changed while exporting rights inventory')
|
|
require(len(json.dumps(result, ensure_ascii=False).encode()) <= MAX_INVENTORY_BYTES, 'Catalog inventory exceeds byte limit')
|
|
return result
|
|
|
|
|
|
def _bind_observed_bytes(document, catalog):
|
|
"""Preserve exact matching bytes/aliases through later title or URL renames.
|
|
|
|
Never promote a package-only scope to an entire title/submission. No fuzzy
|
|
name matching or new rights authority is inferred from a catalog.
|
|
"""
|
|
from submission_intake import applies
|
|
observed = inventory(catalog)
|
|
rows = []
|
|
for item in document['suppressions']:
|
|
scope = item['scope']
|
|
hashes, urls = set(scope.get('package_sha256', [])), set(scope.get('source_urls', []))
|
|
for row in observed['releases']:
|
|
if applies(scope, row):
|
|
hashes.add(row['package_sha256']); urls.update(row['source_urls'])
|
|
for row in observed['assets']:
|
|
if set(row['source_urls']) & set(scope.get('source_urls', [])) or {row.get('sha256'), row.get('thumbnail_sha256')} & set(scope.get('package_sha256', [])):
|
|
hashes.update(h for h in (row.get('sha256'), row.get('thumbnail_sha256')) if h); urls.update(row['source_urls'])
|
|
# Scope arrays have a 100-value limit. Keep the original exact scope and
|
|
# retain bounded additional records rather than dropping observed aliases.
|
|
rows.append(item)
|
|
for key, values in [('package_sha256', sorted(hashes)), ('source_urls', sorted(urls))]:
|
|
for start in range(0, len(values), 100):
|
|
rows.append({**item, 'scope': {key: values[start:start+100]}})
|
|
return combine({'schema_version': 1, 'suppressions': rows})
|
|
|
|
|
|
def refresh(snapshot, cache_path=None, *, catalog=None, as_of=None):
|
|
document = normalize(snapshot, as_of=as_of, fresh=True)
|
|
path = Path(cache_path) if cache_path is not None else DEFAULT_CACHE
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
lock = path.with_name('.' + path.name + '.lock')
|
|
with os.fdopen(os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600), 'a+b') as stream:
|
|
fcntl.flock(stream, fcntl.LOCK_EX)
|
|
retained = combine(cached(path), document)
|
|
if catalog is not None:
|
|
retained = _bind_observed_bytes(retained, catalog)
|
|
retained['generated_at'] = document['generated_at']
|
|
_atomic_json(path, retained, MAX_CACHE_BYTES, replace=True)
|
|
return retained
|
|
|
|
|
|
def export_inventory(catalog, output):
|
|
result = inventory(catalog)
|
|
_atomic_json(output, result, MAX_INVENTORY_BYTES)
|
|
return {'catalog_sha256': result['catalog_sha256'], 'inventory_sha256': hashlib.sha256(Path(output).read_bytes()).hexdigest(),
|
|
'releases': len(result['releases']), 'assets': len(result['assets'])}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
sub = parser.add_subparsers(dest='command', required=True)
|
|
update = sub.add_parser('refresh', help='Monotonically retain a fresh authenticated suppression snapshot')
|
|
update.add_argument('--snapshot', required=True, type=Path)
|
|
update.add_argument('--cache', type=Path, default=DEFAULT_CACHE)
|
|
update.add_argument('--catalog', type=Path, help='Current app catalog for binding matching bytes and aliases; defaults to the prepared catalog when present')
|
|
export = sub.add_parser('inventory', help='Export a SHA-bound rights inventory from an actual app catalog')
|
|
export.add_argument('catalog', type=Path); export.add_argument('--output', required=True, type=Path)
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.command == 'refresh':
|
|
catalog = args.catalog if args.catalog is not None else DEFAULT_CATALOG if DEFAULT_CATALOG.exists() else None
|
|
result = refresh(read_json(args.snapshot), args.cache, catalog=catalog)
|
|
print(json.dumps({'retained_suppressions': len(result['suppressions'])}))
|
|
else:
|
|
print(json.dumps(export_inventory(args.catalog, args.output)))
|
|
except (ValueError, OSError, sqlite3.DatabaseError) as error:
|
|
parser.exit(1, 'error: Rights catalog: ' + str(error) + '\n')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|