Sync source edition with VCC Plus 3.7.0 build 63
This commit is contained in:
1 parent
c652a5276d
commit
b2ecec4d87
86 files changed
+10519
-472
No files matched your search
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Retain rights exclusions offline and inventory an actual portable app catalog.
|
||||
|
||||
Refresh consumes an explicitly obtained authenticated snapshot; this tool does
|
||||
not authenticate, contact servers, publish content or infer rights authority.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timezone
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_CACHE = ROOT / 'media/software-library-full/curation/rights-suppressions.json'
|
||||
DEFAULT_CATALOG = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite'
|
||||
MAX_CACHE_BYTES = 8 * 1024 * 1024
|
||||
MAX_INVENTORY_BYTES = 16 * 1024 * 1024
|
||||
MAX_CASES = 10000
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def read_json(path, maximum=MAX_CACHE_BYTES):
|
||||
path = Path(path)
|
||||
require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum,
|
||||
'Rights input must be a bounded regular nonsymlink file')
|
||||
with path.open('rb') as stream:
|
||||
data = stream.read(maximum + 1)
|
||||
require(len(data) <= maximum, 'Rights input exceeds its byte limit')
|
||||
return json.loads(data)
|
||||
|
||||
|
||||
def normalize(value, *, as_of=None, fresh=False):
|
||||
from submission_intake import suppressions, time_value
|
||||
require(isinstance(value, dict), 'Invalid rights snapshot')
|
||||
records = value.get('suppressions')
|
||||
require(isinstance(records, list) and len(records) <= MAX_CASES, 'Rights case count exceeds bounds')
|
||||
require(all(isinstance(item, dict) for item in records), 'Invalid rights case')
|
||||
suppressions(value, as_of, require_fresh=fresh)
|
||||
result = []
|
||||
for item in records:
|
||||
scope = {}
|
||||
for key, values in item['scope'].items():
|
||||
require(all(len(x.encode('utf-8')) <= 4096 and '\0' not in x for x in values), 'Rights scope identity exceeds bounds')
|
||||
if values:
|
||||
scope[key] = sorted(set(values))
|
||||
result.append({'case_reference': item['case_reference'],
|
||||
'effective_at': time_value(item['effective_at']).isoformat(), 'scope': scope})
|
||||
unique = {json.dumps(item, sort_keys=True, separators=(',', ':')): item for item in result}
|
||||
document = {'schema_version': 1, 'suppressions': [unique[key] for key in sorted(unique)]}
|
||||
if 'generated_at' in value:
|
||||
document['generated_at'] = time_value(value['generated_at']).isoformat()
|
||||
require(len(json.dumps(document, ensure_ascii=False).encode()) <= MAX_CACHE_BYTES, 'Rights cache exceeds byte limit')
|
||||
return document
|
||||
|
||||
|
||||
def cached(path=None):
|
||||
path = Path(path) if path is not None else DEFAULT_CACHE
|
||||
if not path.exists() and not path.is_symlink():
|
||||
return {'schema_version': 1, 'suppressions': []}
|
||||
return normalize(read_json(path))
|
||||
|
||||
|
||||
def combine(*documents):
|
||||
records = {}
|
||||
for document in documents:
|
||||
for item in normalize(document)['suppressions']:
|
||||
records[json.dumps(item, sort_keys=True, separators=(',', ':'))] = item
|
||||
# No omission, changed timestamp or narrowed replacement scope revokes an
|
||||
# earlier exclusion. Future cases remain retained until they take effect.
|
||||
return normalize({'schema_version': 1, 'suppressions': list(records.values())})
|
||||
|
||||
|
||||
def apply_cached(db, *, cache_path=None, extra=None, as_of=None):
|
||||
from submission_intake import apply_suppressions
|
||||
document = combine(cached(cache_path), *([extra] if extra is not None else []))
|
||||
return apply_suppressions(db, document, as_of)
|
||||
|
||||
|
||||
def _atomic_json(path, value, maximum, *, replace=False):
|
||||
path = Path(path)
|
||||
require(not path.is_symlink() and (replace or not path.exists()), 'Rights output already exists or is a symlink')
|
||||
data = (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + '\n').encode()
|
||||
require(len(data) <= maximum, 'Rights output exceeds byte limit')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix='.rights-', dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(data); stream.flush(); os.fsync(stream.fileno())
|
||||
if replace:
|
||||
require(not path.is_symlink(), 'Rights output became a symlink')
|
||||
os.replace(temporary, path)
|
||||
else:
|
||||
os.link(temporary, path) # Exclusive promotion, including racing writers.
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _hash(value, *, optional=False):
|
||||
import re
|
||||
require((optional and value is None) or isinstance(value, str) and re.fullmatch('[a-f0-9]{64}', value), 'Invalid catalog inventory hash')
|
||||
return value
|
||||
|
||||
|
||||
def _identity(value):
|
||||
require(isinstance(value, str) and 0 < len(value.encode('utf-8')) <= 4096 and '\0' not in value,
|
||||
'Invalid catalog inventory identity')
|
||||
return value
|
||||
|
||||
|
||||
def _urls(values):
|
||||
result = sorted(set(value for value in values if value))
|
||||
require(len(result) <= 64, 'Catalog inventory URL count exceeds bounds')
|
||||
for url in result:
|
||||
_identity(url)
|
||||
parsed = urlsplit(url)
|
||||
require(parsed.scheme in ('http', 'https') and parsed.hostname and parsed.username is None and parsed.password is None
|
||||
and not any(ord(c) < 32 for c in url), 'Catalog inventory requires public HTTP references without credentials')
|
||||
return result
|
||||
|
||||
|
||||
def inventory(catalog):
|
||||
"""Read only the actual app export, never a maintainer DB or media corpus."""
|
||||
from package import validate
|
||||
from submission_intake import digest
|
||||
catalog = Path(catalog)
|
||||
validate(catalog)
|
||||
before = digest(catalog)
|
||||
result = {'schema_version': 1, 'catalog_sha256': before, 'releases': [], 'assets': []}
|
||||
with closing(sqlite3.connect(catalog.resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
||||
db.execute('PRAGMA query_only=ON')
|
||||
require(db.execute('SELECT count(*) FROM release').fetchone()[0] <= 50000
|
||||
and db.execute('SELECT count(*) FROM asset').fetchone()[0] <= 100000, 'Catalog inventory row count exceeds bounds')
|
||||
for title, identity, sha, url, metadata in db.execute('SELECT title_id,id,package_sha256,source_url,metadata_json FROM release ORDER BY id'):
|
||||
reference = json.loads(metadata).get('submission_reference')
|
||||
if reference is not None:
|
||||
from submission_intake import REFERENCE
|
||||
require(isinstance(reference, str) and REFERENCE.fullmatch(reference), 'Invalid inventory submission reference')
|
||||
urls = _urls([url] + [r[0] for r in db.execute('SELECT url FROM download_location WHERE package_sha256=? ORDER BY url', (sha,))])
|
||||
result['releases'].append({'title_id': _identity(title), 'id': _identity(identity), 'package_sha256': _hash(sha),
|
||||
'source_url': url, 'source_urls': urls, 'submission_reference': reference})
|
||||
for identity, sha, source, manual, thumbnail, thumbnail_sha in db.execute(
|
||||
'SELECT a.id,a.sha256,a.source_url,d.manual_url,d.thumbnail_url,d.thumbnail_sha256 FROM asset a LEFT JOIN asset_delivery d ON d.asset_id=a.id ORDER BY a.id'):
|
||||
row = {'id': _identity(identity), 'sha256': _hash(sha, optional=True), 'source_urls': _urls([source, manual, thumbnail])}
|
||||
if thumbnail_sha is not None:
|
||||
row['thumbnail_sha256'] = _hash(thumbnail_sha)
|
||||
result['assets'].append(row)
|
||||
require(digest(catalog) == before, 'Catalog changed while exporting rights inventory')
|
||||
require(len(json.dumps(result, ensure_ascii=False).encode()) <= MAX_INVENTORY_BYTES, 'Catalog inventory exceeds byte limit')
|
||||
return result
|
||||
|
||||
|
||||
def _bind_observed_bytes(document, catalog):
|
||||
"""Preserve exact matching bytes/aliases through later title or URL renames.
|
||||
|
||||
Never promote a package-only scope to an entire title/submission. No fuzzy
|
||||
name matching or new rights authority is inferred from a catalog.
|
||||
"""
|
||||
from submission_intake import applies
|
||||
observed = inventory(catalog)
|
||||
rows = []
|
||||
for item in document['suppressions']:
|
||||
scope = item['scope']
|
||||
hashes, urls = set(scope.get('package_sha256', [])), set(scope.get('source_urls', []))
|
||||
for row in observed['releases']:
|
||||
if applies(scope, row):
|
||||
hashes.add(row['package_sha256']); urls.update(row['source_urls'])
|
||||
for row in observed['assets']:
|
||||
if set(row['source_urls']) & set(scope.get('source_urls', [])) or {row.get('sha256'), row.get('thumbnail_sha256')} & set(scope.get('package_sha256', [])):
|
||||
hashes.update(h for h in (row.get('sha256'), row.get('thumbnail_sha256')) if h); urls.update(row['source_urls'])
|
||||
# Scope arrays have a 100-value limit. Keep the original exact scope and
|
||||
# retain bounded additional records rather than dropping observed aliases.
|
||||
rows.append(item)
|
||||
for key, values in [('package_sha256', sorted(hashes)), ('source_urls', sorted(urls))]:
|
||||
for start in range(0, len(values), 100):
|
||||
rows.append({**item, 'scope': {key: values[start:start+100]}})
|
||||
return combine({'schema_version': 1, 'suppressions': rows})
|
||||
|
||||
|
||||
def refresh(snapshot, cache_path=None, *, catalog=None, as_of=None):
|
||||
document = normalize(snapshot, as_of=as_of, fresh=True)
|
||||
path = Path(cache_path) if cache_path is not None else DEFAULT_CACHE
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock = path.with_name('.' + path.name + '.lock')
|
||||
with os.fdopen(os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600), 'a+b') as stream:
|
||||
fcntl.flock(stream, fcntl.LOCK_EX)
|
||||
retained = combine(cached(path), document)
|
||||
if catalog is not None:
|
||||
retained = _bind_observed_bytes(retained, catalog)
|
||||
retained['generated_at'] = document['generated_at']
|
||||
_atomic_json(path, retained, MAX_CACHE_BYTES, replace=True)
|
||||
return retained
|
||||
|
||||
|
||||
def export_inventory(catalog, output):
|
||||
result = inventory(catalog)
|
||||
_atomic_json(output, result, MAX_INVENTORY_BYTES)
|
||||
return {'catalog_sha256': result['catalog_sha256'], 'inventory_sha256': hashlib.sha256(Path(output).read_bytes()).hexdigest(),
|
||||
'releases': len(result['releases']), 'assets': len(result['assets'])}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest='command', required=True)
|
||||
update = sub.add_parser('refresh', help='Monotonically retain a fresh authenticated suppression snapshot')
|
||||
update.add_argument('--snapshot', required=True, type=Path)
|
||||
update.add_argument('--cache', type=Path, default=DEFAULT_CACHE)
|
||||
update.add_argument('--catalog', type=Path, help='Current app catalog for binding matching bytes and aliases; defaults to the prepared catalog when present')
|
||||
export = sub.add_parser('inventory', help='Export a SHA-bound rights inventory from an actual app catalog')
|
||||
export.add_argument('catalog', type=Path); export.add_argument('--output', required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == 'refresh':
|
||||
catalog = args.catalog if args.catalog is not None else DEFAULT_CATALOG if DEFAULT_CATALOG.exists() else None
|
||||
result = refresh(read_json(args.snapshot), args.cache, catalog=catalog)
|
||||
print(json.dumps({'retained_suppressions': len(result['suppressions'])}))
|
||||
else:
|
||||
print(json.dumps(export_inventory(args.catalog, args.output)))
|
||||
except (ValueError, OSError, sqlite3.DatabaseError) as error:
|
||||
parser.exit(1, 'error: Rights catalog: ' + str(error) + '\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user