Files

315 lines
21 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Explicit, offline intake of exact approved software into a NEW app catalog.
No download, execution, extraction into a project, publication or installed-media
mutation. Fresh approval/suppression snapshots come from the authenticated website
maintenance channel. Originals and permission records remain in ignored storage.
"""
import argparse
from contextlib import closing
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import sqlite3
import stat
import tempfile
from urllib.parse import urlsplit, unquote
import zipfile
HASH = re.compile(r'[a-f0-9]{64}')
REFERENCE = re.compile(r'SW-[0-9]{4}-[0-9]+')
MAX_PACKAGE = 256 * 1024 * 1024
MAX_EXPANDED = 256 * 1024 * 1024
MAX_MEMBERS = 4096
def require(condition, message):
if not condition:
raise ValueError(message)
def encoded(value):
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), sort_keys=True)
def digest(path):
h = hashlib.sha256()
with Path(path).open('rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
h.update(chunk)
return h.hexdigest()
def time_value(value):
require(isinstance(value, str), 'A timezone-aware timestamp is required')
result = datetime.fromisoformat(value.replace('Z', '+00:00'))
require(result.tzinfo is not None, 'Timestamp requires a timezone')
return result
def load(path, maximum=8 * 1024 * 1024):
path = Path(path)
require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum,
'Use a bounded regular JSON file')
return json.loads(path.read_text())
def approval(manifest, current, as_of=None):
now = as_of or datetime.now(timezone.utc)
require(manifest.get('schema_version') == 1 and REFERENCE.fullmatch(manifest.get('submission_reference', '')),
'Unsupported approved submission manifest')
snapshot = manifest.get('approval_snapshot_json')
require(isinstance(snapshot, str) and HASH.fullmatch(manifest.get('approval_sha256', '')),
'Exact immutable approval snapshot is required')
require(hashlib.sha256(snapshot.encode()).hexdigest() == manifest['approval_sha256'], 'Approval snapshot changed')
original = json.loads(snapshot)
require({k: v for k, v in manifest.items() if k not in ('approval_snapshot_json', 'approval_sha256')} == original,
'Manifest disagrees with its approved snapshot')
require(current.get('state') in ('claimed', 'ready') and current.get('manifest', {}).get('approval_sha256') == manifest['approval_sha256']
and current['manifest'].get('submission_reference') == manifest['submission_reference']
and current['manifest'].get('approval_revision') == manifest.get('approval_revision'),
'Current claimed approval must match; withdrawn/stale approvals are refused')
checked = time_value(current.get('checked_at'))
require(0 <= (now - checked).total_seconds() <= 900, 'Approval check expired; refresh the authenticated claim')
permission = manifest.get('permission', {})
require(permission.get('operator') == 'Dezota LLC' and permission.get('record_reference')
and HASH.fullmatch(permission.get('terms_sha256', '')) and isinstance(permission.get('terms_text'), str)
and hashlib.sha256(permission['terms_text'].encode()).hexdigest() == permission['terms_sha256'],
'Missing or changed permission record')
assets = manifest.get('assets')
require(isinstance(assets, list) and 1 <= len(assets) <= 4, 'Approved asset descriptors required')
require(sum(a.get('role') == 'package' for a in assets) == 1, 'Exactly one approved package is required')
for asset in assets:
require(asset.get('role') in ('package', 'manual', 'image') and HASH.fullmatch(asset.get('sha256', ''))
and type(asset.get('byte_size')) is int and asset['byte_size'] > 0, 'Invalid approved asset')
return manifest
def suppressions(value, as_of=None, require_fresh=False):
now = as_of or datetime.now(timezone.utc)
require(value.get('schema_version') == 1 and isinstance(value.get('suppressions'), list), 'Unsupported suppression manifest')
if require_fresh:
age = (now - time_value(value.get('generated_at'))).total_seconds()
require(0 <= age <= 900, 'Suppression check expired; refresh it before intake')
result = []
for item in value['suppressions']:
require(isinstance(item.get('case_reference'), str) and re.fullmatch(r'FB-[0-9]{4}-[0-9]+',item['case_reference']), 'Rights case reference required')
scope = item.get('scope')
require(isinstance(scope, dict) and set(scope) <= {'title_ids', 'release_ids', 'package_sha256', 'source_urls', 'submission_references'}, 'Invalid withdrawal scope')
require(any(scope.values()) and all(isinstance(v, list) and len(v) <= 100 and all(isinstance(x, str) and x for x in v) for v in scope.values()), 'Empty or invalid withdrawal scope')
require(all(HASH.fullmatch(v) for v in scope.get('package_sha256', [])), 'Invalid withdrawal hash')
if time_value(item.get('effective_at')) <= now:
result.append(scope)
return result
def applies(scope, row):
return (row.get('title_id') in scope.get('title_ids', [])
or row.get('id') in scope.get('release_ids', [])
or row.get('package_sha256') in scope.get('package_sha256', [])
or row.get('source_url') in scope.get('source_urls', [])
or bool(set(row.get('source_urls', [])) & set(scope.get('source_urls', [])))
or row.get('submission_reference') in scope.get('submission_references', []))
def apply_suppressions(db, manifest, as_of=None):
"""Remove only scoped catalog rows in caller transaction; never touch files.
Requires explicit suppression data. Feed this at each candidate preparation;
publication additionally requires a freshly authenticated suppression check.
Future-dated exclusions remain in the manifest and become effective by time.
"""
scopes = suppressions(manifest, as_of)
removed = []
affected_titles = set()
affected_assets = set()
columns = [row[1] for row in db.execute('PRAGMA table_info(release)')]
for raw in db.execute('SELECT * FROM release').fetchall():
row = dict(zip(columns, raw))
meta = json.loads(row.get('metadata_json') or '{}')
row['submission_reference'] = meta.get('submission_reference')
row['source_urls'] = [u[0] for u in db.execute('SELECT url FROM download_location WHERE package_sha256=?', (row['package_sha256'],))]
if any(applies(scope, row) for scope in scopes):
release_id = row['id']
affected_titles.add(row['title_id'])
affected_assets.update(a[0] for a in db.execute('SELECT asset_id FROM release_asset WHERE release_id=?', (release_id,)))
for table in ('release_asset', 'setup', 'member', 'source_hint'):
if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone():
db.execute(f'DELETE FROM {table} WHERE release_id=?', (release_id,))
db.execute('DELETE FROM release WHERE id=?', (release_id,))
removed.append(release_id)
empty = [title_id for title_id in sorted(affected_titles) if not db.execute('SELECT 1 FROM release WHERE title_id=?', (title_id,)).fetchone()]
for title_id in empty:
db.execute('DELETE FROM title WHERE id=?', (title_id,))
# Asset-only exclusions remove only supporting assets, retaining unrelated software.
# Match both upstream/original and hosted derivative URLs, including old catalogs.
asset_columns = [r[1] for r in db.execute('PRAGMA table_info(asset)')]
for raw in db.execute('SELECT * FROM asset').fetchall():
asset = dict(zip(asset_columns, raw))
delivery = db.execute('SELECT manual_url,thumbnail_url,thumbnail_sha256 FROM asset_delivery WHERE asset_id=?', (asset['id'],)).fetchone()
urls = {asset['source_url'], *(delivery[:2] if delivery else [])}
hashes = {asset.get('sha256'), delivery[2] if delivery else None}
if any(urls & set(scope.get('source_urls', [])) or hashes & set(scope.get('package_sha256', [])) for scope in scopes):
db.execute('DELETE FROM release_asset WHERE asset_id=?', (asset['id'],))
affected_assets.add(asset['id'])
removed_assets = []
for asset_id in sorted(affected_assets):
if not db.execute('SELECT 1 FROM release_asset WHERE asset_id=?', (asset_id,)).fetchone():
db.execute('DELETE FROM asset_delivery WHERE asset_id=?', (asset_id,))
db.execute('DELETE FROM asset WHERE id=?', (asset_id,))
removed_assets.append(asset_id)
return {'removed_release_ids': removed, 'removed_title_ids': empty, 'removed_asset_ids': removed_assets}
def inspect_zip(path):
require(path.stat().st_size <= MAX_PACKAGE, 'Package exceeds size limit')
members = []
seen = set()
with zipfile.ZipFile(path) as archive:
infos = archive.infolist()
require(1 <= len(infos) <= MAX_MEMBERS, 'ZIP member count is outside bounds')
require(sum(i.file_size for i in infos) <= MAX_EXPANDED, 'ZIP expansion exceeds bounds')
for info in infos:
name = PurePosixPath(info.filename)
require(not name.is_absolute() and '\\' not in info.filename and not any(p in ('..', '.', '') for p in name.parts)
and not any(p in ('', '.', '..') for p in info.filename.rstrip('/').split('/')) and '\x00' not in info.filename and ':' not in info.filename and not info.flag_bits & 1, 'Unsafe ZIP member path or encrypted entry')
require(not stat.S_ISLNK(info.external_attr >> 16), 'ZIP links are refused')
key = info.filename.casefold()
require(key not in seen, 'ZIP contains colliding member names')
seen.add(key)
if info.is_dir():
continue
require(info.file_size <= MAX_EXPANDED and (not info.file_size or info.file_size <= max(1, info.compress_size) * 4096), 'ZIP expansion ratio exceeds bounds')
h = hashlib.sha256()
count = 0
with archive.open(info) as stream:
while chunk := stream.read(1024 * 1024):
count += len(chunk)
require(count <= info.file_size, 'ZIP expanded beyond declared member size')
h.update(chunk)
require(count == info.file_size, 'ZIP member is truncated')
kind = {'.dsk': 'disk', '.os9': 'disk', '.vhd': 'hard_disk', '.img': 'hard_disk', '.ccc': 'cartridge', '.cas': 'cassette', '.wav': 'cassette', '.bin': 'binary'}.get(name.suffix.lower(), 'supporting')
members.append({'path': info.filename, 'sha256': h.hexdigest(), 'byte_size': count, 'kind': kind})
require(any(m['kind'] != 'supporting' for m in members), 'Package contains no recognized CoCo media')
return members
def hosted_url(url, role):
parsed = urlsplit(url)
path = unquote(parsed.path)
require(parsed.scheme == 'https' and parsed.netloc == 'vccplus.brianholman.com' and not parsed.query and not parsed.fragment and '\\' not in path and '\x00' not in path and all(p not in ('.', '..') for p in path.split('/')), 'Invalid hosted software location')
base = '/dl/software/CoCoLibrary/'
prefix, extensions = {'package': (base, ('.zip',)), 'manual': (base + 'Documents/Manuals/', ('.pdf',)), 'image': (base + 'Artwork/Originals/', ('.png', '.jpg', '.jpeg')), 'thumbnail': (base + 'Artwork/Thumbnails/', ('.png',))}[role]
require(path.startswith(prefix) and path.lower().endswith(extensions), 'Invalid hosted software location for ' + role)
return url
def verified_locations(receipt, assets):
require(receipt.get('state') == 'verified' and isinstance(receipt.get('artifacts'), list), 'Verified hosting receipt required')
result = {}
for asset in assets:
matches = [r for r in receipt['artifacts'] if r.get('sha256') == asset['sha256'] and r.get('byte_size') == asset['byte_size'] and r.get('status') == 200]
require(len(matches) == 1, 'Missing or ambiguous verified publication for an approved asset')
result[asset['sha256']] = {'url': hosted_url(matches[0].get('url', ''), asset['role'])}
derived = receipt.get('derived_artifacts', [])
require(isinstance(derived, list), 'Invalid derived artwork receipt')
for artifact in derived:
source = artifact.get('source_sha256')
require(any(a['role'] == 'image' and a['sha256'] == source for a in assets), 'Derived thumbnail must identify approved original artwork')
require(artifact.get('role') == 'thumbnail' and artifact.get('status') == 200 and isinstance(artifact.get('sha256'), str) and HASH.fullmatch(artifact['sha256']) and type(artifact.get('byte_size')) is int and 0 < artifact['byte_size'] <= 2 * 1024 * 1024, 'Invalid prepared PNG thumbnail receipt')
require('thumbnail' not in result[source], 'Ambiguous prepared thumbnail receipt')
result[source]['thumbnail'] = {**artifact, 'url': hosted_url(artifact.get('url', ''), 'thumbnail')}
return result
def prepare_catalog(source, destination, manifest, current, suppression_manifest, files, hosting_receipt, as_of=None, *, rights_cache=None):
manifest = approval(manifest, current, as_of)
suppressions(suppression_manifest, as_of, require_fresh=True)
from rights import cached, combine
retained_suppressions = combine(cached(rights_cache), suppression_manifest)
require(Path(source).is_file() and not Path(source).is_symlink(), 'Use an existing regular prepared app catalog')
destination = Path(destination)
require(destination.resolve() != Path(source).resolve() and not destination.exists(), 'Create a NEW catalog; existing source/output is never replaced')
assets = manifest['assets']
paths = {}
for asset in assets:
path = Path(files.get(asset['sha256'], ''))
require(path.is_file() and not path.is_symlink() and path.stat().st_size == asset['byte_size'] and digest(path) == asset['sha256'], 'Approved original bytes are missing or changed')
paths[asset['sha256']] = path
package = next(a for a in assets if a['role'] == 'package')
members = inspect_zip(paths[package['sha256']])
locations = verified_locations(hosting_receipt, assets)
ref = manifest['submission_reference']
title_id = 'direct-' + ref.lower()
release_id = title_id + '-r' + str(manifest['approval_revision'])
row = {'id': release_id, 'title_id': title_id, 'package_sha256': package['sha256'], 'source_url': locations[package['sha256']]['url'], 'submission_reference': ref}
require(not any(applies(s, row) for s in suppressions(retained_suppressions, as_of)), 'A rights withdrawal blocks this submission')
destination.parent.mkdir(parents=True, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix='.submission-', suffix='.sqlite', dir=destination.parent)
os.close(fd)
try:
with closing(sqlite3.connect(f'file:{Path(source).resolve()}?mode=ro', uri=True)) as original, closing(sqlite3.connect(temporary)) as db:
original.backup(db)
db.execute('PRAGMA foreign_keys=ON')
require(db.execute('PRAGMA user_version').fetchone()[0] == 1, 'Unsupported app catalog version')
with db:
prior = db.execute('SELECT package_sha256 FROM release WHERE id=?', (release_id,)).fetchone()
require(prior is None or prior[0] == package['sha256'], 'Existing submission identity conflicts')
if prior is None:
primary = next(m for m in members if m['kind'] != 'supporting')
media = {'hard_disk': 'hard-disk'}.get(primary['kind'], primary['kind'])
metadata = {'origin': 'direct_submission', 'submission_reference': ref, 'approval_revision': manifest['approval_revision'], 'author': manifest['author'], 'publisher': manifest['author'], 'software_version': manifest['software_version'], 'license': manifest['metadata']['end_user_terms'], 'project_url': manifest['metadata'].get('project_url', ''), 'description': manifest['description'], 'permission_operator': 'Dezota LLC'}
db.execute('INSERT OR IGNORE INTO title VALUES(?,?,?)', (title_id, manifest['title'], 'reviewed-direct-submission'))
db.execute('INSERT OR IGNORE INTO package VALUES(?,?)', (package['sha256'], package['byte_size']))
db.execute('INSERT INTO release VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)', (release_id,title_id,manifest['software_version'],'Uncategorized',media,'software',locations[package['sha256']]['url'],None,None,None,package['sha256'],'unknown',encoded(metadata)))
db.execute('INSERT OR IGNORE INTO download_location VALUES(?,?,?)', (package['sha256'],locations[package['sha256']]['url'],0))
for member in members:
db.execute('INSERT INTO member VALUES(?,?,?,?,?,?)', (release_id,member['path'],member['sha256'],member['byte_size'],member['kind'],'software' if member['kind'] != 'supporting' else 'supporting'))
db.execute('INSERT INTO setup VALUES(?,?,?,?,?,?,?,?)', (release_id,'manual-setup','{}','[]',None,manifest['metadata'].get('setup') or 'Review media and configure the CoCo manually.',encoded(['Direct submission: automatic setup is not runtime-qualified.']),encoded({'scope':'manual-setup','approval_reference':ref})))
for number, asset in enumerate(a for a in assets if a['role'] != 'package'):
asset_id = release_id + '-asset-' + str(number)
kind = 'manual' if asset['role'] == 'manual' else 'cover'
db.execute('INSERT INTO asset VALUES(?,?,?,?,?,?,?)',(asset_id,asset['name'],kind,locations[asset['sha256']]['url'],asset['sha256'],asset['byte_size'],manifest['author']))
db.execute('INSERT INTO release_asset VALUES(?,?,?,?)',(release_id,asset_id,'reviewed','Approved direct-submission supporting asset'))
thumb = locations[asset['sha256']].get('thumbnail', {})
db.execute('INSERT INTO asset_delivery VALUES(?,?,?,?,?,?)',(asset_id,locations[asset['sha256']]['url'] if kind=='manual' else None,thumb.get('url'),thumb.get('sha256'),thumb.get('byte_size'),0))
exclusions = apply_suppressions(db,retained_suppressions,as_of)
db.execute('INSERT OR REPLACE INTO metadata VALUES(?,?)',('submission_approval:'+ref,manifest['approval_sha256']))
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'Candidate catalog integrity failed')
# Exclusive hardlink promotion never replaces an output created by a racing operation.
os.link(temporary,destination)
finally:
Path(temporary).unlink(missing_ok=True)
return {'schema_version':1,'submission_reference':ref,'approval_revision':manifest['approval_revision'],'title_id':title_id,'release_id':release_id,'catalog_sha256':digest(destination),'hosting_receipt':hosting_receipt,'target_release':manifest.get('target_release',''),'qualification_record':'Prepared with manual setup; runtime qualification pending.','suppression_result':exclusions}
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('source',type=Path)
parser.add_argument('--output',required=True,type=Path)
parser.add_argument('--manifest',required=True,type=Path)
parser.add_argument('--approval-check',required=True,type=Path)
parser.add_argument('--suppressions',required=True,type=Path)
parser.add_argument('--files',required=True,type=Path,help='Private JSON mapping approved SHA-256 to local retained file path')
parser.add_argument('--hosting-receipt',required=True,type=Path)
parser.add_argument('--result',required=True,type=Path)
parser.add_argument('--rights-cache',type=Path,help='Persistent exclusion cache override; refresh merges instead of replacing previous exclusions')
args=parser.parse_args()
from rights import refresh
snapshot = load(args.suppressions)
# This explicit file/CLI boundary retains authenticated exclusions. The pure
# preparation helper above reads the cache but never writes global state.
refresh(snapshot,args.rights_cache,catalog=args.source)
result=prepare_catalog(args.source,args.output,load(args.manifest),load(args.approval_check),snapshot,load(args.files),load(args.hosting_receipt),rights_cache=args.rights_cache)
args.result.parent.mkdir(parents=True,exist_ok=True)
with os.fdopen(os.open(args.result, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as stream:stream.write(json.dumps(result,indent=2)+'\n')
print('New catalog candidate prepared; runtime qualification/publication remain separate.')
if __name__=='__main__':main()