Sync source edition with VCC Plus 3.7.0 build 63
This commit is contained in:
1 parent
c652a5276d
commit
b2ecec4d87
86 files changed
+10519
-472
No files matched your search
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Explicit, offline intake of exact approved software into a NEW app catalog.
|
||||
|
||||
No download, execution, extraction into a project, publication or installed-media
|
||||
mutation. Fresh approval/suppression snapshots come from the authenticated website
|
||||
maintenance channel. Originals and permission records remain in ignored storage.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import stat
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit, unquote
|
||||
import zipfile
|
||||
|
||||
HASH = re.compile(r'[a-f0-9]{64}')
|
||||
REFERENCE = re.compile(r'SW-[0-9]{4}-[0-9]+')
|
||||
MAX_PACKAGE = 256 * 1024 * 1024
|
||||
MAX_EXPANDED = 256 * 1024 * 1024
|
||||
MAX_MEMBERS = 4096
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def encoded(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), sort_keys=True)
|
||||
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def time_value(value):
|
||||
require(isinstance(value, str), 'A timezone-aware timestamp is required')
|
||||
result = datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
require(result.tzinfo is not None, 'Timestamp requires a timezone')
|
||||
return result
|
||||
|
||||
|
||||
def load(path, maximum=8 * 1024 * 1024):
|
||||
path = Path(path)
|
||||
require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum,
|
||||
'Use a bounded regular JSON file')
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def approval(manifest, current, as_of=None):
|
||||
now = as_of or datetime.now(timezone.utc)
|
||||
require(manifest.get('schema_version') == 1 and REFERENCE.fullmatch(manifest.get('submission_reference', '')),
|
||||
'Unsupported approved submission manifest')
|
||||
snapshot = manifest.get('approval_snapshot_json')
|
||||
require(isinstance(snapshot, str) and HASH.fullmatch(manifest.get('approval_sha256', '')),
|
||||
'Exact immutable approval snapshot is required')
|
||||
require(hashlib.sha256(snapshot.encode()).hexdigest() == manifest['approval_sha256'], 'Approval snapshot changed')
|
||||
original = json.loads(snapshot)
|
||||
require({k: v for k, v in manifest.items() if k not in ('approval_snapshot_json', 'approval_sha256')} == original,
|
||||
'Manifest disagrees with its approved snapshot')
|
||||
require(current.get('state') in ('claimed', 'ready') and current.get('manifest', {}).get('approval_sha256') == manifest['approval_sha256']
|
||||
and current['manifest'].get('submission_reference') == manifest['submission_reference']
|
||||
and current['manifest'].get('approval_revision') == manifest.get('approval_revision'),
|
||||
'Current claimed approval must match; withdrawn/stale approvals are refused')
|
||||
checked = time_value(current.get('checked_at'))
|
||||
require(0 <= (now - checked).total_seconds() <= 900, 'Approval check expired; refresh the authenticated claim')
|
||||
permission = manifest.get('permission', {})
|
||||
require(permission.get('operator') == 'Dezota LLC' and permission.get('record_reference')
|
||||
and HASH.fullmatch(permission.get('terms_sha256', '')) and isinstance(permission.get('terms_text'), str)
|
||||
and hashlib.sha256(permission['terms_text'].encode()).hexdigest() == permission['terms_sha256'],
|
||||
'Missing or changed permission record')
|
||||
assets = manifest.get('assets')
|
||||
require(isinstance(assets, list) and 1 <= len(assets) <= 4, 'Approved asset descriptors required')
|
||||
require(sum(a.get('role') == 'package' for a in assets) == 1, 'Exactly one approved package is required')
|
||||
for asset in assets:
|
||||
require(asset.get('role') in ('package', 'manual', 'image') and HASH.fullmatch(asset.get('sha256', ''))
|
||||
and type(asset.get('byte_size')) is int and asset['byte_size'] > 0, 'Invalid approved asset')
|
||||
return manifest
|
||||
|
||||
|
||||
def suppressions(value, as_of=None, require_fresh=False):
|
||||
now = as_of or datetime.now(timezone.utc)
|
||||
require(value.get('schema_version') == 1 and isinstance(value.get('suppressions'), list), 'Unsupported suppression manifest')
|
||||
if require_fresh:
|
||||
age = (now - time_value(value.get('generated_at'))).total_seconds()
|
||||
require(0 <= age <= 900, 'Suppression check expired; refresh it before intake')
|
||||
result = []
|
||||
for item in value['suppressions']:
|
||||
require(isinstance(item.get('case_reference'), str) and re.fullmatch(r'FB-[0-9]{4}-[0-9]+',item['case_reference']), 'Rights case reference required')
|
||||
scope = item.get('scope')
|
||||
require(isinstance(scope, dict) and set(scope) <= {'title_ids', 'release_ids', 'package_sha256', 'source_urls', 'submission_references'}, 'Invalid withdrawal scope')
|
||||
require(any(scope.values()) and all(isinstance(v, list) and len(v) <= 100 and all(isinstance(x, str) and x for x in v) for v in scope.values()), 'Empty or invalid withdrawal scope')
|
||||
require(all(HASH.fullmatch(v) for v in scope.get('package_sha256', [])), 'Invalid withdrawal hash')
|
||||
if time_value(item.get('effective_at')) <= now:
|
||||
result.append(scope)
|
||||
return result
|
||||
|
||||
|
||||
def applies(scope, row):
|
||||
return (row.get('title_id') in scope.get('title_ids', [])
|
||||
or row.get('id') in scope.get('release_ids', [])
|
||||
or row.get('package_sha256') in scope.get('package_sha256', [])
|
||||
or row.get('source_url') in scope.get('source_urls', [])
|
||||
or bool(set(row.get('source_urls', [])) & set(scope.get('source_urls', [])))
|
||||
or row.get('submission_reference') in scope.get('submission_references', []))
|
||||
|
||||
|
||||
def apply_suppressions(db, manifest, as_of=None):
|
||||
"""Remove only scoped catalog rows in caller transaction; never touch files.
|
||||
|
||||
Requires explicit suppression data. Feed this at each candidate preparation;
|
||||
publication additionally requires a freshly authenticated suppression check.
|
||||
Future-dated exclusions remain in the manifest and become effective by time.
|
||||
"""
|
||||
scopes = suppressions(manifest, as_of)
|
||||
removed = []
|
||||
affected_titles = set()
|
||||
affected_assets = set()
|
||||
columns = [row[1] for row in db.execute('PRAGMA table_info(release)')]
|
||||
for raw in db.execute('SELECT * FROM release').fetchall():
|
||||
row = dict(zip(columns, raw))
|
||||
meta = json.loads(row.get('metadata_json') or '{}')
|
||||
row['submission_reference'] = meta.get('submission_reference')
|
||||
row['source_urls'] = [u[0] for u in db.execute('SELECT url FROM download_location WHERE package_sha256=?', (row['package_sha256'],))]
|
||||
if any(applies(scope, row) for scope in scopes):
|
||||
release_id = row['id']
|
||||
affected_titles.add(row['title_id'])
|
||||
affected_assets.update(a[0] for a in db.execute('SELECT asset_id FROM release_asset WHERE release_id=?', (release_id,)))
|
||||
for table in ('release_asset', 'setup', 'member', 'source_hint'):
|
||||
if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone():
|
||||
db.execute(f'DELETE FROM {table} WHERE release_id=?', (release_id,))
|
||||
db.execute('DELETE FROM release WHERE id=?', (release_id,))
|
||||
removed.append(release_id)
|
||||
empty = [title_id for title_id in sorted(affected_titles) if not db.execute('SELECT 1 FROM release WHERE title_id=?', (title_id,)).fetchone()]
|
||||
for title_id in empty:
|
||||
db.execute('DELETE FROM title WHERE id=?', (title_id,))
|
||||
# Asset-only exclusions remove only supporting assets, retaining unrelated software.
|
||||
# Match both upstream/original and hosted derivative URLs, including old catalogs.
|
||||
asset_columns = [r[1] for r in db.execute('PRAGMA table_info(asset)')]
|
||||
for raw in db.execute('SELECT * FROM asset').fetchall():
|
||||
asset = dict(zip(asset_columns, raw))
|
||||
delivery = db.execute('SELECT manual_url,thumbnail_url,thumbnail_sha256 FROM asset_delivery WHERE asset_id=?', (asset['id'],)).fetchone()
|
||||
urls = {asset['source_url'], *(delivery[:2] if delivery else [])}
|
||||
hashes = {asset.get('sha256'), delivery[2] if delivery else None}
|
||||
if any(urls & set(scope.get('source_urls', [])) or hashes & set(scope.get('package_sha256', [])) for scope in scopes):
|
||||
db.execute('DELETE FROM release_asset WHERE asset_id=?', (asset['id'],))
|
||||
affected_assets.add(asset['id'])
|
||||
removed_assets = []
|
||||
for asset_id in sorted(affected_assets):
|
||||
if not db.execute('SELECT 1 FROM release_asset WHERE asset_id=?', (asset_id,)).fetchone():
|
||||
db.execute('DELETE FROM asset_delivery WHERE asset_id=?', (asset_id,))
|
||||
db.execute('DELETE FROM asset WHERE id=?', (asset_id,))
|
||||
removed_assets.append(asset_id)
|
||||
return {'removed_release_ids': removed, 'removed_title_ids': empty, 'removed_asset_ids': removed_assets}
|
||||
|
||||
|
||||
def inspect_zip(path):
|
||||
require(path.stat().st_size <= MAX_PACKAGE, 'Package exceeds size limit')
|
||||
members = []
|
||||
seen = set()
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
infos = archive.infolist()
|
||||
require(1 <= len(infos) <= MAX_MEMBERS, 'ZIP member count is outside bounds')
|
||||
require(sum(i.file_size for i in infos) <= MAX_EXPANDED, 'ZIP expansion exceeds bounds')
|
||||
for info in infos:
|
||||
name = PurePosixPath(info.filename)
|
||||
require(not name.is_absolute() and '\\' not in info.filename and not any(p in ('..', '.', '') for p in name.parts)
|
||||
and not any(p in ('', '.', '..') for p in info.filename.rstrip('/').split('/')) and '\x00' not in info.filename and ':' not in info.filename and not info.flag_bits & 1, 'Unsafe ZIP member path or encrypted entry')
|
||||
require(not stat.S_ISLNK(info.external_attr >> 16), 'ZIP links are refused')
|
||||
key = info.filename.casefold()
|
||||
require(key not in seen, 'ZIP contains colliding member names')
|
||||
seen.add(key)
|
||||
if info.is_dir():
|
||||
continue
|
||||
require(info.file_size <= MAX_EXPANDED and (not info.file_size or info.file_size <= max(1, info.compress_size) * 4096), 'ZIP expansion ratio exceeds bounds')
|
||||
h = hashlib.sha256()
|
||||
count = 0
|
||||
with archive.open(info) as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
count += len(chunk)
|
||||
require(count <= info.file_size, 'ZIP expanded beyond declared member size')
|
||||
h.update(chunk)
|
||||
require(count == info.file_size, 'ZIP member is truncated')
|
||||
kind = {'.dsk': 'disk', '.os9': 'disk', '.vhd': 'hard_disk', '.img': 'hard_disk', '.ccc': 'cartridge', '.cas': 'cassette', '.wav': 'cassette', '.bin': 'binary'}.get(name.suffix.lower(), 'supporting')
|
||||
members.append({'path': info.filename, 'sha256': h.hexdigest(), 'byte_size': count, 'kind': kind})
|
||||
require(any(m['kind'] != 'supporting' for m in members), 'Package contains no recognized CoCo media')
|
||||
return members
|
||||
|
||||
|
||||
def hosted_url(url, role):
|
||||
parsed = urlsplit(url)
|
||||
path = unquote(parsed.path)
|
||||
require(parsed.scheme == 'https' and parsed.netloc == 'vccplus.brianholman.com' and not parsed.query and not parsed.fragment and '\\' not in path and '\x00' not in path and all(p not in ('.', '..') for p in path.split('/')), 'Invalid hosted software location')
|
||||
base = '/dl/software/CoCoLibrary/'
|
||||
prefix, extensions = {'package': (base, ('.zip',)), 'manual': (base + 'Documents/Manuals/', ('.pdf',)), 'image': (base + 'Artwork/Originals/', ('.png', '.jpg', '.jpeg')), 'thumbnail': (base + 'Artwork/Thumbnails/', ('.png',))}[role]
|
||||
require(path.startswith(prefix) and path.lower().endswith(extensions), 'Invalid hosted software location for ' + role)
|
||||
return url
|
||||
|
||||
|
||||
def verified_locations(receipt, assets):
|
||||
require(receipt.get('state') == 'verified' and isinstance(receipt.get('artifacts'), list), 'Verified hosting receipt required')
|
||||
result = {}
|
||||
for asset in assets:
|
||||
matches = [r for r in receipt['artifacts'] if r.get('sha256') == asset['sha256'] and r.get('byte_size') == asset['byte_size'] and r.get('status') == 200]
|
||||
require(len(matches) == 1, 'Missing or ambiguous verified publication for an approved asset')
|
||||
result[asset['sha256']] = {'url': hosted_url(matches[0].get('url', ''), asset['role'])}
|
||||
derived = receipt.get('derived_artifacts', [])
|
||||
require(isinstance(derived, list), 'Invalid derived artwork receipt')
|
||||
for artifact in derived:
|
||||
source = artifact.get('source_sha256')
|
||||
require(any(a['role'] == 'image' and a['sha256'] == source for a in assets), 'Derived thumbnail must identify approved original artwork')
|
||||
require(artifact.get('role') == 'thumbnail' and artifact.get('status') == 200 and isinstance(artifact.get('sha256'), str) and HASH.fullmatch(artifact['sha256']) and type(artifact.get('byte_size')) is int and 0 < artifact['byte_size'] <= 2 * 1024 * 1024, 'Invalid prepared PNG thumbnail receipt')
|
||||
require('thumbnail' not in result[source], 'Ambiguous prepared thumbnail receipt')
|
||||
result[source]['thumbnail'] = {**artifact, 'url': hosted_url(artifact.get('url', ''), 'thumbnail')}
|
||||
return result
|
||||
|
||||
|
||||
def prepare_catalog(source, destination, manifest, current, suppression_manifest, files, hosting_receipt, as_of=None, *, rights_cache=None):
|
||||
manifest = approval(manifest, current, as_of)
|
||||
suppressions(suppression_manifest, as_of, require_fresh=True)
|
||||
from rights import cached, combine
|
||||
retained_suppressions = combine(cached(rights_cache), suppression_manifest)
|
||||
require(Path(source).is_file() and not Path(source).is_symlink(), 'Use an existing regular prepared app catalog')
|
||||
destination = Path(destination)
|
||||
require(destination.resolve() != Path(source).resolve() and not destination.exists(), 'Create a NEW catalog; existing source/output is never replaced')
|
||||
assets = manifest['assets']
|
||||
paths = {}
|
||||
for asset in assets:
|
||||
path = Path(files.get(asset['sha256'], ''))
|
||||
require(path.is_file() and not path.is_symlink() and path.stat().st_size == asset['byte_size'] and digest(path) == asset['sha256'], 'Approved original bytes are missing or changed')
|
||||
paths[asset['sha256']] = path
|
||||
package = next(a for a in assets if a['role'] == 'package')
|
||||
members = inspect_zip(paths[package['sha256']])
|
||||
locations = verified_locations(hosting_receipt, assets)
|
||||
ref = manifest['submission_reference']
|
||||
title_id = 'direct-' + ref.lower()
|
||||
release_id = title_id + '-r' + str(manifest['approval_revision'])
|
||||
row = {'id': release_id, 'title_id': title_id, 'package_sha256': package['sha256'], 'source_url': locations[package['sha256']]['url'], 'submission_reference': ref}
|
||||
require(not any(applies(s, row) for s in suppressions(retained_suppressions, as_of)), 'A rights withdrawal blocks this submission')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix='.submission-', suffix='.sqlite', dir=destination.parent)
|
||||
os.close(fd)
|
||||
try:
|
||||
with closing(sqlite3.connect(f'file:{Path(source).resolve()}?mode=ro', uri=True)) as original, closing(sqlite3.connect(temporary)) as db:
|
||||
original.backup(db)
|
||||
db.execute('PRAGMA foreign_keys=ON')
|
||||
require(db.execute('PRAGMA user_version').fetchone()[0] == 1, 'Unsupported app catalog version')
|
||||
with db:
|
||||
prior = db.execute('SELECT package_sha256 FROM release WHERE id=?', (release_id,)).fetchone()
|
||||
require(prior is None or prior[0] == package['sha256'], 'Existing submission identity conflicts')
|
||||
if prior is None:
|
||||
primary = next(m for m in members if m['kind'] != 'supporting')
|
||||
media = {'hard_disk': 'hard-disk'}.get(primary['kind'], primary['kind'])
|
||||
metadata = {'origin': 'direct_submission', 'submission_reference': ref, 'approval_revision': manifest['approval_revision'], 'author': manifest['author'], 'publisher': manifest['author'], 'software_version': manifest['software_version'], 'license': manifest['metadata']['end_user_terms'], 'project_url': manifest['metadata'].get('project_url', ''), 'description': manifest['description'], 'permission_operator': 'Dezota LLC'}
|
||||
db.execute('INSERT OR IGNORE INTO title VALUES(?,?,?)', (title_id, manifest['title'], 'reviewed-direct-submission'))
|
||||
db.execute('INSERT OR IGNORE INTO package VALUES(?,?)', (package['sha256'], package['byte_size']))
|
||||
db.execute('INSERT INTO release VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)', (release_id,title_id,manifest['software_version'],'Uncategorized',media,'software',locations[package['sha256']]['url'],None,None,None,package['sha256'],'unknown',encoded(metadata)))
|
||||
db.execute('INSERT OR IGNORE INTO download_location VALUES(?,?,?)', (package['sha256'],locations[package['sha256']]['url'],0))
|
||||
for member in members:
|
||||
db.execute('INSERT INTO member VALUES(?,?,?,?,?,?)', (release_id,member['path'],member['sha256'],member['byte_size'],member['kind'],'software' if member['kind'] != 'supporting' else 'supporting'))
|
||||
db.execute('INSERT INTO setup VALUES(?,?,?,?,?,?,?,?)', (release_id,'manual-setup','{}','[]',None,manifest['metadata'].get('setup') or 'Review media and configure the CoCo manually.',encoded(['Direct submission: automatic setup is not runtime-qualified.']),encoded({'scope':'manual-setup','approval_reference':ref})))
|
||||
for number, asset in enumerate(a for a in assets if a['role'] != 'package'):
|
||||
asset_id = release_id + '-asset-' + str(number)
|
||||
kind = 'manual' if asset['role'] == 'manual' else 'cover'
|
||||
db.execute('INSERT INTO asset VALUES(?,?,?,?,?,?,?)',(asset_id,asset['name'],kind,locations[asset['sha256']]['url'],asset['sha256'],asset['byte_size'],manifest['author']))
|
||||
db.execute('INSERT INTO release_asset VALUES(?,?,?,?)',(release_id,asset_id,'reviewed','Approved direct-submission supporting asset'))
|
||||
thumb = locations[asset['sha256']].get('thumbnail', {})
|
||||
db.execute('INSERT INTO asset_delivery VALUES(?,?,?,?,?,?)',(asset_id,locations[asset['sha256']]['url'] if kind=='manual' else None,thumb.get('url'),thumb.get('sha256'),thumb.get('byte_size'),0))
|
||||
exclusions = apply_suppressions(db,retained_suppressions,as_of)
|
||||
db.execute('INSERT OR REPLACE INTO metadata VALUES(?,?)',('submission_approval:'+ref,manifest['approval_sha256']))
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'Candidate catalog integrity failed')
|
||||
# Exclusive hardlink promotion never replaces an output created by a racing operation.
|
||||
os.link(temporary,destination)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
return {'schema_version':1,'submission_reference':ref,'approval_revision':manifest['approval_revision'],'title_id':title_id,'release_id':release_id,'catalog_sha256':digest(destination),'hosting_receipt':hosting_receipt,'target_release':manifest.get('target_release',''),'qualification_record':'Prepared with manual setup; runtime qualification pending.','suppression_result':exclusions}
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('source',type=Path)
|
||||
parser.add_argument('--output',required=True,type=Path)
|
||||
parser.add_argument('--manifest',required=True,type=Path)
|
||||
parser.add_argument('--approval-check',required=True,type=Path)
|
||||
parser.add_argument('--suppressions',required=True,type=Path)
|
||||
parser.add_argument('--files',required=True,type=Path,help='Private JSON mapping approved SHA-256 to local retained file path')
|
||||
parser.add_argument('--hosting-receipt',required=True,type=Path)
|
||||
parser.add_argument('--result',required=True,type=Path)
|
||||
parser.add_argument('--rights-cache',type=Path,help='Persistent exclusion cache override; refresh merges instead of replacing previous exclusions')
|
||||
args=parser.parse_args()
|
||||
from rights import refresh
|
||||
snapshot = load(args.suppressions)
|
||||
# This explicit file/CLI boundary retains authenticated exclusions. The pure
|
||||
# preparation helper above reads the cache but never writes global state.
|
||||
refresh(snapshot,args.rights_cache,catalog=args.source)
|
||||
result=prepare_catalog(args.source,args.output,load(args.manifest),load(args.approval_check),snapshot,load(args.files),load(args.hosting_receipt),rights_cache=args.rights_cache)
|
||||
args.result.parent.mkdir(parents=True,exist_ok=True)
|
||||
with os.fdopen(os.open(args.result, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as stream:stream.write(json.dumps(result,indent=2)+'\n')
|
||||
print('New catalog candidate prepared; runtime qualification/publication remain separate.')
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
Reference in new issue
Block a user