Files

306 lines
16 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Stage the reviewed software catalog as a read-only app resource, never media.
Ordinary source builds get an empty schema if the private catalog is absent.
Release packaging passes --require-full and fails rather than shipping it empty.
The explicit source content profile generates an empty schema without reading
maintainer inputs; normal remains the default.
"""
import argparse
from contextlib import closing
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import tempfile
import struct
import subprocess
import zipfile
import sys
from urllib.parse import urlsplit
from concurrent.futures import ThreadPoolExecutor
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tools'))
from build_content import add_content_profile_argument, atomic_empty_database, content_profile as resolve_content_profile
SCHEMA = Path(__file__).with_name('app-schema.sql')
DEFAULT_SOURCE = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite'
DEFAULT_ARTWORK_CACHE = ROOT / 'media/software-library-full/cache'
MAX_BYTES = 256 * 1024 * 1024
PRIVATE = re.compile(r'(?:file://|/Users/|/home/|/private/|(?<![A-Za-z])[A-Za-z]:[\\/]|(?:^|[\s"\'])\.build/|software-library-full/)', re.I)
def schema_signature(db):
return {(kind, name): ' '.join(sql.split()) for kind, name, sql in db.execute(
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'") if sql}
def validate(path, require_full=False):
"""Validate a standalone finalized export without allowing writes or extensions."""
path = Path(path)
if path.is_symlink() or not path.is_file():
raise ValueError('Catalog input must be a regular, nonsymlink file')
if not 0 < path.stat().st_size <= MAX_BYTES:
raise ValueError('Catalog input has an invalid size')
for suffix in ('-wal', '-journal'):
sidecar = Path(str(path) + suffix)
if sidecar.exists() and sidecar.stat().st_size:
raise ValueError('Catalog has an active transaction sidecar; finalize it before packaging')
with closing(sqlite3.connect(path.resolve().as_uri() + '?mode=ro', uri=True)) as db:
db.execute('PRAGMA query_only=ON')
db.execute('PRAGMA trusted_schema=OFF')
if db.execute('PRAGMA user_version').fetchone()[0] != 1:
raise ValueError('Unsupported catalog schema version')
with closing(sqlite3.connect(':memory:')) as expected:
expected.executescript(SCHEMA.read_text())
if schema_signature(db) != schema_signature(expected):
raise ValueError('Catalog does not match the approved app export schema')
# Older macOS SQLite marks json_valid() unsafe with trusted_schema=OFF.
# The exact schema was just compared to our trusted, checked-in schema.
db.execute('PRAGMA trusted_schema=ON')
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
raise ValueError('Catalog integrity check failed')
if db.execute('PRAGMA foreign_key_check').fetchall():
raise ValueError('Catalog contains broken relationships')
metadata = dict(db.execute('SELECT key,value FROM metadata'))
if metadata.get('schema_version') != '1' or metadata.get('recipe_contract') != '2':
raise ValueError('Unsupported catalog metadata or loading contract')
releases = db.execute('SELECT count(*) FROM release').fetchone()[0]
if require_full and releases == 0:
raise ValueError('A populated catalog is required for release packaging')
if db.execute('SELECT count(*) FROM release LEFT JOIN setup ON release.id=setup.release_id WHERE setup.release_id IS NULL').fetchone()[0]:
raise ValueError('Catalog release is missing loading metadata')
for table, in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"):
for row in db.execute('SELECT * FROM "' + table + '"'):
for value in row:
if isinstance(value, bytes):
raise ValueError('Catalog must not contain bundled media blobs')
if isinstance(value, str) and PRIVATE.search(value):
raise ValueError('Catalog contains a private filesystem reference')
return {'titles': db.execute('SELECT count(*) FROM title').fetchone()[0],
'releases': releases, 'assets': db.execute('SELECT count(*) FROM asset').fetchone()[0]}
def empty_catalog(path):
with closing(sqlite3.connect(path)) as db:
db.executescript(SCHEMA.read_text())
db.executemany('INSERT INTO metadata VALUES(?,?)', [
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-development'),
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')])
db.commit()
def source_catalog(output):
result = atomic_empty_database(output, SCHEMA, (
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-source'),
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')),
('title', 'package', 'release', 'member', 'download_location', 'asset', 'asset_delivery',
'release_asset', 'setup', 'source_hint'), expected_name='catalog.sqlite',
expected_parent='SoftwareLibrary', user_version=1)
return dict(result, schema_version=1, recipe_contract=2, stage='empty-source', titles=0,
releases=0, assets=0, thumbnails=0, empty=True)
def package(source, output, require_full=False, *, content_profile='normal', suppression_manifest=None, rights_cache=None):
if resolve_content_profile(content_profile) == 'source':
if suppression_manifest is not None:
raise ValueError('A rights manifest applies to a full candidate, not the empty source-content profile')
if require_full:
raise ValueError('--require-full cannot be combined with the source content profile')
return source_catalog(output)
source, output = Path(source), Path(output)
if output.is_symlink():
raise ValueError('Catalog destination must not be a symlink')
if source.resolve() == output.resolve():
raise ValueError('Catalog output must differ from the input')
output.parent.mkdir(parents=True, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix='.catalog-', suffix='.sqlite', dir=output.parent)
os.close(fd)
temporary = Path(temporary)
try:
if source.exists() or source.is_symlink():
validate(source, require_full=require_full)
shutil.copyfile(source, temporary)
elif require_full:
raise ValueError('Full catalog is missing. Prepare media/software-library-full/prepared/app-catalog.sqlite before packaging.')
else:
print('warning: Full Software Library catalog is absent; bundling an empty development catalog. See tools/catalog/README.md.')
empty_catalog(temporary)
# Derived metadata is prepared from tracked reviewed associations; the
# source catalog remains unchanged. Retained rights exclusions never
# expire during offline builds; a fresh LIVE gate governs publication.
from related_archive import apply_curated
from rights import apply_cached, read_json
with closing(sqlite3.connect(temporary)) as candidate:
apply_curated(candidate)
manifest = read_json(suppression_manifest) if isinstance(suppression_manifest, (str, Path)) else suppression_manifest
apply_cached(candidate, cache_path=rights_cache, extra=manifest)
candidate.commit()
counts = validate(temporary, require_full=require_full)
digest = hashlib.sha256(temporary.read_bytes()).hexdigest()
os.chmod(temporary, 0o444)
# Avoid touching a resource that is byte-identical across incremental builds.
if output.is_file() and hashlib.sha256(output.read_bytes()).hexdigest() == digest:
os.chmod(output, 0o444)
else:
os.replace(temporary, output)
return dict(counts, sha256=digest, empty=counts['releases'] == 0)
finally:
temporary.unlink(missing_ok=True)
def artwork_key(url, sha):
# Keep in sync with SoftwareLibraryArtworkCache.cacheKey(for:).
return hashlib.sha256(('thumbnail-v1-640\n' + url + '\n' + (sha or 'unverified')).encode()).hexdigest()
def png_thumbnail(path):
data = path.read_bytes()
if not 24 <= len(data) <= 2 * 1024 * 1024 or data[:8] != b'\x89PNG\r\n\x1a\n' or data[12:16] != b'IHDR':
return False
width, height = struct.unpack('>II', data[16:24])
return 0 < width <= 640 and 0 < height <= 640
def screenshot_member(archive):
"""Read one bounded image, never extract archive-supplied paths to disk."""
with zipfile.ZipFile(archive) as images:
members = images.infolist()
if len(members) > 256:
raise ValueError('Artwork archive has too many members')
candidates = [m for m in members if not m.is_dir()
and Path(m.filename).suffix.lower() in ('.png', '.jpg', '.jpeg', '.gif', '.webp')
and not any(part.startswith('.') or part == '__MACOSX' for part in Path(m.filename).parts)
and not (m.flag_bits & 1)
and 0 < m.file_size <= 8 * 1024 * 1024
and (m.external_attr >> 16) & 0o170000 != 0o120000]
def rank(member):
stem = Path(member.filename).stem.lower()
preferred = {'cover': 0, 'title': 1, 'intro': 2}
return (preferred.get(stem, 3), member.filename.casefold(), member.filename)
if not candidates:
return None
member = min(candidates, key=rank)
with images.open(member) as stream:
data = stream.read(8 * 1024 * 1024 + 1)
if len(data) != member.file_size or len(data) > 8 * 1024 * 1024:
raise ValueError('Artwork archive image exceeds its declared size')
return data
def package_artwork(catalog, cache, destination):
"""Generate maintainer thumbnails from previously verified local art.
The content-addressed crawl cache is optional for a developer checkout.
No network requests, original artwork or documents are copied into the app.
"""
cache, destination = Path(cache), Path(destination)
if destination.is_symlink():
raise ValueError('Artwork destination must not be a symlink')
with closing(sqlite3.connect(Path(catalog).resolve().as_uri() + '?mode=ro', uri=True)) as db:
rows = db.execute("SELECT DISTINCT a.source_url,a.sha256,a.byte_size FROM asset a JOIN release_asset ra ON ra.asset_id=a.id WHERE a.kind IN ('replacement-label','screenshot') ORDER BY a.id").fetchall()
destination.mkdir(parents=True, exist_ok=True)
def convert(row):
url, sha, size = row
parts = urlsplit(url)
suffix = Path(parts.path).suffix.lower()
if (parts.scheme != 'https' or parts.hostname != 'colorcomputerarchive.com' or not parts.path.startswith('/repo/')
or suffix not in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.zip') or not sha or not re.fullmatch('[a-f0-9]{64}', sha)):
return None
source = cache / (sha + suffix)
if not source.is_file() or source.is_symlink() or not 0 < source.stat().st_size <= 8 * 1024 * 1024:
return None
original = source.read_bytes()
if len(original) != size or hashlib.sha256(original).hexdigest() != sha:
raise ValueError('Cached artwork does not match its catalog fingerprint')
name = artwork_key(url, sha) + '.png'
target = destination / name
# A generated output is already bounded; its input identity is in its name.
if target.is_file() and not target.is_symlink() and png_thumbnail(target):
return name
# ZIP assets retain their original URL/fingerprint and attribution. Only
# a single selected image becomes a thumbnail; no archive is shipped.
image_data = screenshot_member(source) if suffix == '.zip' else None
if suffix == '.zip' and image_data is None:
return None
fd, temporary = tempfile.mkstemp(prefix='.artwork-', suffix='.png', dir=destination)
os.close(fd)
temporary = Path(temporary)
try:
with tempfile.TemporaryDirectory(prefix='vcc-artwork-') as scratch:
input_path = source
if image_data is not None:
input_path = Path(scratch) / 'image'
input_path.write_bytes(image_data)
subprocess.run(['/usr/bin/sips', '-s', 'format', 'png', '-Z', '640', str(input_path), '--out', str(temporary)],
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
if not png_thumbnail(temporary):
raise ValueError('Generated artwork thumbnail is not a bounded PNG')
os.chmod(temporary, 0o444)
os.replace(temporary, target)
finally:
temporary.unlink(missing_ok=True)
return name
with ThreadPoolExecutor(max_workers=4) as workers:
names = {name for name in workers.map(convert, rows) if name}
# Clear only our generated PNG namespace after a successful complete pass.
for path in destination.iterdir():
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and path.name not in names and not path.is_symlink():
path.unlink()
return len(names)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=Path)
add_content_profile_argument(parser)
parser.add_argument('--output', type=Path)
parser.add_argument('--check-only', action='store_true', help='Validate input without staging a resource')
parser.add_argument('--require-full', action='store_true')
parser.add_argument('--suppression-manifest', type=Path, help='Explicit fresh reviewed rights-withdrawal manifest for candidate preparation')
parser.add_argument('--rights-cache', type=Path, help='Retained exclusion cache override; ordinary full preparation uses the persistent maintainer cache')
args = parser.parse_args()
if not args.check_only and args.output is None:
parser.error('--output is required unless --check-only is used')
try:
profile = resolve_content_profile(args.content_profile)
if profile == 'source':
if args.suppression_manifest is not None:
raise ValueError('A rights manifest applies to a full candidate, not the empty source-content profile')
if args.require_full:
raise ValueError('--require-full cannot be combined with the source content profile')
if args.check_only:
with tempfile.TemporaryDirectory(prefix='vcc-source-catalog-check-') as temporary:
result = source_catalog(Path(temporary) / 'SoftwareLibrary/catalog.sqlite')
else:
result = source_catalog(args.output)
else:
source = args.source if args.source is not None else DEFAULT_SOURCE
if args.check_only:
validate(source, args.require_full)
with tempfile.TemporaryDirectory(prefix='vcc-suppression-check-') as temporary:
result = package(source, Path(temporary) / 'catalog.sqlite', args.require_full,
suppression_manifest=args.suppression_manifest, rights_cache=args.rights_cache)
result['suppression_checked'] = True
else:
result = package(source, args.output, args.require_full, suppression_manifest=args.suppression_manifest, rights_cache=args.rights_cache)
if not args.check_only:
artwork = args.output.parent / 'Artwork'
if artwork.is_symlink(): raise ValueError('Artwork destination must not be a symlink')
if artwork.is_dir():
for path in artwork.iterdir():
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and not path.is_symlink(): path.unlink()
result['thumbnails'] = 0
except (ValueError, OSError, sqlite3.DatabaseError, subprocess.CalledProcessError) as error:
parser.exit(1, 'error: Software Library catalog: ' + str(error) + '\n')
print('Software Library catalog: ' + json.dumps(result, sort_keys=True))
if __name__ == '__main__':
main()