306 lines
16 KiB
Python
306 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Stage the reviewed software catalog as a read-only app resource, never media.
|
|
|
|
Ordinary source builds get an empty schema if the private catalog is absent.
|
|
Release packaging passes --require-full and fails rather than shipping it empty.
|
|
The explicit source content profile generates an empty schema without reading
|
|
maintainer inputs; normal remains the default.
|
|
"""
|
|
import argparse
|
|
from contextlib import closing
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import tempfile
|
|
import struct
|
|
import subprocess
|
|
import zipfile
|
|
import sys
|
|
from urllib.parse import urlsplit
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'tools'))
|
|
from build_content import add_content_profile_argument, atomic_empty_database, content_profile as resolve_content_profile
|
|
SCHEMA = Path(__file__).with_name('app-schema.sql')
|
|
DEFAULT_SOURCE = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite'
|
|
DEFAULT_ARTWORK_CACHE = ROOT / 'media/software-library-full/cache'
|
|
MAX_BYTES = 256 * 1024 * 1024
|
|
PRIVATE = re.compile(r'(?:file://|/Users/|/home/|/private/|(?<![A-Za-z])[A-Za-z]:[\\/]|(?:^|[\s"\'])\.build/|software-library-full/)', re.I)
|
|
|
|
|
|
def schema_signature(db):
|
|
return {(kind, name): ' '.join(sql.split()) for kind, name, sql in db.execute(
|
|
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'") if sql}
|
|
|
|
|
|
def validate(path, require_full=False):
|
|
"""Validate a standalone finalized export without allowing writes or extensions."""
|
|
path = Path(path)
|
|
if path.is_symlink() or not path.is_file():
|
|
raise ValueError('Catalog input must be a regular, nonsymlink file')
|
|
if not 0 < path.stat().st_size <= MAX_BYTES:
|
|
raise ValueError('Catalog input has an invalid size')
|
|
for suffix in ('-wal', '-journal'):
|
|
sidecar = Path(str(path) + suffix)
|
|
if sidecar.exists() and sidecar.stat().st_size:
|
|
raise ValueError('Catalog has an active transaction sidecar; finalize it before packaging')
|
|
with closing(sqlite3.connect(path.resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
|
db.execute('PRAGMA query_only=ON')
|
|
db.execute('PRAGMA trusted_schema=OFF')
|
|
if db.execute('PRAGMA user_version').fetchone()[0] != 1:
|
|
raise ValueError('Unsupported catalog schema version')
|
|
with closing(sqlite3.connect(':memory:')) as expected:
|
|
expected.executescript(SCHEMA.read_text())
|
|
if schema_signature(db) != schema_signature(expected):
|
|
raise ValueError('Catalog does not match the approved app export schema')
|
|
# Older macOS SQLite marks json_valid() unsafe with trusted_schema=OFF.
|
|
# The exact schema was just compared to our trusted, checked-in schema.
|
|
db.execute('PRAGMA trusted_schema=ON')
|
|
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
|
|
raise ValueError('Catalog integrity check failed')
|
|
if db.execute('PRAGMA foreign_key_check').fetchall():
|
|
raise ValueError('Catalog contains broken relationships')
|
|
metadata = dict(db.execute('SELECT key,value FROM metadata'))
|
|
if metadata.get('schema_version') != '1' or metadata.get('recipe_contract') != '2':
|
|
raise ValueError('Unsupported catalog metadata or loading contract')
|
|
releases = db.execute('SELECT count(*) FROM release').fetchone()[0]
|
|
if require_full and releases == 0:
|
|
raise ValueError('A populated catalog is required for release packaging')
|
|
if db.execute('SELECT count(*) FROM release LEFT JOIN setup ON release.id=setup.release_id WHERE setup.release_id IS NULL').fetchone()[0]:
|
|
raise ValueError('Catalog release is missing loading metadata')
|
|
for table, in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"):
|
|
for row in db.execute('SELECT * FROM "' + table + '"'):
|
|
for value in row:
|
|
if isinstance(value, bytes):
|
|
raise ValueError('Catalog must not contain bundled media blobs')
|
|
if isinstance(value, str) and PRIVATE.search(value):
|
|
raise ValueError('Catalog contains a private filesystem reference')
|
|
return {'titles': db.execute('SELECT count(*) FROM title').fetchone()[0],
|
|
'releases': releases, 'assets': db.execute('SELECT count(*) FROM asset').fetchone()[0]}
|
|
|
|
|
|
def empty_catalog(path):
|
|
with closing(sqlite3.connect(path)) as db:
|
|
db.executescript(SCHEMA.read_text())
|
|
db.executemany('INSERT INTO metadata VALUES(?,?)', [
|
|
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-development'),
|
|
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')])
|
|
db.commit()
|
|
|
|
|
|
def source_catalog(output):
|
|
result = atomic_empty_database(output, SCHEMA, (
|
|
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-source'),
|
|
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')),
|
|
('title', 'package', 'release', 'member', 'download_location', 'asset', 'asset_delivery',
|
|
'release_asset', 'setup', 'source_hint'), expected_name='catalog.sqlite',
|
|
expected_parent='SoftwareLibrary', user_version=1)
|
|
return dict(result, schema_version=1, recipe_contract=2, stage='empty-source', titles=0,
|
|
releases=0, assets=0, thumbnails=0, empty=True)
|
|
|
|
|
|
def package(source, output, require_full=False, *, content_profile='normal', suppression_manifest=None, rights_cache=None):
|
|
if resolve_content_profile(content_profile) == 'source':
|
|
if suppression_manifest is not None:
|
|
raise ValueError('A rights manifest applies to a full candidate, not the empty source-content profile')
|
|
if require_full:
|
|
raise ValueError('--require-full cannot be combined with the source content profile')
|
|
return source_catalog(output)
|
|
source, output = Path(source), Path(output)
|
|
if output.is_symlink():
|
|
raise ValueError('Catalog destination must not be a symlink')
|
|
if source.resolve() == output.resolve():
|
|
raise ValueError('Catalog output must differ from the input')
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, temporary = tempfile.mkstemp(prefix='.catalog-', suffix='.sqlite', dir=output.parent)
|
|
os.close(fd)
|
|
temporary = Path(temporary)
|
|
try:
|
|
if source.exists() or source.is_symlink():
|
|
validate(source, require_full=require_full)
|
|
shutil.copyfile(source, temporary)
|
|
elif require_full:
|
|
raise ValueError('Full catalog is missing. Prepare media/software-library-full/prepared/app-catalog.sqlite before packaging.')
|
|
else:
|
|
print('warning: Full Software Library catalog is absent; bundling an empty development catalog. See tools/catalog/README.md.')
|
|
empty_catalog(temporary)
|
|
# Derived metadata is prepared from tracked reviewed associations; the
|
|
# source catalog remains unchanged. Retained rights exclusions never
|
|
# expire during offline builds; a fresh LIVE gate governs publication.
|
|
from related_archive import apply_curated
|
|
from rights import apply_cached, read_json
|
|
with closing(sqlite3.connect(temporary)) as candidate:
|
|
apply_curated(candidate)
|
|
manifest = read_json(suppression_manifest) if isinstance(suppression_manifest, (str, Path)) else suppression_manifest
|
|
apply_cached(candidate, cache_path=rights_cache, extra=manifest)
|
|
candidate.commit()
|
|
counts = validate(temporary, require_full=require_full)
|
|
digest = hashlib.sha256(temporary.read_bytes()).hexdigest()
|
|
os.chmod(temporary, 0o444)
|
|
# Avoid touching a resource that is byte-identical across incremental builds.
|
|
if output.is_file() and hashlib.sha256(output.read_bytes()).hexdigest() == digest:
|
|
os.chmod(output, 0o444)
|
|
else:
|
|
os.replace(temporary, output)
|
|
return dict(counts, sha256=digest, empty=counts['releases'] == 0)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
|
|
def artwork_key(url, sha):
|
|
# Keep in sync with SoftwareLibraryArtworkCache.cacheKey(for:).
|
|
return hashlib.sha256(('thumbnail-v1-640\n' + url + '\n' + (sha or 'unverified')).encode()).hexdigest()
|
|
|
|
|
|
def png_thumbnail(path):
|
|
data = path.read_bytes()
|
|
if not 24 <= len(data) <= 2 * 1024 * 1024 or data[:8] != b'\x89PNG\r\n\x1a\n' or data[12:16] != b'IHDR':
|
|
return False
|
|
width, height = struct.unpack('>II', data[16:24])
|
|
return 0 < width <= 640 and 0 < height <= 640
|
|
|
|
|
|
def screenshot_member(archive):
|
|
"""Read one bounded image, never extract archive-supplied paths to disk."""
|
|
with zipfile.ZipFile(archive) as images:
|
|
members = images.infolist()
|
|
if len(members) > 256:
|
|
raise ValueError('Artwork archive has too many members')
|
|
candidates = [m for m in members if not m.is_dir()
|
|
and Path(m.filename).suffix.lower() in ('.png', '.jpg', '.jpeg', '.gif', '.webp')
|
|
and not any(part.startswith('.') or part == '__MACOSX' for part in Path(m.filename).parts)
|
|
and not (m.flag_bits & 1)
|
|
and 0 < m.file_size <= 8 * 1024 * 1024
|
|
and (m.external_attr >> 16) & 0o170000 != 0o120000]
|
|
def rank(member):
|
|
stem = Path(member.filename).stem.lower()
|
|
preferred = {'cover': 0, 'title': 1, 'intro': 2}
|
|
return (preferred.get(stem, 3), member.filename.casefold(), member.filename)
|
|
if not candidates:
|
|
return None
|
|
member = min(candidates, key=rank)
|
|
with images.open(member) as stream:
|
|
data = stream.read(8 * 1024 * 1024 + 1)
|
|
if len(data) != member.file_size or len(data) > 8 * 1024 * 1024:
|
|
raise ValueError('Artwork archive image exceeds its declared size')
|
|
return data
|
|
|
|
|
|
def package_artwork(catalog, cache, destination):
|
|
"""Generate maintainer thumbnails from previously verified local art.
|
|
|
|
The content-addressed crawl cache is optional for a developer checkout.
|
|
No network requests, original artwork or documents are copied into the app.
|
|
"""
|
|
cache, destination = Path(cache), Path(destination)
|
|
if destination.is_symlink():
|
|
raise ValueError('Artwork destination must not be a symlink')
|
|
with closing(sqlite3.connect(Path(catalog).resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
|
rows = db.execute("SELECT DISTINCT a.source_url,a.sha256,a.byte_size FROM asset a JOIN release_asset ra ON ra.asset_id=a.id WHERE a.kind IN ('replacement-label','screenshot') ORDER BY a.id").fetchall()
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
def convert(row):
|
|
url, sha, size = row
|
|
parts = urlsplit(url)
|
|
suffix = Path(parts.path).suffix.lower()
|
|
if (parts.scheme != 'https' or parts.hostname != 'colorcomputerarchive.com' or not parts.path.startswith('/repo/')
|
|
or suffix not in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.zip') or not sha or not re.fullmatch('[a-f0-9]{64}', sha)):
|
|
return None
|
|
source = cache / (sha + suffix)
|
|
if not source.is_file() or source.is_symlink() or not 0 < source.stat().st_size <= 8 * 1024 * 1024:
|
|
return None
|
|
original = source.read_bytes()
|
|
if len(original) != size or hashlib.sha256(original).hexdigest() != sha:
|
|
raise ValueError('Cached artwork does not match its catalog fingerprint')
|
|
name = artwork_key(url, sha) + '.png'
|
|
target = destination / name
|
|
# A generated output is already bounded; its input identity is in its name.
|
|
if target.is_file() and not target.is_symlink() and png_thumbnail(target):
|
|
return name
|
|
# ZIP assets retain their original URL/fingerprint and attribution. Only
|
|
# a single selected image becomes a thumbnail; no archive is shipped.
|
|
image_data = screenshot_member(source) if suffix == '.zip' else None
|
|
if suffix == '.zip' and image_data is None:
|
|
return None
|
|
fd, temporary = tempfile.mkstemp(prefix='.artwork-', suffix='.png', dir=destination)
|
|
os.close(fd)
|
|
temporary = Path(temporary)
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix='vcc-artwork-') as scratch:
|
|
input_path = source
|
|
if image_data is not None:
|
|
input_path = Path(scratch) / 'image'
|
|
input_path.write_bytes(image_data)
|
|
subprocess.run(['/usr/bin/sips', '-s', 'format', 'png', '-Z', '640', str(input_path), '--out', str(temporary)],
|
|
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
|
if not png_thumbnail(temporary):
|
|
raise ValueError('Generated artwork thumbnail is not a bounded PNG')
|
|
os.chmod(temporary, 0o444)
|
|
os.replace(temporary, target)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
return name
|
|
with ThreadPoolExecutor(max_workers=4) as workers:
|
|
names = {name for name in workers.map(convert, rows) if name}
|
|
# Clear only our generated PNG namespace after a successful complete pass.
|
|
for path in destination.iterdir():
|
|
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and path.name not in names and not path.is_symlink():
|
|
path.unlink()
|
|
return len(names)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--source', type=Path)
|
|
add_content_profile_argument(parser)
|
|
parser.add_argument('--output', type=Path)
|
|
parser.add_argument('--check-only', action='store_true', help='Validate input without staging a resource')
|
|
parser.add_argument('--require-full', action='store_true')
|
|
parser.add_argument('--suppression-manifest', type=Path, help='Explicit fresh reviewed rights-withdrawal manifest for candidate preparation')
|
|
parser.add_argument('--rights-cache', type=Path, help='Retained exclusion cache override; ordinary full preparation uses the persistent maintainer cache')
|
|
args = parser.parse_args()
|
|
if not args.check_only and args.output is None:
|
|
parser.error('--output is required unless --check-only is used')
|
|
try:
|
|
profile = resolve_content_profile(args.content_profile)
|
|
if profile == 'source':
|
|
if args.suppression_manifest is not None:
|
|
raise ValueError('A rights manifest applies to a full candidate, not the empty source-content profile')
|
|
if args.require_full:
|
|
raise ValueError('--require-full cannot be combined with the source content profile')
|
|
if args.check_only:
|
|
with tempfile.TemporaryDirectory(prefix='vcc-source-catalog-check-') as temporary:
|
|
result = source_catalog(Path(temporary) / 'SoftwareLibrary/catalog.sqlite')
|
|
else:
|
|
result = source_catalog(args.output)
|
|
else:
|
|
source = args.source if args.source is not None else DEFAULT_SOURCE
|
|
if args.check_only:
|
|
validate(source, args.require_full)
|
|
with tempfile.TemporaryDirectory(prefix='vcc-suppression-check-') as temporary:
|
|
result = package(source, Path(temporary) / 'catalog.sqlite', args.require_full,
|
|
suppression_manifest=args.suppression_manifest, rights_cache=args.rights_cache)
|
|
result['suppression_checked'] = True
|
|
else:
|
|
result = package(source, args.output, args.require_full, suppression_manifest=args.suppression_manifest, rights_cache=args.rights_cache)
|
|
if not args.check_only:
|
|
artwork = args.output.parent / 'Artwork'
|
|
if artwork.is_symlink(): raise ValueError('Artwork destination must not be a symlink')
|
|
if artwork.is_dir():
|
|
for path in artwork.iterdir():
|
|
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and not path.is_symlink(): path.unlink()
|
|
result['thumbnails'] = 0
|
|
except (ValueError, OSError, sqlite3.DatabaseError, subprocess.CalledProcessError) as error:
|
|
parser.exit(1, 'error: Software Library catalog: ' + str(error) + '\n')
|
|
print('Software Library catalog: ' + json.dumps(result, sort_keys=True))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|