Publish VCC Plus OSS source distribution for 3.6.2 build 60
This commit is contained in:
commit
c652a5276d
1192 files changed
+272368
No files matched your search
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Stage the reviewed software catalog as a read-only app resource, never media.
|
||||
|
||||
Ordinary source builds get an empty schema if the private catalog is absent.
|
||||
Release packaging passes --require-full and fails rather than shipping it empty.
|
||||
The explicit source content profile generates an empty schema without reading
|
||||
maintainer inputs; normal remains the default.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import struct
|
||||
import subprocess
|
||||
import zipfile
|
||||
import sys
|
||||
from urllib.parse import urlsplit
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
from build_content import add_content_profile_argument, atomic_empty_database, content_profile as resolve_content_profile
|
||||
SCHEMA = Path(__file__).with_name('app-schema.sql')
|
||||
DEFAULT_SOURCE = ROOT / 'media/software-library-full/prepared/app-catalog.sqlite'
|
||||
DEFAULT_ARTWORK_CACHE = ROOT / 'media/software-library-full/cache'
|
||||
MAX_BYTES = 256 * 1024 * 1024
|
||||
PRIVATE = re.compile(r'(?:file://|/Users/|/home/|/private/|(?<![A-Za-z])[A-Za-z]:[\\/]|(?:^|[\s"\'])\.build/|software-library-full/)', re.I)
|
||||
|
||||
|
||||
def schema_signature(db):
|
||||
return {(kind, name): ' '.join(sql.split()) for kind, name, sql in db.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'") if sql}
|
||||
|
||||
|
||||
def validate(path, require_full=False):
|
||||
"""Validate a standalone finalized export without allowing writes or extensions."""
|
||||
path = Path(path)
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError('Catalog input must be a regular, nonsymlink file')
|
||||
if not 0 < path.stat().st_size <= MAX_BYTES:
|
||||
raise ValueError('Catalog input has an invalid size')
|
||||
for suffix in ('-wal', '-journal'):
|
||||
sidecar = Path(str(path) + suffix)
|
||||
if sidecar.exists() and sidecar.stat().st_size:
|
||||
raise ValueError('Catalog has an active transaction sidecar; finalize it before packaging')
|
||||
with closing(sqlite3.connect(path.resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
||||
db.execute('PRAGMA query_only=ON')
|
||||
db.execute('PRAGMA trusted_schema=OFF')
|
||||
if db.execute('PRAGMA user_version').fetchone()[0] != 1:
|
||||
raise ValueError('Unsupported catalog schema version')
|
||||
with closing(sqlite3.connect(':memory:')) as expected:
|
||||
expected.executescript(SCHEMA.read_text())
|
||||
if schema_signature(db) != schema_signature(expected):
|
||||
raise ValueError('Catalog does not match the approved app export schema')
|
||||
# Older macOS SQLite marks json_valid() unsafe with trusted_schema=OFF.
|
||||
# The exact schema was just compared to our trusted, checked-in schema.
|
||||
db.execute('PRAGMA trusted_schema=ON')
|
||||
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
|
||||
raise ValueError('Catalog integrity check failed')
|
||||
if db.execute('PRAGMA foreign_key_check').fetchall():
|
||||
raise ValueError('Catalog contains broken relationships')
|
||||
metadata = dict(db.execute('SELECT key,value FROM metadata'))
|
||||
if metadata.get('schema_version') != '1' or metadata.get('recipe_contract') != '2':
|
||||
raise ValueError('Unsupported catalog metadata or loading contract')
|
||||
releases = db.execute('SELECT count(*) FROM release').fetchone()[0]
|
||||
if require_full and releases == 0:
|
||||
raise ValueError('A populated catalog is required for release packaging')
|
||||
if db.execute('SELECT count(*) FROM release LEFT JOIN setup ON release.id=setup.release_id WHERE setup.release_id IS NULL').fetchone()[0]:
|
||||
raise ValueError('Catalog release is missing loading metadata')
|
||||
for table, in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"):
|
||||
for row in db.execute('SELECT * FROM "' + table + '"'):
|
||||
for value in row:
|
||||
if isinstance(value, bytes):
|
||||
raise ValueError('Catalog must not contain bundled media blobs')
|
||||
if isinstance(value, str) and PRIVATE.search(value):
|
||||
raise ValueError('Catalog contains a private filesystem reference')
|
||||
return {'titles': db.execute('SELECT count(*) FROM title').fetchone()[0],
|
||||
'releases': releases, 'assets': db.execute('SELECT count(*) FROM asset').fetchone()[0]}
|
||||
|
||||
|
||||
def empty_catalog(path):
|
||||
with closing(sqlite3.connect(path)) as db:
|
||||
db.executescript(SCHEMA.read_text())
|
||||
db.executemany('INSERT INTO metadata VALUES(?,?)', [
|
||||
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-development'),
|
||||
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')])
|
||||
db.commit()
|
||||
|
||||
|
||||
def source_catalog(output):
|
||||
result = atomic_empty_database(output, SCHEMA, (
|
||||
('schema_version', '1'), ('recipe_contract', '2'), ('stage', 'empty-source'),
|
||||
('archive_attribution', 'Software catalog sources: Color Computer Archive. Original authors retain their rights.')),
|
||||
('title', 'package', 'release', 'member', 'download_location', 'asset', 'asset_delivery',
|
||||
'release_asset', 'setup', 'source_hint'), expected_name='catalog.sqlite',
|
||||
expected_parent='SoftwareLibrary', user_version=1)
|
||||
return dict(result, schema_version=1, recipe_contract=2, stage='empty-source', titles=0,
|
||||
releases=0, assets=0, thumbnails=0, empty=True)
|
||||
|
||||
|
||||
def package(source, output, require_full=False, *, content_profile='normal'):
|
||||
if resolve_content_profile(content_profile) == 'source':
|
||||
if require_full:
|
||||
raise ValueError('--require-full cannot be combined with the source content profile')
|
||||
return source_catalog(output)
|
||||
source, output = Path(source), Path(output)
|
||||
if output.is_symlink():
|
||||
raise ValueError('Catalog destination must not be a symlink')
|
||||
if source.resolve() == output.resolve():
|
||||
raise ValueError('Catalog output must differ from the input')
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix='.catalog-', suffix='.sqlite', dir=output.parent)
|
||||
os.close(fd)
|
||||
temporary = Path(temporary)
|
||||
try:
|
||||
if source.exists() or source.is_symlink():
|
||||
validate(source, require_full=require_full)
|
||||
shutil.copyfile(source, temporary)
|
||||
elif require_full:
|
||||
raise ValueError('Full catalog is missing. Prepare media/software-library-full/prepared/app-catalog.sqlite before packaging.')
|
||||
else:
|
||||
print('warning: Full Software Library catalog is absent; bundling an empty development catalog. See tools/catalog/README.md.')
|
||||
empty_catalog(temporary)
|
||||
counts = validate(temporary, require_full=require_full)
|
||||
digest = hashlib.sha256(temporary.read_bytes()).hexdigest()
|
||||
os.chmod(temporary, 0o444)
|
||||
# Avoid touching a resource that is byte-identical across incremental builds.
|
||||
if output.is_file() and hashlib.sha256(output.read_bytes()).hexdigest() == digest:
|
||||
os.chmod(output, 0o444)
|
||||
else:
|
||||
os.replace(temporary, output)
|
||||
return dict(counts, sha256=digest, empty=counts['releases'] == 0)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
|
||||
def artwork_key(url, sha):
|
||||
# Keep in sync with SoftwareLibraryArtworkCache.cacheKey(for:).
|
||||
return hashlib.sha256(('thumbnail-v1-640\n' + url + '\n' + (sha or 'unverified')).encode()).hexdigest()
|
||||
|
||||
|
||||
def png_thumbnail(path):
|
||||
data = path.read_bytes()
|
||||
if not 24 <= len(data) <= 2 * 1024 * 1024 or data[:8] != b'\x89PNG\r\n\x1a\n' or data[12:16] != b'IHDR':
|
||||
return False
|
||||
width, height = struct.unpack('>II', data[16:24])
|
||||
return 0 < width <= 640 and 0 < height <= 640
|
||||
|
||||
|
||||
def screenshot_member(archive):
|
||||
"""Read one bounded image, never extract archive-supplied paths to disk."""
|
||||
with zipfile.ZipFile(archive) as images:
|
||||
members = images.infolist()
|
||||
if len(members) > 256:
|
||||
raise ValueError('Artwork archive has too many members')
|
||||
candidates = [m for m in members if not m.is_dir()
|
||||
and Path(m.filename).suffix.lower() in ('.png', '.jpg', '.jpeg', '.gif', '.webp')
|
||||
and not any(part.startswith('.') or part == '__MACOSX' for part in Path(m.filename).parts)
|
||||
and not (m.flag_bits & 1)
|
||||
and 0 < m.file_size <= 8 * 1024 * 1024
|
||||
and (m.external_attr >> 16) & 0o170000 != 0o120000]
|
||||
def rank(member):
|
||||
stem = Path(member.filename).stem.lower()
|
||||
preferred = {'cover': 0, 'title': 1, 'intro': 2}
|
||||
return (preferred.get(stem, 3), member.filename.casefold(), member.filename)
|
||||
if not candidates:
|
||||
return None
|
||||
member = min(candidates, key=rank)
|
||||
with images.open(member) as stream:
|
||||
data = stream.read(8 * 1024 * 1024 + 1)
|
||||
if len(data) != member.file_size or len(data) > 8 * 1024 * 1024:
|
||||
raise ValueError('Artwork archive image exceeds its declared size')
|
||||
return data
|
||||
|
||||
|
||||
def package_artwork(catalog, cache, destination):
|
||||
"""Generate maintainer thumbnails from previously verified local art.
|
||||
|
||||
The content-addressed crawl cache is optional for a developer checkout.
|
||||
No network requests, original artwork or documents are copied into the app.
|
||||
"""
|
||||
cache, destination = Path(cache), Path(destination)
|
||||
if destination.is_symlink():
|
||||
raise ValueError('Artwork destination must not be a symlink')
|
||||
with closing(sqlite3.connect(Path(catalog).resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
||||
rows = db.execute("SELECT DISTINCT a.source_url,a.sha256,a.byte_size FROM asset a JOIN release_asset ra ON ra.asset_id=a.id WHERE a.kind IN ('replacement-label','screenshot') ORDER BY a.id").fetchall()
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
def convert(row):
|
||||
url, sha, size = row
|
||||
parts = urlsplit(url)
|
||||
suffix = Path(parts.path).suffix.lower()
|
||||
if (parts.scheme != 'https' or parts.hostname != 'colorcomputerarchive.com' or not parts.path.startswith('/repo/')
|
||||
or suffix not in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.zip') or not sha or not re.fullmatch('[a-f0-9]{64}', sha)):
|
||||
return None
|
||||
source = cache / (sha + suffix)
|
||||
if not source.is_file() or source.is_symlink() or not 0 < source.stat().st_size <= 8 * 1024 * 1024:
|
||||
return None
|
||||
original = source.read_bytes()
|
||||
if len(original) != size or hashlib.sha256(original).hexdigest() != sha:
|
||||
raise ValueError('Cached artwork does not match its catalog fingerprint')
|
||||
name = artwork_key(url, sha) + '.png'
|
||||
target = destination / name
|
||||
# A generated output is already bounded; its input identity is in its name.
|
||||
if target.is_file() and not target.is_symlink() and png_thumbnail(target):
|
||||
return name
|
||||
# ZIP assets retain their original URL/fingerprint and attribution. Only
|
||||
# a single selected image becomes a thumbnail; no archive is shipped.
|
||||
image_data = screenshot_member(source) if suffix == '.zip' else None
|
||||
if suffix == '.zip' and image_data is None:
|
||||
return None
|
||||
fd, temporary = tempfile.mkstemp(prefix='.artwork-', suffix='.png', dir=destination)
|
||||
os.close(fd)
|
||||
temporary = Path(temporary)
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-artwork-') as scratch:
|
||||
input_path = source
|
||||
if image_data is not None:
|
||||
input_path = Path(scratch) / 'image'
|
||||
input_path.write_bytes(image_data)
|
||||
subprocess.run(['/usr/bin/sips', '-s', 'format', 'png', '-Z', '640', str(input_path), '--out', str(temporary)],
|
||||
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
||||
if not png_thumbnail(temporary):
|
||||
raise ValueError('Generated artwork thumbnail is not a bounded PNG')
|
||||
os.chmod(temporary, 0o444)
|
||||
os.replace(temporary, target)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
return name
|
||||
with ThreadPoolExecutor(max_workers=4) as workers:
|
||||
names = {name for name in workers.map(convert, rows) if name}
|
||||
# Clear only our generated PNG namespace after a successful complete pass.
|
||||
for path in destination.iterdir():
|
||||
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and path.name not in names and not path.is_symlink():
|
||||
path.unlink()
|
||||
return len(names)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--source', type=Path)
|
||||
add_content_profile_argument(parser)
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--check-only', action='store_true', help='Validate input without staging a resource')
|
||||
parser.add_argument('--require-full', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if not args.check_only and args.output is None:
|
||||
parser.error('--output is required unless --check-only is used')
|
||||
try:
|
||||
profile = resolve_content_profile(args.content_profile)
|
||||
if profile == 'source':
|
||||
if args.require_full:
|
||||
raise ValueError('--require-full cannot be combined with the source content profile')
|
||||
if args.check_only:
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-source-catalog-check-') as temporary:
|
||||
result = source_catalog(Path(temporary) / 'SoftwareLibrary/catalog.sqlite')
|
||||
else:
|
||||
result = source_catalog(args.output)
|
||||
else:
|
||||
source = args.source if args.source is not None else DEFAULT_SOURCE
|
||||
result = validate(source, args.require_full) if args.check_only else package(source, args.output, args.require_full)
|
||||
if not args.check_only:
|
||||
artwork = args.output.parent / 'Artwork'
|
||||
if artwork.is_symlink(): raise ValueError('Artwork destination must not be a symlink')
|
||||
if artwork.is_dir():
|
||||
for path in artwork.iterdir():
|
||||
if re.fullmatch(r'[a-f0-9]{64}\.png', path.name) and not path.is_symlink(): path.unlink()
|
||||
result['thumbnails'] = 0
|
||||
except (ValueError, OSError, sqlite3.DatabaseError, subprocess.CalledProcessError) as error:
|
||||
parser.exit(1, 'error: Software Library catalog: ' + str(error) + '\n')
|
||||
print('Software Library catalog: ' + json.dumps(result, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user