Publish VCC Plus OSS source distribution for 3.6.2 build 60
This commit is contained in:
commit
c652a5276d
1192 files changed
+272368
No files matched your search
@@ -0,0 +1,248 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Offline content-profile checks using authored disposable inputs only."""
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
RESOURCES = (
|
||||
('tools/catalog/package.py', 'tools/catalog/app-schema.sql', 'SoftwareLibrary/catalog.sqlite'),
|
||||
('tools/knowledge/package_archive.py', 'tools/knowledge/archive-library.sql', 'ArchiveLibrary/ArchiveLibrary.sqlite'),
|
||||
)
|
||||
|
||||
|
||||
class BuildContentTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory(prefix='vcc-content-profile-')
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.root = Path(temporary.name)
|
||||
self.source = self.root / 'Source With Spaces'
|
||||
self.build = self.root / 'Build With Spaces'
|
||||
for relative in ['tools/build_content.py', *(p for r in RESOURCES for p in r[:2])]:
|
||||
destination = self.source / relative
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(ROOT / relative, destination)
|
||||
self.tempting = []
|
||||
for relative in ('media/software-library-full/prepared/app-catalog.sqlite',
|
||||
'media/app-inputs/archive-library/ArchiveLibrary.sqlite',
|
||||
'media/knowledge-base/archive-library/ArchiveLibrary.sqlite'):
|
||||
target = self.source / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(b'PRIVATE-DATA-TEST-MARKER\x00')
|
||||
self.tempting.append(target)
|
||||
|
||||
def run_script(self, resource, *extra, profile='source', env_profile=None, output=True, check=True):
|
||||
command = [sys.executable, str(self.source / resource[0])]
|
||||
if profile is not None:
|
||||
command += ['--content-profile', profile]
|
||||
if output:
|
||||
command += ['--output', str(self.build / resource[2])]
|
||||
environment = os.environ.copy()
|
||||
environment.pop('VCC_BUILD_CONTENT_PROFILE', None)
|
||||
if env_profile is not None:
|
||||
environment['VCC_BUILD_CONTENT_PROFILE'] = env_profile
|
||||
completed = subprocess.run([*command, *extra], cwd=self.source,
|
||||
env=environment, capture_output=True, text=True)
|
||||
if check:
|
||||
self.assertEqual(completed.returncode, 0, completed.stderr)
|
||||
return completed
|
||||
|
||||
def metadata(self, resource, **kwargs):
|
||||
return json.loads(self.run_script(resource, **kwargs).stdout.split(': ', 1)[1])
|
||||
|
||||
def test_software_schema_and_reader_contract_have_no_content(self):
|
||||
resource = RESOURCES[0]
|
||||
info = self.metadata(resource)
|
||||
self.assertTrue(info['empty'])
|
||||
self.assertEqual((info['titles'], info['releases'], info['assets'], info['thumbnails']), (0, 0, 0, 0))
|
||||
path = self.build / resource[2]
|
||||
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
|
||||
self.assertEqual(db.execute('PRAGMA user_version').fetchone(), (1,))
|
||||
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['stage'], 'empty-source')
|
||||
for table in ('title', 'package', 'release', 'member', 'download_location',
|
||||
'asset', 'asset_delivery', 'release_asset', 'setup', 'source_hint'):
|
||||
self.assertEqual(db.execute('SELECT count(*) FROM ' + table).fetchone(), (0,))
|
||||
self.assertEqual(db.execute('SELECT * FROM browse').fetchall(), [])
|
||||
self.assertEqual(db.execute('SELECT ra.release_id,a.*,d.manual_url FROM release_asset ra '
|
||||
'JOIN asset a ON a.id=ra.asset_id LEFT JOIN asset_delivery d '
|
||||
'ON d.asset_id=a.id').fetchall(), [])
|
||||
self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(), [('ok',)])
|
||||
|
||||
def test_archive_schema_supports_empty_browse_search_and_reader(self):
|
||||
resource = RESOURCES[1]
|
||||
info = self.metadata(resource)
|
||||
self.assertTrue(info['empty'])
|
||||
path = self.build / resource[2]
|
||||
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
|
||||
db.execute('PRAGMA query_only=ON')
|
||||
db.execute('PRAGMA trusted_schema=OFF')
|
||||
self.assertEqual(db.execute('PRAGMA application_id').fetchone(), (0x56434341,))
|
||||
self.assertEqual(db.execute('PRAGMA user_version').fetchone(), (2,))
|
||||
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['source_snapshot'], 'empty-source')
|
||||
for table in ('document', 'page', 'reader_section', 'page_terms', 'reviewed_example'):
|
||||
self.assertEqual(db.execute('SELECT count(*) FROM ' + table).fetchone(), (0,))
|
||||
self.assertEqual(db.execute('SELECT d.* FROM document d ORDER BY d.title COLLATE NOCASE,d.sha256 LIMIT ? OFFSET ?', (50, 0)).fetchall(), [])
|
||||
self.assertEqual(db.execute('SELECT d.*,p.page_number FROM page_terms JOIN page p '
|
||||
'ON p.id=page_terms.rowid JOIN document d ON d.id=p.document_id '
|
||||
'WHERE page_terms MATCH ? ORDER BY bm25(page_terms,1.0,4.0),p.id '
|
||||
'LIMIT ? OFFSET ?', ('"basic"', 50, 0)).fetchall(), [])
|
||||
self.assertEqual(db.execute('SELECT p.page_label,s.text FROM document d JOIN page p '
|
||||
'ON p.document_id=d.id JOIN reader_section s ON s.page_id=p.id '
|
||||
'WHERE d.sha256=? AND p.page_number=?', ('0'*64, 1)).fetchall(), [])
|
||||
self.assertEqual(db.execute('PRAGMA foreign_key_check').fetchall(), [])
|
||||
|
||||
def test_repeat_build_keeps_identical_bytes_and_timestamp(self):
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
first = self.metadata(resource)
|
||||
path = self.build / resource[2]
|
||||
original, stamp = path.read_bytes(), path.stat().st_mtime_ns
|
||||
second = self.metadata(resource)
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
self.assertEqual(path.stat().st_mtime_ns, stamp)
|
||||
self.assertEqual(path.stat().st_mode & 0o777, 0o444)
|
||||
self.assertEqual(first['sha256'], hashlib.sha256(original).hexdigest())
|
||||
self.assertNotIn(b'PRIVATE-DATA-TEST-MARKER', original)
|
||||
|
||||
def test_source_profile_ignores_explicit_and_default_maintainer_inputs(self):
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
self.run_script(resource, '--source', str(self.tempting[0]))
|
||||
for source in self.tempting:
|
||||
self.assertEqual(source.read_bytes(), b'PRIVATE-DATA-TEST-MARKER\x00')
|
||||
self.assertNotIn(b'PRIVATE-DATA-TEST-MARKER', (self.build / resource[2]).read_bytes())
|
||||
|
||||
def test_old_resource_is_replaced_but_invalid_schema_preserves_it(self):
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
path = self.build / resource[2]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(b'OLD-BUILD-PRIVATE-CONTENT')
|
||||
self.metadata(resource)
|
||||
original = path.read_bytes()
|
||||
self.assertNotIn(b'OLD-BUILD-PRIVATE-CONTENT', original)
|
||||
with (self.source / resource[1]).open('a') as stream:
|
||||
stream.write('\nINVALID SQL INTENTIONALLY AUTHORED FOR TEST;\n')
|
||||
self.assertNotEqual(self.run_script(resource, check=False).returncode, 0)
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
self.assertFalse(list(path.parent.glob('.empty-library-*.sqlite')))
|
||||
|
||||
def test_linked_outputs_never_change_the_target(self):
|
||||
for resource in RESOURCES:
|
||||
for kind in ('symlink', 'hardlink', 'dangling'):
|
||||
with self.subTest(resource=resource[0], kind=kind):
|
||||
path = self.build / resource[2]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
target = self.root / 'link-victim'
|
||||
if kind != 'dangling':
|
||||
target.write_bytes(b'unchanged')
|
||||
if kind == 'hardlink':
|
||||
os.link(target, path)
|
||||
else:
|
||||
path.symlink_to(target)
|
||||
self.assertNotEqual(self.run_script(resource, check=False).returncode, 0)
|
||||
if kind != 'dangling':
|
||||
self.assertEqual(target.read_bytes(), b'unchanged')
|
||||
target.unlink()
|
||||
path.unlink()
|
||||
|
||||
def test_linked_output_ancestor_cannot_redirect_a_build_resource(self):
|
||||
target = self.root / 'untouched-directory'
|
||||
target.mkdir()
|
||||
self.build.symlink_to(target, target_is_directory=True)
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
rejected = self.run_script(resource, check=False)
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertIn('linked ancestor', rejected.stderr)
|
||||
self.assertEqual(list(target.iterdir()), [])
|
||||
|
||||
def test_standard_temporary_path_alias_remains_usable(self):
|
||||
if not Path('/tmp').is_symlink() or Path('/tmp').resolve() != Path('/private/tmp'):
|
||||
self.skipTest('macOS temporary-directory alias is absent')
|
||||
with tempfile.TemporaryDirectory(dir='/tmp', prefix='vcc-content-alias-') as temporary:
|
||||
self.build = Path(temporary)
|
||||
for resource in RESOURCES:
|
||||
self.assertTrue(self.metadata(resource)['empty'])
|
||||
|
||||
def test_transaction_sidecars_are_rejected_including_empty_and_dangling(self):
|
||||
for resource in RESOURCES:
|
||||
self.metadata(resource)
|
||||
path = self.build / resource[2]
|
||||
original = path.read_bytes()
|
||||
for suffix in ('-wal', '-shm', '-journal'):
|
||||
for kind in ('empty', 'data', 'dangling'):
|
||||
with self.subTest(resource=resource[0], suffix=suffix, kind=kind):
|
||||
sidecar = Path(str(path) + suffix)
|
||||
if kind == 'dangling':
|
||||
sidecar.symlink_to(self.root / 'missing-sidecar-target')
|
||||
else:
|
||||
sidecar.write_bytes(b'' if kind == 'empty' else b'old transaction')
|
||||
rejected = self.run_script(resource, check=False)
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertIn('sidecars', rejected.stderr)
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
sidecar.unlink()
|
||||
|
||||
def test_source_profile_rejects_full_release_requirement(self):
|
||||
for resource, flag in zip(RESOURCES, ('--require-full', '--require')):
|
||||
self.assertNotEqual(self.run_script(resource, flag, check=False).returncode, 0)
|
||||
self.assertFalse((self.build / resource[2]).exists())
|
||||
|
||||
def test_source_check_only_builds_schema_without_input_or_output(self):
|
||||
self.run_script(RESOURCES[0], '--check-only', output=False)
|
||||
self.run_script(RESOURCES[1], output=False)
|
||||
self.assertFalse(self.build.exists())
|
||||
|
||||
def test_environment_selects_source_and_explicit_cli_overrides_it(self):
|
||||
for resource in RESOURCES:
|
||||
with self.subTest(resource=resource[0]):
|
||||
self.assertTrue(self.metadata(resource, profile=None, env_profile='source')['empty'])
|
||||
# The normal path must read and reject the authored corrupt maintainer input.
|
||||
rejected = self.run_script(resource, profile='normal', env_profile='source', check=False)
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertNotEqual(self.run_script(resource, profile=None, env_profile='unknown', check=False).returncode, 0)
|
||||
self.assertEqual(self.run_script(resource, profile='source', env_profile='unknown').returncode, 0)
|
||||
|
||||
def test_normal_archive_retains_verified_populated_input_exactly(self):
|
||||
source = self.tempting[1]
|
||||
source.unlink()
|
||||
with closing(sqlite3.connect(source)) as db:
|
||||
db.executescript((self.source / RESOURCES[1][1]).read_text())
|
||||
db.execute("INSERT INTO document(id,sha256,title,category,collection,format,source_url,byte_size,page_count) "
|
||||
"VALUES(1,?,'Authored fixture','Books','Books','pdf','https://example.test/fixture.pdf',20,1)", ('a'*64,))
|
||||
db.commit()
|
||||
original = source.read_bytes()
|
||||
digest = hashlib.sha256(original).hexdigest()
|
||||
(source.parent / 'archive-library.json').write_text(json.dumps(dict(schema_version=2,
|
||||
sha256=digest, bytes=len(original), documents=1, reader_sections=0, ranged_members=0)))
|
||||
(source.parent / 'ArchiveLibrary.sqlite.sha256').write_text(digest + ' ArchiveLibrary.sqlite\n')
|
||||
self.run_script(RESOURCES[1], '--require', profile=None)
|
||||
self.assertEqual((self.build / RESOURCES[1][2]).read_bytes(), original)
|
||||
self.assertEqual(source.read_bytes(), original)
|
||||
|
||||
def test_default_normal_behavior_still_uses_existing_fallbacks(self):
|
||||
for path in self.tempting:
|
||||
path.unlink()
|
||||
self.assertIn('absent', self.run_script(RESOURCES[0], profile=None).stdout)
|
||||
path = self.build / RESOURCES[0][2]
|
||||
with closing(sqlite3.connect(path)) as db:
|
||||
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['stage'], 'empty-development')
|
||||
archive = self.build / RESOURCES[1][2]
|
||||
archive.parent.mkdir(parents=True, exist_ok=True)
|
||||
archive.write_bytes(b'old index')
|
||||
self.run_script(RESOURCES[1], profile=None)
|
||||
self.assertFalse(archive.exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user