249 lines
14 KiB
Python
249 lines
14 KiB
Python
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Offline content-profile checks using authored disposable inputs only."""
|
|
from contextlib import closing
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
RESOURCES = (
|
|
('tools/catalog/package.py', 'tools/catalog/app-schema.sql', 'SoftwareLibrary/catalog.sqlite'),
|
|
('tools/knowledge/package_archive.py', 'tools/knowledge/archive-library.sql', 'ArchiveLibrary/ArchiveLibrary.sqlite'),
|
|
)
|
|
|
|
|
|
class BuildContentTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory(prefix='vcc-content-profile-')
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name)
|
|
self.source = self.root / 'Source With Spaces'
|
|
self.build = self.root / 'Build With Spaces'
|
|
for relative in ['tools/build_content.py', *(p for r in RESOURCES for p in r[:2])]:
|
|
destination = self.source / relative
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(ROOT / relative, destination)
|
|
self.tempting = []
|
|
for relative in ('media/software-library-full/prepared/app-catalog.sqlite',
|
|
'media/app-inputs/archive-library/ArchiveLibrary.sqlite',
|
|
'media/knowledge-base/archive-library/ArchiveLibrary.sqlite'):
|
|
target = self.source / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(b'PRIVATE-DATA-TEST-MARKER\x00')
|
|
self.tempting.append(target)
|
|
|
|
def run_script(self, resource, *extra, profile='source', env_profile=None, output=True, check=True):
|
|
command = [sys.executable, str(self.source / resource[0])]
|
|
if profile is not None:
|
|
command += ['--content-profile', profile]
|
|
if output:
|
|
command += ['--output', str(self.build / resource[2])]
|
|
environment = os.environ.copy()
|
|
environment.pop('VCC_BUILD_CONTENT_PROFILE', None)
|
|
if env_profile is not None:
|
|
environment['VCC_BUILD_CONTENT_PROFILE'] = env_profile
|
|
completed = subprocess.run([*command, *extra], cwd=self.source,
|
|
env=environment, capture_output=True, text=True)
|
|
if check:
|
|
self.assertEqual(completed.returncode, 0, completed.stderr)
|
|
return completed
|
|
|
|
def metadata(self, resource, **kwargs):
|
|
return json.loads(self.run_script(resource, **kwargs).stdout.split(': ', 1)[1])
|
|
|
|
def test_software_schema_and_reader_contract_have_no_content(self):
|
|
resource = RESOURCES[0]
|
|
info = self.metadata(resource)
|
|
self.assertTrue(info['empty'])
|
|
self.assertEqual((info['titles'], info['releases'], info['assets'], info['thumbnails']), (0, 0, 0, 0))
|
|
path = self.build / resource[2]
|
|
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
|
|
self.assertEqual(db.execute('PRAGMA user_version').fetchone(), (1,))
|
|
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['stage'], 'empty-source')
|
|
for table in ('title', 'package', 'release', 'member', 'download_location',
|
|
'asset', 'asset_delivery', 'release_asset', 'setup', 'source_hint'):
|
|
self.assertEqual(db.execute('SELECT count(*) FROM ' + table).fetchone(), (0,))
|
|
self.assertEqual(db.execute('SELECT * FROM browse').fetchall(), [])
|
|
self.assertEqual(db.execute('SELECT ra.release_id,a.*,d.manual_url FROM release_asset ra '
|
|
'JOIN asset a ON a.id=ra.asset_id LEFT JOIN asset_delivery d '
|
|
'ON d.asset_id=a.id').fetchall(), [])
|
|
self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(), [('ok',)])
|
|
|
|
def test_archive_schema_supports_empty_browse_search_and_reader(self):
|
|
resource = RESOURCES[1]
|
|
info = self.metadata(resource)
|
|
self.assertTrue(info['empty'])
|
|
path = self.build / resource[2]
|
|
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
|
|
db.execute('PRAGMA query_only=ON')
|
|
db.execute('PRAGMA trusted_schema=OFF')
|
|
self.assertEqual(db.execute('PRAGMA application_id').fetchone(), (0x56434341,))
|
|
self.assertEqual(db.execute('PRAGMA user_version').fetchone(), (2,))
|
|
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['source_snapshot'], 'empty-source')
|
|
for table in ('document', 'page', 'reader_section', 'page_terms', 'reviewed_example'):
|
|
self.assertEqual(db.execute('SELECT count(*) FROM ' + table).fetchone(), (0,))
|
|
self.assertEqual(db.execute('SELECT d.* FROM document d ORDER BY d.title COLLATE NOCASE,d.sha256 LIMIT ? OFFSET ?', (50, 0)).fetchall(), [])
|
|
self.assertEqual(db.execute('SELECT d.*,p.page_number FROM page_terms JOIN page p '
|
|
'ON p.id=page_terms.rowid JOIN document d ON d.id=p.document_id '
|
|
'WHERE page_terms MATCH ? ORDER BY bm25(page_terms,1.0,4.0),p.id '
|
|
'LIMIT ? OFFSET ?', ('"basic"', 50, 0)).fetchall(), [])
|
|
self.assertEqual(db.execute('SELECT p.page_label,s.text FROM document d JOIN page p '
|
|
'ON p.document_id=d.id JOIN reader_section s ON s.page_id=p.id '
|
|
'WHERE d.sha256=? AND p.page_number=?', ('0'*64, 1)).fetchall(), [])
|
|
self.assertEqual(db.execute('PRAGMA foreign_key_check').fetchall(), [])
|
|
|
|
def test_repeat_build_keeps_identical_bytes_and_timestamp(self):
|
|
for resource in RESOURCES:
|
|
with self.subTest(resource=resource[0]):
|
|
first = self.metadata(resource)
|
|
path = self.build / resource[2]
|
|
original, stamp = path.read_bytes(), path.stat().st_mtime_ns
|
|
second = self.metadata(resource)
|
|
self.assertEqual(first, second)
|
|
self.assertEqual(path.read_bytes(), original)
|
|
self.assertEqual(path.stat().st_mtime_ns, stamp)
|
|
self.assertEqual(path.stat().st_mode & 0o777, 0o444)
|
|
self.assertEqual(first['sha256'], hashlib.sha256(original).hexdigest())
|
|
self.assertNotIn(b'PRIVATE-DATA-TEST-MARKER', original)
|
|
|
|
def test_source_profile_ignores_explicit_and_default_maintainer_inputs(self):
|
|
for resource in RESOURCES:
|
|
with self.subTest(resource=resource[0]):
|
|
self.run_script(resource, '--source', str(self.tempting[0]))
|
|
for source in self.tempting:
|
|
self.assertEqual(source.read_bytes(), b'PRIVATE-DATA-TEST-MARKER\x00')
|
|
self.assertNotIn(b'PRIVATE-DATA-TEST-MARKER', (self.build / resource[2]).read_bytes())
|
|
|
|
def test_old_resource_is_replaced_but_invalid_schema_preserves_it(self):
|
|
for resource in RESOURCES:
|
|
with self.subTest(resource=resource[0]):
|
|
path = self.build / resource[2]
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(b'OLD-BUILD-PRIVATE-CONTENT')
|
|
self.metadata(resource)
|
|
original = path.read_bytes()
|
|
self.assertNotIn(b'OLD-BUILD-PRIVATE-CONTENT', original)
|
|
with (self.source / resource[1]).open('a') as stream:
|
|
stream.write('\nINVALID SQL INTENTIONALLY AUTHORED FOR TEST;\n')
|
|
self.assertNotEqual(self.run_script(resource, check=False).returncode, 0)
|
|
self.assertEqual(path.read_bytes(), original)
|
|
self.assertFalse(list(path.parent.glob('.empty-library-*.sqlite')))
|
|
|
|
def test_linked_outputs_never_change_the_target(self):
|
|
for resource in RESOURCES:
|
|
for kind in ('symlink', 'hardlink', 'dangling'):
|
|
with self.subTest(resource=resource[0], kind=kind):
|
|
path = self.build / resource[2]
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
target = self.root / 'link-victim'
|
|
if kind != 'dangling':
|
|
target.write_bytes(b'unchanged')
|
|
if kind == 'hardlink':
|
|
os.link(target, path)
|
|
else:
|
|
path.symlink_to(target)
|
|
self.assertNotEqual(self.run_script(resource, check=False).returncode, 0)
|
|
if kind != 'dangling':
|
|
self.assertEqual(target.read_bytes(), b'unchanged')
|
|
target.unlink()
|
|
path.unlink()
|
|
|
|
def test_linked_output_ancestor_cannot_redirect_a_build_resource(self):
|
|
target = self.root / 'untouched-directory'
|
|
target.mkdir()
|
|
self.build.symlink_to(target, target_is_directory=True)
|
|
for resource in RESOURCES:
|
|
with self.subTest(resource=resource[0]):
|
|
rejected = self.run_script(resource, check=False)
|
|
self.assertNotEqual(rejected.returncode, 0)
|
|
self.assertIn('linked ancestor', rejected.stderr)
|
|
self.assertEqual(list(target.iterdir()), [])
|
|
|
|
def test_standard_temporary_path_alias_remains_usable(self):
|
|
if not Path('/tmp').is_symlink() or Path('/tmp').resolve() != Path('/private/tmp'):
|
|
self.skipTest('macOS temporary-directory alias is absent')
|
|
with tempfile.TemporaryDirectory(dir='/tmp', prefix='vcc-content-alias-') as temporary:
|
|
self.build = Path(temporary)
|
|
for resource in RESOURCES:
|
|
self.assertTrue(self.metadata(resource)['empty'])
|
|
|
|
def test_transaction_sidecars_are_rejected_including_empty_and_dangling(self):
|
|
for resource in RESOURCES:
|
|
self.metadata(resource)
|
|
path = self.build / resource[2]
|
|
original = path.read_bytes()
|
|
for suffix in ('-wal', '-shm', '-journal'):
|
|
for kind in ('empty', 'data', 'dangling'):
|
|
with self.subTest(resource=resource[0], suffix=suffix, kind=kind):
|
|
sidecar = Path(str(path) + suffix)
|
|
if kind == 'dangling':
|
|
sidecar.symlink_to(self.root / 'missing-sidecar-target')
|
|
else:
|
|
sidecar.write_bytes(b'' if kind == 'empty' else b'old transaction')
|
|
rejected = self.run_script(resource, check=False)
|
|
self.assertNotEqual(rejected.returncode, 0)
|
|
self.assertIn('sidecars', rejected.stderr)
|
|
self.assertEqual(path.read_bytes(), original)
|
|
sidecar.unlink()
|
|
|
|
def test_source_profile_rejects_full_release_requirement(self):
|
|
for resource, flag in zip(RESOURCES, ('--require-full', '--require')):
|
|
self.assertNotEqual(self.run_script(resource, flag, check=False).returncode, 0)
|
|
self.assertFalse((self.build / resource[2]).exists())
|
|
|
|
def test_source_check_only_builds_schema_without_input_or_output(self):
|
|
self.run_script(RESOURCES[0], '--check-only', output=False)
|
|
self.run_script(RESOURCES[1], output=False)
|
|
self.assertFalse(self.build.exists())
|
|
|
|
def test_environment_selects_source_and_explicit_cli_overrides_it(self):
|
|
for resource in RESOURCES:
|
|
with self.subTest(resource=resource[0]):
|
|
self.assertTrue(self.metadata(resource, profile=None, env_profile='source')['empty'])
|
|
# The normal path must read and reject the authored corrupt maintainer input.
|
|
rejected = self.run_script(resource, profile='normal', env_profile='source', check=False)
|
|
self.assertNotEqual(rejected.returncode, 0)
|
|
self.assertNotEqual(self.run_script(resource, profile=None, env_profile='unknown', check=False).returncode, 0)
|
|
self.assertEqual(self.run_script(resource, profile='source', env_profile='unknown').returncode, 0)
|
|
|
|
def test_normal_archive_retains_verified_populated_input_exactly(self):
|
|
source = self.tempting[1]
|
|
source.unlink()
|
|
with closing(sqlite3.connect(source)) as db:
|
|
db.executescript((self.source / RESOURCES[1][1]).read_text())
|
|
db.execute("INSERT INTO document(id,sha256,title,category,collection,format,source_url,byte_size,page_count) "
|
|
"VALUES(1,?,'Authored fixture','Books','Books','pdf','https://example.test/fixture.pdf',20,1)", ('a'*64,))
|
|
db.commit()
|
|
original = source.read_bytes()
|
|
digest = hashlib.sha256(original).hexdigest()
|
|
(source.parent / 'archive-library.json').write_text(json.dumps(dict(schema_version=2,
|
|
sha256=digest, bytes=len(original), documents=1, reader_sections=0, ranged_members=0)))
|
|
(source.parent / 'ArchiveLibrary.sqlite.sha256').write_text(digest + ' ArchiveLibrary.sqlite\n')
|
|
self.run_script(RESOURCES[1], '--require', profile=None)
|
|
self.assertEqual((self.build / RESOURCES[1][2]).read_bytes(), original)
|
|
self.assertEqual(source.read_bytes(), original)
|
|
|
|
def test_default_normal_behavior_still_uses_existing_fallbacks(self):
|
|
for path in self.tempting:
|
|
path.unlink()
|
|
self.assertIn('absent', self.run_script(RESOURCES[0], profile=None).stdout)
|
|
path = self.build / RESOURCES[0][2]
|
|
with closing(sqlite3.connect(path)) as db:
|
|
self.assertEqual(dict(db.execute('SELECT key,value FROM metadata'))['stage'], 'empty-development')
|
|
archive = self.build / RESOURCES[1][2]
|
|
archive.parent.mkdir(parents=True, exist_ok=True)
|
|
archive.write_bytes(b'old index')
|
|
self.run_script(RESOURCES[1], profile=None)
|
|
self.assertFalse(archive.exists())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|