Publish VCC Plus OSS source distribution for 3.6.2 build 60
This commit is contained in:
commit
c652a5276d
1192 files changed
+272368
No files matched your search
@@ -0,0 +1,57 @@
|
||||
-- SPDX-License-Identifier: GPL-3.0-or-later
|
||||
PRAGMA user_version=2;
|
||||
PRAGMA application_id=0x56434341;
|
||||
CREATE TABLE metadata(key TEXT PRIMARY KEY,value TEXT NOT NULL) WITHOUT ROWID;
|
||||
CREATE TABLE document(
|
||||
id INTEGER PRIMARY KEY,
|
||||
sha256 TEXT NOT NULL UNIQUE CHECK(length(sha256)=64 AND sha256 NOT GLOB '*[^a-f0-9]*'),
|
||||
title TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
collection TEXT NOT NULL,
|
||||
format TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL,
|
||||
member_path TEXT,
|
||||
delivery_url TEXT,
|
||||
byte_size INTEGER NOT NULL CHECK(byte_size > 0),
|
||||
page_count INTEGER NOT NULL CHECK(page_count > 0),
|
||||
issue_year INTEGER CHECK(issue_year BETWEEN 1900 AND 2099),
|
||||
member_offset INTEGER,
|
||||
member_compressed_bytes INTEGER,
|
||||
member_method INTEGER,
|
||||
archive_bytes INTEGER
|
||||
);
|
||||
CREATE INDEX document_browse ON document(category,collection,title COLLATE NOCASE,sha256);
|
||||
CREATE INDEX document_issue_year ON document(collection,issue_year,title COLLATE NOCASE,sha256)
|
||||
WHERE issue_year IS NOT NULL;
|
||||
CREATE INDEX document_title ON document(title COLLATE NOCASE,sha256);
|
||||
CREATE TABLE page(
|
||||
id INTEGER PRIMARY KEY,
|
||||
document_id INTEGER NOT NULL REFERENCES document(id),
|
||||
page_number INTEGER NOT NULL CHECK(page_number > 0),
|
||||
page_label TEXT,
|
||||
locator_kind TEXT NOT NULL CHECK(locator_kind IN ('pdf_page','worksheet','image','section')),
|
||||
is_ocr INTEGER NOT NULL CHECK(is_ocr IN (0,1)),
|
||||
UNIQUE(document_id,page_number)
|
||||
);
|
||||
-- Rendered text for complex non-PDF formats. PDF and image OCR stays in the research store.
|
||||
CREATE TABLE reader_section(
|
||||
page_id INTEGER PRIMARY KEY REFERENCES page(id),
|
||||
text TEXT NOT NULL
|
||||
);
|
||||
-- Stored positions refer only to the synthetic alphabetical term bag, never to source order.
|
||||
CREATE VIRTUAL TABLE page_terms USING fts5(
|
||||
terms,title_terms,content='',tokenize='unicode61 remove_diacritics 2',detail=full,columnsize=1
|
||||
);
|
||||
CREATE TABLE reviewed_example(
|
||||
original_sha256 TEXT NOT NULL REFERENCES document(sha256),
|
||||
first_page INTEGER NOT NULL CHECK(first_page > 0),
|
||||
last_page INTEGER NOT NULL CHECK(last_page >= first_page),
|
||||
title TEXT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK(kind IN ('snippet','basic_program','source')),
|
||||
language TEXT NOT NULL,
|
||||
author TEXT NOT NULL,
|
||||
provenance TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
source_sha256 TEXT NOT NULL,
|
||||
PRIMARY KEY(original_sha256,first_page,last_page,title)
|
||||
) WITHOUT ROWID;
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Stage the exported public Archive index as an Xcode resource.
|
||||
|
||||
The explicit source content profile stages only the checked-in empty schema.
|
||||
Normal builds retain the existing prepared-index lookup and verification.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import sys
|
||||
|
||||
NAME = 'ArchiveLibrary.sqlite'
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
from build_content import add_content_profile_argument, atomic_empty_database, content_profile as resolve_content_profile
|
||||
DEFAULT = ROOT / 'media/app-inputs/archive-library' / NAME
|
||||
SCHEMA = Path(__file__).with_name('archive-library.sql')
|
||||
|
||||
|
||||
def source_archive(output):
|
||||
result = atomic_empty_database(output, SCHEMA, (
|
||||
('schema_version', '2'), ('application_id', str(0x56434341)), ('source_snapshot', 'empty-source'),
|
||||
('documents', '0'), ('pages', '0'), ('searchable_pages', '0'), ('verified_deliveries', '0'),
|
||||
('ranged_members', '0'), ('reader_sections', '0'), ('reviewed_examples', '0')),
|
||||
('document', 'page', 'reader_section', 'page_terms', 'reviewed_example'),
|
||||
expected_name=NAME, expected_parent='ArchiveLibrary', application_id=0x56434341,
|
||||
user_version=2, fts_integrity=True)
|
||||
return dict(result, schema_version=2, application_id=0x56434341, source_snapshot='empty-source',
|
||||
documents=0, pages=0, searchable_pages=0, verified_deliveries=0, ranged_members=0,
|
||||
reader_sections=0, reviewed_examples=0, empty=True)
|
||||
|
||||
|
||||
def stage(source, output, required=False, *, content_profile='normal'):
|
||||
if resolve_content_profile(content_profile) == 'source':
|
||||
if required:
|
||||
raise ValueError('--require cannot be combined with the source content profile')
|
||||
if output is not None:
|
||||
return source_archive(output)
|
||||
with tempfile.TemporaryDirectory(prefix='vcc-source-archive-check-') as temporary:
|
||||
return source_archive(Path(temporary) / 'ArchiveLibrary' / NAME)
|
||||
if source is None:
|
||||
source = DEFAULT
|
||||
if not source.exists():
|
||||
source = ROOT / 'media/knowledge-base/archive-library' / NAME
|
||||
source = Path(source)
|
||||
if output is not None and (output.name != NAME or output.parent.name != 'ArchiveLibrary' or output.is_symlink()):
|
||||
raise ValueError('Expected an ArchiveLibrary/ArchiveLibrary.sqlite app resource destination')
|
||||
if not source.exists():
|
||||
if required:
|
||||
raise ValueError('Prepare the Archive Library index before packaging this release.')
|
||||
if output is not None:
|
||||
output.unlink(missing_ok=True)
|
||||
return
|
||||
manifest = json.loads((source.parent / 'archive-library.json').read_bytes())
|
||||
checksum = (source.parent / (NAME + '.sha256')).read_text().split()
|
||||
version = manifest['schema_version']
|
||||
if (checksum != [manifest['sha256'], NAME] or manifest['bytes'] != source.stat().st_size
|
||||
or version not in (1, 2)):
|
||||
raise ValueError('The Archive index manifest and sidecar do not match the export')
|
||||
if required and version < 2:
|
||||
raise ValueError('Re-export the Archive Library index with inline reader support before packaging this release.')
|
||||
with closing(sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True)) as db:
|
||||
if (db.execute('PRAGMA application_id').fetchone() != (0x56434341,)
|
||||
or db.execute('PRAGMA user_version').fetchone() != (version,)
|
||||
or db.execute('SELECT COUNT(*) FROM document').fetchone() != (manifest['documents'],)
|
||||
or not manifest['documents']):
|
||||
raise ValueError('Incompatible Archive Library input')
|
||||
if version == 2 and (db.execute('SELECT COUNT(*) FROM reader_section').fetchone() != (manifest['reader_sections'],)
|
||||
or db.execute('SELECT COUNT(*) FROM document WHERE member_offset IS NOT NULL').fetchone() != (manifest['ranged_members'],)):
|
||||
raise ValueError('Archive reader coverage differs from the verified export')
|
||||
if output is not None:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Copy and verify the same stream. Replace only after both checks complete.
|
||||
fd = None
|
||||
temporary = None
|
||||
try:
|
||||
if output is not None:
|
||||
fd = tempfile.NamedTemporaryFile(dir=output.parent, delete=False)
|
||||
temporary = Path(fd.name)
|
||||
checksum = hashlib.sha256()
|
||||
with source.open('rb') as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
||||
checksum.update(chunk)
|
||||
if fd is not None:
|
||||
fd.write(chunk)
|
||||
if checksum.hexdigest() != manifest['sha256']:
|
||||
raise ValueError('Archive Library fingerprint differs from the verified export')
|
||||
if fd is not None:
|
||||
fd.close()
|
||||
os.chmod(temporary, 0o444)
|
||||
os.replace(temporary, output)
|
||||
finally:
|
||||
if fd is not None:
|
||||
fd.close()
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--source', type=Path)
|
||||
add_content_profile_argument(parser)
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--require', action='store_true')
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = stage(args.source, args.output, args.require, content_profile=resolve_content_profile(args.content_profile))
|
||||
except (OSError, ValueError, KeyError, sqlite3.DatabaseError) as error:
|
||||
parser.exit(1, 'error: Archive Library: ' + str(error) + '\n')
|
||||
if result is not None:
|
||||
print('Archive Library index: ' + json.dumps(result, sort_keys=True))
|
||||
Reference in new issue
Block a user