Publish VCC Plus OSS source distribution for 3.6.2 build 60
This commit is contained in:
commit
c652a5276d
1192 files changed
+272368
No files matched your search
@@ -0,0 +1,253 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
import CryptoKit
|
||||
import Foundation
|
||||
|
||||
struct ArchiveMemberRange: Hashable, Sendable {
|
||||
let offset, compressedBytes: Int64
|
||||
let method: Int
|
||||
let archiveBytes: Int64
|
||||
}
|
||||
|
||||
struct ArchiveReaderSection: Sendable {
|
||||
let number: Int
|
||||
let label: String?
|
||||
let text: String
|
||||
}
|
||||
|
||||
struct ArchiveDocument: Identifiable, Hashable, Sendable {
|
||||
let id: String
|
||||
let title: String
|
||||
let collection: String
|
||||
let sourceURL: URL
|
||||
let memberPath: String?
|
||||
let pageCount: Int?
|
||||
let deliveryURL: URL?
|
||||
let deliveryBytes: Int64?
|
||||
var format: String = "pdf"
|
||||
var sourceBytes: Int64?
|
||||
var memberRange: ArchiveMemberRange?
|
||||
var deliverySHA256: String? { deliveryURL == nil ? nil : id }
|
||||
var citation: String { cite(nil, at: sourceURL) }
|
||||
|
||||
func cite(_ locator: String?, at url: URL) -> String {
|
||||
([title, memberPath, locator, "Color Computer Archive", url.absoluteString] as [String?])
|
||||
.compactMap { $0 }.joined(separator: " · ")
|
||||
}
|
||||
}
|
||||
|
||||
struct ArchiveHit: Identifiable, Hashable, Sendable {
|
||||
let id: String
|
||||
let document: ArchiveDocument
|
||||
let page: Int?
|
||||
let locator: String
|
||||
let ocr: Bool
|
||||
let sourceURL: URL
|
||||
var locationNumber: Int = 1
|
||||
var citation: String { document.cite(locator, at: sourceURL) }
|
||||
}
|
||||
|
||||
struct ArchiveExample: Identifiable, Hashable, Sendable {
|
||||
let title: String
|
||||
let kind: String
|
||||
let language: String
|
||||
let author: String
|
||||
let provenance: String
|
||||
let source: String
|
||||
let sourceSHA256: String
|
||||
var id: String { sourceSHA256 }
|
||||
}
|
||||
|
||||
enum ArchiveLibraryError: LocalizedError {
|
||||
case missing, incompatible, input(String), damaged
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .missing: return "This build has no Archive search index. Install a build with the index or prepare it using the knowledge tools before rebuilding."
|
||||
case .incompatible: return "This Archive index belongs to a different database or version. Rebuild with a compatible Archive Library index."
|
||||
case .input(let message): return message
|
||||
case .damaged: return "The Archive index could not be read. Replace it with a verified copy and rebuild the app."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Native and MCP clients use one serialized read-only connection. SQLite periodically
|
||||
/// checks the current task for cancellation, including while ranking a common term.
|
||||
actor ArchiveLibraryCatalog {
|
||||
static let shared = ArchiveLibraryCatalog(url: Bundle.main.url(forResource: "ArchiveLibrary", withExtension: "sqlite", subdirectory: "ArchiveLibrary"))
|
||||
private let url: URL?
|
||||
private var connection: LibrarySQLite?
|
||||
private var hasReaderSections = false
|
||||
|
||||
init(url: URL?) { self.url = url }
|
||||
|
||||
private func database() throws -> LibrarySQLite {
|
||||
try Task.checkCancellation()
|
||||
if let connection { return connection }
|
||||
guard let url, url.isFileURL, FileManager.default.fileExists(atPath: url.path) else { throw ArchiveLibraryError.missing }
|
||||
let db = try LibrarySQLite(url: url, cancellable: true, describeError: { _ in ArchiveLibraryError.damaged })
|
||||
try db.execute("PRAGMA trusted_schema=OFF")
|
||||
try db.execute("PRAGMA query_only=ON")
|
||||
guard try db.rows("PRAGMA application_id").first?["application_id"] == String(0x56434341),
|
||||
let version = try db.rows("PRAGMA user_version").first?["user_version"],
|
||||
["1", "2"].contains(version) else { throw ArchiveLibraryError.incompatible }
|
||||
hasReaderSections = version == "2"
|
||||
connection = db
|
||||
return db
|
||||
}
|
||||
|
||||
func collections() throws -> [String] {
|
||||
try database().rows("""
|
||||
SELECT collection FROM document
|
||||
UNION SELECT collection || '/' || issue_year AS collection FROM document
|
||||
WHERE issue_year IS NOT NULL AND substr(collection,-5) != '/' || issue_year
|
||||
ORDER BY collection COLLATE NOCASE,collection
|
||||
""")
|
||||
.compactMap { $0["collection"] }
|
||||
}
|
||||
|
||||
private func scope(_ collection: String?) throws -> (sql: String, values: [String]) {
|
||||
guard let collection, !collection.isEmpty else { return ("", []) }
|
||||
guard collection.utf8.count <= 2048, !collection.contains("\0") else { throw ArchiveLibraryError.input("Invalid Archive collection.") }
|
||||
return (" AND (d.collection=? OR instr(d.collection, ? || '/')=1 OR d.collection || '/' || d.issue_year=?)", [collection, collection, collection])
|
||||
}
|
||||
|
||||
private func bounds(_ limit: Int, _ offset: Int) throws -> [String] {
|
||||
guard (1...50).contains(limit), (0...250_000).contains(offset) else { throw ArchiveLibraryError.input("Archive pages allow 1–50 results and offsets up to 250,000.") }
|
||||
return [String(limit), String(offset)]
|
||||
}
|
||||
|
||||
func browse(collection: String?, limit: Int, offset: Int) throws -> [ArchiveDocument] {
|
||||
let filter = try scope(collection)
|
||||
let values = try filter.values + bounds(limit, offset)
|
||||
let rows = try database().rows("""
|
||||
SELECT d.* FROM document d WHERE 1=1 \(filter.sql)
|
||||
ORDER BY d.title COLLATE NOCASE,d.sha256 LIMIT ? OFFSET ?
|
||||
""", values)
|
||||
return try rows.map(document)
|
||||
}
|
||||
|
||||
private static let compounds = ["module directory", "width 80", "width 40", "disk basic", "extended basic"]
|
||||
|
||||
private func match(_ query: String) throws -> String {
|
||||
guard query.count <= 256, !query.contains("\0") else { throw ArchiveLibraryError.input("Use at most 256 characters for an Archive search.") }
|
||||
let words = query.lowercased().split { !$0.isLetter && !$0.isNumber }.map(String.init)
|
||||
guard !words.isEmpty, words.count <= 16 else { throw ArchiveLibraryError.input("Enter between one and sixteen words or numbers.") }
|
||||
let required = words.map { "\"\($0)\"" }.joined(separator: " AND ")
|
||||
let pairs = Set(zip(words, words.dropFirst()).map { $0 + " " + $1 })
|
||||
let boosts = Self.compounds.filter(pairs.contains).map { "\"vccphrase\($0.replacingOccurrences(of: " ", with: ""))\"" }
|
||||
guard !boosts.isEmpty else { return required }
|
||||
return "(\(required)) OR ((\(required)) AND (\(boosts.joined(separator: " OR "))))"
|
||||
}
|
||||
|
||||
func search(query: String, collection: String?, limit: Int, offset: Int) throws -> [ArchiveHit] {
|
||||
let expression = try match(query)
|
||||
let filter = try scope(collection)
|
||||
let values = try [expression] + filter.values + bounds(limit, offset)
|
||||
let rows = try database().rows("""
|
||||
SELECT d.*,p.page_number,p.page_label,p.locator_kind,p.is_ocr
|
||||
FROM page_terms JOIN page p ON p.id=page_terms.rowid JOIN document d ON d.id=p.document_id
|
||||
WHERE page_terms MATCH ? \(filter.sql)
|
||||
ORDER BY bm25(page_terms,1.0,4.0),p.id LIMIT ? OFFSET ?
|
||||
""", values)
|
||||
return try rows.map(hit)
|
||||
}
|
||||
|
||||
func citation(id: String) throws -> ArchiveHit? {
|
||||
let fields = id.components(separatedBy: ":")
|
||||
guard fields.count == 4, fields[0] == "sha256", Self.isSHA(fields[1]),
|
||||
let n = Int32(fields[3]), n > 0, String(n) == fields[3] else { return nil }
|
||||
let rows = try database().rows("""
|
||||
SELECT d.*,p.page_number,p.page_label,p.locator_kind,p.is_ocr
|
||||
FROM document d JOIN page p ON p.document_id=d.id
|
||||
WHERE d.sha256=? AND p.locator_kind=? AND p.page_number=?
|
||||
""", Array(fields[1...3]))
|
||||
return try rows.first.map(hit)
|
||||
}
|
||||
|
||||
private static func isSHA(_ value: String) -> Bool {
|
||||
value.utf8.count == 64 && value.utf8.allSatisfy { (48...57).contains($0) || (97...102).contains($0) }
|
||||
}
|
||||
|
||||
private func document(_ row: [String: String]) throws -> ArchiveDocument {
|
||||
guard let sha = row["sha256"], Self.isSHA(sha), let title = row["title"], let collection = row["collection"],
|
||||
let link = row["source_url"].flatMap(URL.init(string:)), link.scheme == "https",
|
||||
link.host == "colorcomputerarchive.com", link.path.hasPrefix("/repo/Documents/"),
|
||||
link.user == nil, link.password == nil, link.port == nil else { throw ArchiveLibraryError.damaged }
|
||||
let delivery = row["delivery_url"].flatMap(URL.init(string:))
|
||||
var memberRange: ArchiveMemberRange?
|
||||
if let offset = row["member_offset"].flatMap(Int64.init),
|
||||
let compressed = row["member_compressed_bytes"].flatMap(Int64.init),
|
||||
let method = row["member_method"].flatMap(Int.init),
|
||||
let bytes = row["archive_bytes"].flatMap(Int64.init) {
|
||||
memberRange = ArchiveMemberRange(offset: offset, compressedBytes: compressed, method: method, archiveBytes: bytes)
|
||||
}
|
||||
return ArchiveDocument(id: sha, title: title, collection: collection, sourceURL: link,
|
||||
memberPath: row["member_path"], pageCount: row["page_count"].flatMap(Int.init),
|
||||
deliveryURL: delivery, deliveryBytes: delivery == nil ? nil : row["byte_size"].flatMap(Int64.init),
|
||||
format: row["format"] ?? "pdf", sourceBytes: row["byte_size"].flatMap(Int64.init), memberRange: memberRange)
|
||||
}
|
||||
|
||||
private func hit(_ row: [String: String]) throws -> ArchiveHit {
|
||||
let doc = try document(row)
|
||||
guard let kind = row["locator_kind"], let number = row["page_number"].flatMap(Int.init), number > 0 else { throw ArchiveLibraryError.damaged }
|
||||
let label = row["page_label"].flatMap { $0.isEmpty ? nil : $0 }
|
||||
let locator: String
|
||||
var source = doc.sourceURL
|
||||
switch kind {
|
||||
case "pdf_page":
|
||||
if let label, label != String(number) {
|
||||
locator = "PDF page \(number) (label \(label))"
|
||||
} else {
|
||||
locator = "PDF page \(number)"
|
||||
}
|
||||
if doc.memberPath == nil, var parts = URLComponents(url: source, resolvingAgainstBaseURL: false) {
|
||||
parts.fragment = "page=\(number)"
|
||||
if let link = parts.url { source = link }
|
||||
}
|
||||
case "worksheet": locator = label ?? "Worksheet \(number)"
|
||||
case "section": locator = label ?? "Section \(number)"
|
||||
case "image": locator = label ?? "Image \(number)"
|
||||
default: throw ArchiveLibraryError.damaged
|
||||
}
|
||||
return ArchiveHit(id: "sha256:\(doc.id):\(kind):\(number)", document: doc,
|
||||
page: kind == "pdf_page" ? number : nil, locator: locator,
|
||||
ocr: row["is_ocr"] == "1", sourceURL: source, locationNumber: number)
|
||||
}
|
||||
|
||||
func readerSection(documentID: String, number: Int) throws -> ArchiveReaderSection? {
|
||||
guard Self.isSHA(documentID), number > 0 else { return nil }
|
||||
let db = try database()
|
||||
guard hasReaderSections else { return nil }
|
||||
let rows = try db.rows("""
|
||||
SELECT p.page_label,s.text FROM document d JOIN page p ON p.document_id=d.id
|
||||
JOIN reader_section s ON s.page_id=p.id WHERE d.sha256=? AND p.page_number=?
|
||||
""", [documentID, String(number)])
|
||||
guard let row = rows.first, let text = row["text"] else { return nil }
|
||||
return ArchiveReaderSection(number: number, label: row["page_label"], text: text)
|
||||
}
|
||||
|
||||
func examples(documentID: String, page: Int) throws -> [ArchiveExample] {
|
||||
guard Self.isSHA(documentID), page > 0 else { return [] }
|
||||
let rows = try database().rows("""
|
||||
SELECT * FROM reviewed_example WHERE original_sha256=? AND ? BETWEEN first_page AND last_page
|
||||
ORDER BY title,source_sha256
|
||||
""", [documentID, String(page)])
|
||||
return try rows.map(reviewedExample)
|
||||
}
|
||||
|
||||
func basicExample(sourceSHA256: String) throws -> ArchiveExample? {
|
||||
guard Self.isSHA(sourceSHA256) else { return nil }
|
||||
let rows = try database().rows("SELECT * FROM reviewed_example WHERE source_sha256=? AND kind='basic_program' LIMIT 1", [sourceSHA256])
|
||||
return try rows.first.map(reviewedExample)
|
||||
}
|
||||
|
||||
private func reviewedExample(_ row: [String: String]) throws -> ArchiveExample {
|
||||
guard let title = row["title"], let kind = row["kind"], let language = row["language"],
|
||||
let author = row["author"], let provenance = row["provenance"], let source = row["source"],
|
||||
let sha = row["source_sha256"],
|
||||
SHA256.hash(data: Data(source.utf8)).map({ String(format: "%02x", $0) }).joined() == sha else {
|
||||
throw ArchiveLibraryError.damaged
|
||||
}
|
||||
return ArchiveExample(title: title, kind: kind, language: language, author: author,
|
||||
provenance: provenance, source: source, sourceSHA256: sha)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user