Files
VCCPlusOSS/platforms/macos/App/ArchiveLibraryCatalog.swift
T

273 lines
14 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// SPDX-License-Identifier: GPL-3.0-or-later
import CryptoKit
import Foundation
struct ArchiveMemberRange: Hashable, Sendable {
let offset, compressedBytes: Int64
let method: Int
let archiveBytes: Int64
}
struct ArchiveReaderSection: Sendable {
let number: Int
let label: String?
let text: String
}
struct ArchiveDocument: Identifiable, Hashable, Sendable {
let id: String
let title: String
let collection: String
let sourceURL: URL
let memberPath: String?
let pageCount: Int?
let deliveryURL: URL?
let deliveryBytes: Int64?
var format: String = "pdf"
var sourceBytes: Int64?
var memberRange: ArchiveMemberRange?
var deliverySHA256: String? { deliveryURL == nil ? nil : id }
var citation: String { cite(nil, at: sourceURL) }
func cite(_ locator: String?, at url: URL) -> String {
([title, memberPath, locator, "Color Computer Archive", url.absoluteString] as [String?])
.compactMap { $0 }.joined(separator: " · ")
}
}
struct ArchiveHit: Identifiable, Hashable, Sendable {
let id: String
let document: ArchiveDocument
let page: Int?
let locator: String
let ocr: Bool
let sourceURL: URL
var locationNumber: Int = 1
var citation: String { document.cite(locator, at: sourceURL) }
}
struct ArchiveExample: Identifiable, Hashable, Sendable {
let title: String
let kind: String
let language: String
let author: String
let provenance: String
let source: String
let sourceSHA256: String
var id: String { sourceSHA256 }
}
enum ArchiveLibraryError: LocalizedError {
case missing, incompatible, input(String), damaged
var errorDescription: String? {
switch self {
case .missing: return "This build has no Archive search index. Install a build with the index or prepare it using the knowledge tools before rebuilding."
case .incompatible: return "This Archive index belongs to a different database or version. Rebuild with a compatible Archive Library index."
case .input(let message): return message
case .damaged: return "The Archive index could not be read. Replace it with a verified copy and rebuild the app."
}
}
}
/// Native and MCP clients use one serialized read-only connection. SQLite periodically
/// checks the current task for cancellation, including while ranking a common term.
actor ArchiveLibraryCatalog {
static let shared = ArchiveLibraryCatalog(url: Bundle.main.url(forResource: "ArchiveLibrary", withExtension: "sqlite", subdirectory: "ArchiveLibrary"))
private let url: URL?
private var connection: LibrarySQLite?
private var hasReaderSections = false
init(url: URL?) { self.url = url }
private func database() throws -> LibrarySQLite {
try Task.checkCancellation()
if let connection { return connection }
guard let url, url.isFileURL, FileManager.default.fileExists(atPath: url.path) else { throw ArchiveLibraryError.missing }
let db = try LibrarySQLite(url: url, cancellable: true, describeError: { _ in ArchiveLibraryError.damaged })
try db.execute("PRAGMA trusted_schema=OFF")
try db.execute("PRAGMA query_only=ON")
guard try db.rows("PRAGMA application_id").first?["application_id"] == String(0x56434341),
let version = try db.rows("PRAGMA user_version").first?["user_version"],
["1", "2"].contains(version) else { throw ArchiveLibraryError.incompatible }
hasReaderSections = version == "2"
connection = db
return db
}
func collections() throws -> [String] {
try database().rows("""
SELECT collection FROM document
UNION SELECT collection || '/' || issue_year AS collection FROM document
WHERE issue_year IS NOT NULL AND substr(collection,-5) != '/' || issue_year
ORDER BY collection COLLATE NOCASE,collection
""")
.compactMap { $0["collection"] }
}
private func scope(_ collection: String?) throws -> (sql: String, values: [String]) {
guard let collection, !collection.isEmpty else { return ("", []) }
guard collection.utf8.count <= 2048, !collection.contains("\0") else { throw ArchiveLibraryError.input("Invalid Archive collection.") }
return (" AND (d.collection=? OR instr(d.collection, ? || '/')=1 OR d.collection || '/' || d.issue_year=?)", [collection, collection, collection])
}
private func bounds(_ limit: Int, _ offset: Int) throws -> [String] {
guard (1...50).contains(limit), (0...250_000).contains(offset) else { throw ArchiveLibraryError.input("Archive pages allow 1–50 results and offsets up to 250,000.") }
return [String(limit), String(offset)]
}
func browse(collection: String?, limit: Int, offset: Int) throws -> [ArchiveDocument] {
let filter = try scope(collection)
let values = try filter.values + bounds(limit, offset)
let rows = try database().rows("""
SELECT d.* FROM document d WHERE 1=1 \(filter.sql)
ORDER BY d.title COLLATE NOCASE,d.sha256 LIMIT ? OFFSET ?
""", values)
return try rows.map(document)
}
func document(id: String) throws -> ArchiveDocument? {
guard Self.isSHA(id) else { return nil }
return try database().rows("SELECT * FROM document WHERE sha256=? LIMIT 1", [id]).first.map(document)
}
/// Locate the exact title in the same deterministic order used by Browse.
/// Deep links must not depend on the title happening to be in its first batch.
func browseOffset(documentID: String, collection: String?, batchSize: Int) throws -> Int? {
guard (1...50).contains(batchSize), let target = try document(id: documentID) else { return nil }
let filter = try scope(collection)
let rows = try database().rows("""
SELECT count(*) AS amount FROM document d WHERE 1=1 \(filter.sql)
AND (d.title COLLATE NOCASE < ? COLLATE NOCASE OR
(d.title COLLATE NOCASE = ? COLLATE NOCASE AND d.sha256 < ?))
""", filter.values + [target.title, target.title, target.id])
guard let amount = rows.first?["amount"].flatMap(Int.init), amount <= 250_000 else { return nil }
return (amount / batchSize) * batchSize
}
private static let compounds = ["module directory", "width 80", "width 40", "disk basic", "extended basic"]
private func match(_ query: String) throws -> String {
guard query.count <= 256, !query.contains("\0") else { throw ArchiveLibraryError.input("Use at most 256 characters for an Archive search.") }
let words = query.lowercased().split { !$0.isLetter && !$0.isNumber }.map(String.init)
guard !words.isEmpty, words.count <= 16 else { throw ArchiveLibraryError.input("Enter between one and sixteen words or numbers.") }
let required = words.map { "\"\($0)\"" }.joined(separator: " AND ")
let pairs = Set(zip(words, words.dropFirst()).map { $0 + " " + $1 })
let boosts = Self.compounds.filter(pairs.contains).map { "\"vccphrase\($0.replacingOccurrences(of: " ", with: ""))\"" }
guard !boosts.isEmpty else { return required }
return "(\(required)) OR ((\(required)) AND (\(boosts.joined(separator: " OR "))))"
}
func search(query: String, collection: String?, limit: Int, offset: Int) throws -> [ArchiveHit] {
let expression = try match(query)
let filter = try scope(collection)
let values = try [expression] + filter.values + bounds(limit, offset)
let rows = try database().rows("""
SELECT d.*,p.page_number,p.page_label,p.locator_kind,p.is_ocr
FROM page_terms JOIN page p ON p.id=page_terms.rowid JOIN document d ON d.id=p.document_id
WHERE page_terms MATCH ? \(filter.sql)
ORDER BY bm25(page_terms,1.0,4.0),p.id LIMIT ? OFFSET ?
""", values)
return try rows.map(hit)
}
func citation(id: String) throws -> ArchiveHit? {
let fields = id.components(separatedBy: ":")
guard fields.count == 4, fields[0] == "sha256", Self.isSHA(fields[1]),
let n = Int32(fields[3]), n > 0, String(n) == fields[3] else { return nil }
let rows = try database().rows("""
SELECT d.*,p.page_number,p.page_label,p.locator_kind,p.is_ocr
FROM document d JOIN page p ON p.document_id=d.id
WHERE d.sha256=? AND p.locator_kind=? AND p.page_number=?
""", Array(fields[1...3]))
return try rows.first.map(hit)
}
private static func isSHA(_ value: String) -> Bool {
value.utf8.count == 64 && value.utf8.allSatisfy { (48...57).contains($0) || (97...102).contains($0) }
}
private func document(_ row: [String: String]) throws -> ArchiveDocument {
guard let sha = row["sha256"], Self.isSHA(sha), let title = row["title"], let collection = row["collection"],
let link = row["source_url"].flatMap(URL.init(string:)), link.scheme == "https",
link.host == "colorcomputerarchive.com", link.path.hasPrefix("/repo/Documents/"),
link.user == nil, link.password == nil, link.port == nil else { throw ArchiveLibraryError.damaged }
let delivery = row["delivery_url"].flatMap(URL.init(string:))
var memberRange: ArchiveMemberRange?
if let offset = row["member_offset"].flatMap(Int64.init),
let compressed = row["member_compressed_bytes"].flatMap(Int64.init),
let method = row["member_method"].flatMap(Int.init),
let bytes = row["archive_bytes"].flatMap(Int64.init) {
memberRange = ArchiveMemberRange(offset: offset, compressedBytes: compressed, method: method, archiveBytes: bytes)
}
return ArchiveDocument(id: sha, title: title, collection: collection, sourceURL: link,
memberPath: row["member_path"], pageCount: row["page_count"].flatMap(Int.init),
deliveryURL: delivery, deliveryBytes: delivery == nil ? nil : row["byte_size"].flatMap(Int64.init),
format: row["format"] ?? "pdf", sourceBytes: row["byte_size"].flatMap(Int64.init), memberRange: memberRange)
}
private func hit(_ row: [String: String]) throws -> ArchiveHit {
let doc = try document(row)
guard let kind = row["locator_kind"], let number = row["page_number"].flatMap(Int.init), number > 0 else { throw ArchiveLibraryError.damaged }
let label = row["page_label"].flatMap { $0.isEmpty ? nil : $0 }
let locator: String
var source = doc.sourceURL
switch kind {
case "pdf_page":
if let label, label != String(number) {
locator = "PDF page \(number) (label \(label))"
} else {
locator = "PDF page \(number)"
}
if doc.memberPath == nil, var parts = URLComponents(url: source, resolvingAgainstBaseURL: false) {
parts.fragment = "page=\(number)"
if let link = parts.url { source = link }
}
case "worksheet": locator = label ?? "Worksheet \(number)"
case "section": locator = label ?? "Section \(number)"
case "image": locator = label ?? "Image \(number)"
default: throw ArchiveLibraryError.damaged
}
return ArchiveHit(id: "sha256:\(doc.id):\(kind):\(number)", document: doc,
page: kind == "pdf_page" ? number : nil, locator: locator,
ocr: row["is_ocr"] == "1", sourceURL: source, locationNumber: number)
}
func readerSection(documentID: String, number: Int) throws -> ArchiveReaderSection? {
guard Self.isSHA(documentID), number > 0 else { return nil }
let db = try database()
guard hasReaderSections else { return nil }
let rows = try db.rows("""
SELECT p.page_label,s.text FROM document d JOIN page p ON p.document_id=d.id
JOIN reader_section s ON s.page_id=p.id WHERE d.sha256=? AND p.page_number=?
""", [documentID, String(number)])
guard let row = rows.first, let text = row["text"] else { return nil }
return ArchiveReaderSection(number: number, label: row["page_label"], text: text)
}
func examples(documentID: String, page: Int) throws -> [ArchiveExample] {
guard Self.isSHA(documentID), page > 0 else { return [] }
let rows = try database().rows("""
SELECT * FROM reviewed_example WHERE original_sha256=? AND ? BETWEEN first_page AND last_page
ORDER BY title,source_sha256
""", [documentID, String(page)])
return try rows.map(reviewedExample)
}
func basicExample(sourceSHA256: String) throws -> ArchiveExample? {
guard Self.isSHA(sourceSHA256) else { return nil }
let rows = try database().rows("SELECT * FROM reviewed_example WHERE source_sha256=? AND kind='basic_program' LIMIT 1", [sourceSHA256])
return try rows.first.map(reviewedExample)
}
private func reviewedExample(_ row: [String: String]) throws -> ArchiveExample {
guard let title = row["title"], let kind = row["kind"], let language = row["language"],
let author = row["author"], let provenance = row["provenance"], let source = row["source"],
let sha = row["source_sha256"],
SHA256.hash(data: Data(source.utf8)).map({ String(format: "%02x", $0) }).joined() == sha else {
throw ArchiveLibraryError.damaged
}
return ArchiveExample(title: title, kind: kind, language: language, author: author,
provenance: provenance, source: source, sourceSHA256: sha)
}
}