273 lines
14 KiB
Swift
273 lines
14 KiB
Swift
// SPDX-License-Identifier: GPL-3.0-or-later
|
||
import CryptoKit
|
||
import Foundation
|
||
|
||
struct ArchiveMemberRange: Hashable, Sendable {
|
||
let offset, compressedBytes: Int64
|
||
let method: Int
|
||
let archiveBytes: Int64
|
||
}
|
||
|
||
struct ArchiveReaderSection: Sendable {
|
||
let number: Int
|
||
let label: String?
|
||
let text: String
|
||
}
|
||
|
||
struct ArchiveDocument: Identifiable, Hashable, Sendable {
|
||
let id: String
|
||
let title: String
|
||
let collection: String
|
||
let sourceURL: URL
|
||
let memberPath: String?
|
||
let pageCount: Int?
|
||
let deliveryURL: URL?
|
||
let deliveryBytes: Int64?
|
||
var format: String = "pdf"
|
||
var sourceBytes: Int64?
|
||
var memberRange: ArchiveMemberRange?
|
||
var deliverySHA256: String? { deliveryURL == nil ? nil : id }
|
||
var citation: String { cite(nil, at: sourceURL) }
|
||
|
||
func cite(_ locator: String?, at url: URL) -> String {
|
||
([title, memberPath, locator, "Color Computer Archive", url.absoluteString] as [String?])
|
||
.compactMap { $0 }.joined(separator: " · ")
|
||
}
|
||
}
|
||
|
||
struct ArchiveHit: Identifiable, Hashable, Sendable {
|
||
let id: String
|
||
let document: ArchiveDocument
|
||
let page: Int?
|
||
let locator: String
|
||
let ocr: Bool
|
||
let sourceURL: URL
|
||
var locationNumber: Int = 1
|
||
var citation: String { document.cite(locator, at: sourceURL) }
|
||
}
|
||
|
||
struct ArchiveExample: Identifiable, Hashable, Sendable {
|
||
let title: String
|
||
let kind: String
|
||
let language: String
|
||
let author: String
|
||
let provenance: String
|
||
let source: String
|
||
let sourceSHA256: String
|
||
var id: String { sourceSHA256 }
|
||
}
|
||
|
||
enum ArchiveLibraryError: LocalizedError {
|
||
case missing, incompatible, input(String), damaged
|
||
var errorDescription: String? {
|
||
switch self {
|
||
case .missing: return "This build has no Archive search index. Install a build with the index or prepare it using the knowledge tools before rebuilding."
|
||
case .incompatible: return "This Archive index belongs to a different database or version. Rebuild with a compatible Archive Library index."
|
||
case .input(let message): return message
|
||
case .damaged: return "The Archive index could not be read. Replace it with a verified copy and rebuild the app."
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Native and MCP clients use one serialized read-only connection. SQLite periodically
|
||
/// checks the current task for cancellation, including while ranking a common term.
|
||
actor ArchiveLibraryCatalog {
|
||
static let shared = ArchiveLibraryCatalog(url: Bundle.main.url(forResource: "ArchiveLibrary", withExtension: "sqlite", subdirectory: "ArchiveLibrary"))
|
||
private let url: URL?
|
||
private var connection: LibrarySQLite?
|
||
private var hasReaderSections = false
|
||
|
||
init(url: URL?) { self.url = url }
|
||
|
||
private func database() throws -> LibrarySQLite {
|
||
try Task.checkCancellation()
|
||
if let connection { return connection }
|
||
guard let url, url.isFileURL, FileManager.default.fileExists(atPath: url.path) else { throw ArchiveLibraryError.missing }
|
||
let db = try LibrarySQLite(url: url, cancellable: true, describeError: { _ in ArchiveLibraryError.damaged })
|
||
try db.execute("PRAGMA trusted_schema=OFF")
|
||
try db.execute("PRAGMA query_only=ON")
|
||
guard try db.rows("PRAGMA application_id").first?["application_id"] == String(0x56434341),
|
||
let version = try db.rows("PRAGMA user_version").first?["user_version"],
|
||
["1", "2"].contains(version) else { throw ArchiveLibraryError.incompatible }
|
||
hasReaderSections = version == "2"
|
||
connection = db
|
||
return db
|
||
}
|
||
|
||
func collections() throws -> [String] {
|
||
try database().rows("""
|
||
SELECT collection FROM document
|
||
UNION SELECT collection || '/' || issue_year AS collection FROM document
|
||
WHERE issue_year IS NOT NULL AND substr(collection,-5) != '/' || issue_year
|
||
ORDER BY collection COLLATE NOCASE,collection
|
||
""")
|
||
.compactMap { $0["collection"] }
|
||
}
|
||
|
||
private func scope(_ collection: String?) throws -> (sql: String, values: [String]) {
|
||
guard let collection, !collection.isEmpty else { return ("", []) }
|
||
guard collection.utf8.count <= 2048, !collection.contains("\0") else { throw ArchiveLibraryError.input("Invalid Archive collection.") }
|
||
return (" AND (d.collection=? OR instr(d.collection, ? || '/')=1 OR d.collection || '/' || d.issue_year=?)", [collection, collection, collection])
|
||
}
|
||
|
||
private func bounds(_ limit: Int, _ offset: Int) throws -> [String] {
|
||
guard (1...50).contains(limit), (0...250_000).contains(offset) else { throw ArchiveLibraryError.input("Archive pages allow 1–50 results and offsets up to 250,000.") }
|
||
return [String(limit), String(offset)]
|
||
}
|
||
|
||
func browse(collection: String?, limit: Int, offset: Int) throws -> [ArchiveDocument] {
|
||
let filter = try scope(collection)
|
||
let values = try filter.values + bounds(limit, offset)
|
||
let rows = try database().rows("""
|
||
SELECT d.* FROM document d WHERE 1=1 \(filter.sql)
|
||
ORDER BY d.title COLLATE NOCASE,d.sha256 LIMIT ? OFFSET ?
|
||
""", values)
|
||
return try rows.map(document)
|
||
}
|
||
|
||
func document(id: String) throws -> ArchiveDocument? {
|
||
guard Self.isSHA(id) else { return nil }
|
||
return try database().rows("SELECT * FROM document WHERE sha256=? LIMIT 1", [id]).first.map(document)
|
||
}
|
||
|
||
/// Locate the exact title in the same deterministic order used by Browse.
|
||
/// Deep links must not depend on the title happening to be in its first batch.
|
||
func browseOffset(documentID: String, collection: String?, batchSize: Int) throws -> Int? {
|
||
guard (1...50).contains(batchSize), let target = try document(id: documentID) else { return nil }
|
||
let filter = try scope(collection)
|
||
let rows = try database().rows("""
|
||
SELECT count(*) AS amount FROM document d WHERE 1=1 \(filter.sql)
|
||
AND (d.title COLLATE NOCASE < ? COLLATE NOCASE OR
|
||
(d.title COLLATE NOCASE = ? COLLATE NOCASE AND d.sha256 < ?))
|
||
""", filter.values + [target.title, target.title, target.id])
|
||
guard let amount = rows.first?["amount"].flatMap(Int.init), amount <= 250_000 else { return nil }
|
||
return (amount / batchSize) * batchSize
|
||
}
|
||
|
||
private static let compounds = ["module directory", "width 80", "width 40", "disk basic", "extended basic"]
|
||
|
||
private func match(_ query: String) throws -> String {
|
||
guard query.count <= 256, !query.contains("\0") else { throw ArchiveLibraryError.input("Use at most 256 characters for an Archive search.") }
|
||
let words = query.lowercased().split { !$0.isLetter && !$0.isNumber }.map(String.init)
|
||
guard !words.isEmpty, words.count <= 16 else { throw ArchiveLibraryError.input("Enter between one and sixteen words or numbers.") }
|
||
let required = words.map { "\"\($0)\"" }.joined(separator: " AND ")
|
||
let pairs = Set(zip(words, words.dropFirst()).map { $0 + " " + $1 })
|
||
let boosts = Self.compounds.filter(pairs.contains).map { "\"vccphrase\($0.replacingOccurrences(of: " ", with: ""))\"" }
|
||
guard !boosts.isEmpty else { return required }
|
||
return "(\(required)) OR ((\(required)) AND (\(boosts.joined(separator: " OR "))))"
|
||
}
|
||
|
||
func search(query: String, collection: String?, limit: Int, offset: Int) throws -> [ArchiveHit] {
|
||
let expression = try match(query)
|
||
let filter = try scope(collection)
|
||
let values = try [expression] + filter.values + bounds(limit, offset)
|
||
let rows = try database().rows("""
|
||
SELECT d.*,p.page_number,p.page_label,p.locator_kind,p.is_ocr
|
||
FROM page_terms JOIN page p ON p.id=page_terms.rowid JOIN document d ON d.id=p.document_id
|
||
WHERE page_terms MATCH ? \(filter.sql)
|
||
ORDER BY bm25(page_terms,1.0,4.0),p.id LIMIT ? OFFSET ?
|
||
""", values)
|
||
return try rows.map(hit)
|
||
}
|
||
|
||
func citation(id: String) throws -> ArchiveHit? {
|
||
let fields = id.components(separatedBy: ":")
|
||
guard fields.count == 4, fields[0] == "sha256", Self.isSHA(fields[1]),
|
||
let n = Int32(fields[3]), n > 0, String(n) == fields[3] else { return nil }
|
||
let rows = try database().rows("""
|
||
SELECT d.*,p.page_number,p.page_label,p.locator_kind,p.is_ocr
|
||
FROM document d JOIN page p ON p.document_id=d.id
|
||
WHERE d.sha256=? AND p.locator_kind=? AND p.page_number=?
|
||
""", Array(fields[1...3]))
|
||
return try rows.first.map(hit)
|
||
}
|
||
|
||
private static func isSHA(_ value: String) -> Bool {
|
||
value.utf8.count == 64 && value.utf8.allSatisfy { (48...57).contains($0) || (97...102).contains($0) }
|
||
}
|
||
|
||
private func document(_ row: [String: String]) throws -> ArchiveDocument {
|
||
guard let sha = row["sha256"], Self.isSHA(sha), let title = row["title"], let collection = row["collection"],
|
||
let link = row["source_url"].flatMap(URL.init(string:)), link.scheme == "https",
|
||
link.host == "colorcomputerarchive.com", link.path.hasPrefix("/repo/Documents/"),
|
||
link.user == nil, link.password == nil, link.port == nil else { throw ArchiveLibraryError.damaged }
|
||
let delivery = row["delivery_url"].flatMap(URL.init(string:))
|
||
var memberRange: ArchiveMemberRange?
|
||
if let offset = row["member_offset"].flatMap(Int64.init),
|
||
let compressed = row["member_compressed_bytes"].flatMap(Int64.init),
|
||
let method = row["member_method"].flatMap(Int.init),
|
||
let bytes = row["archive_bytes"].flatMap(Int64.init) {
|
||
memberRange = ArchiveMemberRange(offset: offset, compressedBytes: compressed, method: method, archiveBytes: bytes)
|
||
}
|
||
return ArchiveDocument(id: sha, title: title, collection: collection, sourceURL: link,
|
||
memberPath: row["member_path"], pageCount: row["page_count"].flatMap(Int.init),
|
||
deliveryURL: delivery, deliveryBytes: delivery == nil ? nil : row["byte_size"].flatMap(Int64.init),
|
||
format: row["format"] ?? "pdf", sourceBytes: row["byte_size"].flatMap(Int64.init), memberRange: memberRange)
|
||
}
|
||
|
||
private func hit(_ row: [String: String]) throws -> ArchiveHit {
|
||
let doc = try document(row)
|
||
guard let kind = row["locator_kind"], let number = row["page_number"].flatMap(Int.init), number > 0 else { throw ArchiveLibraryError.damaged }
|
||
let label = row["page_label"].flatMap { $0.isEmpty ? nil : $0 }
|
||
let locator: String
|
||
var source = doc.sourceURL
|
||
switch kind {
|
||
case "pdf_page":
|
||
if let label, label != String(number) {
|
||
locator = "PDF page \(number) (label \(label))"
|
||
} else {
|
||
locator = "PDF page \(number)"
|
||
}
|
||
if doc.memberPath == nil, var parts = URLComponents(url: source, resolvingAgainstBaseURL: false) {
|
||
parts.fragment = "page=\(number)"
|
||
if let link = parts.url { source = link }
|
||
}
|
||
case "worksheet": locator = label ?? "Worksheet \(number)"
|
||
case "section": locator = label ?? "Section \(number)"
|
||
case "image": locator = label ?? "Image \(number)"
|
||
default: throw ArchiveLibraryError.damaged
|
||
}
|
||
return ArchiveHit(id: "sha256:\(doc.id):\(kind):\(number)", document: doc,
|
||
page: kind == "pdf_page" ? number : nil, locator: locator,
|
||
ocr: row["is_ocr"] == "1", sourceURL: source, locationNumber: number)
|
||
}
|
||
|
||
func readerSection(documentID: String, number: Int) throws -> ArchiveReaderSection? {
|
||
guard Self.isSHA(documentID), number > 0 else { return nil }
|
||
let db = try database()
|
||
guard hasReaderSections else { return nil }
|
||
let rows = try db.rows("""
|
||
SELECT p.page_label,s.text FROM document d JOIN page p ON p.document_id=d.id
|
||
JOIN reader_section s ON s.page_id=p.id WHERE d.sha256=? AND p.page_number=?
|
||
""", [documentID, String(number)])
|
||
guard let row = rows.first, let text = row["text"] else { return nil }
|
||
return ArchiveReaderSection(number: number, label: row["page_label"], text: text)
|
||
}
|
||
|
||
func examples(documentID: String, page: Int) throws -> [ArchiveExample] {
|
||
guard Self.isSHA(documentID), page > 0 else { return [] }
|
||
let rows = try database().rows("""
|
||
SELECT * FROM reviewed_example WHERE original_sha256=? AND ? BETWEEN first_page AND last_page
|
||
ORDER BY title,source_sha256
|
||
""", [documentID, String(page)])
|
||
return try rows.map(reviewedExample)
|
||
}
|
||
|
||
func basicExample(sourceSHA256: String) throws -> ArchiveExample? {
|
||
guard Self.isSHA(sourceSHA256) else { return nil }
|
||
let rows = try database().rows("SELECT * FROM reviewed_example WHERE source_sha256=? AND kind='basic_program' LIMIT 1", [sourceSHA256])
|
||
return try rows.first.map(reviewedExample)
|
||
}
|
||
|
||
private func reviewedExample(_ row: [String: String]) throws -> ArchiveExample {
|
||
guard let title = row["title"], let kind = row["kind"], let language = row["language"],
|
||
let author = row["author"], let provenance = row["provenance"], let source = row["source"],
|
||
let sha = row["source_sha256"],
|
||
SHA256.hash(data: Data(source.utf8)).map({ String(format: "%02x", $0) }).joined() == sha else {
|
||
throw ArchiveLibraryError.damaged
|
||
}
|
||
return ArchiveExample(title: title, kind: kind, language: language, author: author,
|
||
provenance: provenance, source: source, sourceSHA256: sha)
|
||
}
|
||
}
|