// SPDX-License-Identifier: GPL-3.0-or-later import Foundation /// A bounded, non-executing lexical index. This is navigation assistance, not a /// compiler AST: macro expansion, arbitrary Make recipes and expression types /// are deliberately not guessed. Conditional candidates remain visibly marked. struct CMOCProjectIndex: Sendable { enum Kind: String, Sendable { case function = "Function", variable = "Variable", type = "Type", macro = "Macro", parameter = "Parameter", member = "Member", enumerator = "Enumerator" } struct Symbol: Identifiable, Hashable, Sendable { var id: String { "\(path):\(offsetUTF16):\(kind.rawValue)" } let path: String let name: String let line: Int let column: Int let kind: Kind let signature: String let isDefinition: Bool let isStatic: Bool let conditional: Bool let scope: String? let offsetUTF16: Int let lengthUTF16: Int fileprivate let scopeStart: Int fileprivate let scopeEnd: Int } struct Match: Identifiable, Hashable, Sendable { var id: String { "\(path):\(offsetUTF16):\(lengthUTF16)" } let path: String let line: Int let column: Int let lengthUTF16: Int let preview: String let offsetUTF16: Int } struct SearchResults: Sendable { let matches: [Match]; let truncated: Bool } static let maximumFiles = 1024 static let maximumFileBytes = 1_048_576 static let maximumTotalBytes = 16 * 1_048_576 static let maximumTokens = 150_000 static let maximumSymbols = 20_000 let documents: [String: String] let symbols: [Symbol] let issues: [String] private let tokens: [String: [Token]] private struct Include: Sendable { let name: String; let quoted: Bool } private enum Directive: Sendable { case include(Include), define(String), undef(String) } private struct Step: Sendable { let directive: Directive let offset: Int let conditional: Bool } private let steps: [String: [Step]] private let guarded: Set private let undefinedNames: Set private let includePaths: [String] init(documents input: [String: String], defines: [String] = [], includePaths: [String] = [], configurationUncertain: Bool = false) { var accepted: [String: String] = [:], notes: [String] = [], total = 0 for path in input.keys.sorted() { if Task.isCancelled { notes.append("Index cancelled."); break } guard let text = input[path] else { continue } let size = text.utf8.count guard accepted.count < Self.maximumFiles, size <= Self.maximumFileBytes, total + size <= Self.maximumTotalBytes, !text.contains("\0") else { notes.append("Index omitted \(path): file/count/size limit or non-text contents."); continue } accepted[path] = text; total += size } var parsed: [String: [Token]] = [:], all: [Symbol] = [], links: [String: [Step]] = [:] var guards: [String: String] = [:], undefinedNames: Set = [] for path in accepted.keys.sorted() where ["c", "h"].contains((path as NSString).pathExtension.lowercased()) { if Task.isCancelled { notes.append("Index cancelled."); break } let source = accepted[path]! let preparation = Self.prepare(source, path: path, defines: defines, uncertain: configurationUncertain) let lexed = Self.lex(preparation.text, conditionalLines: preparation.conditionalLines) parsed[path] = lexed.tokens links[path] = preparation.steps guards[path] = preparation.guardName for step in preparation.steps { if case .undef(let name) = step.directive { undefinedNames.insert(name) } } if lexed.truncated { notes.append("Symbol scan stopped at \(Self.maximumTokens) tokens in \(path).") } var parser = Parser(path: path, source: source, tokens: lexed.tokens) parser.parse() if parser.limited { notes.append("Declaration scan reached its nesting or symbol limit in \(path).") } let additions = preparation.macros + parser.symbols let remaining = max(0, Self.maximumSymbols - all.count) all += additions.prefix(remaining) if additions.count > remaining { notes.append("Symbol limit reached; some declarations in \(path) were omitted.") } } if configurationUncertain { notes.append("Custom build flags are not evaluated. Definitions are lexical candidates; conditional branches may differ from the compiled program.") } self.documents = accepted; self.symbols = all.sorted { ($0.path, $0.offsetUTF16) < ($1.path, $1.offsetUTF16) } self.issues = Array(notes.prefix(32)); self.tokens = parsed; self.steps = links self.guarded = Set(guards.compactMap { undefinedNames.contains($0.value) ? nil : $0.key }) self.undefinedNames = undefinedNames; self.includePaths = Array(includePaths.prefix(128)) } func outline(path: String) -> [Symbol] { symbols.filter { $0.path == path && $0.scope == nil && $0.kind != .parameter && $0.kind != .member } } func token(atUTF16Offset offset: Int, in path: String) -> String? { guard let values = tokens[path], let token = values.first(where: { $0.identifier && $0.offset <= offset && offset < $0.offset + $0.text.utf16.count }) ?? values.last(where: { $0.identifier && $0.offset + $0.text.utf16.count == offset }) else { return nil } return token.text } /// Locals are visible only after their declaration and inside their lexical /// block. Member access has no inferred C type: return member candidates, /// never accidentally substitute a same-named global. func definitions(named name: String, from path: String, line: Int, column: Int) -> [Symbol] { guard !name.isEmpty, name.utf8.count <= 256 else { return [] } let offset = Self.offset(line: line, column: column, text: documents[path] ?? "") let fileTokens = tokens[path] ?? [] let targetIndex = fileTokens.firstIndex { $0.text == name && $0.offset <= offset && offset < $0.offset + $0.text.utf16.count } ?? fileTokens.lastIndex { $0.text == name && $0.offset + $0.text.utf16.count == offset } let preceding = targetIndex.flatMap { $0 > 0 ? fileTokens[$0 - 1] : nil } ?? fileTokens.last { $0.offset < offset && $0.offset + $0.text.utf16.count <= offset } let member = preceding?.text == "." || preceding?.text == "->" let matches = symbols.filter { $0.name == name } if member { return matches.filter { $0.kind == .member }.map { $0.withConditional() } } let local = matches.filter { $0.path == path && $0.scope != nil && $0.kind != .member && $0.offsetUTF16 <= offset && $0.scopeStart <= offset && offset <= $0.scopeEnd } if let narrowest = local.map(\.scopeStart).max() { return local.filter { $0.scopeStart == narrowest }.sorted { $0.offsetUTF16 > $1.offsetUTF16 } } let visible = visibleFiles(from: path) let overrides = macroOverrides(from: path, offset: offset, names: [name]) let globals = matches.filter { overrides.keeps($0) }.filter { $0.scope == nil && (!$0.isStatic || $0.path == path || visible.contains($0.path)) } let sameFile = globals.filter { $0.path == path && $0.isDefinition } if overrides.uncertain.contains(name) { return globals.filter { visible.contains($0.path) }.sorted { $0.kind != .macro && $1.kind == .macro }.map { $0.withConditional() } } if !sameFile.isEmpty { return sameFile } let visibleDefinitions = globals.filter { visible.contains($0.path) && $0.isDefinition } if !visibleDefinitions.isEmpty { return visibleDefinitions } // External definitions may reside in a different translation unit. Keep // duplicate external definitions visible instead of choosing by filename. let definitions = globals.filter { $0.isDefinition && !$0.isStatic && ($0.kind == .function || $0.kind == .variable) } if !definitions.isEmpty { return definitions } let declarations = globals.filter { visible.contains($0.path) || $0.path == path } return declarations.isEmpty ? globals.map { $0.withConditional() } : declarations } /// Completion is intentionally narrower than definition navigation: only /// declarations in this source and literal included headers participate. func completionSymbols(from path: String, offset: Int, prefix: String, includeLocals: Bool = true) -> [Symbol] { let visible = visibleFiles(from: path) var result = symbols.filter { symbol in guard symbol.name.hasPrefix(prefix), visible.contains(symbol.path), symbol.kind != .member else { return false } if symbol.path == path && symbol.offsetUTF16 > offset { return false } if symbol.scope != nil { return includeLocals && symbol.path == path && symbol.scopeStart <= offset && offset <= symbol.scopeEnd } return true } let overrides = macroOverrides(from: path, offset: offset, names: Set(result.lazy.filter { $0.kind == .macro }.map(\.name))) result = result.filter { overrides.keeps($0) }.map { overrides.uncertain.contains($0.name) ? $0.withConditional() : $0 } let locals = Dictionary(grouping: result.filter { $0.scope != nil }, by: \.name) result.removeAll { symbol in guard let scoped = locals[symbol.name], let innermost = scoped.map(\.scopeStart).max() else { return false } return symbol.scope == nil || symbol.scopeStart != innermost } return result.sorted { let a = ($0.scope == nil ? 1 : 0, $0.kind == .macro && overrides.uncertain.contains($0.name) ? 1 : 0, $0.path == path ? 0 : 1, $0.name, $0.path, $0.offsetUTF16) let b = ($1.scope == nil ? 1 : 0, $1.kind == .macro && overrides.uncertain.contains($1.name) ? 1 : 0, $1.path == path ? 0 : 1, $1.name, $1.path, $1.offsetUTF16) return a < b } } func search(query: String, caseSensitive: Bool = false, limit: Int = 500, paths: Set? = nil) -> SearchResults { let limit = max(1, min(2000, limit)) guard !query.isEmpty, query.utf16.count <= 1024 else { return .init(matches: [], truncated: false) } var found: [Match] = [] for path in documents.keys.sorted() where paths == nil || paths!.contains(path) { if Task.isCancelled { return .init(matches: found, truncated: true) } let text = documents[path]! as NSString var start = 0 while start < text.length { if Task.isCancelled { return .init(matches: found, truncated: true) } let range = text.range(of: query, options: caseSensitive ? [.literal] : [.literal, .caseInsensitive], range: NSRange(location: start, length: text.length - start)) guard range.location != NSNotFound, range.length > 0 else { break } if found.count >= limit { return .init(matches: found, truncated: true) } let before = text.substring(to: range.location) as NSString let lineRange = text.lineRange(for: NSRange(location: range.location, length: 0)) let preview = (text.substring(with: lineRange) as NSString).substring(to: min(240, lineRange.length)).trimmingCharacters(in: .newlines) found.append(.init(path: path, line: before.components(separatedBy: "\n").count, column: range.location - lineRange.location + 1, lengthUTF16: range.length, preview: preview, offsetUTF16: range.location)) start = range.location + range.length } } return .init(matches: found, truncated: false) } private func visibleFiles(from path: String) -> Set { var visible: Set = [path], pending = [path] while let file = pending.popLast(), visible.count <= Self.maximumFiles { for step in steps[file] ?? [] { if case .include(let include) = step.directive, let target = resolve(include, in: file), visible.insert(target).inserted { pending.append(target) } } } return visible } private func resolve(_ include: Include, in file: String) -> String? { let parent = (file as NSString).deletingLastPathComponent let local = parent.isEmpty ? include.name : parent + "/" + include.name let names = (include.quoted ? [local] : []) + includePaths.map { $0.isEmpty ? include.name : $0 + "/" + include.name } return names.lazy.map { ($0 as NSString).standardizingPath }.first { documents[$0] != nil } } private struct MacroOverrides { var live: [String: Set] = [:] var undefined: Set = [] var uncertain: Set = [] func keeps(_ symbol: Symbol) -> Bool { guard undefined.contains(symbol.name) else { return true } let macros = live[symbol.name] ?? [] if symbol.kind == .macro { return macros.contains(symbol.id) } return symbol.kind != .function || macros.isEmpty || uncertain.contains(symbol.name) } } /// Resolve only names with an explicit #undef. Replay literal directives in /// include order up to the caret; unknown branches keep alternative candidates. private func macroOverrides(from path: String, offset: Int, names: Set) -> MacroOverrides { let relevant = names.intersection(undefinedNames) guard !relevant.isEmpty else { return MacroOverrides() } var state = MacroOverrides(), confirmed: Set = [], ancestors: Set = [] var budget = Self.maximumTokens func scan(_ file: String, through limit: Int, certain: Bool) { guard budget > 0, !confirmed.contains(file), !ancestors.contains(file) else { return } guard ancestors.count < 64, !Task.isCancelled else { budget = 0; return } ancestors.insert(file) defer { ancestors.remove(file) } if guarded.contains(file), certain { confirmed.insert(file) } for step in steps[file] ?? [] { guard step.offset <= limit, budget > 0 else { break } budget -= 1 let definite = certain && !step.conditional switch step.directive { case .include(let include): if let nested = resolve(include, in: file) { scan(nested, through: Int.max, certain: definite) } case .define(let name) where relevant.contains(name): let id = "\(file):\(step.offset):\(Kind.macro.rawValue)" if definite { state.live[name] = [id] state.uncertain.remove(name) } else { state.live[name, default: []].insert(id) if state.undefined.contains(name) { state.uncertain.insert(name) } } case .undef(let name) where relevant.contains(name): state.undefined.insert(name) if definite { state.live[name] = [] state.uncertain.remove(name) } else if state.live[name]?.isEmpty == false { state.uncertain.insert(name) } default: break } } } scan(path, through: offset, certain: true) return budget > 0 ? state : MacroOverrides() } private static func offset(line: Int, column: Int, text: String) -> Int { let lines = text.components(separatedBy: "\n") guard line > 0, line <= lines.count else { return 0 } return lines.prefix(line - 1).reduce(0) { $0 + $1.utf16.count + 1 } + max(0, min(column - 1, lines[line - 1].utf16.count)) } private struct Token: Sendable { let text: String, offset: Int, line: Int, column: Int, conditional: Bool var identifier: Bool { guard let first = text.utf8.first else { return false }; return Self.identifierStart(first) } static func identifierStart(_ c: UInt8) -> Bool { c == 95 || (65...90).contains(c) || (97...122).contains(c) } } private static func lex(_ text: String, conditionalLines: Set) -> (tokens: [Token], truncated: Bool) { let bytes = Array(text.utf16); var result: [Token] = [], i = 0, line = 1, column = 1 func idStart(_ c: UInt16) -> Bool { c == 95 || (65...90).contains(c) || (97...122).contains(c) } func advance() { if bytes[i] == 10 { line += 1; column = 1 } else { column += 1 }; i += 1 } while i < bytes.count { if result.count >= maximumTokens || (i % 1024 == 0 && Task.isCancelled) { return (result, true) } let c = bytes[i] if c == 32 || c == 9 || c == 13 || c == 10 { advance(); continue } if c == 34 || c == 39 { let quote = c; advance() while i < bytes.count { if bytes[i] == 92 { advance(); if i < bytes.count { advance() }; continue }; if bytes[i] == quote { advance(); break }; advance() } continue } let start = i, startLine = line, startColumn = column if idStart(c) || (48...57).contains(c) { advance(); while i < bytes.count && (idStart(bytes[i]) || (48...57).contains(bytes[i])) { advance() } } else { advance() if i < bytes.count && ((c == 45 && bytes[i] == 62) || (c == 38 && bytes[i] == 38) || (c == 124 && bytes[i] == 124) || (c == 61 && bytes[i] == 61) || (c == 33 && bytes[i] == 61)) { advance() } } result.append(.init(text: String(decoding: bytes[start.. Self { self == .no || other == .no ? .no : self == .yes && other == .yes ? .yes : .unknown } func or(_ other: Self) -> Self { self == .yes || other == .yes ? .yes : self == .no && other == .no ? .no : .unknown } } private struct Preparation { var text: String; var conditionalLines: Set; var macros: [Symbol]; var steps: [Step]; var guardName: String? } private static func prepare(_ text: String, path: String, defines: [String], uncertain: Bool) -> Preparation { // Blank comments while retaining all UTF-16 coordinates and quoted text. var units = Array(text.utf16), i = 0, quote: UInt16?, block = false, lineComment = false var currentLine = 1, literalContinuationLines: Set = [], lineCommentEscape = false while i < units.count { let c = units[i], next = i + 1 < units.count ? units[i + 1] : 0 if c == 10 { currentLine += 1; if quote != nil { literalContinuationLines.insert(currentLine) } } if block { if c == 42 && next == 47 { units[i] = 32; units[i + 1] = 32; i += 2; block = false; continue } if c != 10 && c != 13 { units[i] = 32 }; i += 1; continue } if lineComment { if c == 10 { lineComment = lineCommentEscape; lineCommentEscape = false } else if c != 13 { lineCommentEscape = c == 92; units[i] = 32 } i += 1; continue } if let q = quote { if c == 92 { if next == 10 { currentLine += 1; literalContinuationLines.insert(currentLine) } i += min(2, units.count - i); continue }; if c == q { quote = nil }; i += 1; continue } if c == 34 || c == 39 { quote = c; i += 1; continue } if c == 47 && next == 42 { units[i] = 32; units[i+1] = 32; i += 2; block = true; continue } if c == 47 && next == 47 { units[i] = 32; units[i+1] = 32; i += 2; lineComment = true; continue } i += 1 } let clean = String(decoding: units, as: UTF16.self).components(separatedBy: "\n") // A conventional outer include guard makes a second literal include inert. let nonempty = clean.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty } let guardName = nonempty.first.flatMap { $0.hasPrefix("#ifndef ") ? String($0.dropFirst(8)) : nil } var guarded = guardName.map { nonempty.dropFirst().first == "#define " + $0 && nonempty.last == "#endif" } ?? false let lastContent = clean.lastIndex { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } var macros: [String: String] = [:] for define in defines.prefix(256) { let pieces = define.split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false); if let name = pieces.first { macros[String(name)] = pieces.count > 1 ? String(pieces[1]) : "1" } } var conditional: Set = [], symbols: [Symbol] = [], steps: [Step] = [], output: [String] = [], offset = 0 var active = Truth.yes, stack: [(parent: Truth, taken: Truth)] = [], unknownMacros: Set = [], included = false, continuing = false func condition(_ expression: String) -> Truth { evaluateCondition(expression, macros: macros, unknownMacros: unknownMacros, unknownAbsent: uncertain || included) } for (number, line) in clean.enumerated() { if Task.isCancelled { break } let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) let blank = String(repeating: " ", count: line.utf16.count) defer { offset += line.utf16.count + 1 } if continuing { continuing = trimmed.hasSuffix("\\"); output.append(blank); continue } guard trimmed.hasPrefix("#"), !literalContinuationLines.contains(number + 1) else { output.append(active == .no ? blank : line) if active == .unknown { conditional.insert(number + 1) }; continue } continuing = trimmed.hasSuffix("\\") output.append(blank) let directive = trimmed.dropFirst().trimmingCharacters(in: .whitespaces) let command = String(directive.prefix { $0.isLetter }), argument = String(directive.dropFirst(directive.prefix { $0.isLetter }.count)).trimmingCharacters(in: .whitespaces) if stack.count == 1, command == "elif" || command == "else" || (command == "endif" && number != lastContent) { guarded = false } switch command { case "if", "ifdef", "ifndef": let test = command == "if" ? condition(argument) : condition("defined(\(argument))") let value = command == "ifndef" ? test.inverted : test stack.append((active, value)); active = active.and(value) case "elif": if let frame = stack.popLast() { let test = condition(argument); active = frame.parent.and(frame.taken.inverted).and(test); stack.append((frame.parent, frame.taken.or(test))) } case "else": if let frame = stack.last { active = frame.parent.and(frame.taken.inverted); stack[stack.count - 1].taken = .yes } case "endif": if let frame = stack.popLast() { active = frame.parent } case "include" where active != .no: if let first = argument.first, first == "\"" || first == "<", let end = argument.dropFirst().firstIndex(of: first == "\"" ? "\"" : ">") { steps.append(.init(directive: .include(.init(name: String(argument[argument.index(after: argument.startIndex).., unknownAbsent: Bool, depth: Int = 0) -> Truth { guard depth < 32, expression.utf16.count <= 8192 else { return .unknown } let expr = expression.trimmingCharacters(in: .whitespaces) if expr.hasPrefix("(") && expr.hasSuffix(")") { var depth = 0, wraps = true for (i, c) in expr.enumerated() { if c == "(" { depth += 1 }; if c == ")" { depth -= 1 }; if depth == 0 && i < expr.count - 1 { wraps = false } } if wraps { return evaluateCondition(String(expr.dropFirst().dropLast()), macros: macros, unknownMacros: unknownMacros, unknownAbsent: unknownAbsent, depth: depth + 1) } } for op in ["||", "&&"] { let chars = Array(expr), operatorChars = Array(op); var depth = 0 if chars.count >= 2 { for i in 0..<(chars.count - 1) { if chars[i] == "(" { depth += 1 }; if chars[i] == ")" { depth -= 1 }; if depth == 0 && chars[i] == operatorChars[0] && chars[i+1] == operatorChars[1] { let left = evaluateCondition(String(chars[.. = ["void", "char", "short", "int", "long", "float", "double", "signed", "unsigned", "byte", "word", "size_t", "ptrdiff_t"] private let qualifiers: Set = ["const", "volatile", "extern", "static", "register", "auto", "typedef", "inline", "interrupt", "__interrupt", "__norts__", "__norts", "__asm__", "asm", "__asm"] private let controls: Set = ["return", "if", "else", "while", "do", "for", "switch", "case", "break", "continue", "goto", "sizeof"] mutating func parse() { parseRange(0.. Int? { let pairs = ["(": ")", "[": "]", "{": "}"] guard let close = pairs[tokens[index].text] else { return nil } var level = 0 for i in index.. [[Token]] { var result: [[Token]] = [], current: [Token] = [], parens = 0, brackets = 0, braces = 0 for token in values { if token.text == separator && parens == 0 && brackets == 0 && braces == 0 { result.append(current); current = []; continue } current.append(token) switch token.text { case "(": parens += 1; case ")": parens -= 1; case "[": brackets += 1; case "]": brackets -= 1; case "{": braces += 1; case "}": braces -= 1; default: break } } if !current.isEmpty { result.append(current) }; return result } private func declaratorStart(_ declaration: [Token]) -> Int? { guard let first = declaration.first, first.identifier, !controls.contains(first.text) else { return nil } var i = 0, consumed = false while i < declaration.count { let name = declaration[i].text if qualifiers.contains(name) || types.contains(name) { consumed = true; i += 1; continue } if ["struct", "union", "enum"].contains(name) { consumed = true; i += 1 if i < declaration.count && declaration[i].identifier { i += 1 } continue } if !consumed && declaration[i].identifier && i + 1 < declaration.count && (declaration[i+1].identifier || declaration[i+1].text == "*") { // A named type from another header is plausible, but it is // only a lexical declaration, never semantic type resolution. consumed = true; i += 1; continue } break } return consumed && i < declaration.count ? i : nil } private func declarator(_ values: [Token]) -> (name: Token, function: Bool, parameters: [Token])? { guard let nameIndex = values.firstIndex(where: { $0.identifier && !qualifiers.contains($0.text) }) else { return nil } let name = values[nameIndex] if nameIndex + 1 < values.count, values[nameIndex+1].text == "(" { var depth = 0 for i in (nameIndex+1).. begin, tokens[i-1].text == "for", let scope, let separator = tokens[(i+1).. Self { .init(path: path, name: name, line: line, column: column, kind: kind, signature: signature, isDefinition: isDefinition, isStatic: isStatic, conditional: true, scope: scope, offsetUTF16: offsetUTF16, lengthUTF16: lengthUTF16, scopeStart: scopeStart, scopeEnd: scopeEnd) } }