diff --git a/Sources/AutoZhuyin/InputController.swift b/Sources/AutoZhuyin/InputController.swift index 3f956a0..116cbcd 100644 --- a/Sources/AutoZhuyin/InputController.swift +++ b/Sources/AutoZhuyin/InputController.swift @@ -11,22 +11,28 @@ enum Shared { static let classifier: Classifier = { let bundle = Bundle.main + let start = Date() let lexicon = Lexicon() if let url = bundle.url(forResource: "lexicon", withExtension: "txt"), - let text = try? String(contentsOf: url, encoding: .utf8) { - lexicon.load(text) + let data = try? Data(contentsOf: url) { + lexicon.load(data: data) } // Optional user phrases: ~/Library/Application Support/AutoZhuyin/user-phrases.txt if let support = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first { let userFile = support.appendingPathComponent("AutoZhuyin/user-phrases.txt") if let text = try? String(contentsOf: userFile, encoding: .utf8) { lexicon.load(text) } } - let english = EnglishModel(commonWordsURL: bundle.url(forResource: "english-common", withExtension: "txt")) + let english = EnglishModel(commonWordsURL: bundle.url(forResource: "english-common", withExtension: "txt"), + dictionaryURL: bundle.url(forResource: "english-dict", withExtension: "txt"), + bigramURL: bundle.url(forResource: "english-bigram", withExtension: "json")) let saved = UserDefaults.standard.dictionary(forKey: learnedKey) as? [String: Int] ?? [:] let learned = LearnedPreferences(weights: saved) { weights in UserDefaults.standard.set(weights, forKey: learnedKey) } let trained = Features.loadWeights(from: bundle.url(forResource: "classifier-weights", withExtension: "json")) + NSLog("AutoZhuyin: data loaded in %.0f ms", Date().timeIntervalSince(start) * 1000) + // Only needed for untoned syllables; build it off the startup path. + DispatchQueue.global(qos: .utility).async { lexicon.prepareTonelessIndex() } return Classifier(lexicon: lexicon, english: english, learned: learned, weights: trained) }() } diff --git a/Sources/AutoZhuyinCore/EnglishModel.swift b/Sources/AutoZhuyinCore/EnglishModel.swift index dc860de..6fe5f81 100644 --- a/Sources/AutoZhuyinCore/EnglishModel.swift +++ b/Sources/AutoZhuyinCore/EnglishModel.swift @@ -8,33 +8,63 @@ import Foundation /// - a letter-bigram model trained on those words, for words not in either list. public final class EnglishModel { private var common = Set() - private var dictionary = Set() - private var sortedWords: [String] = [] private var sortedCommon: [String] = [] + /// Dictionary words (lowercase, > 2 letters), sorted: membership and prefix checks are + /// binary searches, so no 230k-entry Set has to be built at startup. + private var sortedDict: [String] = [] private var bigram: [String: Double] = [:] // "ab" -> log2 P(b|a); "^a", "a$" for boundaries private var unseenBigram = -12.0 - public init(commonWords: [String], dictionaryWords: [String] = []) { + /// - Parameter bigram: precomputed table (english-bigram.json); trained from the words if nil. + public init(commonWords: [String], dictionaryWords: [String] = [], + bigram: (table: [String: Double], unseen: Double)? = nil) { common = Set(commonWords.map { $0.lowercased() }.filter { !$0.isEmpty }) - dictionary = Set(dictionaryWords.map { $0.lowercased() }.filter { $0.count > 2 }) sortedCommon = common.sorted() - sortedWords = common.union(dictionary).sorted() - trainBigram() + var dict = dictionaryWords.map { $0.lowercased() }.filter { $0.count > 2 } + if !zip(dict, dict.dropFirst()).allSatisfy({ $0 < $1 }) { + dict = Array(Set(dict)).sorted() + } + sortedDict = dict + if let bigram { + self.bigram = bigram.table + unseenBigram = bigram.unseen + } else { + trainBigram() + } } - /// Loads common words from a bundle resource file and the system dictionary if present. - public convenience init(commonWordsURL: URL?, systemDictionary: URL? = URL(fileURLWithPath: "/usr/share/dict/words")) { + /// Loads the bundled data files. Falls back to the system word list and to training + /// the bigram table when the precomputed files are missing. + public convenience init(commonWordsURL: URL?, dictionaryURL: URL? = nil, bigramURL: URL? = nil) { func lines(_ url: URL?) -> [String] { guard let url, let text = try? String(contentsOf: url, encoding: .utf8) else { return [] } return text.split(whereSeparator: \.isNewline) .map { $0.trimmingCharacters(in: .whitespaces) } - .filter { !$0.hasPrefix("#") } + .filter { !$0.isEmpty && !$0.hasPrefix("#") } } - self.init(commonWords: lines(commonWordsURL), dictionaryWords: lines(systemDictionary)) + var bigram: (table: [String: Double], unseen: Double)? + if let url = bigramURL, let data = try? Data(contentsOf: url), + let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let table = obj["table"] as? [String: Double], let unseen = obj["unseen"] as? Double { + bigram = (table, unseen) + } + let dictURL = dictionaryURL.flatMap { FileManager.default.fileExists(atPath: $0.path) ? $0 : nil } + ?? URL(fileURLWithPath: "/usr/share/dict/words") + self.init(commonWords: lines(commonWordsURL), dictionaryWords: lines(dictURL), bigram: bigram) } public func isCommon(_ w: String) -> Bool { common.contains(w.lowercased()) } - public func isWord(_ w: String) -> Bool { let l = w.lowercased(); return common.contains(l) || dictionary.contains(l) } + public func isWord(_ w: String) -> Bool { let l = w.lowercased(); return common.contains(l) || inDictionary(l) } + + private func inDictionary(_ w: String) -> Bool { + let i = lowerBound(w, in: sortedDict) + return i < sortedDict.count && sortedDict[i] == w + } + + /// Some word strictly longer than `p` starts with it (common words or dictionary). + private func isPrefixOfWord(_ p: String) -> Bool { + hasPrefix(p, in: sortedCommon) || hasPrefix(p, in: sortedDict) + } /// - Parameter complete: true when the user ended the word (space / punctuation), /// false while still typing (prefixes of words get partial credit). @@ -43,10 +73,10 @@ public final class EnglishModel { let n = Double(w.count) guard !w.isEmpty else { return 0 } if common.contains(w) { return 3 + n } - if dictionary.contains(w) { return 1 + 0.6 * n } + if inDictionary(w) { return 1 + 0.6 * n } if !complete { if hasPrefix(w, in: sortedCommon) { return 0.4 * n } - if hasPrefix(w, in: sortedWords) { return 0.2 * n } + if isPrefixOfWord(w) { return 0.2 * n } } // Unknown word: judge by how English-like its letter sequence is. let plaus = plausibility(w) @@ -63,12 +93,12 @@ public final class EnglishModel { if common.contains(w) { f["lit.common"] = 1 f["lit.common_len"] = n - } else if dictionary.contains(w) { + } else if inDictionary(w) { f["lit.dict"] = 1 f["lit.dict_len"] = n } else if !complete && hasPrefix(w, in: sortedCommon) { f["lit.prefix_common_len"] = n - } else if !complete && hasPrefix(w, in: sortedWords) { + } else if !complete && isPrefixOfWord(w) { f["lit.prefix_dict_len"] = n } else { let plaus = plausibility(w) @@ -85,12 +115,17 @@ public final class EnglishModel { return total / Double(chars.count - 1) } - private func hasPrefix(_ p: String, in words: [String]) -> Bool { + private func lowerBound(_ p: String, in words: [String]) -> Int { var lo = 0, hi = words.count while lo < hi { let mid = (lo + hi) / 2 if words[mid] < p { lo = mid + 1 } else { hi = mid } } + return lo + } + + private func hasPrefix(_ p: String, in words: [String]) -> Bool { + var lo = lowerBound(p, in: words) // words[lo] is the first word >= p; skip an exact match, then test the next one. if lo < words.count && words[lo] == p { lo += 1 } return lo < words.count && words[lo].hasPrefix(p) @@ -99,7 +134,7 @@ public final class EnglishModel { private func trainBigram() { var pair: [String: Double] = [:] var first: [String: Double] = [:] - for w in sortedWords where w.allSatisfy({ $0.isLetter && $0.isASCII }) { + for w in Set(sortedCommon + sortedDict).sorted() where w.allSatisfy({ $0.isLetter && $0.isASCII }) { let weight = common.contains(w) ? 5.0 : 1.0 let chars = ["^"] + w.map(String.init) + ["$"] for i in 1..= 2 else { continue } - let (reading, phrase) = Lexicon.isBopomofo(cols[0]) ? (cols[0], cols[1]) : (cols[1], cols[0]) - guard Lexicon.isBopomofo(reading) else { continue } - let score = cols.count >= 3 ? Double(cols[2]) ?? -6 : -6 - add(reading: reading, phrase: phrase, score: score) - } - for key in exact.keys { exact[key]?.sort { $0.score > $1.score } } - for key in toneless.keys { toneless[key]?.sort { $0.score > $1.score } } + load(data: Data(text.utf8)) } - public func add(reading rawReading: String, phrase: String, score: Double) { - let entry = LexiconEntry(phrase: phrase, score: score) + public func load(data: Data) { + lock.lock() + defer { lock.unlock() } + exact.reserveCapacity(exact.count + data.count / 30) + data.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in + let bytes = raw.bindMemory(to: UInt8.self) + let n = bytes.count + var i = 0 + while i < n { + var end = i + while end < n && bytes[end] != 0x0A { end += 1 } + parseLine(bytes, i, end) + i = end + 1 + } + } + for (key, entries) in exact where entries.count > 1 { + exact[key] = entries.sorted { $0.score > $1.score } + } + tonelessIndex = nil // rebuilt on demand + } + + /// Splits one line into up to three space/tab separated fields. + private func parseLine(_ b: UnsafeBufferPointer, _ start: Int, _ end: Int) { + var fields: [Range] = [] + var i = start + while i < end && fields.count < 3 { + while i < end && (b[i] == 0x20 || b[i] == 0x09 || b[i] == 0x0D) { i += 1 } + if i >= end { break } + if fields.isEmpty && b[i] == 0x23 { return } // "#" comment + let s = i + while i < end && b[i] != 0x20 && b[i] != 0x09 && b[i] != 0x0D { i += 1 } + fields.append(s..= 2 else { return } + func str(_ r: Range) -> String { String(decoding: UnsafeBufferPointer(rebasing: b[r]), as: UTF8.self) } + let (r, p) = Lexicon.isBopomofo(b, fields[0]) ? (fields[0], fields[1]) : (fields[1], fields[0]) + guard Lexicon.isBopomofo(b, r) else { return } + let score = fields.count >= 3 ? Lexicon.parseDouble(b, fields[2]) ?? -6 : -6 + // Neutral tone written as a prefix ("˙ㄉㄜ") needs normalising; detect it on bytes. + var prefixNeutral = b[r.lowerBound] == 0xCB + var k = r.lowerBound + while !prefixNeutral && k + 2 < r.upperBound { + if b[k] == 0x2D && b[k + 1] == 0xCB && b[k + 2] == 0x99 { prefixNeutral = true } + k += 1 + } + addEntry(reading: str(r), phrase: str(p), score: score, normalise: prefixNeutral) + } + + /// Parses "-3.9151"-style decimals without creating a String. + private static func parseDouble(_ b: UnsafeBufferPointer, _ r: Range) -> Double? { + var i = r.lowerBound + var sign = 1.0 + if i < r.upperBound && (b[i] == 0x2D || b[i] == 0x2B) { sign = b[i] == 0x2D ? -1 : 1; i += 1 } + var value = 0.0, scale = 1.0, digits = 0, fraction = false + while i < r.upperBound { + let c = b[i] + if c >= 0x30 && c <= 0x39 { + if fraction { scale /= 10; value += Double(c - 0x30) * scale } else { value = value * 10 + Double(c - 0x30) } + digits += 1 + } else if c == 0x2E && !fraction { + fraction = true + } else { + return Double(String(decoding: UnsafeBufferPointer(rebasing: b[r]), as: UTF8.self)) + } + i += 1 + } + return digits > 0 ? sign * value : nil + } + + /// Adds one entry (call `load` for bulk data; this does not re-sort). + public func add(reading: String, phrase: String, score: Double) { + lock.lock() + defer { lock.unlock() } + let key = addEntry(reading: reading, phrase: phrase, score: score, normalise: reading.contains("˙")) + exact[key]?.sort { $0.score > $1.score } + tonelessIndex = nil + } + + @discardableResult + private func addEntry(reading rawReading: String, phrase: String, score: Double, normalise: Bool) -> String { + var reading = rawReading // Some data sets write the neutral tone as a prefix ("˙ㄉㄜ"); we use a suffix. - let syllables = rawReading.split(separator: "-").map { s -> String in - s.hasPrefix("˙") ? String(s.dropFirst()) + "˙" : String(s) + if normalise { + reading = reading.split(separator: "-").map { s -> String in + s.hasPrefix("˙") ? String(s.dropFirst()) + "˙" : String(s) + }.joined(separator: "-") } - let reading = syllables.joined(separator: "-") - maxPhraseLength = max(maxPhraseLength, syllables.count) - if !(exact[reading]?.contains { $0.phrase == phrase } ?? false) { - exact[reading, default: []].append(entry) + maxPhraseLength = max(maxPhraseLength, reading.utf8.reduce(1) { $0 + ($1 == 0x2D ? 1 : 0) }) + if exact[reading] == nil { + exact[reading] = [LexiconEntry(phrase: phrase, score: score)] + } else if !exact[reading]!.contains(where: { $0.phrase == phrase }) { + exact[reading]!.append(LexiconEntry(phrase: phrase, score: score)) } - let key = Lexicon.stripTones(reading) - if let i = toneless[key]?.firstIndex(where: { $0.phrase == phrase }) { - toneless[key]![i].score = max(toneless[key]![i].score, score) - } else { - toneless[key, default: []].append(entry) + return reading + } + + /// Builds the index that ignores tones (for untoned syllables). Safe to call from a + /// background queue; lookups build it themselves if it is not ready yet. + public func prepareTonelessIndex() { + lock.lock() + defer { lock.unlock() } + _ = buildTonelessIfNeeded() + } + + private func buildTonelessIfNeeded() -> [String: [LexiconEntry]] { + if let index = tonelessIndex { return index } + var best: [String: [String: Double]] = [:] + for (reading, entries) in exact { + let key = Lexicon.stripTones(reading) + for e in entries { + if let s = best[key]?[e.phrase], s >= e.score { continue } + best[key, default: [:]][e.phrase] = e.score + } } + var index: [String: [LexiconEntry]] = [:] + index.reserveCapacity(best.count) + for (key, phrases) in best { + index[key] = phrases.map { LexiconEntry(phrase: $0.key, score: $0.value) } + .sorted { $0.score != $1.score ? $0.score > $1.score : $0.phrase < $1.phrase } + } + tonelessIndex = index + return index } /// Entries for a sequence of readings. Untoned syllables match any tone. public func entries(for syllables: [Syllable]) -> [LexiconEntry] { + lock.lock() + defer { lock.unlock() } if syllables.allSatisfy({ $0.tone != nil }) { return exact[syllables.map(\.reading).joined(separator: "-")] ?? [] } - return toneless[syllables.map(\.toneless).joined(separator: "-")] ?? [] + return buildTonelessIfNeeded()[syllables.map(\.toneless).joined(separator: "-")] ?? [] } public func bestScore(for syllable: Syllable) -> Double? { @@ -81,4 +182,13 @@ public final class Lexicon { guard let first = s.unicodeScalars.first else { return false } return (0x3105...0x312F).contains(first.value) || first == "˙" } + + /// Same test on raw UTF-8: Bopomofo U+3105–U+312F is E3 84 85…E3 84 AF; "˙" is CB 99. + private static func isBopomofo(_ b: UnsafeBufferPointer, _ r: Range) -> Bool { + guard r.count >= 2 else { return false } + if b[r.lowerBound] == 0xCB && b[r.lowerBound + 1] == 0x99 { return true } + guard r.count >= 3, b[r.lowerBound] == 0xE3, b[r.lowerBound + 1] == 0x84 else { return false } + let c = b[r.lowerBound + 2] + return c >= 0x85 && c <= 0xAF + } } diff --git a/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift b/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift index 66cb506..a4c43dc 100644 --- a/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift +++ b/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift @@ -6,7 +6,10 @@ private let resources = URL(fileURLWithPath: #filePath) .appendingPathComponent("Resources") private let sharedLexicon = try! Lexicon(contentsOf: resources.appendingPathComponent("lexicon.txt")) -private let sharedEnglish = EnglishModel(commonWordsURL: resources.appendingPathComponent("english-common.txt")) +private let windowsData = resources.deletingLastPathComponent().appendingPathComponent("windows/autozhuyin/data") +private let sharedEnglish = EnglishModel(commonWordsURL: resources.appendingPathComponent("english-common.txt"), + dictionaryURL: windowsData.appendingPathComponent("english-dict.txt"), + bigramURL: windowsData.appendingPathComponent("english-bigram.json")) private let sharedWeights = Features.loadWeights(from: resources.appendingPathComponent("classifier-weights.json")) final class ParserTests: XCTestCase { @@ -424,3 +427,15 @@ final class FirstToneAfterEnglishTests: XCTestCase { } } } + +final class StartupTimingTests: XCTestCase { + func testStartupTiming() throws { + var t = Date() + func lap(_ name: String) { print(String(format: "STARTUP %@: %.3fs", name, Date().timeIntervalSince(t))); t = Date() } + _ = try Lexicon(contentsOf: resources.appendingPathComponent("lexicon.txt")); lap("lexicon") + _ = EnglishModel(commonWordsURL: resources.appendingPathComponent("english-common.txt"), + dictionaryURL: windowsData.appendingPathComponent("english-dict.txt"), + bigramURL: windowsData.appendingPathComponent("english-bigram.json")); lap("english") + _ = Features.loadWeights(from: resources.appendingPathComponent("classifier-weights.json")); lap("weights") + } +} diff --git a/scripts/build.sh b/scripts/build.sh index 916715d..247b2c1 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -19,6 +19,9 @@ mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources" cp "$BIN" "$APP/Contents/MacOS/AutoZhuyin" cp Resources/Info.plist "$APP/Contents/Info.plist" cp Resources/lexicon.txt Resources/english-common.txt Resources/classifier-weights.json "$APP/Contents/Resources/" +# Precomputed English dictionary and bigram table (shared with the Windows build). +python3 -I windows/tools/build_data.py >/dev/null +cp windows/autozhuyin/data/english-dict.txt windows/autozhuyin/data/english-bigram.json "$APP/Contents/Resources/" if [ ! -f build/icon.tiff ]; then xcrun swift scripts/make-icon.swift build/icon.tiff diff --git a/windows/autozhuyin/core/lexicon.py b/windows/autozhuyin/core/lexicon.py index c685d9d..25f1442 100644 --- a/windows/autozhuyin/core/lexicon.py +++ b/windows/autozhuyin/core/lexicon.py @@ -16,7 +16,7 @@ class Lexicon: def __init__(self): self.exact = {} - self.toneless = {} + self._toneless = None # built on first use (only untoned syllables need it) self.max_phrase_length = 1 @classmethod @@ -42,9 +42,10 @@ class Lexicon: except ValueError: score = -6.0 self.add(reading, phrase, score) - for table in (self.exact, self.toneless): - for v in table.values(): + for v in self.exact.values(): + if len(v) > 1: v.sort(key=lambda e: -e[1]) + self._toneless = None def add(self, reading, phrase, score): syllables = [s[1:] + "˙" if s.startswith("˙") else s for s in reading.split("-")] @@ -53,15 +54,20 @@ class Lexicon: entries = self.exact.setdefault(reading, []) if not any(p == phrase for p, _ in entries): entries.append((phrase, score)) - key = _strip_tones(reading) - tl = self.toneless.setdefault(key, []) - for i, (p, s) in enumerate(tl): - if p == phrase: - if score > s: - tl[i] = (p, score) - break - else: - tl.append((phrase, score)) + self._toneless = None + + @property + def toneless(self): + """Index ignoring tones: toneless reading -> [(phrase, best score)], best first.""" + if self._toneless is None: + best = {} + for reading, entries in self.exact.items(): + bucket = best.setdefault(_strip_tones(reading), {}) + for phrase, score in entries: + if score > bucket.get(phrase, float("-inf")): + bucket[phrase] = score + self._toneless = {k: sorted(v.items(), key=lambda e: -e[1]) for k, v in best.items()} + return self._toneless def entries(self, syllables): if all(s.tone is not None for s in syllables):