Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1dacb8f164 | ||
|
|
bfeed22916 |
@@ -15,9 +15,9 @@
|
|||||||
<key>CFBundlePackageType</key>
|
<key>CFBundlePackageType</key>
|
||||||
<string>APPL</string>
|
<string>APPL</string>
|
||||||
<key>CFBundleShortVersionString</key>
|
<key>CFBundleShortVersionString</key>
|
||||||
<string>0.6.1</string>
|
<string>0.6.2</string>
|
||||||
<key>CFBundleVersion</key>
|
<key>CFBundleVersion</key>
|
||||||
<string>9</string>
|
<string>10</string>
|
||||||
<key>InputMethodConnectionName</key>
|
<key>InputMethodConnectionName</key>
|
||||||
<string>com.autozhuyin.inputmethod.AutoZhuyin_Connection</string>
|
<string>com.autozhuyin.inputmethod.AutoZhuyin_Connection</string>
|
||||||
<key>InputMethodServerControllerClass</key>
|
<key>InputMethodServerControllerClass</key>
|
||||||
|
|||||||
@@ -11,22 +11,28 @@ enum Shared {
|
|||||||
|
|
||||||
static let classifier: Classifier = {
|
static let classifier: Classifier = {
|
||||||
let bundle = Bundle.main
|
let bundle = Bundle.main
|
||||||
|
let start = Date()
|
||||||
let lexicon = Lexicon()
|
let lexicon = Lexicon()
|
||||||
if let url = bundle.url(forResource: "lexicon", withExtension: "txt"),
|
if let url = bundle.url(forResource: "lexicon", withExtension: "txt"),
|
||||||
let text = try? String(contentsOf: url, encoding: .utf8) {
|
let data = try? Data(contentsOf: url) {
|
||||||
lexicon.load(text)
|
lexicon.load(data: data)
|
||||||
}
|
}
|
||||||
// Optional user phrases: ~/Library/Application Support/AutoZhuyin/user-phrases.txt
|
// Optional user phrases: ~/Library/Application Support/AutoZhuyin/user-phrases.txt
|
||||||
if let support = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first {
|
if let support = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first {
|
||||||
let userFile = support.appendingPathComponent("AutoZhuyin/user-phrases.txt")
|
let userFile = support.appendingPathComponent("AutoZhuyin/user-phrases.txt")
|
||||||
if let text = try? String(contentsOf: userFile, encoding: .utf8) { lexicon.load(text) }
|
if let text = try? String(contentsOf: userFile, encoding: .utf8) { lexicon.load(text) }
|
||||||
}
|
}
|
||||||
let english = EnglishModel(commonWordsURL: bundle.url(forResource: "english-common", withExtension: "txt"))
|
let english = EnglishModel(commonWordsURL: bundle.url(forResource: "english-common", withExtension: "txt"),
|
||||||
|
dictionaryURL: bundle.url(forResource: "english-dict", withExtension: "txt"),
|
||||||
|
bigramURL: bundle.url(forResource: "english-bigram", withExtension: "json"))
|
||||||
let saved = UserDefaults.standard.dictionary(forKey: learnedKey) as? [String: Int] ?? [:]
|
let saved = UserDefaults.standard.dictionary(forKey: learnedKey) as? [String: Int] ?? [:]
|
||||||
let learned = LearnedPreferences(weights: saved) { weights in
|
let learned = LearnedPreferences(weights: saved) { weights in
|
||||||
UserDefaults.standard.set(weights, forKey: learnedKey)
|
UserDefaults.standard.set(weights, forKey: learnedKey)
|
||||||
}
|
}
|
||||||
let trained = Features.loadWeights(from: bundle.url(forResource: "classifier-weights", withExtension: "json"))
|
let trained = Features.loadWeights(from: bundle.url(forResource: "classifier-weights", withExtension: "json"))
|
||||||
|
NSLog("AutoZhuyin: data loaded in %.0f ms", Date().timeIntervalSince(start) * 1000)
|
||||||
|
// Only needed for untoned syllables; build it off the startup path.
|
||||||
|
DispatchQueue.global(qos: .utility).async { lexicon.prepareTonelessIndex() }
|
||||||
return Classifier(lexicon: lexicon, english: english, learned: learned, weights: trained)
|
return Classifier(lexicon: lexicon, english: english, learned: learned, weights: trained)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,33 +8,63 @@ import Foundation
|
|||||||
/// - a letter-bigram model trained on those words, for words not in either list.
|
/// - a letter-bigram model trained on those words, for words not in either list.
|
||||||
public final class EnglishModel {
|
public final class EnglishModel {
|
||||||
private var common = Set<String>()
|
private var common = Set<String>()
|
||||||
private var dictionary = Set<String>()
|
|
||||||
private var sortedWords: [String] = []
|
|
||||||
private var sortedCommon: [String] = []
|
private var sortedCommon: [String] = []
|
||||||
|
/// Dictionary words (lowercase, > 2 letters), sorted: membership and prefix checks are
|
||||||
|
/// binary searches, so no 230k-entry Set has to be built at startup.
|
||||||
|
private var sortedDict: [String] = []
|
||||||
private var bigram: [String: Double] = [:] // "ab" -> log2 P(b|a); "^a", "a$" for boundaries
|
private var bigram: [String: Double] = [:] // "ab" -> log2 P(b|a); "^a", "a$" for boundaries
|
||||||
private var unseenBigram = -12.0
|
private var unseenBigram = -12.0
|
||||||
|
|
||||||
public init(commonWords: [String], dictionaryWords: [String] = []) {
|
/// - Parameter bigram: precomputed table (english-bigram.json); trained from the words if nil.
|
||||||
|
public init(commonWords: [String], dictionaryWords: [String] = [],
|
||||||
|
bigram: (table: [String: Double], unseen: Double)? = nil) {
|
||||||
common = Set(commonWords.map { $0.lowercased() }.filter { !$0.isEmpty })
|
common = Set(commonWords.map { $0.lowercased() }.filter { !$0.isEmpty })
|
||||||
dictionary = Set(dictionaryWords.map { $0.lowercased() }.filter { $0.count > 2 })
|
|
||||||
sortedCommon = common.sorted()
|
sortedCommon = common.sorted()
|
||||||
sortedWords = common.union(dictionary).sorted()
|
var dict = dictionaryWords.map { $0.lowercased() }.filter { $0.count > 2 }
|
||||||
trainBigram()
|
if !zip(dict, dict.dropFirst()).allSatisfy({ $0 < $1 }) {
|
||||||
|
dict = Array(Set(dict)).sorted()
|
||||||
|
}
|
||||||
|
sortedDict = dict
|
||||||
|
if let bigram {
|
||||||
|
self.bigram = bigram.table
|
||||||
|
unseenBigram = bigram.unseen
|
||||||
|
} else {
|
||||||
|
trainBigram()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Loads common words from a bundle resource file and the system dictionary if present.
|
/// Loads the bundled data files. Falls back to the system word list and to training
|
||||||
public convenience init(commonWordsURL: URL?, systemDictionary: URL? = URL(fileURLWithPath: "/usr/share/dict/words")) {
|
/// the bigram table when the precomputed files are missing.
|
||||||
|
public convenience init(commonWordsURL: URL?, dictionaryURL: URL? = nil, bigramURL: URL? = nil) {
|
||||||
func lines(_ url: URL?) -> [String] {
|
func lines(_ url: URL?) -> [String] {
|
||||||
guard let url, let text = try? String(contentsOf: url, encoding: .utf8) else { return [] }
|
guard let url, let text = try? String(contentsOf: url, encoding: .utf8) else { return [] }
|
||||||
return text.split(whereSeparator: \.isNewline)
|
return text.split(whereSeparator: \.isNewline)
|
||||||
.map { $0.trimmingCharacters(in: .whitespaces) }
|
.map { $0.trimmingCharacters(in: .whitespaces) }
|
||||||
.filter { !$0.hasPrefix("#") }
|
.filter { !$0.isEmpty && !$0.hasPrefix("#") }
|
||||||
}
|
}
|
||||||
self.init(commonWords: lines(commonWordsURL), dictionaryWords: lines(systemDictionary))
|
var bigram: (table: [String: Double], unseen: Double)?
|
||||||
|
if let url = bigramURL, let data = try? Data(contentsOf: url),
|
||||||
|
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||||
|
let table = obj["table"] as? [String: Double], let unseen = obj["unseen"] as? Double {
|
||||||
|
bigram = (table, unseen)
|
||||||
|
}
|
||||||
|
let dictURL = dictionaryURL.flatMap { FileManager.default.fileExists(atPath: $0.path) ? $0 : nil }
|
||||||
|
?? URL(fileURLWithPath: "/usr/share/dict/words")
|
||||||
|
self.init(commonWords: lines(commonWordsURL), dictionaryWords: lines(dictURL), bigram: bigram)
|
||||||
}
|
}
|
||||||
|
|
||||||
public func isCommon(_ w: String) -> Bool { common.contains(w.lowercased()) }
|
public func isCommon(_ w: String) -> Bool { common.contains(w.lowercased()) }
|
||||||
public func isWord(_ w: String) -> Bool { let l = w.lowercased(); return common.contains(l) || dictionary.contains(l) }
|
public func isWord(_ w: String) -> Bool { let l = w.lowercased(); return common.contains(l) || inDictionary(l) }
|
||||||
|
|
||||||
|
private func inDictionary(_ w: String) -> Bool {
|
||||||
|
let i = lowerBound(w, in: sortedDict)
|
||||||
|
return i < sortedDict.count && sortedDict[i] == w
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Some word strictly longer than `p` starts with it (common words or dictionary).
|
||||||
|
private func isPrefixOfWord(_ p: String) -> Bool {
|
||||||
|
hasPrefix(p, in: sortedCommon) || hasPrefix(p, in: sortedDict)
|
||||||
|
}
|
||||||
|
|
||||||
/// - Parameter complete: true when the user ended the word (space / punctuation),
|
/// - Parameter complete: true when the user ended the word (space / punctuation),
|
||||||
/// false while still typing (prefixes of words get partial credit).
|
/// false while still typing (prefixes of words get partial credit).
|
||||||
@@ -43,10 +73,10 @@ public final class EnglishModel {
|
|||||||
let n = Double(w.count)
|
let n = Double(w.count)
|
||||||
guard !w.isEmpty else { return 0 }
|
guard !w.isEmpty else { return 0 }
|
||||||
if common.contains(w) { return 3 + n }
|
if common.contains(w) { return 3 + n }
|
||||||
if dictionary.contains(w) { return 1 + 0.6 * n }
|
if inDictionary(w) { return 1 + 0.6 * n }
|
||||||
if !complete {
|
if !complete {
|
||||||
if hasPrefix(w, in: sortedCommon) { return 0.4 * n }
|
if hasPrefix(w, in: sortedCommon) { return 0.4 * n }
|
||||||
if hasPrefix(w, in: sortedWords) { return 0.2 * n }
|
if isPrefixOfWord(w) { return 0.2 * n }
|
||||||
}
|
}
|
||||||
// Unknown word: judge by how English-like its letter sequence is.
|
// Unknown word: judge by how English-like its letter sequence is.
|
||||||
let plaus = plausibility(w)
|
let plaus = plausibility(w)
|
||||||
@@ -63,12 +93,12 @@ public final class EnglishModel {
|
|||||||
if common.contains(w) {
|
if common.contains(w) {
|
||||||
f["lit.common"] = 1
|
f["lit.common"] = 1
|
||||||
f["lit.common_len"] = n
|
f["lit.common_len"] = n
|
||||||
} else if dictionary.contains(w) {
|
} else if inDictionary(w) {
|
||||||
f["lit.dict"] = 1
|
f["lit.dict"] = 1
|
||||||
f["lit.dict_len"] = n
|
f["lit.dict_len"] = n
|
||||||
} else if !complete && hasPrefix(w, in: sortedCommon) {
|
} else if !complete && hasPrefix(w, in: sortedCommon) {
|
||||||
f["lit.prefix_common_len"] = n
|
f["lit.prefix_common_len"] = n
|
||||||
} else if !complete && hasPrefix(w, in: sortedWords) {
|
} else if !complete && isPrefixOfWord(w) {
|
||||||
f["lit.prefix_dict_len"] = n
|
f["lit.prefix_dict_len"] = n
|
||||||
} else {
|
} else {
|
||||||
let plaus = plausibility(w)
|
let plaus = plausibility(w)
|
||||||
@@ -85,12 +115,17 @@ public final class EnglishModel {
|
|||||||
return total / Double(chars.count - 1)
|
return total / Double(chars.count - 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
private func hasPrefix(_ p: String, in words: [String]) -> Bool {
|
private func lowerBound(_ p: String, in words: [String]) -> Int {
|
||||||
var lo = 0, hi = words.count
|
var lo = 0, hi = words.count
|
||||||
while lo < hi {
|
while lo < hi {
|
||||||
let mid = (lo + hi) / 2
|
let mid = (lo + hi) / 2
|
||||||
if words[mid] < p { lo = mid + 1 } else { hi = mid }
|
if words[mid] < p { lo = mid + 1 } else { hi = mid }
|
||||||
}
|
}
|
||||||
|
return lo
|
||||||
|
}
|
||||||
|
|
||||||
|
private func hasPrefix(_ p: String, in words: [String]) -> Bool {
|
||||||
|
var lo = lowerBound(p, in: words)
|
||||||
// words[lo] is the first word >= p; skip an exact match, then test the next one.
|
// words[lo] is the first word >= p; skip an exact match, then test the next one.
|
||||||
if lo < words.count && words[lo] == p { lo += 1 }
|
if lo < words.count && words[lo] == p { lo += 1 }
|
||||||
return lo < words.count && words[lo].hasPrefix(p)
|
return lo < words.count && words[lo].hasPrefix(p)
|
||||||
@@ -99,7 +134,7 @@ public final class EnglishModel {
|
|||||||
private func trainBigram() {
|
private func trainBigram() {
|
||||||
var pair: [String: Double] = [:]
|
var pair: [String: Double] = [:]
|
||||||
var first: [String: Double] = [:]
|
var first: [String: Double] = [:]
|
||||||
for w in sortedWords where w.allSatisfy({ $0.isLetter && $0.isASCII }) {
|
for w in Set(sortedCommon + sortedDict).sorted() where w.allSatisfy({ $0.isLetter && $0.isASCII }) {
|
||||||
let weight = common.contains(w) ? 5.0 : 1.0
|
let weight = common.contains(w) ? 5.0 : 1.0
|
||||||
let chars = ["^"] + w.map(String.init) + ["$"]
|
let chars = ["^"] + w.map(String.init) + ["$"]
|
||||||
for i in 1..<chars.count {
|
for i in 1..<chars.count {
|
||||||
|
|||||||
@@ -11,58 +11,159 @@ public struct LexiconEntry: Equatable {
|
|||||||
/// File format, one entry per line (`#` starts a comment):
|
/// File format, one entry per line (`#` starts a comment):
|
||||||
/// ㄋㄧˇ-ㄏㄠˇ 你好 -4.2
|
/// ㄋㄧˇ-ㄏㄠˇ 你好 -4.2
|
||||||
/// The phrase/reading columns may be in either order; the score is optional.
|
/// The phrase/reading columns may be in either order; the score is optional.
|
||||||
|
///
|
||||||
|
/// Loading is on the input method's startup path, so the file is parsed as raw bytes,
|
||||||
|
/// and the tone-ignoring index (only needed for untoned syllables) is built lazily —
|
||||||
|
/// call `prepareTonelessIndex()` from a background queue to have it ready.
|
||||||
public final class Lexicon {
|
public final class Lexicon {
|
||||||
private var exact: [String: [LexiconEntry]] = [:]
|
private var exact: [String: [LexiconEntry]] = [:]
|
||||||
private var toneless: [String: [LexiconEntry]] = [:]
|
private var tonelessIndex: [String: [LexiconEntry]]?
|
||||||
|
private let lock = NSLock()
|
||||||
public private(set) var maxPhraseLength = 1
|
public private(set) var maxPhraseLength = 1
|
||||||
|
|
||||||
public init() {}
|
public init() {}
|
||||||
|
|
||||||
public convenience init(contentsOf url: URL) throws {
|
public convenience init(contentsOf url: URL) throws {
|
||||||
self.init()
|
self.init()
|
||||||
load(try String(contentsOf: url, encoding: .utf8))
|
load(data: try Data(contentsOf: url))
|
||||||
}
|
}
|
||||||
|
|
||||||
public func load(_ text: String) {
|
public func load(_ text: String) {
|
||||||
for line in text.split(whereSeparator: \.isNewline) {
|
load(data: Data(text.utf8))
|
||||||
let trimmed = line.trimmingCharacters(in: .whitespaces)
|
|
||||||
if trimmed.isEmpty || trimmed.hasPrefix("#") { continue }
|
|
||||||
let cols = trimmed.split(whereSeparator: { $0 == " " || $0 == "\t" }).map(String.init)
|
|
||||||
guard cols.count >= 2 else { continue }
|
|
||||||
let (reading, phrase) = Lexicon.isBopomofo(cols[0]) ? (cols[0], cols[1]) : (cols[1], cols[0])
|
|
||||||
guard Lexicon.isBopomofo(reading) else { continue }
|
|
||||||
let score = cols.count >= 3 ? Double(cols[2]) ?? -6 : -6
|
|
||||||
add(reading: reading, phrase: phrase, score: score)
|
|
||||||
}
|
|
||||||
for key in exact.keys { exact[key]?.sort { $0.score > $1.score } }
|
|
||||||
for key in toneless.keys { toneless[key]?.sort { $0.score > $1.score } }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public func add(reading rawReading: String, phrase: String, score: Double) {
|
public func load(data: Data) {
|
||||||
let entry = LexiconEntry(phrase: phrase, score: score)
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
exact.reserveCapacity(exact.count + data.count / 30)
|
||||||
|
data.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in
|
||||||
|
let bytes = raw.bindMemory(to: UInt8.self)
|
||||||
|
let n = bytes.count
|
||||||
|
var i = 0
|
||||||
|
while i < n {
|
||||||
|
var end = i
|
||||||
|
while end < n && bytes[end] != 0x0A { end += 1 }
|
||||||
|
parseLine(bytes, i, end)
|
||||||
|
i = end + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (key, entries) in exact where entries.count > 1 {
|
||||||
|
exact[key] = entries.sorted { $0.score > $1.score }
|
||||||
|
}
|
||||||
|
tonelessIndex = nil // rebuilt on demand
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Splits one line into up to three space/tab separated fields.
|
||||||
|
private func parseLine(_ b: UnsafeBufferPointer<UInt8>, _ start: Int, _ end: Int) {
|
||||||
|
var fields: [Range<Int>] = []
|
||||||
|
var i = start
|
||||||
|
while i < end && fields.count < 3 {
|
||||||
|
while i < end && (b[i] == 0x20 || b[i] == 0x09 || b[i] == 0x0D) { i += 1 }
|
||||||
|
if i >= end { break }
|
||||||
|
if fields.isEmpty && b[i] == 0x23 { return } // "#" comment
|
||||||
|
let s = i
|
||||||
|
while i < end && b[i] != 0x20 && b[i] != 0x09 && b[i] != 0x0D { i += 1 }
|
||||||
|
fields.append(s..<i)
|
||||||
|
}
|
||||||
|
guard fields.count >= 2 else { return }
|
||||||
|
func str(_ r: Range<Int>) -> String { String(decoding: UnsafeBufferPointer(rebasing: b[r]), as: UTF8.self) }
|
||||||
|
let (r, p) = Lexicon.isBopomofo(b, fields[0]) ? (fields[0], fields[1]) : (fields[1], fields[0])
|
||||||
|
guard Lexicon.isBopomofo(b, r) else { return }
|
||||||
|
let score = fields.count >= 3 ? Lexicon.parseDouble(b, fields[2]) ?? -6 : -6
|
||||||
|
// Neutral tone written as a prefix ("˙ㄉㄜ") needs normalising; detect it on bytes.
|
||||||
|
var prefixNeutral = b[r.lowerBound] == 0xCB
|
||||||
|
var k = r.lowerBound
|
||||||
|
while !prefixNeutral && k + 2 < r.upperBound {
|
||||||
|
if b[k] == 0x2D && b[k + 1] == 0xCB && b[k + 2] == 0x99 { prefixNeutral = true }
|
||||||
|
k += 1
|
||||||
|
}
|
||||||
|
addEntry(reading: str(r), phrase: str(p), score: score, normalise: prefixNeutral)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses "-3.9151"-style decimals without creating a String.
|
||||||
|
private static func parseDouble(_ b: UnsafeBufferPointer<UInt8>, _ r: Range<Int>) -> Double? {
|
||||||
|
var i = r.lowerBound
|
||||||
|
var sign = 1.0
|
||||||
|
if i < r.upperBound && (b[i] == 0x2D || b[i] == 0x2B) { sign = b[i] == 0x2D ? -1 : 1; i += 1 }
|
||||||
|
var value = 0.0, scale = 1.0, digits = 0, fraction = false
|
||||||
|
while i < r.upperBound {
|
||||||
|
let c = b[i]
|
||||||
|
if c >= 0x30 && c <= 0x39 {
|
||||||
|
if fraction { scale /= 10; value += Double(c - 0x30) * scale } else { value = value * 10 + Double(c - 0x30) }
|
||||||
|
digits += 1
|
||||||
|
} else if c == 0x2E && !fraction {
|
||||||
|
fraction = true
|
||||||
|
} else {
|
||||||
|
return Double(String(decoding: UnsafeBufferPointer(rebasing: b[r]), as: UTF8.self))
|
||||||
|
}
|
||||||
|
i += 1
|
||||||
|
}
|
||||||
|
return digits > 0 ? sign * value : nil
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Adds one entry (call `load` for bulk data; this does not re-sort).
|
||||||
|
public func add(reading: String, phrase: String, score: Double) {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
let key = addEntry(reading: reading, phrase: phrase, score: score, normalise: reading.contains("˙"))
|
||||||
|
exact[key]?.sort { $0.score > $1.score }
|
||||||
|
tonelessIndex = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@discardableResult
|
||||||
|
private func addEntry(reading rawReading: String, phrase: String, score: Double, normalise: Bool) -> String {
|
||||||
|
var reading = rawReading
|
||||||
// Some data sets write the neutral tone as a prefix ("˙ㄉㄜ"); we use a suffix.
|
// Some data sets write the neutral tone as a prefix ("˙ㄉㄜ"); we use a suffix.
|
||||||
let syllables = rawReading.split(separator: "-").map { s -> String in
|
if normalise {
|
||||||
s.hasPrefix("˙") ? String(s.dropFirst()) + "˙" : String(s)
|
reading = reading.split(separator: "-").map { s -> String in
|
||||||
|
s.hasPrefix("˙") ? String(s.dropFirst()) + "˙" : String(s)
|
||||||
|
}.joined(separator: "-")
|
||||||
}
|
}
|
||||||
let reading = syllables.joined(separator: "-")
|
maxPhraseLength = max(maxPhraseLength, reading.utf8.reduce(1) { $0 + ($1 == 0x2D ? 1 : 0) })
|
||||||
maxPhraseLength = max(maxPhraseLength, syllables.count)
|
if exact[reading] == nil {
|
||||||
if !(exact[reading]?.contains { $0.phrase == phrase } ?? false) {
|
exact[reading] = [LexiconEntry(phrase: phrase, score: score)]
|
||||||
exact[reading, default: []].append(entry)
|
} else if !exact[reading]!.contains(where: { $0.phrase == phrase }) {
|
||||||
|
exact[reading]!.append(LexiconEntry(phrase: phrase, score: score))
|
||||||
}
|
}
|
||||||
let key = Lexicon.stripTones(reading)
|
return reading
|
||||||
if let i = toneless[key]?.firstIndex(where: { $0.phrase == phrase }) {
|
}
|
||||||
toneless[key]![i].score = max(toneless[key]![i].score, score)
|
|
||||||
} else {
|
/// Builds the index that ignores tones (for untoned syllables). Safe to call from a
|
||||||
toneless[key, default: []].append(entry)
|
/// background queue; lookups build it themselves if it is not ready yet.
|
||||||
|
public func prepareTonelessIndex() {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
_ = buildTonelessIfNeeded()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func buildTonelessIfNeeded() -> [String: [LexiconEntry]] {
|
||||||
|
if let index = tonelessIndex { return index }
|
||||||
|
var best: [String: [String: Double]] = [:]
|
||||||
|
for (reading, entries) in exact {
|
||||||
|
let key = Lexicon.stripTones(reading)
|
||||||
|
for e in entries {
|
||||||
|
if let s = best[key]?[e.phrase], s >= e.score { continue }
|
||||||
|
best[key, default: [:]][e.phrase] = e.score
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
var index: [String: [LexiconEntry]] = [:]
|
||||||
|
index.reserveCapacity(best.count)
|
||||||
|
for (key, phrases) in best {
|
||||||
|
index[key] = phrases.map { LexiconEntry(phrase: $0.key, score: $0.value) }
|
||||||
|
.sorted { $0.score != $1.score ? $0.score > $1.score : $0.phrase < $1.phrase }
|
||||||
|
}
|
||||||
|
tonelessIndex = index
|
||||||
|
return index
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Entries for a sequence of readings. Untoned syllables match any tone.
|
/// Entries for a sequence of readings. Untoned syllables match any tone.
|
||||||
public func entries(for syllables: [Syllable]) -> [LexiconEntry] {
|
public func entries(for syllables: [Syllable]) -> [LexiconEntry] {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
if syllables.allSatisfy({ $0.tone != nil }) {
|
if syllables.allSatisfy({ $0.tone != nil }) {
|
||||||
return exact[syllables.map(\.reading).joined(separator: "-")] ?? []
|
return exact[syllables.map(\.reading).joined(separator: "-")] ?? []
|
||||||
}
|
}
|
||||||
return toneless[syllables.map(\.toneless).joined(separator: "-")] ?? []
|
return buildTonelessIfNeeded()[syllables.map(\.toneless).joined(separator: "-")] ?? []
|
||||||
}
|
}
|
||||||
|
|
||||||
public func bestScore(for syllable: Syllable) -> Double? {
|
public func bestScore(for syllable: Syllable) -> Double? {
|
||||||
@@ -81,4 +182,13 @@ public final class Lexicon {
|
|||||||
guard let first = s.unicodeScalars.first else { return false }
|
guard let first = s.unicodeScalars.first else { return false }
|
||||||
return (0x3105...0x312F).contains(first.value) || first == "˙"
|
return (0x3105...0x312F).contains(first.value) || first == "˙"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Same test on raw UTF-8: Bopomofo U+3105–U+312F is E3 84 85…E3 84 AF; "˙" is CB 99.
|
||||||
|
private static func isBopomofo(_ b: UnsafeBufferPointer<UInt8>, _ r: Range<Int>) -> Bool {
|
||||||
|
guard r.count >= 2 else { return false }
|
||||||
|
if b[r.lowerBound] == 0xCB && b[r.lowerBound + 1] == 0x99 { return true }
|
||||||
|
guard r.count >= 3, b[r.lowerBound] == 0xE3, b[r.lowerBound + 1] == 0x84 else { return false }
|
||||||
|
let c = b[r.lowerBound + 2]
|
||||||
|
return c >= 0x85 && c <= 0xAF
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,10 @@ private let resources = URL(fileURLWithPath: #filePath)
|
|||||||
.appendingPathComponent("Resources")
|
.appendingPathComponent("Resources")
|
||||||
|
|
||||||
private let sharedLexicon = try! Lexicon(contentsOf: resources.appendingPathComponent("lexicon.txt"))
|
private let sharedLexicon = try! Lexicon(contentsOf: resources.appendingPathComponent("lexicon.txt"))
|
||||||
private let sharedEnglish = EnglishModel(commonWordsURL: resources.appendingPathComponent("english-common.txt"))
|
private let windowsData = resources.deletingLastPathComponent().appendingPathComponent("windows/autozhuyin/data")
|
||||||
|
private let sharedEnglish = EnglishModel(commonWordsURL: resources.appendingPathComponent("english-common.txt"),
|
||||||
|
dictionaryURL: windowsData.appendingPathComponent("english-dict.txt"),
|
||||||
|
bigramURL: windowsData.appendingPathComponent("english-bigram.json"))
|
||||||
private let sharedWeights = Features.loadWeights(from: resources.appendingPathComponent("classifier-weights.json"))
|
private let sharedWeights = Features.loadWeights(from: resources.appendingPathComponent("classifier-weights.json"))
|
||||||
|
|
||||||
final class ParserTests: XCTestCase {
|
final class ParserTests: XCTestCase {
|
||||||
@@ -424,3 +427,15 @@ final class FirstToneAfterEnglishTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
final class StartupTimingTests: XCTestCase {
|
||||||
|
func testStartupTiming() throws {
|
||||||
|
var t = Date()
|
||||||
|
func lap(_ name: String) { print(String(format: "STARTUP %@: %.3fs", name, Date().timeIntervalSince(t))); t = Date() }
|
||||||
|
_ = try Lexicon(contentsOf: resources.appendingPathComponent("lexicon.txt")); lap("lexicon")
|
||||||
|
_ = EnglishModel(commonWordsURL: resources.appendingPathComponent("english-common.txt"),
|
||||||
|
dictionaryURL: windowsData.appendingPathComponent("english-dict.txt"),
|
||||||
|
bigramURL: windowsData.appendingPathComponent("english-bigram.json")); lap("english")
|
||||||
|
_ = Features.loadWeights(from: resources.appendingPathComponent("classifier-weights.json")); lap("weights")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources"
|
|||||||
cp "$BIN" "$APP/Contents/MacOS/AutoZhuyin"
|
cp "$BIN" "$APP/Contents/MacOS/AutoZhuyin"
|
||||||
cp Resources/Info.plist "$APP/Contents/Info.plist"
|
cp Resources/Info.plist "$APP/Contents/Info.plist"
|
||||||
cp Resources/lexicon.txt Resources/english-common.txt Resources/classifier-weights.json "$APP/Contents/Resources/"
|
cp Resources/lexicon.txt Resources/english-common.txt Resources/classifier-weights.json "$APP/Contents/Resources/"
|
||||||
|
# Precomputed English dictionary and bigram table (shared with the Windows build).
|
||||||
|
python3 -I windows/tools/build_data.py >/dev/null
|
||||||
|
cp windows/autozhuyin/data/english-dict.txt windows/autozhuyin/data/english-bigram.json "$APP/Contents/Resources/"
|
||||||
|
|
||||||
if [ ! -f build/icon.tiff ]; then
|
if [ ! -f build/icon.tiff ]; then
|
||||||
xcrun swift scripts/make-icon.swift build/icon.tiff
|
xcrun swift scripts/make-icon.swift build/icon.tiff
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ class Lexicon:
|
|||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.exact = {}
|
self.exact = {}
|
||||||
self.toneless = {}
|
self._toneless = None # built on first use (only untoned syllables need it)
|
||||||
self.max_phrase_length = 1
|
self.max_phrase_length = 1
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -42,9 +42,10 @@ class Lexicon:
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
score = -6.0
|
score = -6.0
|
||||||
self.add(reading, phrase, score)
|
self.add(reading, phrase, score)
|
||||||
for table in (self.exact, self.toneless):
|
for v in self.exact.values():
|
||||||
for v in table.values():
|
if len(v) > 1:
|
||||||
v.sort(key=lambda e: -e[1])
|
v.sort(key=lambda e: -e[1])
|
||||||
|
self._toneless = None
|
||||||
|
|
||||||
def add(self, reading, phrase, score):
|
def add(self, reading, phrase, score):
|
||||||
syllables = [s[1:] + "˙" if s.startswith("˙") else s for s in reading.split("-")]
|
syllables = [s[1:] + "˙" if s.startswith("˙") else s for s in reading.split("-")]
|
||||||
@@ -53,15 +54,20 @@ class Lexicon:
|
|||||||
entries = self.exact.setdefault(reading, [])
|
entries = self.exact.setdefault(reading, [])
|
||||||
if not any(p == phrase for p, _ in entries):
|
if not any(p == phrase for p, _ in entries):
|
||||||
entries.append((phrase, score))
|
entries.append((phrase, score))
|
||||||
key = _strip_tones(reading)
|
self._toneless = None
|
||||||
tl = self.toneless.setdefault(key, [])
|
|
||||||
for i, (p, s) in enumerate(tl):
|
@property
|
||||||
if p == phrase:
|
def toneless(self):
|
||||||
if score > s:
|
"""Index ignoring tones: toneless reading -> [(phrase, best score)], best first."""
|
||||||
tl[i] = (p, score)
|
if self._toneless is None:
|
||||||
break
|
best = {}
|
||||||
else:
|
for reading, entries in self.exact.items():
|
||||||
tl.append((phrase, score))
|
bucket = best.setdefault(_strip_tones(reading), {})
|
||||||
|
for phrase, score in entries:
|
||||||
|
if score > bucket.get(phrase, float("-inf")):
|
||||||
|
bucket[phrase] = score
|
||||||
|
self._toneless = {k: sorted(v.items(), key=lambda e: -e[1]) for k, v in best.items()}
|
||||||
|
return self._toneless
|
||||||
|
|
||||||
def entries(self, syllables):
|
def entries(self, syllables):
|
||||||
if all(s.tone is not None for s in syllables):
|
if all(s.tone is not None for s in syllables):
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "自動注音 AutoZhuyin",
|
"name": "自動注音 AutoZhuyin",
|
||||||
"version": "0.6.1",
|
"version": "0.6.2",
|
||||||
"guid": "{3895C6E3-6D40-4B4B-B4EA-CEEC75382D23}",
|
"guid": "{3895C6E3-6D40-4B4B-B4EA-CEEC75382D23}",
|
||||||
"locale": "zh-Hant-TW",
|
"locale": "zh-Hant-TW",
|
||||||
"fallbackLocale": "zh-TW",
|
"fallbackLocale": "zh-TW",
|
||||||
|
|||||||
Reference in New Issue
Block a user