Learn classifier weights from synthetic typing data
Scoring is now features x weights (Features.swift / features.py), with the old hand-tuned values as defaults. training/ generates synthetic mixed Chinese/English buffers from the lexicon and word lists and trains the weights with an averaged structured perceptron over the same segmentation DP the input method runs. On 10,000 test buffers commit-time accuracy goes from 97.58% to 99.62%; typing-time (prefix) accuracy from 86.15% to 87.81%. Both apps load Resources/classifier-weights.json. A parity test checks the Swift segmentation matches the Python one on 400 fixtures. Also fixes space after a toned syllable being scored as an extra stray tone, which could turn short Chinese like "u4 " into English. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -50,48 +50,64 @@ public final class LearnedPreferences {
|
||||
}
|
||||
|
||||
/// Decides whether a key sequence is Zhuyin or literal (English / numbers).
|
||||
/// Every segment is scored as features · weights (see Features.swift).
|
||||
public final class Classifier {
|
||||
public let lexicon: Lexicon
|
||||
public let english: EnglishModel
|
||||
public let learned: LearnedPreferences
|
||||
/// Feature weights: the hand-tuned defaults overlaid with trained ones.
|
||||
public private(set) var weights: FeatureVector
|
||||
|
||||
/// Traditional-Chinese users type Zhuyin most of the time: a small prior toward it.
|
||||
public var zhuyinPrior = 0.5
|
||||
/// How much the previous segment's mode pulls the current one.
|
||||
public var contextWeight = 1.5
|
||||
|
||||
public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences()) {
|
||||
public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences(),
|
||||
weights: FeatureVector = [:]) {
|
||||
self.lexicon = lexicon
|
||||
self.english = english
|
||||
self.learned = learned
|
||||
self.weights = Features.defaultWeights.merging(weights) { _, new in new }
|
||||
}
|
||||
|
||||
/// - Parameters:
|
||||
/// - finishing: the user pressed space. Zhuyin is scored as if the space were a
|
||||
/// first-tone key; literal is scored as a completed word.
|
||||
/// - previous: mode of the previously committed segment, if recent.
|
||||
public func classify(_ keys: [Character], finishing: Bool = false, previous: InputMode? = nil) -> Classification {
|
||||
var zh = zhuyinScore(finishing ? keys + [" "] : keys) + zhuyinPrior
|
||||
var lit = literalScore(keys, complete: finishing)
|
||||
/// (Zhuyin features or nil, literal features or nil) for one segment.
|
||||
public func segmentFeatures(_ seg: [Character], isLast: Bool, afterOther: Bool,
|
||||
finishing: Bool) -> (FeatureVector?, FeatureVector?) {
|
||||
(Features.zhuyin(seg, lexicon: lexicon, isLast: isLast, afterOther: afterOther, finishing: finishing),
|
||||
Features.literal(seg, english: english, isLast: isLast, afterOther: afterOther, finishing: finishing))
|
||||
}
|
||||
|
||||
private func scores(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
|
||||
previous: InputMode?, whole: Bool) -> [Double] {
|
||||
let (fz, fl) = segmentFeatures(seg, isLast: isLast, afterOther: afterOther, finishing: finishing)
|
||||
var zh = fz.map { Features.dot(weights, $0) } ?? -.infinity
|
||||
var lit = fl.map { Features.dot(weights, $0) } ?? -.infinity
|
||||
switch previous {
|
||||
case .zhuyin?: zh += contextWeight
|
||||
case .literal?: lit += contextWeight
|
||||
case nil: break
|
||||
}
|
||||
let learnedBias = learned.bias(for: String(keys))
|
||||
if learnedBias > 0 { zh += learnedBias } else { lit -= learnedBias }
|
||||
return Classification(mode: zh >= lit ? .zhuyin : .literal, zhuyinScore: zh, literalScore: lit)
|
||||
if whole {
|
||||
let b = learned.bias(for: String(seg))
|
||||
if b > 0 { zh += b } else { lit -= b }
|
||||
}
|
||||
return [zh, lit]
|
||||
}
|
||||
|
||||
/// The whole buffer as one segment.
|
||||
/// - Parameters:
|
||||
/// - finishing: the user pressed space. An unfinished Zhuyin syllable is scored as
|
||||
/// first tone; literal is scored as a completed word.
|
||||
/// - previous: mode of the previously committed segment, if recent.
|
||||
public func classify(_ keys: [Character], finishing: Bool = false, previous: InputMode? = nil) -> Classification {
|
||||
let s = scores(keys, isLast: true, afterOther: false, finishing: finishing, previous: previous, whole: true)
|
||||
return Classification(mode: s[0] >= s[1] ? .zhuyin : .literal, zhuyinScore: s[0], literalScore: s[1])
|
||||
}
|
||||
|
||||
// MARK: - Segmentation
|
||||
|
||||
/// Cost of each switch between Zhuyin and literal inside one buffer, so that a normal
|
||||
/// word or sentence is not chopped up.
|
||||
public var segmentPenalty = 3.0
|
||||
|
||||
/// Splits a buffer into alternating Zhuyin / literal segments, e.g. "ji3hello" →
|
||||
/// [ji3: zhuyin, hello: literal]. Dynamic programming over every split point.
|
||||
/// A Zhuyin segment followed by another segment must end with a tone key.
|
||||
/// [ji3: zhuyin, hello: literal]. Dynamic programming over every split point; the
|
||||
/// cost of each switch is the "seg.switch" feature.
|
||||
public func segment(_ keys: [Character], finishing: Bool = false, previous: InputMode? = nil) -> [Segment] {
|
||||
let n = keys.count
|
||||
guard n > 0 else { return [] }
|
||||
@@ -102,12 +118,12 @@ public final class Classifier {
|
||||
|
||||
for i in 1...n {
|
||||
for j in 0..<i {
|
||||
let scores = segmentScores(Array(keys[j..<i]), isLast: i == n, afterOther: j > 0,
|
||||
finishing: finishing, previous: j == 0 ? previous : nil,
|
||||
whole: j == 0 && i == n)
|
||||
for m in 0..<2 where scores[m] > -.infinity {
|
||||
let s = scores(Array(keys[j..<i]), isLast: i == n, afterOther: j > 0, finishing: finishing,
|
||||
previous: j == 0 ? previous : nil, whole: j == 0 && i == n)
|
||||
for m in 0..<2 where s[m] > -.infinity {
|
||||
// Adjacent segments must alternate modes; same-mode neighbours are one segment.
|
||||
let total = j == 0 ? scores[m] : best[j][1 - m] + scores[m] - segmentPenalty
|
||||
if j > 0 && best[j][1 - m] == -.infinity { continue }
|
||||
let total = j == 0 ? s[m] : best[j][1 - m] + s[m]
|
||||
if total > best[i][m] {
|
||||
best[i][m] = total
|
||||
back[i][m] = j
|
||||
@@ -130,79 +146,4 @@ public final class Classifier {
|
||||
}
|
||||
return out.reversed()
|
||||
}
|
||||
|
||||
/// Extra cost for a trailing Zhuyin segment after English that has no tone key yet
|
||||
/// ("ji3hel" should stay 我hel, not become 我heㄠ, until a tone confirms the switch).
|
||||
public var unconfirmedSwitchPenalty = 2.0
|
||||
|
||||
private func segmentScores(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
|
||||
previous: InputMode?, whole: Bool) -> [Double] {
|
||||
var zh: Double
|
||||
let parse = ZhuyinParser.parse(seg)
|
||||
if !isLast && parse.pending != nil {
|
||||
zh = -.infinity
|
||||
} else {
|
||||
zh = zhuyinScore(finishing && isLast ? seg + [" "] : seg) + zhuyinPrior
|
||||
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
|
||||
zh -= unconfirmedSwitchPenalty
|
||||
}
|
||||
}
|
||||
var lit = literalScore(seg, complete: !isLast || finishing)
|
||||
switch previous {
|
||||
case .zhuyin?: zh += contextWeight
|
||||
case .literal?: lit += contextWeight
|
||||
case nil: break
|
||||
}
|
||||
if whole {
|
||||
let b = learned.bias(for: String(seg))
|
||||
if b > 0 { zh += b } else { lit -= b }
|
||||
}
|
||||
return [zh, lit]
|
||||
}
|
||||
|
||||
public func zhuyinScore(_ keys: [Character]) -> Double {
|
||||
let parse = ZhuyinParser.parse(keys)
|
||||
if parse.foreignKeys > 0 || parse.syllables.isEmpty { return -.infinity }
|
||||
var s = -5.0 * Double(parse.strayTones)
|
||||
for (i, p) in parse.syllables.enumerated() {
|
||||
let isLast = i == parse.syllables.count - 1
|
||||
// Out-of-order typing is accepted, but is weaker evidence than normal Zhuyin.
|
||||
if p.reordered { s -= 0.5 }
|
||||
let valid = SyllableTable.isValid(p.syllable)
|
||||
if p.toned {
|
||||
if valid {
|
||||
s += 2 + (lexicon.bestScore(for: p.syllable) != nil ? 0.8 : -1.5)
|
||||
} else {
|
||||
s -= 5
|
||||
}
|
||||
} else if isLast {
|
||||
s += SyllableTable.isValidPrefix(p.syllable) ? 0 : -5
|
||||
} else {
|
||||
// Untoned syllable in the middle: allowed (tone omission) but unusual.
|
||||
s += valid ? -2.5 : -5
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
public func literalScore(_ keys: [Character], complete: Bool) -> Double {
|
||||
if keys.isEmpty || keys.contains(" ") { return -.infinity }
|
||||
var score = 0.0
|
||||
if keys.contains(where: { $0.isUppercase }) { score += 20 } // Shift = clearly English
|
||||
|
||||
let letters = String(keys.filter { $0.isLetter })
|
||||
let digits = keys.filter { $0.isNumber }.count
|
||||
let others = keys.count - letters.count - digits
|
||||
|
||||
if letters.isEmpty {
|
||||
// Pure number like "2024" (with possible "." or ",").
|
||||
return score + 0.8 * Double(digits) - 1.0 * Double(others)
|
||||
}
|
||||
score += english.score(letters, complete: complete || digits > 0)
|
||||
score -= 1.5 * Double(digits)
|
||||
// `'` inside a word (don't) is fine; Zhuyin punctuation keys inside a word are not.
|
||||
let apostrophes = keys.filter { $0 == "'" }.count
|
||||
score -= 1.5 * Double(others - apostrophes)
|
||||
return score
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +53,30 @@ public final class EnglishModel {
|
||||
return min(0.5 * n, max(-2 * n, n * (plaus + 4.2)))
|
||||
}
|
||||
|
||||
/// Feature dict for the classifier (see Features.swift); `score` is their dot
|
||||
/// product with the default weights.
|
||||
public func wordFeatures(_ word: String, complete: Bool) -> FeatureVector {
|
||||
let w = word.lowercased()
|
||||
let n = Double(w.count)
|
||||
var f: FeatureVector = [:]
|
||||
if n <= 2 { f["lit.short"] = 1 }
|
||||
if common.contains(w) {
|
||||
f["lit.common"] = 1
|
||||
f["lit.common_len"] = n
|
||||
} else if dictionary.contains(w) {
|
||||
f["lit.dict"] = 1
|
||||
f["lit.dict_len"] = n
|
||||
} else if !complete && hasPrefix(w, in: sortedCommon) {
|
||||
f["lit.prefix_common_len"] = n
|
||||
} else if !complete && hasPrefix(w, in: sortedWords) {
|
||||
f["lit.prefix_dict_len"] = n
|
||||
} else {
|
||||
let plaus = plausibility(w)
|
||||
f["lit.unknown_score"] = min(0.5 * n, max(-2 * n, n * (plaus + 4.2)))
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
/// Mean log2 probability per letter transition (≈ -3.5 for English, lower for gibberish).
|
||||
public func plausibility(_ w: String) -> Double {
|
||||
let chars = ["^"] + w.map(String.init) + ["$"]
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
import Foundation
|
||||
|
||||
/// Segment features for the Zhuyin / literal classifier. A segment's score in each mode
|
||||
/// is the dot product of its features with the weights. `defaultWeights` reproduces the
|
||||
/// original hand-tuned scores; learned weights come from Resources/classifier-weights.json
|
||||
/// (training/train.py).
|
||||
///
|
||||
/// Python twin: windows/autozhuyin/core/features.py — both must produce identical
|
||||
/// features (checked by ParityTests against training/parity-fixtures.json).
|
||||
public typealias FeatureVector = [String: Double]
|
||||
|
||||
public enum Features {
|
||||
public static let defaultWeights: FeatureVector = [
|
||||
// Zhuyin
|
||||
"zh.bias": 0.5,
|
||||
"zh.stray": -5.0,
|
||||
"zh.reordered": -0.5,
|
||||
"zh.toned_valid_lex": 2.8,
|
||||
"zh.toned_valid_nolex": 0.5,
|
||||
"zh.toned_invalid": -5.0,
|
||||
"zh.pending_prefix": 0.0,
|
||||
"zh.pending_invalid": -5.0,
|
||||
"zh.untoned_mid_valid": -2.5,
|
||||
"zh.untoned_mid_invalid": -5.0,
|
||||
"zh.unconfirmed": -2.0,
|
||||
"zh.lexscore": 0.0,
|
||||
"zh.phrase": 0.0,
|
||||
// Literal
|
||||
"lit.bias": 0.0,
|
||||
"lit.upper": 20.0,
|
||||
"lit.num_digits": 0.8,
|
||||
"lit.num_others": -1.0,
|
||||
"lit.common": 3.0,
|
||||
"lit.common_len": 1.0,
|
||||
"lit.dict": 1.0,
|
||||
"lit.dict_len": 0.6,
|
||||
"lit.prefix_common_len": 0.4,
|
||||
"lit.prefix_dict_len": 0.2,
|
||||
"lit.unknown_score": 1.0,
|
||||
"lit.short": 0.0,
|
||||
"lit.digits": -1.5,
|
||||
"lit.others": -1.5,
|
||||
// Segmentation
|
||||
"seg.switch": -3.0,
|
||||
]
|
||||
|
||||
/// Reads {"weights": {...}} from a JSON file; empty if missing or invalid.
|
||||
public static func loadWeights(from url: URL?) -> FeatureVector {
|
||||
guard let url, let data = try? Data(contentsOf: url),
|
||||
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let w = obj["weights"] as? [String: Double] else { return [:] }
|
||||
return w
|
||||
}
|
||||
|
||||
public static func dot(_ w: FeatureVector, _ f: FeatureVector) -> Double {
|
||||
f.reduce(0) { $0 + (w[$1.key] ?? 0) * $1.value }
|
||||
}
|
||||
|
||||
/// Features of `keys` read as Zhuyin, or nil if impossible.
|
||||
public static func zhuyin(_ keys: [Character], lexicon: Lexicon, isLast: Bool = true,
|
||||
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? {
|
||||
let parse = ZhuyinParser.parse(keys)
|
||||
// A Zhuyin segment followed by another must end with a tone.
|
||||
if !isLast && parse.pending != nil { return nil }
|
||||
// Space finishes an untoned last syllable as first tone; after a toned syllable it
|
||||
// just commits, so it must not count as an extra (stray) tone.
|
||||
let scored = finishing && isLast && parse.pending != nil ? ZhuyinParser.parse(keys + [" "]) : parse
|
||||
if scored.foreignKeys > 0 || scored.syllables.isEmpty { return nil }
|
||||
|
||||
var f: FeatureVector = ["zh.bias": 1]
|
||||
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
|
||||
if scored.strayTones > 0 { add("zh.stray", Double(scored.strayTones)) }
|
||||
let last = scored.syllables.count - 1
|
||||
var tonedRun: [Syllable] = []
|
||||
for (i, p) in scored.syllables.enumerated() {
|
||||
if p.reordered { add("zh.reordered") }
|
||||
let valid = SyllableTable.isValid(p.syllable)
|
||||
if p.toned {
|
||||
if valid {
|
||||
if let best = lexicon.bestScore(for: p.syllable) {
|
||||
add("zh.toned_valid_lex")
|
||||
add("zh.lexscore", best + 4)
|
||||
} else {
|
||||
add("zh.toned_valid_nolex")
|
||||
}
|
||||
tonedRun.append(p.syllable)
|
||||
} else {
|
||||
add("zh.toned_invalid")
|
||||
}
|
||||
} else if i == last {
|
||||
add(SyllableTable.isValidPrefix(p.syllable) ? "zh.pending_prefix" : "zh.pending_invalid")
|
||||
} else {
|
||||
add(valid ? "zh.untoned_mid_valid" : "zh.untoned_mid_invalid")
|
||||
}
|
||||
}
|
||||
// Syllables covered by a multi-syllable lexicon phrase (real Chinese forms words).
|
||||
var covered = [Bool](repeating: false, count: tonedRun.count)
|
||||
for i in tonedRun.indices {
|
||||
for n in 2...4 where i + n <= tonedRun.count {
|
||||
if !lexicon.entries(for: Array(tonedRun[i..<(i + n)])).isEmpty {
|
||||
for k in i..<(i + n) { covered[k] = true }
|
||||
}
|
||||
}
|
||||
}
|
||||
let coveredCount = covered.filter { $0 }.count
|
||||
if coveredCount > 0 { add("zh.phrase", Double(coveredCount)) }
|
||||
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
|
||||
add("zh.unconfirmed")
|
||||
}
|
||||
if afterOther { add("seg.switch") }
|
||||
return f
|
||||
}
|
||||
|
||||
/// Features of `keys` output as typed, or nil if impossible.
|
||||
public static func literal(_ keys: [Character], english: EnglishModel, isLast: Bool = true,
|
||||
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? {
|
||||
if keys.isEmpty || keys.contains(" ") { return nil }
|
||||
let complete = !isLast || finishing
|
||||
var f: FeatureVector = ["lit.bias": 1]
|
||||
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
|
||||
if afterOther { add("seg.switch") }
|
||||
if keys.contains(where: { $0.isUppercase }) { add("lit.upper") }
|
||||
|
||||
let letters = String(keys.filter { $0.isLetter })
|
||||
let digits = keys.filter { $0.isNumber }.count
|
||||
let others = keys.count - letters.count - digits
|
||||
if letters.isEmpty {
|
||||
if digits > 0 { add("lit.num_digits", Double(digits)) }
|
||||
if others > 0 { add("lit.num_others", Double(others)) }
|
||||
return f
|
||||
}
|
||||
for (k, v) in english.wordFeatures(letters, complete: complete || digits > 0) { add(k, v) }
|
||||
if digits > 0 { add("lit.digits", Double(digits)) }
|
||||
let nonApostrophe = others - keys.filter { $0 == "'" }.count
|
||||
if nonApostrophe > 0 { add("lit.others", Double(nonApostrophe)) }
|
||||
return f
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user