Learn the previous-word context; fix first-tone characters after English

After an English word, 23 of 50 common first-tone characters (機, 吃, 書…)
were read as English when space was pressed: the fixed +1.5 context bonus
was far too strong for the trained weight scale, and the training data never
contained the "space just pressed on a first-tone syllable" decision.

Context is now the ctx.zh / ctx.lit features, trained with the rest; the
synthetic data includes the previous mode and the first-tone space decision.
Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the
language differs from the previous word 92.0% -> 99.3%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 11:59:57 +08:00
co-authored by Claude Opus 5.5
parent 17f7d6f50d
commit 2c5c70c066
12 changed files with 5257 additions and 3455 deletions
+10 -14
View File
@@ -58,9 +58,6 @@ public final class Classifier {
/// Feature weights: the hand-tuned defaults overlaid with trained ones.
public private(set) var weights: FeatureVector
/// How much the previous segment's mode pulls the current one.
public var contextWeight = 1.5
public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences(),
weights: FeatureVector = [:]) {
self.lexicon = lexicon
@@ -69,23 +66,22 @@ public final class Classifier {
self.weights = Features.defaultWeights.merging(weights) { _, new in new }
}
/// (Zhuyin features or nil, literal features or nil) for one segment.
public func segmentFeatures(_ seg: [Character], isLast: Bool, afterOther: Bool,
finishing: Bool) -> (FeatureVector?, FeatureVector?) {
(Features.zhuyin(seg, lexicon: lexicon, isLast: isLast, afterOther: afterOther, finishing: finishing),
Features.literal(seg, english: english, isLast: isLast, afterOther: afterOther, finishing: finishing))
/// (Zhuyin features or nil, literal features or nil) for one segment. `previous` (mode of
/// the text committed just before) only affects the first segment.
public func segmentFeatures(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
previous: InputMode? = nil) -> (FeatureVector?, FeatureVector?) {
(Features.zhuyin(seg, lexicon: lexicon, isLast: isLast, afterOther: afterOther,
finishing: finishing, previous: previous),
Features.literal(seg, english: english, isLast: isLast, afterOther: afterOther,
finishing: finishing, previous: previous))
}
private func scores(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
previous: InputMode?, whole: Bool) -> [Double] {
let (fz, fl) = segmentFeatures(seg, isLast: isLast, afterOther: afterOther, finishing: finishing)
let (fz, fl) = segmentFeatures(seg, isLast: isLast, afterOther: afterOther,
finishing: finishing, previous: previous)
var zh = fz.map { Features.dot(weights, $0) } ?? -.infinity
var lit = fl.map { Features.dot(weights, $0) } ?? -.infinity
switch previous {
case .zhuyin?: zh += contextWeight
case .literal?: lit += contextWeight
case nil: break
}
if whole {
let b = learned.bias(for: String(seg))
if b > 0 { zh += b } else { lit -= b }
+17 -4
View File
@@ -42,6 +42,9 @@ public enum Features {
"lit.others": -1.5,
// Segmentation
"seg.switch": -3.0,
// Context: the first segment has the same mode as the previously committed text
"ctx.zh": 1.5,
"ctx.lit": 1.5,
]
/// Reads {"weights": {...}} from a JSON file; empty if missing or invalid.
@@ -58,7 +61,8 @@ public enum Features {
/// Features of `keys` read as Zhuyin, or nil if impossible.
public static func zhuyin(_ keys: [Character], lexicon: Lexicon, isLast: Bool = true,
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? {
afterOther: Bool = false, finishing: Bool = false,
previous: InputMode? = nil) -> FeatureVector? {
let parse = ZhuyinParser.parse(keys)
// A Zhuyin segment followed by another must end with a tone.
if !isLast && parse.pending != nil { return nil }
@@ -107,18 +111,27 @@ public enum Features {
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
add("zh.unconfirmed")
}
if afterOther { add("seg.switch") }
if afterOther {
add("seg.switch")
} else if previous == .zhuyin {
add("ctx.zh")
}
return f
}
/// Features of `keys` output as typed, or nil if impossible.
public static func literal(_ keys: [Character], english: EnglishModel, isLast: Bool = true,
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? {
afterOther: Bool = false, finishing: Bool = false,
previous: InputMode? = nil) -> FeatureVector? {
if keys.isEmpty || keys.contains(" ") { return nil }
let complete = !isLast || finishing
var f: FeatureVector = ["lit.bias": 1]
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
if afterOther { add("seg.switch") }
if afterOther {
add("seg.switch")
} else if previous == .literal {
add("ctx.lit")
}
if keys.contains(where: { $0.isUppercase }) { add("lit.upper") }
let letters = String(keys.filter { $0.isLetter })