Learn the previous-word context; fix first-tone characters after English
After an English word, 23 of 50 common first-tone characters (機, 吃, 書…) were read as English when space was pressed: the fixed +1.5 context bonus was far too strong for the trained weight scale, and the training data never contained the "space just pressed on a first-tone syllable" decision. Context is now the ctx.zh / ctx.lit features, trained with the rest; the synthetic data includes the previous mode and the first-tone space decision. Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the language differs from the previous word 92.0% -> 99.3%. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -58,9 +58,6 @@ public final class Classifier {
|
||||
/// Feature weights: the hand-tuned defaults overlaid with trained ones.
|
||||
public private(set) var weights: FeatureVector
|
||||
|
||||
/// How much the previous segment's mode pulls the current one.
|
||||
public var contextWeight = 1.5
|
||||
|
||||
public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences(),
|
||||
weights: FeatureVector = [:]) {
|
||||
self.lexicon = lexicon
|
||||
@@ -69,23 +66,22 @@ public final class Classifier {
|
||||
self.weights = Features.defaultWeights.merging(weights) { _, new in new }
|
||||
}
|
||||
|
||||
/// (Zhuyin features or nil, literal features or nil) for one segment.
|
||||
public func segmentFeatures(_ seg: [Character], isLast: Bool, afterOther: Bool,
|
||||
finishing: Bool) -> (FeatureVector?, FeatureVector?) {
|
||||
(Features.zhuyin(seg, lexicon: lexicon, isLast: isLast, afterOther: afterOther, finishing: finishing),
|
||||
Features.literal(seg, english: english, isLast: isLast, afterOther: afterOther, finishing: finishing))
|
||||
/// (Zhuyin features or nil, literal features or nil) for one segment. `previous` (mode of
|
||||
/// the text committed just before) only affects the first segment.
|
||||
public func segmentFeatures(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
|
||||
previous: InputMode? = nil) -> (FeatureVector?, FeatureVector?) {
|
||||
(Features.zhuyin(seg, lexicon: lexicon, isLast: isLast, afterOther: afterOther,
|
||||
finishing: finishing, previous: previous),
|
||||
Features.literal(seg, english: english, isLast: isLast, afterOther: afterOther,
|
||||
finishing: finishing, previous: previous))
|
||||
}
|
||||
|
||||
private func scores(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
|
||||
previous: InputMode?, whole: Bool) -> [Double] {
|
||||
let (fz, fl) = segmentFeatures(seg, isLast: isLast, afterOther: afterOther, finishing: finishing)
|
||||
let (fz, fl) = segmentFeatures(seg, isLast: isLast, afterOther: afterOther,
|
||||
finishing: finishing, previous: previous)
|
||||
var zh = fz.map { Features.dot(weights, $0) } ?? -.infinity
|
||||
var lit = fl.map { Features.dot(weights, $0) } ?? -.infinity
|
||||
switch previous {
|
||||
case .zhuyin?: zh += contextWeight
|
||||
case .literal?: lit += contextWeight
|
||||
case nil: break
|
||||
}
|
||||
if whole {
|
||||
let b = learned.bias(for: String(seg))
|
||||
if b > 0 { zh += b } else { lit -= b }
|
||||
|
||||
@@ -42,6 +42,9 @@ public enum Features {
|
||||
"lit.others": -1.5,
|
||||
// Segmentation
|
||||
"seg.switch": -3.0,
|
||||
// Context: the first segment has the same mode as the previously committed text
|
||||
"ctx.zh": 1.5,
|
||||
"ctx.lit": 1.5,
|
||||
]
|
||||
|
||||
/// Reads {"weights": {...}} from a JSON file; empty if missing or invalid.
|
||||
@@ -58,7 +61,8 @@ public enum Features {
|
||||
|
||||
/// Features of `keys` read as Zhuyin, or nil if impossible.
|
||||
public static func zhuyin(_ keys: [Character], lexicon: Lexicon, isLast: Bool = true,
|
||||
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? {
|
||||
afterOther: Bool = false, finishing: Bool = false,
|
||||
previous: InputMode? = nil) -> FeatureVector? {
|
||||
let parse = ZhuyinParser.parse(keys)
|
||||
// A Zhuyin segment followed by another must end with a tone.
|
||||
if !isLast && parse.pending != nil { return nil }
|
||||
@@ -107,18 +111,27 @@ public enum Features {
|
||||
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
|
||||
add("zh.unconfirmed")
|
||||
}
|
||||
if afterOther { add("seg.switch") }
|
||||
if afterOther {
|
||||
add("seg.switch")
|
||||
} else if previous == .zhuyin {
|
||||
add("ctx.zh")
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
/// Features of `keys` output as typed, or nil if impossible.
|
||||
public static func literal(_ keys: [Character], english: EnglishModel, isLast: Bool = true,
|
||||
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? {
|
||||
afterOther: Bool = false, finishing: Bool = false,
|
||||
previous: InputMode? = nil) -> FeatureVector? {
|
||||
if keys.isEmpty || keys.contains(" ") { return nil }
|
||||
let complete = !isLast || finishing
|
||||
var f: FeatureVector = ["lit.bias": 1]
|
||||
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
|
||||
if afterOther { add("seg.switch") }
|
||||
if afterOther {
|
||||
add("seg.switch")
|
||||
} else if previous == .literal {
|
||||
add("ctx.lit")
|
||||
}
|
||||
if keys.contains(where: { $0.isUppercase }) { add("lit.upper") }
|
||||
|
||||
let letters = String(keys.filter { $0.isLetter })
|
||||
|
||||
Reference in New Issue
Block a user