After an English word, 23 of 50 common first-tone characters (機, 吃, 書…) were read as English when space was pressed: the fixed +1.5 context bonus was far too strong for the trained weight scale, and the training data never contained the "space just pressed on a first-tone syllable" decision. Context is now the ctx.zh / ctx.lit features, trained with the rest; the synthetic data includes the previous mode and the first-tone space decision. Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the language differs from the previous word 92.0% -> 99.3%. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
152 lines
6.4 KiB
Swift
152 lines
6.4 KiB
Swift
import Foundation
|
|
|
|
/// Segment features for the Zhuyin / literal classifier. A segment's score in each mode
|
|
/// is the dot product of its features with the weights. `defaultWeights` reproduces the
|
|
/// original hand-tuned scores; learned weights come from Resources/classifier-weights.json
|
|
/// (training/train.py).
|
|
///
|
|
/// Python twin: windows/autozhuyin/core/features.py — both must produce identical
|
|
/// features (checked by ParityTests against training/parity-fixtures.json).
|
|
public typealias FeatureVector = [String: Double]
|
|
|
|
public enum Features {
|
|
public static let defaultWeights: FeatureVector = [
|
|
// Zhuyin
|
|
"zh.bias": 0.5,
|
|
"zh.stray": -5.0,
|
|
"zh.reordered": -0.5,
|
|
"zh.toned_valid_lex": 2.8,
|
|
"zh.toned_valid_nolex": 0.5,
|
|
"zh.toned_invalid": -5.0,
|
|
"zh.pending_prefix": 0.0,
|
|
"zh.pending_invalid": -5.0,
|
|
"zh.untoned_mid_valid": -2.5,
|
|
"zh.untoned_mid_invalid": -5.0,
|
|
"zh.unconfirmed": -2.0,
|
|
"zh.lexscore": 0.0,
|
|
"zh.phrase": 0.0,
|
|
// Literal
|
|
"lit.bias": 0.0,
|
|
"lit.upper": 20.0,
|
|
"lit.num_digits": 0.8,
|
|
"lit.num_others": -1.0,
|
|
"lit.common": 3.0,
|
|
"lit.common_len": 1.0,
|
|
"lit.dict": 1.0,
|
|
"lit.dict_len": 0.6,
|
|
"lit.prefix_common_len": 0.4,
|
|
"lit.prefix_dict_len": 0.2,
|
|
"lit.unknown_score": 1.0,
|
|
"lit.short": 0.0,
|
|
"lit.digits": -1.5,
|
|
"lit.others": -1.5,
|
|
// Segmentation
|
|
"seg.switch": -3.0,
|
|
// Context: the first segment has the same mode as the previously committed text
|
|
"ctx.zh": 1.5,
|
|
"ctx.lit": 1.5,
|
|
]
|
|
|
|
/// Reads {"weights": {...}} from a JSON file; empty if missing or invalid.
|
|
public static func loadWeights(from url: URL?) -> FeatureVector {
|
|
guard let url, let data = try? Data(contentsOf: url),
|
|
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
|
let w = obj["weights"] as? [String: Double] else { return [:] }
|
|
return w
|
|
}
|
|
|
|
public static func dot(_ w: FeatureVector, _ f: FeatureVector) -> Double {
|
|
f.reduce(0) { $0 + (w[$1.key] ?? 0) * $1.value }
|
|
}
|
|
|
|
/// Features of `keys` read as Zhuyin, or nil if impossible.
|
|
public static func zhuyin(_ keys: [Character], lexicon: Lexicon, isLast: Bool = true,
|
|
afterOther: Bool = false, finishing: Bool = false,
|
|
previous: InputMode? = nil) -> FeatureVector? {
|
|
let parse = ZhuyinParser.parse(keys)
|
|
// A Zhuyin segment followed by another must end with a tone.
|
|
if !isLast && parse.pending != nil { return nil }
|
|
// Space finishes an untoned last syllable as first tone; after a toned syllable it
|
|
// just commits, so it must not count as an extra (stray) tone.
|
|
let scored = finishing && isLast && parse.pending != nil ? ZhuyinParser.parse(keys + [" "]) : parse
|
|
if scored.foreignKeys > 0 || scored.syllables.isEmpty { return nil }
|
|
|
|
var f: FeatureVector = ["zh.bias": 1]
|
|
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
|
|
if scored.strayTones > 0 { add("zh.stray", Double(scored.strayTones)) }
|
|
let last = scored.syllables.count - 1
|
|
var tonedRun: [Syllable] = []
|
|
for (i, p) in scored.syllables.enumerated() {
|
|
if p.reordered { add("zh.reordered") }
|
|
let valid = SyllableTable.isValid(p.syllable)
|
|
if p.toned {
|
|
if valid {
|
|
if let best = lexicon.bestScore(for: p.syllable) {
|
|
add("zh.toned_valid_lex")
|
|
add("zh.lexscore", best + 4)
|
|
} else {
|
|
add("zh.toned_valid_nolex")
|
|
}
|
|
tonedRun.append(p.syllable)
|
|
} else {
|
|
add("zh.toned_invalid")
|
|
}
|
|
} else if i == last {
|
|
add(SyllableTable.isValidPrefix(p.syllable) ? "zh.pending_prefix" : "zh.pending_invalid")
|
|
} else {
|
|
add(valid ? "zh.untoned_mid_valid" : "zh.untoned_mid_invalid")
|
|
}
|
|
}
|
|
// Syllables covered by a multi-syllable lexicon phrase (real Chinese forms words).
|
|
var covered = [Bool](repeating: false, count: tonedRun.count)
|
|
for i in tonedRun.indices {
|
|
for n in 2...4 where i + n <= tonedRun.count {
|
|
if !lexicon.entries(for: Array(tonedRun[i..<(i + n)])).isEmpty {
|
|
for k in i..<(i + n) { covered[k] = true }
|
|
}
|
|
}
|
|
}
|
|
let coveredCount = covered.filter { $0 }.count
|
|
if coveredCount > 0 { add("zh.phrase", Double(coveredCount)) }
|
|
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
|
|
add("zh.unconfirmed")
|
|
}
|
|
if afterOther {
|
|
add("seg.switch")
|
|
} else if previous == .zhuyin {
|
|
add("ctx.zh")
|
|
}
|
|
return f
|
|
}
|
|
|
|
/// Features of `keys` output as typed, or nil if impossible.
|
|
public static func literal(_ keys: [Character], english: EnglishModel, isLast: Bool = true,
|
|
afterOther: Bool = false, finishing: Bool = false,
|
|
previous: InputMode? = nil) -> FeatureVector? {
|
|
if keys.isEmpty || keys.contains(" ") { return nil }
|
|
let complete = !isLast || finishing
|
|
var f: FeatureVector = ["lit.bias": 1]
|
|
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
|
|
if afterOther {
|
|
add("seg.switch")
|
|
} else if previous == .literal {
|
|
add("ctx.lit")
|
|
}
|
|
if keys.contains(where: { $0.isUppercase }) { add("lit.upper") }
|
|
|
|
let letters = String(keys.filter { $0.isLetter })
|
|
let digits = keys.filter { $0.isNumber }.count
|
|
let others = keys.count - letters.count - digits
|
|
if letters.isEmpty {
|
|
if digits > 0 { add("lit.num_digits", Double(digits)) }
|
|
if others > 0 { add("lit.num_others", Double(others)) }
|
|
return f
|
|
}
|
|
for (k, v) in english.wordFeatures(letters, complete: complete || digits > 0) { add(k, v) }
|
|
if digits > 0 { add("lit.digits", Double(digits)) }
|
|
let nonApostrophe = others - keys.filter { $0 == "'" }.count
|
|
if nonApostrophe > 0 { add("lit.others", Double(nonApostrophe)) }
|
|
return f
|
|
}
|
|
}
|