Scoring is now features x weights (Features.swift / features.py), with the old hand-tuned values as defaults. training/ generates synthetic mixed Chinese/English buffers from the lexicon and word lists and trains the weights with an averaged structured perceptron over the same segmentation DP the input method runs. On 10,000 test buffers commit-time accuracy goes from 97.58% to 99.62%; typing-time (prefix) accuracy from 86.15% to 87.81%. Both apps load Resources/classifier-weights.json. A parity test checks the Swift segmentation matches the Python one on 400 fixtures. Also fixes space after a toned syllable being scored as an extra stray tone, which could turn short Chinese like "u4 " into English. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
139 lines
6.0 KiB
Swift
139 lines
6.0 KiB
Swift
import Foundation
|
|
|
|
/// Segment features for the Zhuyin / literal classifier. A segment's score in each mode
|
|
/// is the dot product of its features with the weights. `defaultWeights` reproduces the
|
|
/// original hand-tuned scores; learned weights come from Resources/classifier-weights.json
|
|
/// (training/train.py).
|
|
///
|
|
/// Python twin: windows/autozhuyin/core/features.py — both must produce identical
|
|
/// features (checked by ParityTests against training/parity-fixtures.json).
|
|
public typealias FeatureVector = [String: Double]
|
|
|
|
public enum Features {
|
|
public static let defaultWeights: FeatureVector = [
|
|
// Zhuyin
|
|
"zh.bias": 0.5,
|
|
"zh.stray": -5.0,
|
|
"zh.reordered": -0.5,
|
|
"zh.toned_valid_lex": 2.8,
|
|
"zh.toned_valid_nolex": 0.5,
|
|
"zh.toned_invalid": -5.0,
|
|
"zh.pending_prefix": 0.0,
|
|
"zh.pending_invalid": -5.0,
|
|
"zh.untoned_mid_valid": -2.5,
|
|
"zh.untoned_mid_invalid": -5.0,
|
|
"zh.unconfirmed": -2.0,
|
|
"zh.lexscore": 0.0,
|
|
"zh.phrase": 0.0,
|
|
// Literal
|
|
"lit.bias": 0.0,
|
|
"lit.upper": 20.0,
|
|
"lit.num_digits": 0.8,
|
|
"lit.num_others": -1.0,
|
|
"lit.common": 3.0,
|
|
"lit.common_len": 1.0,
|
|
"lit.dict": 1.0,
|
|
"lit.dict_len": 0.6,
|
|
"lit.prefix_common_len": 0.4,
|
|
"lit.prefix_dict_len": 0.2,
|
|
"lit.unknown_score": 1.0,
|
|
"lit.short": 0.0,
|
|
"lit.digits": -1.5,
|
|
"lit.others": -1.5,
|
|
// Segmentation
|
|
"seg.switch": -3.0,
|
|
]
|
|
|
|
/// Reads {"weights": {...}} from a JSON file; empty if missing or invalid.
|
|
public static func loadWeights(from url: URL?) -> FeatureVector {
|
|
guard let url, let data = try? Data(contentsOf: url),
|
|
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
|
let w = obj["weights"] as? [String: Double] else { return [:] }
|
|
return w
|
|
}
|
|
|
|
public static func dot(_ w: FeatureVector, _ f: FeatureVector) -> Double {
|
|
f.reduce(0) { $0 + (w[$1.key] ?? 0) * $1.value }
|
|
}
|
|
|
|
/// Features of `keys` read as Zhuyin, or nil if impossible.
|
|
public static func zhuyin(_ keys: [Character], lexicon: Lexicon, isLast: Bool = true,
|
|
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? {
|
|
let parse = ZhuyinParser.parse(keys)
|
|
// A Zhuyin segment followed by another must end with a tone.
|
|
if !isLast && parse.pending != nil { return nil }
|
|
// Space finishes an untoned last syllable as first tone; after a toned syllable it
|
|
// just commits, so it must not count as an extra (stray) tone.
|
|
let scored = finishing && isLast && parse.pending != nil ? ZhuyinParser.parse(keys + [" "]) : parse
|
|
if scored.foreignKeys > 0 || scored.syllables.isEmpty { return nil }
|
|
|
|
var f: FeatureVector = ["zh.bias": 1]
|
|
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
|
|
if scored.strayTones > 0 { add("zh.stray", Double(scored.strayTones)) }
|
|
let last = scored.syllables.count - 1
|
|
var tonedRun: [Syllable] = []
|
|
for (i, p) in scored.syllables.enumerated() {
|
|
if p.reordered { add("zh.reordered") }
|
|
let valid = SyllableTable.isValid(p.syllable)
|
|
if p.toned {
|
|
if valid {
|
|
if let best = lexicon.bestScore(for: p.syllable) {
|
|
add("zh.toned_valid_lex")
|
|
add("zh.lexscore", best + 4)
|
|
} else {
|
|
add("zh.toned_valid_nolex")
|
|
}
|
|
tonedRun.append(p.syllable)
|
|
} else {
|
|
add("zh.toned_invalid")
|
|
}
|
|
} else if i == last {
|
|
add(SyllableTable.isValidPrefix(p.syllable) ? "zh.pending_prefix" : "zh.pending_invalid")
|
|
} else {
|
|
add(valid ? "zh.untoned_mid_valid" : "zh.untoned_mid_invalid")
|
|
}
|
|
}
|
|
// Syllables covered by a multi-syllable lexicon phrase (real Chinese forms words).
|
|
var covered = [Bool](repeating: false, count: tonedRun.count)
|
|
for i in tonedRun.indices {
|
|
for n in 2...4 where i + n <= tonedRun.count {
|
|
if !lexicon.entries(for: Array(tonedRun[i..<(i + n)])).isEmpty {
|
|
for k in i..<(i + n) { covered[k] = true }
|
|
}
|
|
}
|
|
}
|
|
let coveredCount = covered.filter { $0 }.count
|
|
if coveredCount > 0 { add("zh.phrase", Double(coveredCount)) }
|
|
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
|
|
add("zh.unconfirmed")
|
|
}
|
|
if afterOther { add("seg.switch") }
|
|
return f
|
|
}
|
|
|
|
/// Features of `keys` output as typed, or nil if impossible.
|
|
public static func literal(_ keys: [Character], english: EnglishModel, isLast: Bool = true,
|
|
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? {
|
|
if keys.isEmpty || keys.contains(" ") { return nil }
|
|
let complete = !isLast || finishing
|
|
var f: FeatureVector = ["lit.bias": 1]
|
|
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
|
|
if afterOther { add("seg.switch") }
|
|
if keys.contains(where: { $0.isUppercase }) { add("lit.upper") }
|
|
|
|
let letters = String(keys.filter { $0.isLetter })
|
|
let digits = keys.filter { $0.isNumber }.count
|
|
let others = keys.count - letters.count - digits
|
|
if letters.isEmpty {
|
|
if digits > 0 { add("lit.num_digits", Double(digits)) }
|
|
if others > 0 { add("lit.num_others", Double(others)) }
|
|
return f
|
|
}
|
|
for (k, v) in english.wordFeatures(letters, complete: complete || digits > 0) { add(k, v) }
|
|
if digits > 0 { add("lit.digits", Double(digits)) }
|
|
let nonApostrophe = others - keys.filter { $0 == "'" }.count
|
|
if nonApostrophe > 0 { add("lit.others", Double(nonApostrophe)) }
|
|
return f
|
|
}
|
|
}
|