Learn the previous-word context; fix first-tone characters after English

After an English word, 23 of 50 common first-tone characters (機, 吃, 書…)
were read as English when space was pressed: the fixed +1.5 context bonus
was far too strong for the trained weight scale, and the training data never
contained the "space just pressed on a first-tone syllable" decision.

Context is now the ctx.zh / ctx.lit features, trained with the rest; the
synthetic data includes the previous mode and the first-tone space decision.
Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the
language differs from the previous word 92.0% -> 99.3%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 11:59:57 +08:00
co-authored by Claude Opus 5.5
parent 17f7d6f50d
commit 2c5c70c066
12 changed files with 5257 additions and 3455 deletions
@@ -265,10 +265,13 @@ final class ParityTests: XCTestCase {
for fx in fixtures {
let keys = Array(fx["keys"] as! String)
let expected = (fx["segments"] as! [[Any]]).map { "\($0[0])-\($0[1]):\($0[2])" }
let got = c.segment(keys, finishing: fx["finishing"] as! Bool).map {
let previous = (fx["previous"] as? String).flatMap(InputMode.init(rawValue:))
let got = c.segment(keys, finishing: fx["finishing"] as! Bool, previous: previous).map {
"\($0.range.lowerBound)-\($0.range.upperBound):\($0.mode.rawValue)"
}
if got != expected { mismatches.append("\(String(keys)) finishing=\(fx["finishing"]!): \(got) vs \(expected)") }
if got != expected {
mismatches.append("\(String(keys)) finishing=\(fx["finishing"]!) previous=\(String(describing: previous)): \(got) vs \(expected)")
}
}
XCTAssertGreaterThan(fixtures.count, 300)
XCTAssertTrue(mismatches.isEmpty, "\(mismatches.count) mismatches:\n" + mismatches.prefix(10).joined(separator: "\n"))
@@ -408,3 +411,16 @@ final class CursorTests: XCTestCase {
XCTAssertTrue(e.isShowingCandidates)
}
}
final class FirstToneAfterEnglishTests: XCTestCase {
/// After an English word, a first-tone syllable + space must stay Chinese (機, 吃, 書…).
func testFirstToneAfterEnglish() {
for keys in ["ru", "t", "gj", "vup", "tj", "el"] {
let e = CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights))
var out = ""
for c in "hello " + keys + " " { if let t = e.handle(.character(c)).commit { out += t } }
XCTAssertEqual(out, "hello ", keys)
XCTAssertEqual(e.markedText.count, 1, keys) // still composing one Chinese character
}
}
}