Hold ambiguous English words (up = 因) until the next word decides

"up " also reads as ㄧㄣ (因). Instead of committing English on space, keep the
word composing: "up jo4" becomes 因為, "up the" stays English. Only applies when
the reading is a common single syllable (lexicon score >= -3.5): up, url.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 16:10:47 +08:00
co-authored by Claude Opus 5.5
parent 1dacb8f164
commit ef9f91a6b7
4 changed files with 126 additions and 2 deletions
@@ -51,6 +51,12 @@ public final class CompositionEngine {
public var candidateIndex = 0
public var isShowingCandidates: Bool { !candidates.isEmpty }
/// Index of a held space: an English word that is also a common one-syllable Chinese
/// reading ("up " = ㄧㄣ 因) is not committed on space; the next word decides
/// ("up jo4" = 因為, "up the" = up the).
private var held: Int?
static let holdMinScore = -3.5
private var previousMode: InputMode?
private var previousCommitTime = Date.distantPast
/// How long the previous segment's mode keeps influencing the next one.
@@ -71,9 +77,33 @@ public final class CompositionEngine {
private func segments(of keys: [Character], finishing: Bool = false) -> [Segment] {
guard !keys.isEmpty else { return [] }
if let forced = forcedMode { return [Segment(range: 0..<keys.count, mode: forced)] }
if let h = held, h < keys.count, keys[h] == " " {
return heldSegments(keys, held: h, finishing: finishing)
}
return classifier.segment(keys, finishing: finishing, previous: context)
}
/// The held word is Chinese if the words typed after it start with Chinese.
private func heldSegments(_ keys: [Character], held h: Int, finishing: Bool) -> [Segment] {
let base = h + 1
let rest = classifier.segment(Array(keys[base...]), finishing: finishing, previous: nil).map {
Segment(range: ($0.range.lowerBound + base)..<($0.range.upperBound + base), mode: $0.mode)
}
if let first = rest.first, first.mode == .zhuyin {
return [Segment(range: 0..<first.range.upperBound, mode: .zhuyin)] + rest.dropFirst()
}
return [Segment(range: 0..<base, mode: .literal)] + rest
}
/// Whether keys + space also read as a common Chinese syllable (see `held`).
private func isAmbiguousWord(_ keys: [Character]) -> Bool {
guard keys.allSatisfy({ $0.isASCII && $0.isLetter }) else { return false }
let p = ZhuyinParser.parse(keys + [" "])
guard p.pending == nil, p.syllables.count == 1 else { return false }
guard let best = classifier.lexicon.entries(for: p.syllables.map(\.syllable)).first else { return false }
return best.score >= CompositionEngine.holdMinScore
}
/// Current split of the buffer into Zhuyin / literal runs.
public var segments: [Segment] { segments(of: keys) }
@@ -164,6 +194,7 @@ public final class CompositionEngine {
guard !keys.isEmpty else { return .passThrough }
cursor = nil
keys.removeLast()
if let h = held, keys.count <= h { held = nil }
fixed = fixed.filter { $0.key < keys.count }
if keys.isEmpty { forcedMode = nil; fixed = [:] }
return .consumed
@@ -218,6 +249,11 @@ public final class CompositionEngine {
guard !keys.isEmpty else { return .passThrough }
let segs = segments(of: keys, finishing: true)
if segs.last?.mode == .literal {
if held == nil, segs.count == 1, forcedMode == nil, isAmbiguousWord(keys) {
held = keys.count
keys.append(" ")
return .consumed
}
return KeyResult(handled: true, commit: commitCurrent(segs) + " ")
}
if parse(ofLast: segs)?.pending != nil {
@@ -326,6 +362,7 @@ public final class CompositionEngine {
public func reset() {
keys = []
held = nil
cursor = nil
forcedMode = nil
fixed = [:]
@@ -428,6 +428,40 @@ final class FirstToneAfterEnglishTests: XCTestCase {
}
}
final class HeldWordTests: XCTestCase {
func makeEngine() -> CompositionEngine {
CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights))
}
func type(_ s: String, into e: CompositionEngine) -> String {
var out = ""
for c in s {
let key: KeyInput = c == "\n" ? .enter : .character(c)
let r = e.handle(key)
if let t = r.commit { out += t } else if !r.handled { out.append(c) }
}
return out
}
/// "up " is also ㄧㄣ (因): the next word decides.
func testNextWordDecides() {
let cases = ["up jo4 ": "因為", "up the ": "up the ", "pick up the ": "pick up the ",
"up\n": "up", "i love ": "i love "]
for (keys, want) in cases {
XCTAssertEqual(type(keys, into: makeEngine()), want, keys)
}
}
func testHeldWordStaysComposing() {
let e = makeEngine()
XCTAssertEqual(type("up ", into: e), "")
XCTAssertEqual(e.markedText, "up ")
_ = e.handle(.backspace)
XCTAssertEqual(e.markedText, "up")
XCTAssertEqual(type(" the ", into: e), "up the ")
}
}
final class StartupTimingTests: XCTestCase {
func testStartupTiming() throws {
var t = Date()
+39 -2
View File
@@ -20,6 +20,10 @@ FULLWIDTH = {
"{": "『", "}": "』", "[": "「", "]": "」", "~": "~", "_": "——",
}
# An English word that is also a common one-syllable Chinese reading ("up " = ㄧㄣ 因)
# is not committed on space; the next word decides ("up jo4" = 因為, "up the" = up the).
HOLD_MIN_SCORE = -3.5
PASS = (False, None)
CONSUMED = (True, None)
@@ -43,6 +47,8 @@ class CompositionEngine:
self._candidate_segment_start = 0
# Composition cursor for choosing candidates: index into _cursor_stops(), None = end.
self.cursor = None
# Index of a held space (see HOLD_MIN_SCORE), or None.
self.held = None
self._previous_mode = None
self._previous_time = float("-inf")
self._cache_key = None
@@ -60,12 +66,35 @@ class CompositionEngine:
if self.forced_mode:
return [(0, len(keys), self.forced_mode)]
ctx = self.context
cache_key = (tuple(keys), finishing, ctx)
held = self.held if self.held is not None and self.held < len(keys) and keys[self.held] == " " else None
cache_key = (tuple(keys), finishing, ctx, held)
if cache_key != self._cache_key:
self._cache_key = cache_key
self._cache_value = self.classifier.segment(keys, finishing, ctx)
if held is None:
self._cache_value = self.classifier.segment(keys, finishing, ctx)
else:
self._cache_value = self._held_segments(keys, held, finishing)
return self._cache_value
def _held_segments(self, keys, held, finishing):
"""The held word is Chinese if the words typed after it start with Chinese."""
rest = keys[held + 1:]
base = held + 1
rest_segs = [(a + base, b + base, m) for a, b, m in self.classifier.segment(rest, finishing, None)]
if rest_segs and rest_segs[0][2] == ZHUYIN:
return [(0, rest_segs[0][1], ZHUYIN)] + rest_segs[1:]
return [(0, base, LITERAL)] + rest_segs
def _is_ambiguous_word(self, keys):
"""Whether keys + space also read as a common Chinese syllable (see HOLD_MIN_SCORE)."""
if not all(k.isascii() and k.isalpha() for k in keys):
return False
p = bopomofo.parse(list(keys) + [" "])
if p.pending is not None or len(p.syllables) != 1:
return False
entries = self.classifier.lexicon.entries([s.syllable for s in p.syllables])
return bool(entries) and entries[0][1] >= HOLD_MIN_SCORE
@property
def segments(self):
return self._segments(self.keys)
@@ -178,6 +207,8 @@ class CompositionEngine:
return PASS
self.cursor = None
self.keys.pop()
if self.held is not None and len(self.keys) <= self.held:
self.held = None
self.fixed = {k: v for k, v in self.fixed.items() if k < len(self.keys)}
if not self.keys:
self.forced_mode = None
@@ -235,6 +266,11 @@ class CompositionEngine:
return PASS
segs = self._segments(self.keys, finishing=True)
if segs[-1][2] == LITERAL:
if (self.held is None and len(segs) == 1 and not self.forced_mode
and self._is_ambiguous_word(self.keys)):
self.held = len(self.keys)
self.keys.append(" ")
return CONSUMED
return True, self._commit(segs) + " "
p = self._last_parse(segs)
if p is not None and p.pending is not None:
@@ -319,6 +355,7 @@ class CompositionEngine:
def reset(self):
self.keys = []
self.held = None
self.cursor = None
self.forced_mode = None
self.fixed = {}
+16
View File
@@ -279,5 +279,21 @@ class FirstToneAfterEnglishTests(unittest.TestCase):
self.assertEqual(len(e.marked_text), 1, keys)
class HeldWordTests(unittest.TestCase):
def test_next_word_decides(self):
cases = {"up jo4 ": "因為", "up the ": "up the ", "pick up the ": "pick up the ",
"up\n": "up", "i love ": "i love "}
for keys, want in cases.items():
self.assertEqual(type_keys(keys, make_engine()), want, keys)
def test_held_word_stays_composing(self):
e = make_engine()
self.assertEqual(type_keys("up ", e), "")
self.assertEqual(e.marked_text, "up ")
e.handle(BACKSPACE)
self.assertEqual(e.marked_text, "up")
self.assertEqual(type_keys(" the ", e), "up the ")
if __name__ == "__main__":
unittest.main()