Hold the first word on space when it also reads as Zhuyin
At the start of a composition (not right after English), a word that reads as valid Zhuyin is no longer committed as English on space. The next word decides, by comparing "Chinese all the way" with "English word + Chinese" using the classifier's features: el ji3 = 高我, no su3cl3 = no 你好, up the = up the. After English text only common one-syllable readings (up, url) are held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,7 @@ import time
|
||||
|
||||
from . import bopomofo
|
||||
from .classifier import LITERAL, ZHUYIN
|
||||
from .features import dot
|
||||
from .lexicon import Converter
|
||||
|
||||
BACKSPACE, ENTER, ESCAPE, TAB = "<backspace>", "<enter>", "<escape>", "<tab>"
|
||||
@@ -20,8 +21,10 @@ FULLWIDTH = {
|
||||
"{": "『", "}": "』", "[": "「", "]": "」", "~": "~", "_": "——",
|
||||
}
|
||||
|
||||
# An English word that is also a common one-syllable Chinese reading ("up " = ㄧㄣ 因)
|
||||
# is not committed on space; the next word decides ("up jo4" = 因為, "up the" = up the).
|
||||
# The first word of a composition that also reads as Zhuyin is not committed as English
|
||||
# on space; the next word decides ("up jo4" = 因為, "up the" = up the, "no su3" = no 你).
|
||||
# Right after English text only common one-syllable readings are held (up = 因, url = 教),
|
||||
# so normal English typing is not delayed.
|
||||
HOLD_MIN_SCORE = -3.5
|
||||
|
||||
# "`" then one of these keys always gives the full-width symbol, whatever the context.
|
||||
@@ -94,19 +97,39 @@ class CompositionEngine:
|
||||
rest = keys[held + 1:]
|
||||
base = held + 1
|
||||
rest_segs = [(a + base, b + base, m) for a, b, m in self.classifier.segment(rest, finishing, None)]
|
||||
if rest_segs and rest_segs[0][2] == ZHUYIN:
|
||||
if rest_segs and rest_segs[0][2] == ZHUYIN and self._held_reads_as_chinese(keys, held, rest_segs[0][1], finishing):
|
||||
return [(0, rest_segs[0][1], ZHUYIN)] + rest_segs[1:]
|
||||
return [(0, base, LITERAL)] + rest_segs
|
||||
|
||||
def _held_reads_as_chinese(self, keys, held, end, finishing):
|
||||
"""Chinese all the way to `end` vs. the held English word followed by Chinese."""
|
||||
c, w, ctx = self.classifier, self.classifier.weights, self.context
|
||||
is_last = end == len(keys)
|
||||
zh_whole = c.segment_features(keys[:end], is_last, False, finishing, ctx)[0]
|
||||
word = c.segment_features(keys[:held], False, False, True, ctx)[1]
|
||||
zh_rest = c.segment_features(keys[held + 1:end], is_last, True, finishing, None)[0]
|
||||
if zh_whole is None:
|
||||
return False
|
||||
if word is None or zh_rest is None:
|
||||
return True
|
||||
return dot(w, zh_whole) >= dot(w, word) + dot(w, zh_rest)
|
||||
|
||||
def _is_ambiguous_word(self, keys):
|
||||
"""Whether keys + space also read as a common Chinese syllable (see HOLD_MIN_SCORE)."""
|
||||
if not all(k.isascii() and k.isalpha() for k in keys):
|
||||
return False
|
||||
p = bopomofo.parse(list(keys) + [" "])
|
||||
if p.pending is not None or len(p.syllables) != 1:
|
||||
if p.pending is not None or not p.syllables or p.stray_tones or p.foreign_keys:
|
||||
return False
|
||||
entries = self.classifier.lexicon.entries([s.syllable for s in p.syllables])
|
||||
return bool(entries) and entries[0][1] >= HOLD_MIN_SCORE
|
||||
best = []
|
||||
for ps in p.syllables:
|
||||
entries = self.classifier.lexicon.entries([ps.syllable])
|
||||
if not entries:
|
||||
return False
|
||||
best.append(entries[0][1])
|
||||
if self.context != LITERAL:
|
||||
return True # first word: hold anything that reads as Zhuyin
|
||||
return len(best) == 1 and best[0] >= HOLD_MIN_SCORE
|
||||
|
||||
@property
|
||||
def segments(self):
|
||||
|
||||
@@ -308,8 +308,10 @@ class FirstToneAfterEnglishTests(unittest.TestCase):
|
||||
def test_first_tone_after_english(self):
|
||||
for keys in ["ru", "t", "gj", "vup", "tj", "el"]:
|
||||
e = make_engine()
|
||||
self.assertEqual(type_keys("hello " + keys + " ", e), "hello ", keys)
|
||||
self.assertEqual(len(e.marked_text), 1, keys)
|
||||
out = type_keys("hello " + keys + " \n", e)
|
||||
self.assertEqual(out[:6], "hello ", keys)
|
||||
self.assertEqual(len(out), 7, keys) # one Chinese character after it
|
||||
self.assertFalse(out[6].isascii(), keys)
|
||||
|
||||
|
||||
class HeldWordTests(unittest.TestCase):
|
||||
@@ -348,5 +350,19 @@ class BacktickSymbolTests(unittest.TestCase):
|
||||
self.assertEqual(e.marked_text, "你")
|
||||
|
||||
|
||||
class FirstWordHoldTests(unittest.TestCase):
|
||||
"""The first word is not committed as English on space if it also reads as Zhuyin."""
|
||||
def test_next_word_decides(self):
|
||||
cases = {"el ji3 ": "高我", "no su3cl3 ": "no 你好", "ok ji3 ": "ok 我",
|
||||
"i love you ": "i love you ", "up the ": "up the "}
|
||||
for keys, want in cases.items():
|
||||
self.assertEqual(type_keys(keys, make_engine()), want, keys)
|
||||
|
||||
def test_not_held_after_english(self):
|
||||
e = make_engine()
|
||||
self.assertEqual(type_keys("hello world ", e), "hello world ")
|
||||
self.assertEqual(type_keys("so ", e), "so ") # right after English: commits at once
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user