Learn the previous-word context; fix first-tone characters after English
After an English word, 23 of 50 common first-tone characters (機, 吃, 書…) were read as English when space was pressed: the fixed +1.5 context bonus was far too strong for the trained weight scale, and the training data never contained the "space just pressed on a first-tone syllable" decision. Context is now the ctx.zh / ctx.lit features, trained with the rest; the synthetic data includes the previous mode and the first-tone space decision. Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the language differs from the previous word 92.0% -> 99.3%. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -30,7 +30,6 @@ class LearnedPreferences:
|
||||
|
||||
|
||||
class Classifier:
|
||||
context_weight = 1.5
|
||||
|
||||
def __init__(self, lexicon, english, learned=None, weights=None):
|
||||
self.lexicon = lexicon
|
||||
@@ -49,20 +48,17 @@ class Classifier:
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
def segment_features(self, seg, is_last, after_other, finishing):
|
||||
"""(zhuyin features or None, literal features or None) for one segment."""
|
||||
def segment_features(self, seg, is_last, after_other, finishing, previous=None):
|
||||
"""(zhuyin features or None, literal features or None) for one segment.
|
||||
`previous` (mode of the text committed just before) only affects the first segment."""
|
||||
seg = list(seg)
|
||||
return (zhuyin_features(seg, self.lexicon, is_last, after_other, finishing),
|
||||
literal_features(seg, self.english, is_last, after_other, finishing))
|
||||
return (zhuyin_features(seg, self.lexicon, is_last, after_other, finishing, previous),
|
||||
literal_features(seg, self.english, is_last, after_other, finishing, previous))
|
||||
|
||||
def _scores(self, seg, is_last, after_other, finishing, previous, whole, weights):
|
||||
fz, fl = self.segment_features(seg, is_last, after_other, finishing)
|
||||
fz, fl = self.segment_features(seg, is_last, after_other, finishing, previous)
|
||||
zh = dot(weights, fz) if fz is not None else NEG_INF
|
||||
lit = dot(weights, fl) if fl is not None else NEG_INF
|
||||
if previous == ZHUYIN:
|
||||
zh += self.context_weight
|
||||
elif previous == LITERAL:
|
||||
lit += self.context_weight
|
||||
if whole:
|
||||
b = self.learned.bias("".join(seg))
|
||||
if b > 0:
|
||||
|
||||
@@ -41,6 +41,9 @@ DEFAULT_WEIGHTS = {
|
||||
"lit.others": -1.5,
|
||||
# Segmentation
|
||||
"seg.switch": -3.0,
|
||||
# Context: the first segment has the same mode as the previously committed text
|
||||
"ctx.zh": 1.5,
|
||||
"ctx.lit": 1.5,
|
||||
}
|
||||
|
||||
|
||||
@@ -52,7 +55,7 @@ def _add(f, k, v=1.0):
|
||||
f[k] = f.get(k, 0.0) + v
|
||||
|
||||
|
||||
def zhuyin_features(keys, lexicon, is_last=True, after_other=False, finishing=False):
|
||||
def zhuyin_features(keys, lexicon, is_last=True, after_other=False, finishing=False, previous=None):
|
||||
"""Features of `keys` read as Zhuyin, or None if impossible."""
|
||||
parse = bopomofo.parse(keys)
|
||||
if not is_last and parse.pending is not None:
|
||||
@@ -99,10 +102,12 @@ def zhuyin_features(keys, lexicon, is_last=True, after_other=False, finishing=Fa
|
||||
_add(f, "zh.unconfirmed")
|
||||
if after_other:
|
||||
_add(f, "seg.switch")
|
||||
elif previous == "zhuyin":
|
||||
_add(f, "ctx.zh")
|
||||
return f
|
||||
|
||||
|
||||
def literal_features(keys, english, is_last=True, after_other=False, finishing=False):
|
||||
def literal_features(keys, english, is_last=True, after_other=False, finishing=False, previous=None):
|
||||
"""Features of `keys` output as typed, or None if impossible."""
|
||||
if not keys or " " in keys:
|
||||
return None
|
||||
@@ -110,6 +115,8 @@ def literal_features(keys, english, is_last=True, after_other=False, finishing=F
|
||||
f = {"lit.bias": 1.0}
|
||||
if after_other:
|
||||
_add(f, "seg.switch")
|
||||
elif previous == "literal":
|
||||
_add(f, "ctx.lit")
|
||||
if any(c.isupper() for c in keys):
|
||||
_add(f, "lit.upper")
|
||||
letters = "".join(c for c in keys if c.isalpha())
|
||||
|
||||
@@ -271,5 +271,13 @@ class CursorTests(unittest.TestCase):
|
||||
self.assertTrue(e.showing_candidates)
|
||||
|
||||
|
||||
class FirstToneAfterEnglishTests(unittest.TestCase):
|
||||
def test_first_tone_after_english(self):
|
||||
for keys in ["ru", "t", "gj", "vup", "tj", "el"]:
|
||||
e = make_engine()
|
||||
self.assertEqual(type_keys("hello " + keys + " ", e), "hello ", keys)
|
||||
self.assertEqual(len(e.marked_text), 1, keys)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user