Files
ays953andClaude Opus 5.5 2c5c70c066 Learn the previous-word context; fix first-tone characters after English
After an English word, 23 of 50 common first-tone characters (機, 吃, 書…)
were read as English when space was pressed: the fixed +1.5 context bonus
was far too strong for the trained weight scale, and the training data never
contained the "space just pressed on a first-tone syllable" decision.

Context is now the ctx.zh / ctx.lit features, trained with the rest; the
synthetic data includes the previous mode and the first-tone space decision.
Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the
language differs from the previous word 92.0% -> 99.3%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 11:59:57 +08:00

139 lines
4.8 KiB
Python

"""Segment features for the Zhuyin / literal classifier.
A segment's score in each mode is the dot product of its feature dict with a weight
dict. DEFAULT_WEIGHTS reproduces the original hand-tuned scores; trained weights
(training/train.py → Resources/classifier-weights.json) replace them.
Swift port: Sources/AutoZhuyinCore/Features.swift. Both must produce identical
features; Tests/.../ParityTests checks this against training/parity-fixtures.json.
"""
from . import bopomofo
DEFAULT_WEIGHTS = {
# Zhuyin
"zh.bias": 0.5,
"zh.stray": -5.0,
"zh.reordered": -0.5,
"zh.toned_valid_lex": 2.8,
"zh.toned_valid_nolex": 0.5,
"zh.toned_invalid": -5.0,
"zh.pending_prefix": 0.0,
"zh.pending_invalid": -5.0,
"zh.untoned_mid_valid": -2.5,
"zh.untoned_mid_invalid": -5.0,
"zh.unconfirmed": -2.0,
"zh.lexscore": 0.0,
"zh.phrase": 0.0,
# Literal
"lit.bias": 0.0,
"lit.upper": 20.0,
"lit.num_digits": 0.8,
"lit.num_others": -1.0,
"lit.common": 3.0,
"lit.common_len": 1.0,
"lit.dict": 1.0,
"lit.dict_len": 0.6,
"lit.prefix_common_len": 0.4,
"lit.prefix_dict_len": 0.2,
"lit.unknown_score": 1.0,
"lit.short": 0.0,
"lit.digits": -1.5,
"lit.others": -1.5,
# Segmentation
"seg.switch": -3.0,
# Context: the first segment has the same mode as the previously committed text
"ctx.zh": 1.5,
"ctx.lit": 1.5,
}
def dot(weights, feats):
return sum(weights.get(k, 0.0) * v for k, v in feats.items())
def _add(f, k, v=1.0):
f[k] = f.get(k, 0.0) + v
def zhuyin_features(keys, lexicon, is_last=True, after_other=False, finishing=False, previous=None):
"""Features of `keys` read as Zhuyin, or None if impossible."""
parse = bopomofo.parse(keys)
if not is_last and parse.pending is not None:
return None # a Zhuyin segment followed by another must end with a tone
# Pressing space finishes an untoned last syllable as first tone; after a toned
# syllable it just commits, so it must not count as an extra (stray) tone.
scored = bopomofo.parse(keys + [" "]) if finishing and is_last and parse.pending is not None else parse
if scored.foreign_keys > 0 or not scored.syllables:
return None
f = {"zh.bias": 1.0}
if scored.stray_tones:
_add(f, "zh.stray", scored.stray_tones)
last = len(scored.syllables) - 1
toned_run = []
for i, ps in enumerate(scored.syllables):
if ps.reordered:
_add(f, "zh.reordered")
valid = bopomofo.is_valid(ps.syllable)
if ps.toned:
if valid:
best = lexicon.best_score(ps.syllable)
if best is not None:
_add(f, "zh.toned_valid_lex")
_add(f, "zh.lexscore", best + 4.0)
else:
_add(f, "zh.toned_valid_nolex")
toned_run.append(ps.syllable)
else:
_add(f, "zh.toned_invalid")
elif i == last:
_add(f, "zh.pending_prefix" if bopomofo.is_valid_prefix(ps.syllable) else "zh.pending_invalid")
else:
_add(f, "zh.untoned_mid_valid" if valid else "zh.untoned_mid_invalid")
# Syllables covered by a multi-syllable lexicon phrase (real Chinese forms words).
covered = [False] * len(toned_run)
for i in range(len(toned_run)):
for n in (2, 3, 4):
if i + n <= len(toned_run) and lexicon.entries(toned_run[i:i + n]):
for k in range(i, i + n):
covered[k] = True
if any(covered):
_add(f, "zh.phrase", sum(covered))
if is_last and after_other and not finishing and not any(ps.toned for ps in parse.syllables):
_add(f, "zh.unconfirmed")
if after_other:
_add(f, "seg.switch")
elif previous == "zhuyin":
_add(f, "ctx.zh")
return f
def literal_features(keys, english, is_last=True, after_other=False, finishing=False, previous=None):
"""Features of `keys` output as typed, or None if impossible."""
if not keys or " " in keys:
return None
complete = (not is_last) or finishing
f = {"lit.bias": 1.0}
if after_other:
_add(f, "seg.switch")
elif previous == "literal":
_add(f, "ctx.lit")
if any(c.isupper() for c in keys):
_add(f, "lit.upper")
letters = "".join(c for c in keys if c.isalpha())
digits = sum(1 for c in keys if c.isdigit())
others = len(keys) - len(letters) - digits
if not letters:
if digits:
_add(f, "lit.num_digits", digits)
if others:
_add(f, "lit.num_others", others)
return f
for k, v in english.word_features(letters, complete or digits > 0).items():
_add(f, k, v)
if digits:
_add(f, "lit.digits", digits)
non_apostrophe = others - keys.count("'")
if non_apostrophe:
_add(f, "lit.others", non_apostrophe)
return f