After an English word, 23 of 50 common first-tone characters (機, 吃, 書…) were read as English when space was pressed: the fixed +1.5 context bonus was far too strong for the trained weight scale, and the training data never contained the "space just pressed on a first-tone syllable" decision. Context is now the ctx.zh / ctx.lit features, trained with the rest; the synthetic data includes the previous mode and the first-tone space decision. Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the language differs from the previous word 92.0% -> 99.3%. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
4.4 KiB
Python
117 lines
4.4 KiB
Python
"""Zhuyin-vs-literal classifier with learned corrections. Port of Classifier.swift."""
|
|
import json
|
|
|
|
from .features import DEFAULT_WEIGHTS, dot, literal_features, zhuyin_features
|
|
|
|
ZHUYIN = "zhuyin"
|
|
LITERAL = "literal"
|
|
|
|
NEG_INF = float("-inf")
|
|
|
|
|
|
class LearnedPreferences:
|
|
"""Per key-sequence corrections. Positive weight = prefers Zhuyin."""
|
|
|
|
def __init__(self, weights=None, on_change=None):
|
|
self.weights = dict(weights or {})
|
|
self._on_change = on_change or (lambda w: None)
|
|
|
|
def record(self, keys, mode):
|
|
delta = 1 if mode == ZHUYIN else -1
|
|
self.weights[keys] = max(-3, min(3, self.weights.get(keys, 0) + delta))
|
|
self._on_change(self.weights)
|
|
|
|
def clear(self):
|
|
self.weights = {}
|
|
self._on_change(self.weights)
|
|
|
|
def bias(self, keys):
|
|
return self.weights.get(keys, 0) * 4.0
|
|
|
|
|
|
class Classifier:
|
|
|
|
def __init__(self, lexicon, english, learned=None, weights=None):
|
|
self.lexicon = lexicon
|
|
self.english = english
|
|
self.learned = learned or LearnedPreferences()
|
|
self.weights = dict(DEFAULT_WEIGHTS)
|
|
if weights:
|
|
self.weights.update(weights)
|
|
|
|
@classmethod
|
|
def load_weights(cls, path):
|
|
"""Reads a weights JSON file ({"weights": {...}}); returns {} if missing."""
|
|
try:
|
|
with open(path, encoding="utf-8") as f:
|
|
return json.load(f).get("weights", {})
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
def segment_features(self, seg, is_last, after_other, finishing, previous=None):
|
|
"""(zhuyin features or None, literal features or None) for one segment.
|
|
`previous` (mode of the text committed just before) only affects the first segment."""
|
|
seg = list(seg)
|
|
return (zhuyin_features(seg, self.lexicon, is_last, after_other, finishing, previous),
|
|
literal_features(seg, self.english, is_last, after_other, finishing, previous))
|
|
|
|
def _scores(self, seg, is_last, after_other, finishing, previous, whole, weights):
|
|
fz, fl = self.segment_features(seg, is_last, after_other, finishing, previous)
|
|
zh = dot(weights, fz) if fz is not None else NEG_INF
|
|
lit = dot(weights, fl) if fl is not None else NEG_INF
|
|
if whole:
|
|
b = self.learned.bias("".join(seg))
|
|
if b > 0:
|
|
zh += b
|
|
else:
|
|
lit -= b
|
|
return zh, lit
|
|
|
|
def classify(self, keys, finishing=False, previous=None):
|
|
"""Whole buffer as one segment. Returns (mode, zhuyin_score, literal_score)."""
|
|
zh, lit = self._scores(list(keys), True, False, finishing, previous, True, self.weights)
|
|
return (ZHUYIN if zh >= lit else LITERAL), zh, lit
|
|
|
|
def mode(self, keys, finishing=False, previous=None):
|
|
return self.classify(keys, finishing, previous)[0]
|
|
|
|
# --- segmentation ---
|
|
|
|
def segment(self, keys, finishing=False, previous=None, weights=None):
|
|
"""Splits keys into alternating segments: [(start, end, mode)], e.g.
|
|
"ji3hello" -> [(0, 3, zhuyin), (3, 8, literal)]. Dynamic programming over
|
|
split points; each segment is scored by features x weights."""
|
|
weights = weights or self.weights
|
|
keys = list(keys)
|
|
n = len(keys)
|
|
if n == 0:
|
|
return []
|
|
modes = (ZHUYIN, LITERAL)
|
|
best = [[NEG_INF, NEG_INF] for _ in range(n + 1)]
|
|
back = [[-1, -1] for _ in range(n + 1)]
|
|
for i in range(1, n + 1):
|
|
for j in range(i):
|
|
scores = self._scores(keys[j:i], i == n, j > 0, finishing,
|
|
previous if j == 0 else None, j == 0 and i == n, weights)
|
|
for m in (0, 1):
|
|
if scores[m] == NEG_INF:
|
|
continue
|
|
if j == 0:
|
|
total = scores[m]
|
|
else:
|
|
if best[j][1 - m] == NEG_INF:
|
|
continue
|
|
total = best[j][1 - m] + scores[m]
|
|
if total > best[i][m]:
|
|
best[i][m] = total
|
|
back[i][m] = j
|
|
if best[n][0] == NEG_INF and best[n][1] == NEG_INF:
|
|
return [(0, n, self.mode(keys, finishing, previous))]
|
|
m = 0 if best[n][0] >= best[n][1] else 1
|
|
out, i = [], n
|
|
while i > 0:
|
|
j = back[i][m]
|
|
out.append((j, i, modes[m]))
|
|
i, m = j, 1 - m
|
|
return out[::-1]
|