Files
AutoZhuyin/windows/autozhuyin/autozhuyin_ime.py
T
ays953andClaude Opus 5.5 476858d841 Learn classifier weights from synthetic typing data
Scoring is now features x weights (Features.swift / features.py), with the
old hand-tuned values as defaults. training/ generates synthetic mixed
Chinese/English buffers from the lexicon and word lists and trains the
weights with an averaged structured perceptron over the same segmentation
DP the input method runs. On 10,000 test buffers commit-time accuracy goes
from 97.58% to 99.62%; typing-time (prefix) accuracy from 86.15% to 87.81%.

Both apps load Resources/classifier-weights.json. A parity test checks the
Swift segmentation matches the Python one on 400 fixtures.

Also fixes space after a toned syllable being scored as an extra stray tone,
which could turn short Chinese like "u4 " into English.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 08:58:08 +08:00

139 lines
4.7 KiB
Python

"""PIME text service for AutoZhuyin (自動注音).
PIME imports this as `input_methods.autozhuyin.autozhuyin_ime` and creates one
AutoZhuyinTextService per client. All the logic lives in .core (shared with the
tests); this file only translates PIME key events and replies.
"""
import json
import os
from keycodes import * # noqa: F401,F403 (VK_* constants, provided by PIME)
from textService import TextService
from .core.classifier import Classifier, LearnedPreferences
from .core.engine import BACKSPACE, DOWN, ENTER, ESCAPE, LEFT, RIGHT, TAB, UP, CompositionEngine
from .core.english import EnglishModel
from .core.lexicon import Lexicon
_HERE = os.path.dirname(os.path.abspath(__file__))
_DATA = os.path.join(_HERE, "data")
_USER_DIR = os.path.join(os.environ.get("APPDATA", _HERE), "PIME", "autozhuyin")
_LEARNED_FILE = os.path.join(_USER_DIR, "learned.json")
_USER_PHRASES = os.path.join(_USER_DIR, "user-phrases.txt")
_classifier = None
def _save_learned(weights):
try:
os.makedirs(_USER_DIR, exist_ok=True)
with open(_LEARNED_FILE, "w", encoding="utf-8") as f:
json.dump(weights, f, ensure_ascii=False)
except OSError:
pass
def _shared_classifier():
"""Loads the lexicon and English model once per PIME server process."""
global _classifier
if _classifier is None:
lexicon = Lexicon.from_file(os.path.join(_DATA, "lexicon.txt"))
if os.path.exists(_USER_PHRASES):
with open(_USER_PHRASES, encoding="utf-8") as f:
lexicon.load(f)
english = EnglishModel.from_files(
os.path.join(_DATA, "english-common.txt"),
os.path.join(_DATA, "english-dict.txt"),
os.path.join(_DATA, "english-bigram.json"),
)
weights = {}
try:
with open(_LEARNED_FILE, encoding="utf-8") as f:
weights = json.load(f)
except (OSError, ValueError):
pass
trained = Classifier.load_weights(os.path.join(_DATA, "classifier-weights.json"))
_classifier = Classifier(lexicon, english, LearnedPreferences(weights, _save_learned), trained)
return _classifier
_SPECIAL = {
VK_BACK: BACKSPACE, VK_RETURN: ENTER, VK_ESCAPE: ESCAPE, VK_TAB: TAB, # noqa: F405
VK_UP: UP, VK_DOWN: DOWN, VK_LEFT: LEFT, VK_RIGHT: RIGHT, # noqa: F405
}
class AutoZhuyinTextService(TextService):
def __init__(self, client):
TextService.__init__(self, client)
self.engine = CompositionEngine(_shared_classifier())
# --- lifecycle ---
def onActivate(self):
TextService.onActivate(self)
self.setSelKeys("123456789")
self.customizeUI(candFontSize=16, candPerRow=1, candUseCursor=True)
def onDeactivate(self):
self.engine.reset()
TextService.onDeactivate(self)
def onCompositionTerminated(self, forced):
# The app ended the composition (focus change, mouse click): the shown text stays.
self.engine.reset()
if self.showCandidates:
self.setShowCandidates(False)
TextService.onCompositionTerminated(self, forced)
# --- keys ---
def _key(self, ev):
if ev.keyCode in _SPECIAL:
return _SPECIAL[ev.keyCode]
if ev.keyCode == VK_SPACE: # noqa: F405
return " "
if not ev.isPrintableChar():
return None
c = chr(ev.charCode)
# Caps Lock without Shift: keep Zhuyin working by using lowercase.
if ev.isKeyToggled(VK_CAPITAL) and not ev.isKeyDown(VK_SHIFT): # noqa: F405
c = c.lower()
return c
def _busy(self):
return bool(self.engine.keys or self.engine.candidates)
def filterKeyDown(self, ev):
# Shortcuts (Ctrl/Alt) go to the app.
if ev.isKeyDown(VK_CONTROL) or ev.isKeyDown(VK_MENU): # noqa: F405
return False
key = self._key(ev)
if key is None:
return self._busy() # swallow other keys (Delete, Home…) while composing
return self.engine.wants(key)
def onKeyDown(self, ev):
key = self._key(ev)
if key is None:
return self._busy()
handled, commit = self.engine.handle(key)
self._update(commit)
return handled
# --- replies ---
def _update(self, commit):
if commit:
self.setCommitString(commit)
marked = self.engine.marked_text
self.setCompositionString(marked)
self.setCompositionCursor(len(marked))
if self.engine.showing_candidates:
page, cursor = self.engine.page()
self.setCandidateList(page)
self.setCandidateCursor(cursor)
self.setShowCandidates(True)
elif self.showCandidates:
self.setShowCandidates(False)