Remember picked candidates (user history)

Candidates chosen with the arrow-down list are recorded on commit as
toneless reading -> phrase with a count. The converter lifts a picked phrase just
above the best lexicon entry for that reading (more for more picks), so it
comes first next time without overriding longer phrases. Up to 2000 pairs,
least recently used evicted. Stored as user-history.json on every platform; the
Mac menu's clear item clears it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-13 14:30:55 +08:00
co-authored by Claude Opus 5.5
parent 815dcf979d
commit 2cebf0e700
19 changed files with 414 additions and 39 deletions
@@ -0,0 +1 @@
{"ㄋㄧ": {"妳": [1, 1]}}
+23 -10
View File
@@ -11,6 +11,7 @@ import time
from keycodes import * # noqa: F401,F403 (VK_* constants, provided by PIME)
from textService import TextService
from .core.history import UserHistory
from .core.classifier import Classifier, LearnedPreferences
from .core.engine import BACKSPACE, DOWN, ENTER, ESCAPE, LEFT, RIGHT, SHIFT_TAP, TAB, UP, CompositionEngine
from .core.english import EnglishModel
@@ -22,19 +23,36 @@ _DATA = os.path.join(_HERE, "data")
_USER_DIR = os.path.join(os.environ.get("APPDATA", _HERE), "PIME", "autozhuyin")
_LEARNED_FILE = os.path.join(_USER_DIR, "learned.json")
_USER_PHRASES = os.path.join(_USER_DIR, "user-phrases.txt")
_HISTORY_FILE = os.path.join(_USER_DIR, "user-history.json")
_classifier = None
def _save_learned(weights):
def _save_json(path, data):
try:
os.makedirs(_USER_DIR, exist_ok=True)
with open(_LEARNED_FILE, "w", encoding="utf-8") as f:
json.dump(weights, f, ensure_ascii=False)
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False)
except OSError:
pass
def _load_json(path):
try:
with open(path, encoding="utf-8") as f:
return json.load(f)
except (OSError, ValueError):
return {}
def _save_learned(weights):
_save_json(_LEARNED_FILE, weights)
def _save_history(data):
_save_json(_HISTORY_FILE, data)
def _shared_classifier():
"""Loads the lexicon and English model once per PIME server process."""
global _classifier
@@ -48,14 +66,9 @@ def _shared_classifier():
os.path.join(_DATA, "english-dict.txt"),
os.path.join(_DATA, "english-bigram.json"),
)
weights = {}
try:
with open(_LEARNED_FILE, encoding="utf-8") as f:
weights = json.load(f)
except (OSError, ValueError):
pass
trained = Classifier.load_weights(os.path.join(_DATA, "classifier-weights.json"))
_classifier = Classifier(lexicon, english, LearnedPreferences(weights, _save_learned), trained)
_classifier = Classifier(lexicon, english, LearnedPreferences(_load_json(_LEARNED_FILE), _save_learned),
trained, UserHistory(_load_json(_HISTORY_FILE), _save_history))
return _classifier
+3 -1
View File
@@ -1,6 +1,7 @@
"""Zhuyin-vs-literal classifier with learned corrections. Port of Classifier.swift."""
import json
from .history import UserHistory
from .features import DEFAULT_WEIGHTS, dot, literal_features, zhuyin_features
ZHUYIN = "zhuyin"
@@ -31,10 +32,11 @@ class LearnedPreferences:
class Classifier:
def __init__(self, lexicon, english, learned=None, weights=None):
def __init__(self, lexicon, english, learned=None, weights=None, history=None):
self.lexicon = lexicon
self.english = english
self.learned = learned or LearnedPreferences()
self.history = history or UserHistory() # picked candidates (shared by all engines)
self.weights = dict(DEFAULT_WEIGHTS)
if weights:
self.weights.update(weights)
+13 -1
View File
@@ -46,7 +46,7 @@ class CompositionEngine:
def __init__(self, classifier, now=time.monotonic):
self.classifier = classifier
self.converter = Converter(classifier.lexicon)
self.converter = Converter(classifier.lexicon, classifier.history)
self.now = now
self.keys = []
self.forced_mode = None
@@ -552,6 +552,7 @@ class CompositionEngine:
def _commit(self, segs=None):
segs = segs if segs is not None else self.segments
text = "".join(self._render(s, True) for s in segs)
self._remember_picks(segs)
if self.forced_mode:
self.classifier.learned.record("".join(self.keys), self.forced_mode)
self._previous_mode = segs[-1][2] if segs else None
@@ -559,6 +560,17 @@ class CompositionEngine:
self.reset()
return text
def _remember_picks(self, segs):
"""Candidates the user picked are remembered and come first next time."""
for i, seg in enumerate(segs):
picks = self.fixed.get(seg[0])
if seg[2] != ZHUYIN or not picks:
continue
syl = self._convertible(seg, True)[0]
for (a, b), text in picks.items():
if b <= len(syl):
self.classifier.history.record(syl[a:b], text)
def commit_all(self):
return self._commit() if self.keys else ""
+65
View File
@@ -0,0 +1,65 @@
"""Remembers the candidates the user picked. Port of UserHistory.swift.
Keys are toneless readings ("ㄊㄧㄢ", "ㄐㄧㄣ-ㄊㄧㄢ"); a picked phrase is boosted only
where the lexicon has it for the reading being converted, so a toneless key is safe.
"""
class UserHistory:
max_entries = 2000 # (reading, phrase) pairs kept; the least recently used go first
max_count = 10
def __init__(self, data=None, on_change=None):
# reading -> {phrase: [count, last use]}
self.data = {}
self.on_change = on_change or (lambda data: None)
self._clock = 0
for reading, phrases in (data or {}).items():
if not isinstance(phrases, dict):
continue
for phrase, value in phrases.items():
try:
count, used = int(value[0]), int(value[1])
except (TypeError, ValueError, IndexError):
continue
self.data.setdefault(reading, {})[phrase] = [count, used]
self._clock = max(self._clock, used)
@staticmethod
def key(syllables):
return "-".join(s.toneless for s in syllables)
def record(self, syllables, phrase):
if not syllables or not phrase:
return
self._clock += 1
entry = self.data.setdefault(self.key(syllables), {}).setdefault(phrase, [0, 0])
entry[0] = min(self.max_count, entry[0] + 1)
entry[1] = self._clock
self._evict()
self.on_change(self.data)
def _evict(self):
pairs = [(v[1], r, p) for r, ps in self.data.items() for p, v in ps.items()]
if len(pairs) <= self.max_entries:
return
pairs.sort()
for _, r, p in pairs[:len(pairs) - self.max_entries]:
del self.data[r][p]
if not self.data[r]:
del self.data[r]
def clear(self):
self.data = {}
self.on_change(self.data)
def adjust(self, syllables, entries):
"""Lexicon entries [(phrase, score)] with picked phrases moved to the top: just above
the best entry, more so the more often they were picked. Returns a new list."""
picked = self.data.get(self.key(syllables))
if not picked or not entries:
return entries
top = entries[0][1]
out = [(p, max(s, top + 0.01 + 0.1 * picked[p][0]) if p in picked else s) for p, s in entries]
out.sort(key=lambda e: -e[1])
return out
+8 -3
View File
@@ -84,8 +84,13 @@ TONELESS_PENALTY = -1.5
class Converter:
def __init__(self, lexicon):
def __init__(self, lexicon, history=None):
self.lexicon = lexicon
self.history = history # UserHistory: picked candidates come first
def _entries(self, syllables):
entries = self.lexicon.entries(syllables)
return self.history.adjust(syllables, entries) if self.history else entries
def _options(self, syllables, i, j, fixed):
if (i, j) in fixed:
@@ -94,7 +99,7 @@ class Converter:
return []
sl = syllables[i:j]
penalty = TONELESS_PENALTY if any(s.tone is None for s in sl) else 0.0
out = [(p, s + penalty) for p, s in self.lexicon.entries(sl)]
out = [(p, s + penalty) for p, s in self._entries(sl)]
if not out and j - i == 1:
out = [(sl[0].reading, UNKNOWN_SCORE)]
return out
@@ -132,7 +137,7 @@ class Converter:
out, seen = [], set()
for length in range(min(self.lexicon.max_phrase_length, end), 0, -1):
i = end - length
for phrase, _ in self.lexicon.entries(syllables[i:end]):
for phrase, _ in self._entries(syllables[i:end]):
if phrase not in seen:
seen.add(phrase)
out.append(((i, end), phrase))
+2 -1
View File
@@ -28,7 +28,8 @@
Tab 判斷錯了?按 Tab 在中/英之間翻轉,輸入法會記住這次修正
← → 在組字區移動游標,選要改哪個字;游標在中間時直接打注音
會插入新字,Backspace 刪掉游標前的字
↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字
↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字;
選過的字會被記住,下次優先出現
Esc 取消組字
單按 Shift 切換英文模式(按鍵直接送出、保持小寫、沒有底線),
再單按一次回到中英自動判斷,會出現「英文」/「中英自動」小提示;
+49
View File
@@ -15,6 +15,7 @@ from autozhuyin.core.engine import DOWN, ENTER, LEFT, RIGHT, SHIFT_TAP, TAB, BAC
from autozhuyin.core.shift_tap import ShiftTapDetector # noqa: E402
from autozhuyin.core.english import EnglishModel # noqa: E402
from autozhuyin.core.lexicon import Converter, Lexicon # noqa: E402
from autozhuyin.core.history import UserHistory # noqa: E402
DATA = os.path.join(os.path.dirname(HERE), "autozhuyin", "data")
_start = time.time()
@@ -364,5 +365,53 @@ class FirstWordHoldTests(unittest.TestCase):
self.assertEqual(type_keys("so ", e), "so ") # right after English: commits at once
class UserHistoryTests(unittest.TestCase):
def setUp(self):
self.saved = []
self.c = Classifier(LEX, ENG, weights=WEIGHTS, history=UserHistory(on_change=self.saved.append))
def pick(self, keys, phrase):
e = CompositionEngine(self.c)
type_keys(keys + "↓", e)
e.select_candidate([t for _, t in e.candidates].index(phrase))
return type_keys("\n", e)
def test_picked_candidate_comes_first(self):
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "天")
self.assertEqual(self.pick("wu0 ", "添"), "添")
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "添")
e = CompositionEngine(self.c)
type_keys("wu0 ↓", e)
self.assertEqual(e.candidates[0][1], "添")
self.assertEqual(self.saved[-1], {"ㄊㄧㄢ": {"添": [1, 1]}})
def test_phrases_still_win(self):
self.pick("wu0 ", "添")
self.assertEqual(type_keys("rup wu0 \n", CompositionEngine(self.c)), "今天")
def test_most_picked_wins(self):
self.pick("wu0 ", "添")
self.pick("wu0 ", "添")
self.pick("wu0 ", "天")
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "添")
def test_round_trip_and_clear(self):
self.pick("wu0 ", "添")
h = UserHistory(self.saved[-1])
self.assertEqual(h.data, {"ㄊㄧㄢ": {"添": [1, 1]}})
h.record([bopomofo.parse(list("wu0 ")).syllables[0].syllable], "天")
self.assertEqual(h.data["ㄊㄧㄢ"]["天"], [1, 2])
h.clear()
self.assertEqual(h.data, {})
def test_evicts_least_recent(self):
h = UserHistory()
h.max_entries = 2
syl = [bopomofo.parse(list("wu0 ")).syllables[0].syllable]
for p in ["天", "添", "田"]:
h.record(syl, p)
self.assertEqual(set(h.data["ㄊㄧㄢ"]), {"添", "田"})
if __name__ == "__main__":
unittest.main()
+8 -1
View File
@@ -6,6 +6,9 @@ import types
import unittest
HERE = os.path.dirname(os.path.abspath(__file__))
import tempfile # noqa: E402
os.environ["APPDATA"] = tempfile.mkdtemp(prefix="autozhuyin-test-") # learned / history files
sys.path.insert(0, os.path.dirname(HERE))
# --- stand-ins for PIME modules (only what the wrapper uses) ---
@@ -46,7 +49,7 @@ class FakeTextService:
textService.TextService = FakeTextService
sys.modules["textService"] = textService
from autozhuyin.autozhuyin_ime import AutoZhuyinTextService # noqa: E402
from autozhuyin.autozhuyin_ime import AutoZhuyinTextService, _shared_classifier # noqa: E402
VK = {"\n": 0x0D, "\t": 0x09, " ": 0x20, "\b": 0x08, "↓": 0x28}
@@ -99,6 +102,8 @@ def type_text(svc, text):
class WrapperTests(unittest.TestCase):
def setUp(self):
_shared_classifier().history.clear()
_shared_classifier().learned.clear()
self.svc = AutoZhuyinTextService(client=None)
self.svc.onActivate()
@@ -142,6 +147,8 @@ class WrapperTests(unittest.TestCase):
class ShiftTapWrapperTests(unittest.TestCase):
def setUp(self):
_shared_classifier().history.clear()
_shared_classifier().learned.clear()
self.svc = AutoZhuyinTextService(client=None)
self.svc.onActivate()