Remember picked candidates (user history)
Candidates chosen with the arrow-down list are recorded on commit as toneless reading -> phrase with a count. The converter lifts a picked phrase just above the best lexicon entry for that reading (more for more picks), so it comes first next time without overriding longer phrases. Up to 2000 pairs, least recently used evicted. Stored as user-history.json on every platform; the Mac menu's clear item clears it too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1 @@
|
||||
{"ㄋㄧ": {"妳": [1, 1]}}
|
||||
@@ -11,6 +11,7 @@ import time
|
||||
from keycodes import * # noqa: F401,F403 (VK_* constants, provided by PIME)
|
||||
from textService import TextService
|
||||
|
||||
from .core.history import UserHistory
|
||||
from .core.classifier import Classifier, LearnedPreferences
|
||||
from .core.engine import BACKSPACE, DOWN, ENTER, ESCAPE, LEFT, RIGHT, SHIFT_TAP, TAB, UP, CompositionEngine
|
||||
from .core.english import EnglishModel
|
||||
@@ -22,19 +23,36 @@ _DATA = os.path.join(_HERE, "data")
|
||||
_USER_DIR = os.path.join(os.environ.get("APPDATA", _HERE), "PIME", "autozhuyin")
|
||||
_LEARNED_FILE = os.path.join(_USER_DIR, "learned.json")
|
||||
_USER_PHRASES = os.path.join(_USER_DIR, "user-phrases.txt")
|
||||
_HISTORY_FILE = os.path.join(_USER_DIR, "user-history.json")
|
||||
|
||||
_classifier = None
|
||||
|
||||
|
||||
def _save_learned(weights):
|
||||
def _save_json(path, data):
|
||||
try:
|
||||
os.makedirs(_USER_DIR, exist_ok=True)
|
||||
with open(_LEARNED_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(weights, f, ensure_ascii=False)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _load_json(path):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def _save_learned(weights):
|
||||
_save_json(_LEARNED_FILE, weights)
|
||||
|
||||
|
||||
def _save_history(data):
|
||||
_save_json(_HISTORY_FILE, data)
|
||||
|
||||
|
||||
def _shared_classifier():
|
||||
"""Loads the lexicon and English model once per PIME server process."""
|
||||
global _classifier
|
||||
@@ -48,14 +66,9 @@ def _shared_classifier():
|
||||
os.path.join(_DATA, "english-dict.txt"),
|
||||
os.path.join(_DATA, "english-bigram.json"),
|
||||
)
|
||||
weights = {}
|
||||
try:
|
||||
with open(_LEARNED_FILE, encoding="utf-8") as f:
|
||||
weights = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
trained = Classifier.load_weights(os.path.join(_DATA, "classifier-weights.json"))
|
||||
_classifier = Classifier(lexicon, english, LearnedPreferences(weights, _save_learned), trained)
|
||||
_classifier = Classifier(lexicon, english, LearnedPreferences(_load_json(_LEARNED_FILE), _save_learned),
|
||||
trained, UserHistory(_load_json(_HISTORY_FILE), _save_history))
|
||||
return _classifier
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Zhuyin-vs-literal classifier with learned corrections. Port of Classifier.swift."""
|
||||
import json
|
||||
|
||||
from .history import UserHistory
|
||||
from .features import DEFAULT_WEIGHTS, dot, literal_features, zhuyin_features
|
||||
|
||||
ZHUYIN = "zhuyin"
|
||||
@@ -31,10 +32,11 @@ class LearnedPreferences:
|
||||
|
||||
class Classifier:
|
||||
|
||||
def __init__(self, lexicon, english, learned=None, weights=None):
|
||||
def __init__(self, lexicon, english, learned=None, weights=None, history=None):
|
||||
self.lexicon = lexicon
|
||||
self.english = english
|
||||
self.learned = learned or LearnedPreferences()
|
||||
self.history = history or UserHistory() # picked candidates (shared by all engines)
|
||||
self.weights = dict(DEFAULT_WEIGHTS)
|
||||
if weights:
|
||||
self.weights.update(weights)
|
||||
|
||||
@@ -46,7 +46,7 @@ class CompositionEngine:
|
||||
|
||||
def __init__(self, classifier, now=time.monotonic):
|
||||
self.classifier = classifier
|
||||
self.converter = Converter(classifier.lexicon)
|
||||
self.converter = Converter(classifier.lexicon, classifier.history)
|
||||
self.now = now
|
||||
self.keys = []
|
||||
self.forced_mode = None
|
||||
@@ -552,6 +552,7 @@ class CompositionEngine:
|
||||
def _commit(self, segs=None):
|
||||
segs = segs if segs is not None else self.segments
|
||||
text = "".join(self._render(s, True) for s in segs)
|
||||
self._remember_picks(segs)
|
||||
if self.forced_mode:
|
||||
self.classifier.learned.record("".join(self.keys), self.forced_mode)
|
||||
self._previous_mode = segs[-1][2] if segs else None
|
||||
@@ -559,6 +560,17 @@ class CompositionEngine:
|
||||
self.reset()
|
||||
return text
|
||||
|
||||
def _remember_picks(self, segs):
|
||||
"""Candidates the user picked are remembered and come first next time."""
|
||||
for i, seg in enumerate(segs):
|
||||
picks = self.fixed.get(seg[0])
|
||||
if seg[2] != ZHUYIN or not picks:
|
||||
continue
|
||||
syl = self._convertible(seg, True)[0]
|
||||
for (a, b), text in picks.items():
|
||||
if b <= len(syl):
|
||||
self.classifier.history.record(syl[a:b], text)
|
||||
|
||||
def commit_all(self):
|
||||
return self._commit() if self.keys else ""
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Remembers the candidates the user picked. Port of UserHistory.swift.
|
||||
|
||||
Keys are toneless readings ("ㄊㄧㄢ", "ㄐㄧㄣ-ㄊㄧㄢ"); a picked phrase is boosted only
|
||||
where the lexicon has it for the reading being converted, so a toneless key is safe.
|
||||
"""
|
||||
|
||||
|
||||
class UserHistory:
|
||||
max_entries = 2000 # (reading, phrase) pairs kept; the least recently used go first
|
||||
max_count = 10
|
||||
|
||||
def __init__(self, data=None, on_change=None):
|
||||
# reading -> {phrase: [count, last use]}
|
||||
self.data = {}
|
||||
self.on_change = on_change or (lambda data: None)
|
||||
self._clock = 0
|
||||
for reading, phrases in (data or {}).items():
|
||||
if not isinstance(phrases, dict):
|
||||
continue
|
||||
for phrase, value in phrases.items():
|
||||
try:
|
||||
count, used = int(value[0]), int(value[1])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
continue
|
||||
self.data.setdefault(reading, {})[phrase] = [count, used]
|
||||
self._clock = max(self._clock, used)
|
||||
|
||||
@staticmethod
|
||||
def key(syllables):
|
||||
return "-".join(s.toneless for s in syllables)
|
||||
|
||||
def record(self, syllables, phrase):
|
||||
if not syllables or not phrase:
|
||||
return
|
||||
self._clock += 1
|
||||
entry = self.data.setdefault(self.key(syllables), {}).setdefault(phrase, [0, 0])
|
||||
entry[0] = min(self.max_count, entry[0] + 1)
|
||||
entry[1] = self._clock
|
||||
self._evict()
|
||||
self.on_change(self.data)
|
||||
|
||||
def _evict(self):
|
||||
pairs = [(v[1], r, p) for r, ps in self.data.items() for p, v in ps.items()]
|
||||
if len(pairs) <= self.max_entries:
|
||||
return
|
||||
pairs.sort()
|
||||
for _, r, p in pairs[:len(pairs) - self.max_entries]:
|
||||
del self.data[r][p]
|
||||
if not self.data[r]:
|
||||
del self.data[r]
|
||||
|
||||
def clear(self):
|
||||
self.data = {}
|
||||
self.on_change(self.data)
|
||||
|
||||
def adjust(self, syllables, entries):
|
||||
"""Lexicon entries [(phrase, score)] with picked phrases moved to the top: just above
|
||||
the best entry, more so the more often they were picked. Returns a new list."""
|
||||
picked = self.data.get(self.key(syllables))
|
||||
if not picked or not entries:
|
||||
return entries
|
||||
top = entries[0][1]
|
||||
out = [(p, max(s, top + 0.01 + 0.1 * picked[p][0]) if p in picked else s) for p, s in entries]
|
||||
out.sort(key=lambda e: -e[1])
|
||||
return out
|
||||
@@ -84,8 +84,13 @@ TONELESS_PENALTY = -1.5
|
||||
|
||||
|
||||
class Converter:
|
||||
def __init__(self, lexicon):
|
||||
def __init__(self, lexicon, history=None):
|
||||
self.lexicon = lexicon
|
||||
self.history = history # UserHistory: picked candidates come first
|
||||
|
||||
def _entries(self, syllables):
|
||||
entries = self.lexicon.entries(syllables)
|
||||
return self.history.adjust(syllables, entries) if self.history else entries
|
||||
|
||||
def _options(self, syllables, i, j, fixed):
|
||||
if (i, j) in fixed:
|
||||
@@ -94,7 +99,7 @@ class Converter:
|
||||
return []
|
||||
sl = syllables[i:j]
|
||||
penalty = TONELESS_PENALTY if any(s.tone is None for s in sl) else 0.0
|
||||
out = [(p, s + penalty) for p, s in self.lexicon.entries(sl)]
|
||||
out = [(p, s + penalty) for p, s in self._entries(sl)]
|
||||
if not out and j - i == 1:
|
||||
out = [(sl[0].reading, UNKNOWN_SCORE)]
|
||||
return out
|
||||
@@ -132,7 +137,7 @@ class Converter:
|
||||
out, seen = [], set()
|
||||
for length in range(min(self.lexicon.max_phrase_length, end), 0, -1):
|
||||
i = end - length
|
||||
for phrase, _ in self.lexicon.entries(syllables[i:end]):
|
||||
for phrase, _ in self._entries(syllables[i:end]):
|
||||
if phrase not in seen:
|
||||
seen.add(phrase)
|
||||
out.append(((i, end), phrase))
|
||||
|
||||
@@ -28,7 +28,8 @@
|
||||
Tab 判斷錯了?按 Tab 在中/英之間翻轉,輸入法會記住這次修正
|
||||
← → 在組字區移動游標,選要改哪個字;游標在中間時直接打注音
|
||||
會插入新字,Backspace 刪掉游標前的字
|
||||
↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字
|
||||
↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字;
|
||||
選過的字會被記住,下次優先出現
|
||||
Esc 取消組字
|
||||
單按 Shift 切換英文模式(按鍵直接送出、保持小寫、沒有底線),
|
||||
再單按一次回到中英自動判斷,會出現「英文」/「中英自動」小提示;
|
||||
|
||||
@@ -15,6 +15,7 @@ from autozhuyin.core.engine import DOWN, ENTER, LEFT, RIGHT, SHIFT_TAP, TAB, BAC
|
||||
from autozhuyin.core.shift_tap import ShiftTapDetector # noqa: E402
|
||||
from autozhuyin.core.english import EnglishModel # noqa: E402
|
||||
from autozhuyin.core.lexicon import Converter, Lexicon # noqa: E402
|
||||
from autozhuyin.core.history import UserHistory # noqa: E402
|
||||
|
||||
DATA = os.path.join(os.path.dirname(HERE), "autozhuyin", "data")
|
||||
_start = time.time()
|
||||
@@ -364,5 +365,53 @@ class FirstWordHoldTests(unittest.TestCase):
|
||||
self.assertEqual(type_keys("so ", e), "so ") # right after English: commits at once
|
||||
|
||||
|
||||
class UserHistoryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.saved = []
|
||||
self.c = Classifier(LEX, ENG, weights=WEIGHTS, history=UserHistory(on_change=self.saved.append))
|
||||
|
||||
def pick(self, keys, phrase):
|
||||
e = CompositionEngine(self.c)
|
||||
type_keys(keys + "↓", e)
|
||||
e.select_candidate([t for _, t in e.candidates].index(phrase))
|
||||
return type_keys("\n", e)
|
||||
|
||||
def test_picked_candidate_comes_first(self):
|
||||
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "天")
|
||||
self.assertEqual(self.pick("wu0 ", "添"), "添")
|
||||
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "添")
|
||||
e = CompositionEngine(self.c)
|
||||
type_keys("wu0 ↓", e)
|
||||
self.assertEqual(e.candidates[0][1], "添")
|
||||
self.assertEqual(self.saved[-1], {"ㄊㄧㄢ": {"添": [1, 1]}})
|
||||
|
||||
def test_phrases_still_win(self):
|
||||
self.pick("wu0 ", "添")
|
||||
self.assertEqual(type_keys("rup wu0 \n", CompositionEngine(self.c)), "今天")
|
||||
|
||||
def test_most_picked_wins(self):
|
||||
self.pick("wu0 ", "添")
|
||||
self.pick("wu0 ", "添")
|
||||
self.pick("wu0 ", "天")
|
||||
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "添")
|
||||
|
||||
def test_round_trip_and_clear(self):
|
||||
self.pick("wu0 ", "添")
|
||||
h = UserHistory(self.saved[-1])
|
||||
self.assertEqual(h.data, {"ㄊㄧㄢ": {"添": [1, 1]}})
|
||||
h.record([bopomofo.parse(list("wu0 ")).syllables[0].syllable], "天")
|
||||
self.assertEqual(h.data["ㄊㄧㄢ"]["天"], [1, 2])
|
||||
h.clear()
|
||||
self.assertEqual(h.data, {})
|
||||
|
||||
def test_evicts_least_recent(self):
|
||||
h = UserHistory()
|
||||
h.max_entries = 2
|
||||
syl = [bopomofo.parse(list("wu0 ")).syllables[0].syllable]
|
||||
for p in ["天", "添", "田"]:
|
||||
h.record(syl, p)
|
||||
self.assertEqual(set(h.data["ㄊㄧㄢ"]), {"添", "田"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -6,6 +6,9 @@ import types
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
import tempfile # noqa: E402
|
||||
|
||||
os.environ["APPDATA"] = tempfile.mkdtemp(prefix="autozhuyin-test-") # learned / history files
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
|
||||
# --- stand-ins for PIME modules (only what the wrapper uses) ---
|
||||
@@ -46,7 +49,7 @@ class FakeTextService:
|
||||
textService.TextService = FakeTextService
|
||||
sys.modules["textService"] = textService
|
||||
|
||||
from autozhuyin.autozhuyin_ime import AutoZhuyinTextService # noqa: E402
|
||||
from autozhuyin.autozhuyin_ime import AutoZhuyinTextService, _shared_classifier # noqa: E402
|
||||
|
||||
VK = {"\n": 0x0D, "\t": 0x09, " ": 0x20, "\b": 0x08, "↓": 0x28}
|
||||
|
||||
@@ -99,6 +102,8 @@ def type_text(svc, text):
|
||||
|
||||
class WrapperTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
_shared_classifier().history.clear()
|
||||
_shared_classifier().learned.clear()
|
||||
self.svc = AutoZhuyinTextService(client=None)
|
||||
self.svc.onActivate()
|
||||
|
||||
@@ -142,6 +147,8 @@ class WrapperTests(unittest.TestCase):
|
||||
|
||||
class ShiftTapWrapperTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
_shared_classifier().history.clear()
|
||||
_shared_classifier().learned.clear()
|
||||
self.svc = AutoZhuyinTextService(client=None)
|
||||
self.svc.onActivate()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user