diff --git a/README.md b/README.md index c3185d2..cf63c52 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - **注音順序打錯也能修正**:`ㄧㄒㄠˇ`、`ㄒㄠㄧˇ` 都會變成 `ㄒㄧㄠˇ`(小) - **可省略聲調**:`jiap7`(ㄨㄛ ㄇㄣ˙)一樣能打出「我們」 - **候選字**:用 ← → 把游標移到要改的字後面,按 ↓ 選字;選過的會固定下來 +- **記住選字**:從候選字選過的字詞會被記住,下次打同樣的讀音會優先出現(選越多次越優先) - **中間補字、刪字**:游標移到前面後直接打注音,會插在游標處;Backspace 刪掉游標前的那個字 - **全形標點**:中文語境下 `?` `!` `<` 自動變成 `?` `!` `,`;任何時候按 `` ` `` 再按標點鍵,一定得到全形標點(例如 `` `< `` = `,`) - **完整詞庫**:約 17 萬筆,來自小麥注音 McBopomofo(MIT 授權) @@ -100,10 +101,12 @@ SmartScreen 出現警告時,按「其他資訊 → 仍要執行」。解除安 | | macOS | Windows | Linux | |---|---|---|---| | 學習紀錄(Tab 修正) | `UserDefaults`(`com.autozhuyin.inputmethod.AutoZhuyin`) | `%APPDATA%\PIME\autozhuyin\learned.json` | `~/.config/autozhuyin/learned.json` | +| 選字紀錄 | `~/Library/Application Support/AutoZhuyin/user-history.json` | `%APPDATA%\PIME\autozhuyin\user-history.json` | `~/.config/autozhuyin/user-history.json` | | 自訂詞 | `~/Library/Application Support/AutoZhuyin/user-phrases.txt` | `%APPDATA%\PIME\autozhuyin\user-phrases.txt` | `~/.config/autozhuyin/user-phrases.txt` | 自訂詞的格式和 `lexicon.txt` 一樣,每行一個詞:`ㄗˋ-ㄉㄨㄥˋ-ㄓㄨˋ-ㄧㄣ 自動注音 -4`(讀音、詞、分數;分數越大越優先)。 -Mac 版可以在輸入法選單選「清除學習紀錄」。 +選字紀錄最多保留 2000 筆,太久沒用的會先淘汰。被選過的字只會排到同讀音第一順位,不會蓋過整個詞組(例如選過「添」之後,ㄊㄧㄢ 單打會出「添」,但「今天」還是「今天」)。 +Mac 版可以在輸入法選單選「清除學習紀錄」(中英判斷和選字紀錄一起清除);Windows、Linux 刪除上面的檔案即可。 ## 開發 @@ -261,7 +264,7 @@ THIRD_PARTY_LICENSES.txt 第三方授權聲明(會打包進兩個安裝 - [ ] 加入真實打字資料一起訓練 - [ ] 已送出的英文單字若判斷錯誤,可以「事後翻轉」 - [ ] 用 Core ML 小模型取代手調分數,參考整句語境 -- [ ] 使用者詞頻學習(選過的候選字優先) +- [x] 使用者詞頻學習(選過的候選字優先) - [ ] 偏好設定視窗 ## 致謝與授權 diff --git a/Sources/AutoZhuyin/InputController.swift b/Sources/AutoZhuyin/InputController.swift index 116cbcd..53ebe29 100644 --- a/Sources/AutoZhuyin/InputController.swift +++ b/Sources/AutoZhuyin/InputController.swift @@ -9,6 +9,11 @@ enum Shared { static let learnedKey = "LearnedPreferences" + /// ~/Library/Application Support/AutoZhuyin (user phrases, picked-candidate history). + static let supportDir: URL? = FileManager.default + .urls(for: .applicationSupportDirectory, in: .userDomainMask).first? + .appendingPathComponent("AutoZhuyin") + static let classifier: Classifier = { let bundle = Bundle.main let start = Date() @@ -18,9 +23,9 @@ enum Shared { lexicon.load(data: data) } // Optional user phrases: ~/Library/Application Support/AutoZhuyin/user-phrases.txt - if let support = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first { - let userFile = support.appendingPathComponent("AutoZhuyin/user-phrases.txt") - if let text = try? String(contentsOf: userFile, encoding: .utf8) { lexicon.load(text) } + if let userFile = supportDir?.appendingPathComponent("user-phrases.txt"), + let text = try? String(contentsOf: userFile, encoding: .utf8) { + lexicon.load(text) } let english = EnglishModel(commonWordsURL: bundle.url(forResource: "english-common", withExtension: "txt"), dictionaryURL: bundle.url(forResource: "english-dict", withExtension: "txt"), @@ -29,11 +34,17 @@ enum Shared { let learned = LearnedPreferences(weights: saved) { weights in UserDefaults.standard.set(weights, forKey: learnedKey) } + let historyFile = supportDir?.appendingPathComponent("user-history.json") + let history = UserHistory(data: historyFile.map(UserHistory.load(from:)) ?? [:]) { data in + guard let url = historyFile, let json = try? JSONEncoder().encode(data) else { return } + try? FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try? json.write(to: url, options: .atomic) + } let trained = Features.loadWeights(from: bundle.url(forResource: "classifier-weights", withExtension: "json")) NSLog("AutoZhuyin: data loaded in %.0f ms", Date().timeIntervalSince(start) * 1000) // Only needed for untoned syllables; build it off the startup path. DispatchQueue.global(qos: .utility).async { lexicon.prepareTonelessIndex() } - return Classifier(lexicon: lexicon, english: english, learned: learned, weights: trained) + return Classifier(lexicon: lexicon, english: english, learned: learned, weights: trained, history: history) }() } @@ -177,5 +188,6 @@ final class InputController: IMKInputController { @objc private func clearLearned(_ sender: Any?) { Shared.classifier.learned.clear() + Shared.classifier.history.clear() } } diff --git a/Sources/AutoZhuyinCore/Classifier.swift b/Sources/AutoZhuyinCore/Classifier.swift index dbb8eee..bd9d229 100644 --- a/Sources/AutoZhuyinCore/Classifier.swift +++ b/Sources/AutoZhuyinCore/Classifier.swift @@ -55,14 +55,17 @@ public final class Classifier { public let lexicon: Lexicon public let english: EnglishModel public let learned: LearnedPreferences + /// Picked candidates (shared by every engine using this classifier). + public let history: UserHistory /// Feature weights: the hand-tuned defaults overlaid with trained ones. public private(set) var weights: FeatureVector public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences(), - weights: FeatureVector = [:]) { + weights: FeatureVector = [:], history: UserHistory = UserHistory()) { self.lexicon = lexicon self.english = english self.learned = learned + self.history = history self.weights = Features.defaultWeights.merging(weights) { _, new in new } } diff --git a/Sources/AutoZhuyinCore/CompositionEngine.swift b/Sources/AutoZhuyinCore/CompositionEngine.swift index 6332123..3b30179 100644 --- a/Sources/AutoZhuyinCore/CompositionEngine.swift +++ b/Sources/AutoZhuyinCore/CompositionEngine.swift @@ -71,7 +71,7 @@ public final class CompositionEngine { public init(classifier: Classifier) { self.classifier = classifier - self.converter = Converter(lexicon: classifier.lexicon) + self.converter = Converter(lexicon: classifier.lexicon, history: classifier.history) } // MARK: - Derived state @@ -537,6 +537,7 @@ public final class CompositionEngine { private func commitCurrent(_ given: [Segment]? = nil) -> String { let segs = given ?? segments let text = segs.map { render($0, final: true) }.joined() + rememberPicks(segs) if let forced = forcedMode { classifier.learned.record(String(keys), as: forced) } @@ -546,6 +547,17 @@ public final class CompositionEngine { return text } + /// Candidates the user picked are remembered and come first next time. + private func rememberPicks(_ segs: [Segment]) { + for seg in segs where seg.mode == .zhuyin { + guard let picks = fixed[seg.range.lowerBound], !picks.isEmpty else { continue } + let syl = convertibleSyllables(seg, final: true).syllables + for (range, text) in picks where range.upperBound <= syl.count { + classifier.history.record(Array(syl[range]), phrase: text) + } + } + } + /// Commits whatever is composed (e.g. the app lost focus). Returns "" when empty. public func commitAll() -> String { keys.isEmpty ? "" : commitCurrent() diff --git a/Sources/AutoZhuyinCore/Converter.swift b/Sources/AutoZhuyinCore/Converter.swift index de6b38b..88d4133 100644 --- a/Sources/AutoZhuyinCore/Converter.swift +++ b/Sources/AutoZhuyinCore/Converter.swift @@ -13,12 +13,22 @@ public struct Candidate: Equatable { /// Turns a syllable sequence into the most likely Chinese text (Viterbi over the lexicon). public struct Converter { public let lexicon: Lexicon + /// Picked candidates come first. + public let history: UserHistory? /// Score used when a syllable has no lexicon entry; the Bopomofo itself is emitted. static let unknownScore = -30.0 /// Penalty for matching an untoned syllable against all tones. static let tonelessPenalty = -1.5 - public init(lexicon: Lexicon) { self.lexicon = lexicon } + public init(lexicon: Lexicon, history: UserHistory? = nil) { + self.lexicon = lexicon + self.history = history + } + + private func entries(_ syllables: [Syllable]) -> [LexiconEntry] { + let e = lexicon.entries(for: syllables) + return history?.adjust(syllables, e) ?? e + } /// - Parameter fixed: user-chosen phrases by syllable range; they are always kept. public func convert(_ syllables: [Syllable], fixed: [Range: String] = [:]) -> [ConvertedSegment] { @@ -57,7 +67,7 @@ public struct Converter { if fixed.keys.contains(where: { $0.overlaps(range) }) { return [] } let slice = Array(syllables[range]) - var result = lexicon.entries(for: slice).map { entry -> (String, Double) in + var result = entries(slice).map { entry -> (String, Double) in let penalty = slice.contains { $0.tone == nil } ? Converter.tonelessPenalty : 0 return (entry.phrase, entry.score + penalty) } @@ -74,7 +84,7 @@ public struct Converter { var seen = Set() for len in stride(from: min(lexicon.maxPhraseLength, end), through: 1, by: -1) { let range = (end - len).. Void + + public init(data: Data = [:], onChange: @escaping (Data) -> Void = { _ in }) { + self.onChange = onChange + for (reading, phrases) in data { + for (phrase, value) in phrases where value.count >= 2 { + self.data[reading, default: [:]][phrase] = [value[0], value[1]] + clock = max(clock, value[1]) + } + } + } + + /// Loads the JSON file written through `onChange`; empty if missing or unreadable. + public static func load(from url: URL) -> Data { + guard let raw = try? Foundation.Data(contentsOf: url), + let data = try? JSONDecoder().decode(Data.self, from: raw) else { return [:] } + return data + } + + static func key(_ syllables: [Syllable]) -> String { + syllables.map(\.toneless).joined(separator: "-") + } + + public func record(_ syllables: [Syllable], phrase: String) { + guard !syllables.isEmpty, !phrase.isEmpty else { return } + clock += 1 + let key = UserHistory.key(syllables) + let count = data[key]?[phrase]?.first ?? 0 + data[key, default: [:]][phrase] = [min(UserHistory.maxCount, count + 1), clock] + evict() + onChange(data) + } + + private func evict() { + var pairs: [(used: Int, reading: String, phrase: String)] = [] + for (r, phrases) in data { for (p, v) in phrases { pairs.append((v[1], r, p)) } } + guard pairs.count > maxEntries else { return } + pairs.sort { $0.used < $1.used } + for pair in pairs.prefix(pairs.count - maxEntries) { + data[pair.reading]?[pair.phrase] = nil + if data[pair.reading]?.isEmpty == true { data[pair.reading] = nil } + } + } + + public func clear() { + data = [:] + onChange(data) + } + + /// Lexicon entries with picked phrases moved to the top: just above the best entry, + /// more so the more often they were picked. + func adjust(_ syllables: [Syllable], _ entries: [LexiconEntry]) -> [LexiconEntry] { + guard let picked = data[UserHistory.key(syllables)], let top = entries.first?.score else { return entries } + var out = entries.map { e -> LexiconEntry in + guard let count = picked[e.phrase]?.first else { return e } + return LexiconEntry(phrase: e.phrase, score: max(e.score, top + 0.01 + 0.1 * Double(count))) + } + // Stable sort (keep lexicon order among equal scores), like Python's sort. + out = out.enumerated().sorted { $0.element.score != $1.element.score + ? $0.element.score > $1.element.score : $0.offset < $1.offset }.map(\.element) + return out + } +} diff --git a/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift b/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift index b95f8d1..3cdca21 100644 --- a/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift +++ b/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift @@ -569,6 +569,76 @@ final class FirstWordHoldTests: XCTestCase { } } +final class UserHistoryTests: XCTestCase { + var saved: [UserHistory.Data] = [] + var classifier: Classifier! + + override func setUp() { + saved = [] + classifier = Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights, + history: UserHistory { [unowned self] in self.saved.append($0) }) + } + + func type(_ s: String, into e: CompositionEngine) -> String { + var out = "" + for c in s { + let key: KeyInput = c == "↓" ? .down : c == "\n" ? .enter : .character(c) + let r = e.handle(key) + if let t = r.commit { out += t } else if !r.handled { out.append(c) } + } + return out + } + + @discardableResult + func pick(_ keys: String, _ phrase: String) -> String { + let e = CompositionEngine(classifier: classifier) + _ = type(keys + "↓", into: e) + e.selectCandidate(e.candidates.firstIndex { $0.text == phrase }!) + return type("\n", into: e) + } + + func testPickedCandidateComesFirst() { + XCTAssertEqual(type("wu0 \n", into: CompositionEngine(classifier: classifier)), "天") + XCTAssertEqual(pick("wu0 ", "添"), "添") + XCTAssertEqual(type("wu0 \n", into: CompositionEngine(classifier: classifier)), "添") + let e = CompositionEngine(classifier: classifier) + _ = type("wu0 ↓", into: e) + XCTAssertEqual(e.candidates.first?.text, "添") + XCTAssertEqual(saved.last, ["ㄊㄧㄢ": ["添": [1, 1]]]) + } + + func testPhrasesStillWin() { + pick("wu0 ", "添") + XCTAssertEqual(type("rup wu0 \n", into: CompositionEngine(classifier: classifier)), "今天") + } + + func testMostPickedWins() { + pick("wu0 ", "添") + pick("wu0 ", "添") + pick("wu0 ", "天") + XCTAssertEqual(type("wu0 \n", into: CompositionEngine(classifier: classifier)), "添") + } + + func testRoundTripAndClear() throws { + pick("wu0 ", "添") + let json = try JSONEncoder().encode(saved.last!) + let h = UserHistory(data: try JSONDecoder().decode(UserHistory.Data.self, from: json)) + XCTAssertEqual(h.data, ["ㄊㄧㄢ": ["添": [1, 1]]]) + h.record(ZhuyinParser.parse(Array("wu0 ")).syllables.map(\.syllable), phrase: "天") + XCTAssertEqual(h.data["ㄊㄧㄢ"]?["天"], [1, 2]) + h.clear() + XCTAssertTrue(h.data.isEmpty) + } + + func testEvictsLeastRecent() { + let h = UserHistory() + h.maxEntries = 2 + let syl = ZhuyinParser.parse(Array("wu0 ")).syllables.map(\.syllable) + for p in ["天", "添", "田"] { h.record(syl, phrase: p) } + XCTAssertEqual(Set(h.data["ㄊㄧㄢ"]!.keys), ["添", "田"]) + } +} + final class StartupTimingTests: XCTestCase { func testStartupTiming() throws { var t = Date() diff --git a/linux/autozhuyin_ibus.py b/linux/autozhuyin_ibus.py index 6bedd25..7d37916 100644 --- a/linux/autozhuyin_ibus.py +++ b/linux/autozhuyin_ibus.py @@ -24,6 +24,7 @@ gi.require_version("IBus", "1.0") from gi.repository import GLib, GObject, IBus # noqa: E402 import autozhuyin # noqa: E402 +from autozhuyin.core.history import UserHistory # noqa: E402 from autozhuyin.core.classifier import LITERAL, Classifier, LearnedPreferences # noqa: E402 from autozhuyin.core.engine import ( # noqa: E402 BACKSPACE, DOWN, ENTER, ESCAPE, LEFT, RIGHT, SHIFT_TAP, TAB, UP, CompositionEngine, @@ -40,6 +41,7 @@ _DATA = os.path.join(os.path.dirname(os.path.abspath(autozhuyin.__file__)), "dat _USER_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config"), "autozhuyin") _LEARNED_FILE = os.path.join(_USER_DIR, "learned.json") _USER_PHRASES = os.path.join(_USER_DIR, "user-phrases.txt") +_HISTORY_FILE = os.path.join(_USER_DIR, "user-history.json") try: with open(os.path.join(HERE, "version.txt"), encoding="utf-8") as _f: @@ -50,15 +52,31 @@ except OSError: _classifier = None -def _save_learned(weights): +def _save_json(path, data): try: os.makedirs(_USER_DIR, exist_ok=True) - with open(_LEARNED_FILE, "w", encoding="utf-8") as f: - json.dump(weights, f, ensure_ascii=False) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False) except OSError: pass +def _load_json(path): + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError): + return {} + + +def _save_learned(weights): + _save_json(_LEARNED_FILE, weights) + + +def _save_history(data): + _save_json(_HISTORY_FILE, data) + + def shared_classifier(): """Loads the lexicon and English model once per process.""" global _classifier @@ -72,14 +90,9 @@ def shared_classifier(): os.path.join(_DATA, "english-dict.txt"), os.path.join(_DATA, "english-bigram.json"), ) - weights = {} - try: - with open(_LEARNED_FILE, encoding="utf-8") as f: - weights = json.load(f) - except (OSError, ValueError): - pass trained = Classifier.load_weights(os.path.join(_DATA, "classifier-weights.json")) - _classifier = Classifier(lexicon, english, LearnedPreferences(weights, _save_learned), trained) + _classifier = Classifier(lexicon, english, LearnedPreferences(_load_json(_LEARNED_FILE), _save_learned), + trained, UserHistory(_load_json(_HISTORY_FILE), _save_history)) return _classifier diff --git a/linux/installer/README.txt b/linux/installer/README.txt index 8838192..1b68f2f 100644 --- a/linux/installer/README.txt +++ b/linux/installer/README.txt @@ -30,7 +30,8 @@ Tab 判斷錯了?按 Tab 在中/英之間翻轉,輸入法會記住這次修正 ← → 在組字區移動游標;游標在中間時直接打注音會插入新字, Backspace 刪掉游標前的字 - ↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字 + ↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字; + 選過的字會被記住,下次優先出現 Esc 取消組字 單按 Shift 切換英文模式(按鍵直接送出),再單按一次回到中英自動判斷 Shift+字母 打大寫字母(也會強制英文) @@ -47,6 +48,7 @@ 個人資料(學習紀錄、自訂詞)在 ~/.config/autozhuyin/: learned.json 按 Tab 修正的學習紀錄 + user-history.json 選字紀錄(選過的候選字) user-phrases.txt 自訂詞,一行一個,例如:ㄗˋ-ㄉㄨㄥˋ 自動 -1 diff --git a/linux/tests/test_ibus_engine.py b/linux/tests/test_ibus_engine.py index 8145670..246095b 100644 --- a/linux/tests/test_ibus_engine.py +++ b/linux/tests/test_ibus_engine.py @@ -8,6 +8,9 @@ import types import unittest LINUX = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +import tempfile # noqa: E402 + +os.environ["XDG_CONFIG_HOME"] = tempfile.mkdtemp(prefix="autozhuyin-test-") # learned / history files sys.path.insert(0, os.path.join(os.path.dirname(LINUX), "windows")) # autozhuyin package sys.path.insert(0, LINUX) @@ -107,6 +110,8 @@ def press(e, keys): class IBusEngineTests(unittest.TestCase): def setUp(self): + autozhuyin_ibus.shared_classifier().history.clear() + autozhuyin_ibus.shared_classifier().learned.clear() self.e = autozhuyin_ibus.AutoZhuyinEngine() def test_mixed_sentence(self): @@ -157,6 +162,17 @@ class IBusEngineTests(unittest.TestCase): press(self.e, "rup fu4←wu0 \n") self.assertEqual(self.e.committed, "今天氣") + def test_picked_candidate_saved(self): + press(self.e, "su3↓") + second = self.e.shown[1] + self.e.do_candidate_clicked(1, 1, 0) + press(self.e, "\n") + path = os.path.join(os.environ["XDG_CONFIG_HOME"], "autozhuyin", "user-history.json") + with open(path, encoding="utf-8") as f: + self.assertIn(second, f.read()) + press(self.e, "su3 \n") + self.assertTrue(self.e.committed.endswith(second)) + def test_focus_out_resets(self): press(self.e, "su3") self.e.do_focus_out() diff --git a/windows/autozhuyin/PIME/autozhuyin/user-history.json b/windows/autozhuyin/PIME/autozhuyin/user-history.json new file mode 100644 index 0000000..b6e45bf --- /dev/null +++ b/windows/autozhuyin/PIME/autozhuyin/user-history.json @@ -0,0 +1 @@ +{"ㄋㄧ": {"妳": [1, 1]}} \ No newline at end of file diff --git a/windows/autozhuyin/autozhuyin_ime.py b/windows/autozhuyin/autozhuyin_ime.py index 6601d66..8c35bc7 100644 --- a/windows/autozhuyin/autozhuyin_ime.py +++ b/windows/autozhuyin/autozhuyin_ime.py @@ -11,6 +11,7 @@ import time from keycodes import * # noqa: F401,F403 (VK_* constants, provided by PIME) from textService import TextService +from .core.history import UserHistory from .core.classifier import Classifier, LearnedPreferences from .core.engine import BACKSPACE, DOWN, ENTER, ESCAPE, LEFT, RIGHT, SHIFT_TAP, TAB, UP, CompositionEngine from .core.english import EnglishModel @@ -22,19 +23,36 @@ _DATA = os.path.join(_HERE, "data") _USER_DIR = os.path.join(os.environ.get("APPDATA", _HERE), "PIME", "autozhuyin") _LEARNED_FILE = os.path.join(_USER_DIR, "learned.json") _USER_PHRASES = os.path.join(_USER_DIR, "user-phrases.txt") +_HISTORY_FILE = os.path.join(_USER_DIR, "user-history.json") _classifier = None -def _save_learned(weights): +def _save_json(path, data): try: os.makedirs(_USER_DIR, exist_ok=True) - with open(_LEARNED_FILE, "w", encoding="utf-8") as f: - json.dump(weights, f, ensure_ascii=False) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False) except OSError: pass +def _load_json(path): + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError): + return {} + + +def _save_learned(weights): + _save_json(_LEARNED_FILE, weights) + + +def _save_history(data): + _save_json(_HISTORY_FILE, data) + + def _shared_classifier(): """Loads the lexicon and English model once per PIME server process.""" global _classifier @@ -48,14 +66,9 @@ def _shared_classifier(): os.path.join(_DATA, "english-dict.txt"), os.path.join(_DATA, "english-bigram.json"), ) - weights = {} - try: - with open(_LEARNED_FILE, encoding="utf-8") as f: - weights = json.load(f) - except (OSError, ValueError): - pass trained = Classifier.load_weights(os.path.join(_DATA, "classifier-weights.json")) - _classifier = Classifier(lexicon, english, LearnedPreferences(weights, _save_learned), trained) + _classifier = Classifier(lexicon, english, LearnedPreferences(_load_json(_LEARNED_FILE), _save_learned), + trained, UserHistory(_load_json(_HISTORY_FILE), _save_history)) return _classifier diff --git a/windows/autozhuyin/core/classifier.py b/windows/autozhuyin/core/classifier.py index 6c64191..c3126c3 100644 --- a/windows/autozhuyin/core/classifier.py +++ b/windows/autozhuyin/core/classifier.py @@ -1,6 +1,7 @@ """Zhuyin-vs-literal classifier with learned corrections. Port of Classifier.swift.""" import json +from .history import UserHistory from .features import DEFAULT_WEIGHTS, dot, literal_features, zhuyin_features ZHUYIN = "zhuyin" @@ -31,10 +32,11 @@ class LearnedPreferences: class Classifier: - def __init__(self, lexicon, english, learned=None, weights=None): + def __init__(self, lexicon, english, learned=None, weights=None, history=None): self.lexicon = lexicon self.english = english self.learned = learned or LearnedPreferences() + self.history = history or UserHistory() # picked candidates (shared by all engines) self.weights = dict(DEFAULT_WEIGHTS) if weights: self.weights.update(weights) diff --git a/windows/autozhuyin/core/engine.py b/windows/autozhuyin/core/engine.py index 1684458..b16de96 100644 --- a/windows/autozhuyin/core/engine.py +++ b/windows/autozhuyin/core/engine.py @@ -46,7 +46,7 @@ class CompositionEngine: def __init__(self, classifier, now=time.monotonic): self.classifier = classifier - self.converter = Converter(classifier.lexicon) + self.converter = Converter(classifier.lexicon, classifier.history) self.now = now self.keys = [] self.forced_mode = None @@ -552,6 +552,7 @@ class CompositionEngine: def _commit(self, segs=None): segs = segs if segs is not None else self.segments text = "".join(self._render(s, True) for s in segs) + self._remember_picks(segs) if self.forced_mode: self.classifier.learned.record("".join(self.keys), self.forced_mode) self._previous_mode = segs[-1][2] if segs else None @@ -559,6 +560,17 @@ class CompositionEngine: self.reset() return text + def _remember_picks(self, segs): + """Candidates the user picked are remembered and come first next time.""" + for i, seg in enumerate(segs): + picks = self.fixed.get(seg[0]) + if seg[2] != ZHUYIN or not picks: + continue + syl = self._convertible(seg, True)[0] + for (a, b), text in picks.items(): + if b <= len(syl): + self.classifier.history.record(syl[a:b], text) + def commit_all(self): return self._commit() if self.keys else "" diff --git a/windows/autozhuyin/core/history.py b/windows/autozhuyin/core/history.py new file mode 100644 index 0000000..3457768 --- /dev/null +++ b/windows/autozhuyin/core/history.py @@ -0,0 +1,65 @@ +"""Remembers the candidates the user picked. Port of UserHistory.swift. + +Keys are toneless readings ("ㄊㄧㄢ", "ㄐㄧㄣ-ㄊㄧㄢ"); a picked phrase is boosted only +where the lexicon has it for the reading being converted, so a toneless key is safe. +""" + + +class UserHistory: + max_entries = 2000 # (reading, phrase) pairs kept; the least recently used go first + max_count = 10 + + def __init__(self, data=None, on_change=None): + # reading -> {phrase: [count, last use]} + self.data = {} + self.on_change = on_change or (lambda data: None) + self._clock = 0 + for reading, phrases in (data or {}).items(): + if not isinstance(phrases, dict): + continue + for phrase, value in phrases.items(): + try: + count, used = int(value[0]), int(value[1]) + except (TypeError, ValueError, IndexError): + continue + self.data.setdefault(reading, {})[phrase] = [count, used] + self._clock = max(self._clock, used) + + @staticmethod + def key(syllables): + return "-".join(s.toneless for s in syllables) + + def record(self, syllables, phrase): + if not syllables or not phrase: + return + self._clock += 1 + entry = self.data.setdefault(self.key(syllables), {}).setdefault(phrase, [0, 0]) + entry[0] = min(self.max_count, entry[0] + 1) + entry[1] = self._clock + self._evict() + self.on_change(self.data) + + def _evict(self): + pairs = [(v[1], r, p) for r, ps in self.data.items() for p, v in ps.items()] + if len(pairs) <= self.max_entries: + return + pairs.sort() + for _, r, p in pairs[:len(pairs) - self.max_entries]: + del self.data[r][p] + if not self.data[r]: + del self.data[r] + + def clear(self): + self.data = {} + self.on_change(self.data) + + def adjust(self, syllables, entries): + """Lexicon entries [(phrase, score)] with picked phrases moved to the top: just above + the best entry, more so the more often they were picked. Returns a new list.""" + picked = self.data.get(self.key(syllables)) + if not picked or not entries: + return entries + top = entries[0][1] + out = [(p, max(s, top + 0.01 + 0.1 * picked[p][0]) if p in picked else s) for p, s in entries] + out.sort(key=lambda e: -e[1]) + return out diff --git a/windows/autozhuyin/core/lexicon.py b/windows/autozhuyin/core/lexicon.py index 25f1442..be88ece 100644 --- a/windows/autozhuyin/core/lexicon.py +++ b/windows/autozhuyin/core/lexicon.py @@ -84,8 +84,13 @@ TONELESS_PENALTY = -1.5 class Converter: - def __init__(self, lexicon): + def __init__(self, lexicon, history=None): self.lexicon = lexicon + self.history = history # UserHistory: picked candidates come first + + def _entries(self, syllables): + entries = self.lexicon.entries(syllables) + return self.history.adjust(syllables, entries) if self.history else entries def _options(self, syllables, i, j, fixed): if (i, j) in fixed: @@ -94,7 +99,7 @@ class Converter: return [] sl = syllables[i:j] penalty = TONELESS_PENALTY if any(s.tone is None for s in sl) else 0.0 - out = [(p, s + penalty) for p, s in self.lexicon.entries(sl)] + out = [(p, s + penalty) for p, s in self._entries(sl)] if not out and j - i == 1: out = [(sl[0].reading, UNKNOWN_SCORE)] return out @@ -132,7 +137,7 @@ class Converter: out, seen = [], set() for length in range(min(self.lexicon.max_phrase_length, end), 0, -1): i = end - length - for phrase, _ in self.lexicon.entries(syllables[i:end]): + for phrase, _ in self._entries(syllables[i:end]): if phrase not in seen: seen.add(phrase) out.append(((i, end), phrase)) diff --git a/windows/installer/README.txt b/windows/installer/README.txt index c351a92..f5c0e18 100644 --- a/windows/installer/README.txt +++ b/windows/installer/README.txt @@ -28,7 +28,8 @@ Tab 判斷錯了?按 Tab 在中/英之間翻轉,輸入法會記住這次修正 ← → 在組字區移動游標,選要改哪個字;游標在中間時直接打注音 會插入新字,Backspace 刪掉游標前的字 - ↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字 + ↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字; + 選過的字會被記住,下次優先出現 Esc 取消組字 單按 Shift 切換英文模式(按鍵直接送出、保持小寫、沒有底線), 再單按一次回到中英自動判斷,會出現「英文」/「中英自動」小提示; diff --git a/windows/tests/test_core.py b/windows/tests/test_core.py index d1f1682..3687def 100644 --- a/windows/tests/test_core.py +++ b/windows/tests/test_core.py @@ -15,6 +15,7 @@ from autozhuyin.core.engine import DOWN, ENTER, LEFT, RIGHT, SHIFT_TAP, TAB, BAC from autozhuyin.core.shift_tap import ShiftTapDetector # noqa: E402 from autozhuyin.core.english import EnglishModel # noqa: E402 from autozhuyin.core.lexicon import Converter, Lexicon # noqa: E402 +from autozhuyin.core.history import UserHistory # noqa: E402 DATA = os.path.join(os.path.dirname(HERE), "autozhuyin", "data") _start = time.time() @@ -364,5 +365,53 @@ class FirstWordHoldTests(unittest.TestCase): self.assertEqual(type_keys("so ", e), "so ") # right after English: commits at once +class UserHistoryTests(unittest.TestCase): + def setUp(self): + self.saved = [] + self.c = Classifier(LEX, ENG, weights=WEIGHTS, history=UserHistory(on_change=self.saved.append)) + + def pick(self, keys, phrase): + e = CompositionEngine(self.c) + type_keys(keys + "↓", e) + e.select_candidate([t for _, t in e.candidates].index(phrase)) + return type_keys("\n", e) + + def test_picked_candidate_comes_first(self): + self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "天") + self.assertEqual(self.pick("wu0 ", "添"), "添") + self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "添") + e = CompositionEngine(self.c) + type_keys("wu0 ↓", e) + self.assertEqual(e.candidates[0][1], "添") + self.assertEqual(self.saved[-1], {"ㄊㄧㄢ": {"添": [1, 1]}}) + + def test_phrases_still_win(self): + self.pick("wu0 ", "添") + self.assertEqual(type_keys("rup wu0 \n", CompositionEngine(self.c)), "今天") + + def test_most_picked_wins(self): + self.pick("wu0 ", "添") + self.pick("wu0 ", "添") + self.pick("wu0 ", "天") + self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "添") + + def test_round_trip_and_clear(self): + self.pick("wu0 ", "添") + h = UserHistory(self.saved[-1]) + self.assertEqual(h.data, {"ㄊㄧㄢ": {"添": [1, 1]}}) + h.record([bopomofo.parse(list("wu0 ")).syllables[0].syllable], "天") + self.assertEqual(h.data["ㄊㄧㄢ"]["天"], [1, 2]) + h.clear() + self.assertEqual(h.data, {}) + + def test_evicts_least_recent(self): + h = UserHistory() + h.max_entries = 2 + syl = [bopomofo.parse(list("wu0 ")).syllables[0].syllable] + for p in ["天", "添", "田"]: + h.record(syl, p) + self.assertEqual(set(h.data["ㄊㄧㄢ"]), {"添", "田"}) + + if __name__ == "__main__": unittest.main() diff --git a/windows/tests/test_pime_wrapper.py b/windows/tests/test_pime_wrapper.py index b865070..027456d 100644 --- a/windows/tests/test_pime_wrapper.py +++ b/windows/tests/test_pime_wrapper.py @@ -6,6 +6,9 @@ import types import unittest HERE = os.path.dirname(os.path.abspath(__file__)) +import tempfile # noqa: E402 + +os.environ["APPDATA"] = tempfile.mkdtemp(prefix="autozhuyin-test-") # learned / history files sys.path.insert(0, os.path.dirname(HERE)) # --- stand-ins for PIME modules (only what the wrapper uses) --- @@ -46,7 +49,7 @@ class FakeTextService: textService.TextService = FakeTextService sys.modules["textService"] = textService -from autozhuyin.autozhuyin_ime import AutoZhuyinTextService # noqa: E402 +from autozhuyin.autozhuyin_ime import AutoZhuyinTextService, _shared_classifier # noqa: E402 VK = {"\n": 0x0D, "\t": 0x09, " ": 0x20, "\b": 0x08, "↓": 0x28} @@ -99,6 +102,8 @@ def type_text(svc, text): class WrapperTests(unittest.TestCase): def setUp(self): + _shared_classifier().history.clear() + _shared_classifier().learned.clear() self.svc = AutoZhuyinTextService(client=None) self.svc.onActivate() @@ -142,6 +147,8 @@ class WrapperTests(unittest.TestCase): class ShiftTapWrapperTests(unittest.TestCase): def setUp(self): + _shared_classifier().history.clear() + _shared_classifier().learned.clear() self.svc = AutoZhuyinTextService(client=None) self.svc.onActivate()