Remember picked candidates (user history)
Candidates chosen with the arrow-down list are recorded on commit as toneless reading -> phrase with a count. The converter lifts a picked phrase just above the best lexicon entry for that reading (more for more picks), so it comes first next time without overriding longer phrases. Up to 2000 pairs, least recently used evicted. Stored as user-history.json on every platform; the Mac menu's clear item clears it too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@
|
||||
- **注音順序打錯也能修正**:`ㄧㄒㄠˇ`、`ㄒㄠㄧˇ` 都會變成 `ㄒㄧㄠˇ`(小)
|
||||
- **可省略聲調**:`jiap7`(ㄨㄛ ㄇㄣ˙)一樣能打出「我們」
|
||||
- **候選字**:用 ← → 把游標移到要改的字後面,按 ↓ 選字;選過的會固定下來
|
||||
- **記住選字**:從候選字選過的字詞會被記住,下次打同樣的讀音會優先出現(選越多次越優先)
|
||||
- **中間補字、刪字**:游標移到前面後直接打注音,會插在游標處;Backspace 刪掉游標前的那個字
|
||||
- **全形標點**:中文語境下 `?` `!` `<` 自動變成 `?` `!` `,`;任何時候按 `` ` `` 再按標點鍵,一定得到全形標點(例如 `` `< `` = `,`)
|
||||
- **完整詞庫**:約 17 萬筆,來自小麥注音 McBopomofo(MIT 授權)
|
||||
@@ -100,10 +101,12 @@ SmartScreen 出現警告時,按「其他資訊 → 仍要執行」。解除安
|
||||
| | macOS | Windows | Linux |
|
||||
|---|---|---|---|
|
||||
| 學習紀錄(Tab 修正) | `UserDefaults`(`com.autozhuyin.inputmethod.AutoZhuyin`) | `%APPDATA%\PIME\autozhuyin\learned.json` | `~/.config/autozhuyin/learned.json` |
|
||||
| 選字紀錄 | `~/Library/Application Support/AutoZhuyin/user-history.json` | `%APPDATA%\PIME\autozhuyin\user-history.json` | `~/.config/autozhuyin/user-history.json` |
|
||||
| 自訂詞 | `~/Library/Application Support/AutoZhuyin/user-phrases.txt` | `%APPDATA%\PIME\autozhuyin\user-phrases.txt` | `~/.config/autozhuyin/user-phrases.txt` |
|
||||
|
||||
自訂詞的格式和 `lexicon.txt` 一樣,每行一個詞:`ㄗˋ-ㄉㄨㄥˋ-ㄓㄨˋ-ㄧㄣ 自動注音 -4`(讀音、詞、分數;分數越大越優先)。
|
||||
Mac 版可以在輸入法選單選「清除學習紀錄」。
|
||||
選字紀錄最多保留 2000 筆,太久沒用的會先淘汰。被選過的字只會排到同讀音第一順位,不會蓋過整個詞組(例如選過「添」之後,ㄊㄧㄢ 單打會出「添」,但「今天」還是「今天」)。
|
||||
Mac 版可以在輸入法選單選「清除學習紀錄」(中英判斷和選字紀錄一起清除);Windows、Linux 刪除上面的檔案即可。
|
||||
|
||||
## 開發
|
||||
|
||||
@@ -261,7 +264,7 @@ THIRD_PARTY_LICENSES.txt 第三方授權聲明(會打包進兩個安裝
|
||||
- [ ] 加入真實打字資料一起訓練
|
||||
- [ ] 已送出的英文單字若判斷錯誤,可以「事後翻轉」
|
||||
- [ ] 用 Core ML 小模型取代手調分數,參考整句語境
|
||||
- [ ] 使用者詞頻學習(選過的候選字優先)
|
||||
- [x] 使用者詞頻學習(選過的候選字優先)
|
||||
- [ ] 偏好設定視窗
|
||||
|
||||
## 致謝與授權
|
||||
|
||||
@@ -9,6 +9,11 @@ enum Shared {
|
||||
|
||||
static let learnedKey = "LearnedPreferences"
|
||||
|
||||
/// ~/Library/Application Support/AutoZhuyin (user phrases, picked-candidate history).
|
||||
static let supportDir: URL? = FileManager.default
|
||||
.urls(for: .applicationSupportDirectory, in: .userDomainMask).first?
|
||||
.appendingPathComponent("AutoZhuyin")
|
||||
|
||||
static let classifier: Classifier = {
|
||||
let bundle = Bundle.main
|
||||
let start = Date()
|
||||
@@ -18,9 +23,9 @@ enum Shared {
|
||||
lexicon.load(data: data)
|
||||
}
|
||||
// Optional user phrases: ~/Library/Application Support/AutoZhuyin/user-phrases.txt
|
||||
if let support = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first {
|
||||
let userFile = support.appendingPathComponent("AutoZhuyin/user-phrases.txt")
|
||||
if let text = try? String(contentsOf: userFile, encoding: .utf8) { lexicon.load(text) }
|
||||
if let userFile = supportDir?.appendingPathComponent("user-phrases.txt"),
|
||||
let text = try? String(contentsOf: userFile, encoding: .utf8) {
|
||||
lexicon.load(text)
|
||||
}
|
||||
let english = EnglishModel(commonWordsURL: bundle.url(forResource: "english-common", withExtension: "txt"),
|
||||
dictionaryURL: bundle.url(forResource: "english-dict", withExtension: "txt"),
|
||||
@@ -29,11 +34,17 @@ enum Shared {
|
||||
let learned = LearnedPreferences(weights: saved) { weights in
|
||||
UserDefaults.standard.set(weights, forKey: learnedKey)
|
||||
}
|
||||
let historyFile = supportDir?.appendingPathComponent("user-history.json")
|
||||
let history = UserHistory(data: historyFile.map(UserHistory.load(from:)) ?? [:]) { data in
|
||||
guard let url = historyFile, let json = try? JSONEncoder().encode(data) else { return }
|
||||
try? FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)
|
||||
try? json.write(to: url, options: .atomic)
|
||||
}
|
||||
let trained = Features.loadWeights(from: bundle.url(forResource: "classifier-weights", withExtension: "json"))
|
||||
NSLog("AutoZhuyin: data loaded in %.0f ms", Date().timeIntervalSince(start) * 1000)
|
||||
// Only needed for untoned syllables; build it off the startup path.
|
||||
DispatchQueue.global(qos: .utility).async { lexicon.prepareTonelessIndex() }
|
||||
return Classifier(lexicon: lexicon, english: english, learned: learned, weights: trained)
|
||||
return Classifier(lexicon: lexicon, english: english, learned: learned, weights: trained, history: history)
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -177,5 +188,6 @@ final class InputController: IMKInputController {
|
||||
|
||||
@objc private func clearLearned(_ sender: Any?) {
|
||||
Shared.classifier.learned.clear()
|
||||
Shared.classifier.history.clear()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,14 +55,17 @@ public final class Classifier {
|
||||
public let lexicon: Lexicon
|
||||
public let english: EnglishModel
|
||||
public let learned: LearnedPreferences
|
||||
/// Picked candidates (shared by every engine using this classifier).
|
||||
public let history: UserHistory
|
||||
/// Feature weights: the hand-tuned defaults overlaid with trained ones.
|
||||
public private(set) var weights: FeatureVector
|
||||
|
||||
public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences(),
|
||||
weights: FeatureVector = [:]) {
|
||||
weights: FeatureVector = [:], history: UserHistory = UserHistory()) {
|
||||
self.lexicon = lexicon
|
||||
self.english = english
|
||||
self.learned = learned
|
||||
self.history = history
|
||||
self.weights = Features.defaultWeights.merging(weights) { _, new in new }
|
||||
}
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ public final class CompositionEngine {
|
||||
|
||||
public init(classifier: Classifier) {
|
||||
self.classifier = classifier
|
||||
self.converter = Converter(lexicon: classifier.lexicon)
|
||||
self.converter = Converter(lexicon: classifier.lexicon, history: classifier.history)
|
||||
}
|
||||
|
||||
// MARK: - Derived state
|
||||
@@ -537,6 +537,7 @@ public final class CompositionEngine {
|
||||
private func commitCurrent(_ given: [Segment]? = nil) -> String {
|
||||
let segs = given ?? segments
|
||||
let text = segs.map { render($0, final: true) }.joined()
|
||||
rememberPicks(segs)
|
||||
if let forced = forcedMode {
|
||||
classifier.learned.record(String(keys), as: forced)
|
||||
}
|
||||
@@ -546,6 +547,17 @@ public final class CompositionEngine {
|
||||
return text
|
||||
}
|
||||
|
||||
/// Candidates the user picked are remembered and come first next time.
|
||||
private func rememberPicks(_ segs: [Segment]) {
|
||||
for seg in segs where seg.mode == .zhuyin {
|
||||
guard let picks = fixed[seg.range.lowerBound], !picks.isEmpty else { continue }
|
||||
let syl = convertibleSyllables(seg, final: true).syllables
|
||||
for (range, text) in picks where range.upperBound <= syl.count {
|
||||
classifier.history.record(Array(syl[range]), phrase: text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Commits whatever is composed (e.g. the app lost focus). Returns "" when empty.
|
||||
public func commitAll() -> String {
|
||||
keys.isEmpty ? "" : commitCurrent()
|
||||
|
||||
@@ -13,12 +13,22 @@ public struct Candidate: Equatable {
|
||||
/// Turns a syllable sequence into the most likely Chinese text (Viterbi over the lexicon).
|
||||
public struct Converter {
|
||||
public let lexicon: Lexicon
|
||||
/// Picked candidates come first.
|
||||
public let history: UserHistory?
|
||||
/// Score used when a syllable has no lexicon entry; the Bopomofo itself is emitted.
|
||||
static let unknownScore = -30.0
|
||||
/// Penalty for matching an untoned syllable against all tones.
|
||||
static let tonelessPenalty = -1.5
|
||||
|
||||
public init(lexicon: Lexicon) { self.lexicon = lexicon }
|
||||
public init(lexicon: Lexicon, history: UserHistory? = nil) {
|
||||
self.lexicon = lexicon
|
||||
self.history = history
|
||||
}
|
||||
|
||||
private func entries(_ syllables: [Syllable]) -> [LexiconEntry] {
|
||||
let e = lexicon.entries(for: syllables)
|
||||
return history?.adjust(syllables, e) ?? e
|
||||
}
|
||||
|
||||
/// - Parameter fixed: user-chosen phrases by syllable range; they are always kept.
|
||||
public func convert(_ syllables: [Syllable], fixed: [Range<Int>: String] = [:]) -> [ConvertedSegment] {
|
||||
@@ -57,7 +67,7 @@ public struct Converter {
|
||||
if fixed.keys.contains(where: { $0.overlaps(range) }) { return [] }
|
||||
|
||||
let slice = Array(syllables[range])
|
||||
var result = lexicon.entries(for: slice).map { entry -> (String, Double) in
|
||||
var result = entries(slice).map { entry -> (String, Double) in
|
||||
let penalty = slice.contains { $0.tone == nil } ? Converter.tonelessPenalty : 0
|
||||
return (entry.phrase, entry.score + penalty)
|
||||
}
|
||||
@@ -74,7 +84,7 @@ public struct Converter {
|
||||
var seen = Set<String>()
|
||||
for len in stride(from: min(lexicon.maxPhraseLength, end), through: 1, by: -1) {
|
||||
let range = (end - len)..<end
|
||||
for entry in lexicon.entries(for: Array(syllables[range])) where !seen.contains(entry.phrase) {
|
||||
for entry in entries(Array(syllables[range])) where !seen.contains(entry.phrase) {
|
||||
seen.insert(entry.phrase)
|
||||
out.append(Candidate(range: range, text: entry.phrase))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import Foundation
|
||||
|
||||
/// Remembers the candidates the user picked, so they come first next time.
|
||||
///
|
||||
/// Keys are toneless readings ("ㄊㄧㄢ", "ㄐㄧㄣ-ㄊㄧㄢ"); a picked phrase is boosted only
|
||||
/// where the lexicon has it for the reading being converted, so a toneless key is safe.
|
||||
/// Stored as JSON: {reading: {phrase: [count, last use]}} (same format on every platform).
|
||||
public final class UserHistory {
|
||||
public typealias Data = [String: [String: [Int]]]
|
||||
|
||||
/// (reading, phrase) pairs kept; the least recently used go first.
|
||||
public var maxEntries = 2000
|
||||
static let maxCount = 10
|
||||
|
||||
public private(set) var data: Data = [:]
|
||||
private var clock = 0
|
||||
private let onChange: (Data) -> Void
|
||||
|
||||
public init(data: Data = [:], onChange: @escaping (Data) -> Void = { _ in }) {
|
||||
self.onChange = onChange
|
||||
for (reading, phrases) in data {
|
||||
for (phrase, value) in phrases where value.count >= 2 {
|
||||
self.data[reading, default: [:]][phrase] = [value[0], value[1]]
|
||||
clock = max(clock, value[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads the JSON file written through `onChange`; empty if missing or unreadable.
|
||||
public static func load(from url: URL) -> Data {
|
||||
guard let raw = try? Foundation.Data(contentsOf: url),
|
||||
let data = try? JSONDecoder().decode(Data.self, from: raw) else { return [:] }
|
||||
return data
|
||||
}
|
||||
|
||||
static func key(_ syllables: [Syllable]) -> String {
|
||||
syllables.map(\.toneless).joined(separator: "-")
|
||||
}
|
||||
|
||||
public func record(_ syllables: [Syllable], phrase: String) {
|
||||
guard !syllables.isEmpty, !phrase.isEmpty else { return }
|
||||
clock += 1
|
||||
let key = UserHistory.key(syllables)
|
||||
let count = data[key]?[phrase]?.first ?? 0
|
||||
data[key, default: [:]][phrase] = [min(UserHistory.maxCount, count + 1), clock]
|
||||
evict()
|
||||
onChange(data)
|
||||
}
|
||||
|
||||
private func evict() {
|
||||
var pairs: [(used: Int, reading: String, phrase: String)] = []
|
||||
for (r, phrases) in data { for (p, v) in phrases { pairs.append((v[1], r, p)) } }
|
||||
guard pairs.count > maxEntries else { return }
|
||||
pairs.sort { $0.used < $1.used }
|
||||
for pair in pairs.prefix(pairs.count - maxEntries) {
|
||||
data[pair.reading]?[pair.phrase] = nil
|
||||
if data[pair.reading]?.isEmpty == true { data[pair.reading] = nil }
|
||||
}
|
||||
}
|
||||
|
||||
public func clear() {
|
||||
data = [:]
|
||||
onChange(data)
|
||||
}
|
||||
|
||||
/// Lexicon entries with picked phrases moved to the top: just above the best entry,
|
||||
/// more so the more often they were picked.
|
||||
func adjust(_ syllables: [Syllable], _ entries: [LexiconEntry]) -> [LexiconEntry] {
|
||||
guard let picked = data[UserHistory.key(syllables)], let top = entries.first?.score else { return entries }
|
||||
var out = entries.map { e -> LexiconEntry in
|
||||
guard let count = picked[e.phrase]?.first else { return e }
|
||||
return LexiconEntry(phrase: e.phrase, score: max(e.score, top + 0.01 + 0.1 * Double(count)))
|
||||
}
|
||||
// Stable sort (keep lexicon order among equal scores), like Python's sort.
|
||||
out = out.enumerated().sorted { $0.element.score != $1.element.score
|
||||
? $0.element.score > $1.element.score : $0.offset < $1.offset }.map(\.element)
|
||||
return out
|
||||
}
|
||||
}
|
||||
@@ -569,6 +569,76 @@ final class FirstWordHoldTests: XCTestCase {
|
||||
}
|
||||
}
|
||||
|
||||
final class UserHistoryTests: XCTestCase {
|
||||
var saved: [UserHistory.Data] = []
|
||||
var classifier: Classifier!
|
||||
|
||||
override func setUp() {
|
||||
saved = []
|
||||
classifier = Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights,
|
||||
history: UserHistory { [unowned self] in self.saved.append($0) })
|
||||
}
|
||||
|
||||
func type(_ s: String, into e: CompositionEngine) -> String {
|
||||
var out = ""
|
||||
for c in s {
|
||||
let key: KeyInput = c == "↓" ? .down : c == "\n" ? .enter : .character(c)
|
||||
let r = e.handle(key)
|
||||
if let t = r.commit { out += t } else if !r.handled { out.append(c) }
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
func pick(_ keys: String, _ phrase: String) -> String {
|
||||
let e = CompositionEngine(classifier: classifier)
|
||||
_ = type(keys + "↓", into: e)
|
||||
e.selectCandidate(e.candidates.firstIndex { $0.text == phrase }!)
|
||||
return type("\n", into: e)
|
||||
}
|
||||
|
||||
func testPickedCandidateComesFirst() {
|
||||
XCTAssertEqual(type("wu0 \n", into: CompositionEngine(classifier: classifier)), "天")
|
||||
XCTAssertEqual(pick("wu0 ", "添"), "添")
|
||||
XCTAssertEqual(type("wu0 \n", into: CompositionEngine(classifier: classifier)), "添")
|
||||
let e = CompositionEngine(classifier: classifier)
|
||||
_ = type("wu0 ↓", into: e)
|
||||
XCTAssertEqual(e.candidates.first?.text, "添")
|
||||
XCTAssertEqual(saved.last, ["ㄊㄧㄢ": ["添": [1, 1]]])
|
||||
}
|
||||
|
||||
func testPhrasesStillWin() {
|
||||
pick("wu0 ", "添")
|
||||
XCTAssertEqual(type("rup wu0 \n", into: CompositionEngine(classifier: classifier)), "今天")
|
||||
}
|
||||
|
||||
func testMostPickedWins() {
|
||||
pick("wu0 ", "添")
|
||||
pick("wu0 ", "添")
|
||||
pick("wu0 ", "天")
|
||||
XCTAssertEqual(type("wu0 \n", into: CompositionEngine(classifier: classifier)), "添")
|
||||
}
|
||||
|
||||
func testRoundTripAndClear() throws {
|
||||
pick("wu0 ", "添")
|
||||
let json = try JSONEncoder().encode(saved.last!)
|
||||
let h = UserHistory(data: try JSONDecoder().decode(UserHistory.Data.self, from: json))
|
||||
XCTAssertEqual(h.data, ["ㄊㄧㄢ": ["添": [1, 1]]])
|
||||
h.record(ZhuyinParser.parse(Array("wu0 ")).syllables.map(\.syllable), phrase: "天")
|
||||
XCTAssertEqual(h.data["ㄊㄧㄢ"]?["天"], [1, 2])
|
||||
h.clear()
|
||||
XCTAssertTrue(h.data.isEmpty)
|
||||
}
|
||||
|
||||
func testEvictsLeastRecent() {
|
||||
let h = UserHistory()
|
||||
h.maxEntries = 2
|
||||
let syl = ZhuyinParser.parse(Array("wu0 ")).syllables.map(\.syllable)
|
||||
for p in ["天", "添", "田"] { h.record(syl, phrase: p) }
|
||||
XCTAssertEqual(Set(h.data["ㄊㄧㄢ"]!.keys), ["添", "田"])
|
||||
}
|
||||
}
|
||||
|
||||
final class StartupTimingTests: XCTestCase {
|
||||
func testStartupTiming() throws {
|
||||
var t = Date()
|
||||
|
||||
+23
-10
@@ -24,6 +24,7 @@ gi.require_version("IBus", "1.0")
|
||||
from gi.repository import GLib, GObject, IBus # noqa: E402
|
||||
|
||||
import autozhuyin # noqa: E402
|
||||
from autozhuyin.core.history import UserHistory # noqa: E402
|
||||
from autozhuyin.core.classifier import LITERAL, Classifier, LearnedPreferences # noqa: E402
|
||||
from autozhuyin.core.engine import ( # noqa: E402
|
||||
BACKSPACE, DOWN, ENTER, ESCAPE, LEFT, RIGHT, SHIFT_TAP, TAB, UP, CompositionEngine,
|
||||
@@ -40,6 +41,7 @@ _DATA = os.path.join(os.path.dirname(os.path.abspath(autozhuyin.__file__)), "dat
|
||||
_USER_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config"), "autozhuyin")
|
||||
_LEARNED_FILE = os.path.join(_USER_DIR, "learned.json")
|
||||
_USER_PHRASES = os.path.join(_USER_DIR, "user-phrases.txt")
|
||||
_HISTORY_FILE = os.path.join(_USER_DIR, "user-history.json")
|
||||
|
||||
try:
|
||||
with open(os.path.join(HERE, "version.txt"), encoding="utf-8") as _f:
|
||||
@@ -50,15 +52,31 @@ except OSError:
|
||||
_classifier = None
|
||||
|
||||
|
||||
def _save_learned(weights):
|
||||
def _save_json(path, data):
|
||||
try:
|
||||
os.makedirs(_USER_DIR, exist_ok=True)
|
||||
with open(_LEARNED_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(weights, f, ensure_ascii=False)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _load_json(path):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def _save_learned(weights):
|
||||
_save_json(_LEARNED_FILE, weights)
|
||||
|
||||
|
||||
def _save_history(data):
|
||||
_save_json(_HISTORY_FILE, data)
|
||||
|
||||
|
||||
def shared_classifier():
|
||||
"""Loads the lexicon and English model once per process."""
|
||||
global _classifier
|
||||
@@ -72,14 +90,9 @@ def shared_classifier():
|
||||
os.path.join(_DATA, "english-dict.txt"),
|
||||
os.path.join(_DATA, "english-bigram.json"),
|
||||
)
|
||||
weights = {}
|
||||
try:
|
||||
with open(_LEARNED_FILE, encoding="utf-8") as f:
|
||||
weights = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
trained = Classifier.load_weights(os.path.join(_DATA, "classifier-weights.json"))
|
||||
_classifier = Classifier(lexicon, english, LearnedPreferences(weights, _save_learned), trained)
|
||||
_classifier = Classifier(lexicon, english, LearnedPreferences(_load_json(_LEARNED_FILE), _save_learned),
|
||||
trained, UserHistory(_load_json(_HISTORY_FILE), _save_history))
|
||||
return _classifier
|
||||
|
||||
|
||||
|
||||
@@ -30,7 +30,8 @@
|
||||
Tab 判斷錯了?按 Tab 在中/英之間翻轉,輸入法會記住這次修正
|
||||
← → 在組字區移動游標;游標在中間時直接打注音會插入新字,
|
||||
Backspace 刪掉游標前的字
|
||||
↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字
|
||||
↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字;
|
||||
選過的字會被記住,下次優先出現
|
||||
Esc 取消組字
|
||||
單按 Shift 切換英文模式(按鍵直接送出),再單按一次回到中英自動判斷
|
||||
Shift+字母 打大寫字母(也會強制英文)
|
||||
@@ -47,6 +48,7 @@
|
||||
|
||||
個人資料(學習紀錄、自訂詞)在 ~/.config/autozhuyin/:
|
||||
learned.json 按 Tab 修正的學習紀錄
|
||||
user-history.json 選字紀錄(選過的候選字)
|
||||
user-phrases.txt 自訂詞,一行一個,例如:ㄗˋ-ㄉㄨㄥˋ 自動 -1
|
||||
|
||||
|
||||
|
||||
@@ -8,6 +8,9 @@ import types
|
||||
import unittest
|
||||
|
||||
LINUX = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
import tempfile # noqa: E402
|
||||
|
||||
os.environ["XDG_CONFIG_HOME"] = tempfile.mkdtemp(prefix="autozhuyin-test-") # learned / history files
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(LINUX), "windows")) # autozhuyin package
|
||||
sys.path.insert(0, LINUX)
|
||||
|
||||
@@ -107,6 +110,8 @@ def press(e, keys):
|
||||
|
||||
class IBusEngineTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
autozhuyin_ibus.shared_classifier().history.clear()
|
||||
autozhuyin_ibus.shared_classifier().learned.clear()
|
||||
self.e = autozhuyin_ibus.AutoZhuyinEngine()
|
||||
|
||||
def test_mixed_sentence(self):
|
||||
@@ -157,6 +162,17 @@ class IBusEngineTests(unittest.TestCase):
|
||||
press(self.e, "rup fu4←wu0 \n")
|
||||
self.assertEqual(self.e.committed, "今天氣")
|
||||
|
||||
def test_picked_candidate_saved(self):
|
||||
press(self.e, "su3↓")
|
||||
second = self.e.shown[1]
|
||||
self.e.do_candidate_clicked(1, 1, 0)
|
||||
press(self.e, "\n")
|
||||
path = os.path.join(os.environ["XDG_CONFIG_HOME"], "autozhuyin", "user-history.json")
|
||||
with open(path, encoding="utf-8") as f:
|
||||
self.assertIn(second, f.read())
|
||||
press(self.e, "su3 \n")
|
||||
self.assertTrue(self.e.committed.endswith(second))
|
||||
|
||||
def test_focus_out_resets(self):
|
||||
press(self.e, "su3")
|
||||
self.e.do_focus_out()
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"ㄋㄧ": {"妳": [1, 1]}}
|
||||
@@ -11,6 +11,7 @@ import time
|
||||
from keycodes import * # noqa: F401,F403 (VK_* constants, provided by PIME)
|
||||
from textService import TextService
|
||||
|
||||
from .core.history import UserHistory
|
||||
from .core.classifier import Classifier, LearnedPreferences
|
||||
from .core.engine import BACKSPACE, DOWN, ENTER, ESCAPE, LEFT, RIGHT, SHIFT_TAP, TAB, UP, CompositionEngine
|
||||
from .core.english import EnglishModel
|
||||
@@ -22,19 +23,36 @@ _DATA = os.path.join(_HERE, "data")
|
||||
_USER_DIR = os.path.join(os.environ.get("APPDATA", _HERE), "PIME", "autozhuyin")
|
||||
_LEARNED_FILE = os.path.join(_USER_DIR, "learned.json")
|
||||
_USER_PHRASES = os.path.join(_USER_DIR, "user-phrases.txt")
|
||||
_HISTORY_FILE = os.path.join(_USER_DIR, "user-history.json")
|
||||
|
||||
_classifier = None
|
||||
|
||||
|
||||
def _save_learned(weights):
|
||||
def _save_json(path, data):
|
||||
try:
|
||||
os.makedirs(_USER_DIR, exist_ok=True)
|
||||
with open(_LEARNED_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(weights, f, ensure_ascii=False)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _load_json(path):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def _save_learned(weights):
|
||||
_save_json(_LEARNED_FILE, weights)
|
||||
|
||||
|
||||
def _save_history(data):
|
||||
_save_json(_HISTORY_FILE, data)
|
||||
|
||||
|
||||
def _shared_classifier():
|
||||
"""Loads the lexicon and English model once per PIME server process."""
|
||||
global _classifier
|
||||
@@ -48,14 +66,9 @@ def _shared_classifier():
|
||||
os.path.join(_DATA, "english-dict.txt"),
|
||||
os.path.join(_DATA, "english-bigram.json"),
|
||||
)
|
||||
weights = {}
|
||||
try:
|
||||
with open(_LEARNED_FILE, encoding="utf-8") as f:
|
||||
weights = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
trained = Classifier.load_weights(os.path.join(_DATA, "classifier-weights.json"))
|
||||
_classifier = Classifier(lexicon, english, LearnedPreferences(weights, _save_learned), trained)
|
||||
_classifier = Classifier(lexicon, english, LearnedPreferences(_load_json(_LEARNED_FILE), _save_learned),
|
||||
trained, UserHistory(_load_json(_HISTORY_FILE), _save_history))
|
||||
return _classifier
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Zhuyin-vs-literal classifier with learned corrections. Port of Classifier.swift."""
|
||||
import json
|
||||
|
||||
from .history import UserHistory
|
||||
from .features import DEFAULT_WEIGHTS, dot, literal_features, zhuyin_features
|
||||
|
||||
ZHUYIN = "zhuyin"
|
||||
@@ -31,10 +32,11 @@ class LearnedPreferences:
|
||||
|
||||
class Classifier:
|
||||
|
||||
def __init__(self, lexicon, english, learned=None, weights=None):
|
||||
def __init__(self, lexicon, english, learned=None, weights=None, history=None):
|
||||
self.lexicon = lexicon
|
||||
self.english = english
|
||||
self.learned = learned or LearnedPreferences()
|
||||
self.history = history or UserHistory() # picked candidates (shared by all engines)
|
||||
self.weights = dict(DEFAULT_WEIGHTS)
|
||||
if weights:
|
||||
self.weights.update(weights)
|
||||
|
||||
@@ -46,7 +46,7 @@ class CompositionEngine:
|
||||
|
||||
def __init__(self, classifier, now=time.monotonic):
|
||||
self.classifier = classifier
|
||||
self.converter = Converter(classifier.lexicon)
|
||||
self.converter = Converter(classifier.lexicon, classifier.history)
|
||||
self.now = now
|
||||
self.keys = []
|
||||
self.forced_mode = None
|
||||
@@ -552,6 +552,7 @@ class CompositionEngine:
|
||||
def _commit(self, segs=None):
|
||||
segs = segs if segs is not None else self.segments
|
||||
text = "".join(self._render(s, True) for s in segs)
|
||||
self._remember_picks(segs)
|
||||
if self.forced_mode:
|
||||
self.classifier.learned.record("".join(self.keys), self.forced_mode)
|
||||
self._previous_mode = segs[-1][2] if segs else None
|
||||
@@ -559,6 +560,17 @@ class CompositionEngine:
|
||||
self.reset()
|
||||
return text
|
||||
|
||||
def _remember_picks(self, segs):
|
||||
"""Candidates the user picked are remembered and come first next time."""
|
||||
for i, seg in enumerate(segs):
|
||||
picks = self.fixed.get(seg[0])
|
||||
if seg[2] != ZHUYIN or not picks:
|
||||
continue
|
||||
syl = self._convertible(seg, True)[0]
|
||||
for (a, b), text in picks.items():
|
||||
if b <= len(syl):
|
||||
self.classifier.history.record(syl[a:b], text)
|
||||
|
||||
def commit_all(self):
|
||||
return self._commit() if self.keys else ""
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Remembers the candidates the user picked. Port of UserHistory.swift.
|
||||
|
||||
Keys are toneless readings ("ㄊㄧㄢ", "ㄐㄧㄣ-ㄊㄧㄢ"); a picked phrase is boosted only
|
||||
where the lexicon has it for the reading being converted, so a toneless key is safe.
|
||||
"""
|
||||
|
||||
|
||||
class UserHistory:
|
||||
max_entries = 2000 # (reading, phrase) pairs kept; the least recently used go first
|
||||
max_count = 10
|
||||
|
||||
def __init__(self, data=None, on_change=None):
|
||||
# reading -> {phrase: [count, last use]}
|
||||
self.data = {}
|
||||
self.on_change = on_change or (lambda data: None)
|
||||
self._clock = 0
|
||||
for reading, phrases in (data or {}).items():
|
||||
if not isinstance(phrases, dict):
|
||||
continue
|
||||
for phrase, value in phrases.items():
|
||||
try:
|
||||
count, used = int(value[0]), int(value[1])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
continue
|
||||
self.data.setdefault(reading, {})[phrase] = [count, used]
|
||||
self._clock = max(self._clock, used)
|
||||
|
||||
@staticmethod
|
||||
def key(syllables):
|
||||
return "-".join(s.toneless for s in syllables)
|
||||
|
||||
def record(self, syllables, phrase):
|
||||
if not syllables or not phrase:
|
||||
return
|
||||
self._clock += 1
|
||||
entry = self.data.setdefault(self.key(syllables), {}).setdefault(phrase, [0, 0])
|
||||
entry[0] = min(self.max_count, entry[0] + 1)
|
||||
entry[1] = self._clock
|
||||
self._evict()
|
||||
self.on_change(self.data)
|
||||
|
||||
def _evict(self):
|
||||
pairs = [(v[1], r, p) for r, ps in self.data.items() for p, v in ps.items()]
|
||||
if len(pairs) <= self.max_entries:
|
||||
return
|
||||
pairs.sort()
|
||||
for _, r, p in pairs[:len(pairs) - self.max_entries]:
|
||||
del self.data[r][p]
|
||||
if not self.data[r]:
|
||||
del self.data[r]
|
||||
|
||||
def clear(self):
|
||||
self.data = {}
|
||||
self.on_change(self.data)
|
||||
|
||||
def adjust(self, syllables, entries):
|
||||
"""Lexicon entries [(phrase, score)] with picked phrases moved to the top: just above
|
||||
the best entry, more so the more often they were picked. Returns a new list."""
|
||||
picked = self.data.get(self.key(syllables))
|
||||
if not picked or not entries:
|
||||
return entries
|
||||
top = entries[0][1]
|
||||
out = [(p, max(s, top + 0.01 + 0.1 * picked[p][0]) if p in picked else s) for p, s in entries]
|
||||
out.sort(key=lambda e: -e[1])
|
||||
return out
|
||||
@@ -84,8 +84,13 @@ TONELESS_PENALTY = -1.5
|
||||
|
||||
|
||||
class Converter:
|
||||
def __init__(self, lexicon):
|
||||
def __init__(self, lexicon, history=None):
|
||||
self.lexicon = lexicon
|
||||
self.history = history # UserHistory: picked candidates come first
|
||||
|
||||
def _entries(self, syllables):
|
||||
entries = self.lexicon.entries(syllables)
|
||||
return self.history.adjust(syllables, entries) if self.history else entries
|
||||
|
||||
def _options(self, syllables, i, j, fixed):
|
||||
if (i, j) in fixed:
|
||||
@@ -94,7 +99,7 @@ class Converter:
|
||||
return []
|
||||
sl = syllables[i:j]
|
||||
penalty = TONELESS_PENALTY if any(s.tone is None for s in sl) else 0.0
|
||||
out = [(p, s + penalty) for p, s in self.lexicon.entries(sl)]
|
||||
out = [(p, s + penalty) for p, s in self._entries(sl)]
|
||||
if not out and j - i == 1:
|
||||
out = [(sl[0].reading, UNKNOWN_SCORE)]
|
||||
return out
|
||||
@@ -132,7 +137,7 @@ class Converter:
|
||||
out, seen = [], set()
|
||||
for length in range(min(self.lexicon.max_phrase_length, end), 0, -1):
|
||||
i = end - length
|
||||
for phrase, _ in self.lexicon.entries(syllables[i:end]):
|
||||
for phrase, _ in self._entries(syllables[i:end]):
|
||||
if phrase not in seen:
|
||||
seen.add(phrase)
|
||||
out.append(((i, end), phrase))
|
||||
|
||||
@@ -28,7 +28,8 @@
|
||||
Tab 判斷錯了?按 Tab 在中/英之間翻轉,輸入法會記住這次修正
|
||||
← → 在組字區移動游標,選要改哪個字;游標在中間時直接打注音
|
||||
會插入新字,Backspace 刪掉游標前的字
|
||||
↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字
|
||||
↓ 開啟游標前那個字或詞的候選字,按數字鍵 1–9 選字;
|
||||
選過的字會被記住,下次優先出現
|
||||
Esc 取消組字
|
||||
單按 Shift 切換英文模式(按鍵直接送出、保持小寫、沒有底線),
|
||||
再單按一次回到中英自動判斷,會出現「英文」/「中英自動」小提示;
|
||||
|
||||
@@ -15,6 +15,7 @@ from autozhuyin.core.engine import DOWN, ENTER, LEFT, RIGHT, SHIFT_TAP, TAB, BAC
|
||||
from autozhuyin.core.shift_tap import ShiftTapDetector # noqa: E402
|
||||
from autozhuyin.core.english import EnglishModel # noqa: E402
|
||||
from autozhuyin.core.lexicon import Converter, Lexicon # noqa: E402
|
||||
from autozhuyin.core.history import UserHistory # noqa: E402
|
||||
|
||||
DATA = os.path.join(os.path.dirname(HERE), "autozhuyin", "data")
|
||||
_start = time.time()
|
||||
@@ -364,5 +365,53 @@ class FirstWordHoldTests(unittest.TestCase):
|
||||
self.assertEqual(type_keys("so ", e), "so ") # right after English: commits at once
|
||||
|
||||
|
||||
class UserHistoryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.saved = []
|
||||
self.c = Classifier(LEX, ENG, weights=WEIGHTS, history=UserHistory(on_change=self.saved.append))
|
||||
|
||||
def pick(self, keys, phrase):
|
||||
e = CompositionEngine(self.c)
|
||||
type_keys(keys + "↓", e)
|
||||
e.select_candidate([t for _, t in e.candidates].index(phrase))
|
||||
return type_keys("\n", e)
|
||||
|
||||
def test_picked_candidate_comes_first(self):
|
||||
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "天")
|
||||
self.assertEqual(self.pick("wu0 ", "添"), "添")
|
||||
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "添")
|
||||
e = CompositionEngine(self.c)
|
||||
type_keys("wu0 ↓", e)
|
||||
self.assertEqual(e.candidates[0][1], "添")
|
||||
self.assertEqual(self.saved[-1], {"ㄊㄧㄢ": {"添": [1, 1]}})
|
||||
|
||||
def test_phrases_still_win(self):
|
||||
self.pick("wu0 ", "添")
|
||||
self.assertEqual(type_keys("rup wu0 \n", CompositionEngine(self.c)), "今天")
|
||||
|
||||
def test_most_picked_wins(self):
|
||||
self.pick("wu0 ", "添")
|
||||
self.pick("wu0 ", "添")
|
||||
self.pick("wu0 ", "天")
|
||||
self.assertEqual(type_keys("wu0 \n", CompositionEngine(self.c)), "添")
|
||||
|
||||
def test_round_trip_and_clear(self):
|
||||
self.pick("wu0 ", "添")
|
||||
h = UserHistory(self.saved[-1])
|
||||
self.assertEqual(h.data, {"ㄊㄧㄢ": {"添": [1, 1]}})
|
||||
h.record([bopomofo.parse(list("wu0 ")).syllables[0].syllable], "天")
|
||||
self.assertEqual(h.data["ㄊㄧㄢ"]["天"], [1, 2])
|
||||
h.clear()
|
||||
self.assertEqual(h.data, {})
|
||||
|
||||
def test_evicts_least_recent(self):
|
||||
h = UserHistory()
|
||||
h.max_entries = 2
|
||||
syl = [bopomofo.parse(list("wu0 ")).syllables[0].syllable]
|
||||
for p in ["天", "添", "田"]:
|
||||
h.record(syl, p)
|
||||
self.assertEqual(set(h.data["ㄊㄧㄢ"]), {"添", "田"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -6,6 +6,9 @@ import types
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
import tempfile # noqa: E402
|
||||
|
||||
os.environ["APPDATA"] = tempfile.mkdtemp(prefix="autozhuyin-test-") # learned / history files
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
|
||||
# --- stand-ins for PIME modules (only what the wrapper uses) ---
|
||||
@@ -46,7 +49,7 @@ class FakeTextService:
|
||||
textService.TextService = FakeTextService
|
||||
sys.modules["textService"] = textService
|
||||
|
||||
from autozhuyin.autozhuyin_ime import AutoZhuyinTextService # noqa: E402
|
||||
from autozhuyin.autozhuyin_ime import AutoZhuyinTextService, _shared_classifier # noqa: E402
|
||||
|
||||
VK = {"\n": 0x0D, "\t": 0x09, " ": 0x20, "\b": 0x08, "↓": 0x28}
|
||||
|
||||
@@ -99,6 +102,8 @@ def type_text(svc, text):
|
||||
|
||||
class WrapperTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
_shared_classifier().history.clear()
|
||||
_shared_classifier().learned.clear()
|
||||
self.svc = AutoZhuyinTextService(client=None)
|
||||
self.svc.onActivate()
|
||||
|
||||
@@ -142,6 +147,8 @@ class WrapperTests(unittest.TestCase):
|
||||
|
||||
class ShiftTapWrapperTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
_shared_classifier().history.clear()
|
||||
_shared_classifier().learned.clear()
|
||||
self.svc = AutoZhuyinTextService(client=None)
|
||||
self.svc.onActivate()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user