Remember picked candidates (user history)
Candidates chosen with the arrow-down list are recorded on commit as toneless reading -> phrase with a count. The converter lifts a picked phrase just above the best lexicon entry for that reading (more for more picks), so it comes first next time without overriding longer phrases. Up to 2000 pairs, least recently used evicted. Stored as user-history.json on every platform; the Mac menu's clear item clears it too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -55,14 +55,17 @@ public final class Classifier {
|
||||
public let lexicon: Lexicon
|
||||
public let english: EnglishModel
|
||||
public let learned: LearnedPreferences
|
||||
/// Picked candidates (shared by every engine using this classifier).
|
||||
public let history: UserHistory
|
||||
/// Feature weights: the hand-tuned defaults overlaid with trained ones.
|
||||
public private(set) var weights: FeatureVector
|
||||
|
||||
public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences(),
|
||||
weights: FeatureVector = [:]) {
|
||||
weights: FeatureVector = [:], history: UserHistory = UserHistory()) {
|
||||
self.lexicon = lexicon
|
||||
self.english = english
|
||||
self.learned = learned
|
||||
self.history = history
|
||||
self.weights = Features.defaultWeights.merging(weights) { _, new in new }
|
||||
}
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ public final class CompositionEngine {
|
||||
|
||||
public init(classifier: Classifier) {
|
||||
self.classifier = classifier
|
||||
self.converter = Converter(lexicon: classifier.lexicon)
|
||||
self.converter = Converter(lexicon: classifier.lexicon, history: classifier.history)
|
||||
}
|
||||
|
||||
// MARK: - Derived state
|
||||
@@ -537,6 +537,7 @@ public final class CompositionEngine {
|
||||
private func commitCurrent(_ given: [Segment]? = nil) -> String {
|
||||
let segs = given ?? segments
|
||||
let text = segs.map { render($0, final: true) }.joined()
|
||||
rememberPicks(segs)
|
||||
if let forced = forcedMode {
|
||||
classifier.learned.record(String(keys), as: forced)
|
||||
}
|
||||
@@ -546,6 +547,17 @@ public final class CompositionEngine {
|
||||
return text
|
||||
}
|
||||
|
||||
/// Candidates the user picked are remembered and come first next time.
|
||||
private func rememberPicks(_ segs: [Segment]) {
|
||||
for seg in segs where seg.mode == .zhuyin {
|
||||
guard let picks = fixed[seg.range.lowerBound], !picks.isEmpty else { continue }
|
||||
let syl = convertibleSyllables(seg, final: true).syllables
|
||||
for (range, text) in picks where range.upperBound <= syl.count {
|
||||
classifier.history.record(Array(syl[range]), phrase: text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Commits whatever is composed (e.g. the app lost focus). Returns "" when empty.
|
||||
public func commitAll() -> String {
|
||||
keys.isEmpty ? "" : commitCurrent()
|
||||
|
||||
@@ -13,12 +13,22 @@ public struct Candidate: Equatable {
|
||||
/// Turns a syllable sequence into the most likely Chinese text (Viterbi over the lexicon).
|
||||
public struct Converter {
|
||||
public let lexicon: Lexicon
|
||||
/// Picked candidates come first.
|
||||
public let history: UserHistory?
|
||||
/// Score used when a syllable has no lexicon entry; the Bopomofo itself is emitted.
|
||||
static let unknownScore = -30.0
|
||||
/// Penalty for matching an untoned syllable against all tones.
|
||||
static let tonelessPenalty = -1.5
|
||||
|
||||
public init(lexicon: Lexicon) { self.lexicon = lexicon }
|
||||
public init(lexicon: Lexicon, history: UserHistory? = nil) {
|
||||
self.lexicon = lexicon
|
||||
self.history = history
|
||||
}
|
||||
|
||||
private func entries(_ syllables: [Syllable]) -> [LexiconEntry] {
|
||||
let e = lexicon.entries(for: syllables)
|
||||
return history?.adjust(syllables, e) ?? e
|
||||
}
|
||||
|
||||
/// - Parameter fixed: user-chosen phrases by syllable range; they are always kept.
|
||||
public func convert(_ syllables: [Syllable], fixed: [Range<Int>: String] = [:]) -> [ConvertedSegment] {
|
||||
@@ -57,7 +67,7 @@ public struct Converter {
|
||||
if fixed.keys.contains(where: { $0.overlaps(range) }) { return [] }
|
||||
|
||||
let slice = Array(syllables[range])
|
||||
var result = lexicon.entries(for: slice).map { entry -> (String, Double) in
|
||||
var result = entries(slice).map { entry -> (String, Double) in
|
||||
let penalty = slice.contains { $0.tone == nil } ? Converter.tonelessPenalty : 0
|
||||
return (entry.phrase, entry.score + penalty)
|
||||
}
|
||||
@@ -74,7 +84,7 @@ public struct Converter {
|
||||
var seen = Set<String>()
|
||||
for len in stride(from: min(lexicon.maxPhraseLength, end), through: 1, by: -1) {
|
||||
let range = (end - len)..<end
|
||||
for entry in lexicon.entries(for: Array(syllables[range])) where !seen.contains(entry.phrase) {
|
||||
for entry in entries(Array(syllables[range])) where !seen.contains(entry.phrase) {
|
||||
seen.insert(entry.phrase)
|
||||
out.append(Candidate(range: range, text: entry.phrase))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import Foundation
|
||||
|
||||
/// Remembers the candidates the user picked, so they come first next time.
|
||||
///
|
||||
/// Keys are toneless readings ("ㄊㄧㄢ", "ㄐㄧㄣ-ㄊㄧㄢ"); a picked phrase is boosted only
|
||||
/// where the lexicon has it for the reading being converted, so a toneless key is safe.
|
||||
/// Stored as JSON: {reading: {phrase: [count, last use]}} (same format on every platform).
|
||||
public final class UserHistory {
|
||||
public typealias Data = [String: [String: [Int]]]
|
||||
|
||||
/// (reading, phrase) pairs kept; the least recently used go first.
|
||||
public var maxEntries = 2000
|
||||
static let maxCount = 10
|
||||
|
||||
public private(set) var data: Data = [:]
|
||||
private var clock = 0
|
||||
private let onChange: (Data) -> Void
|
||||
|
||||
public init(data: Data = [:], onChange: @escaping (Data) -> Void = { _ in }) {
|
||||
self.onChange = onChange
|
||||
for (reading, phrases) in data {
|
||||
for (phrase, value) in phrases where value.count >= 2 {
|
||||
self.data[reading, default: [:]][phrase] = [value[0], value[1]]
|
||||
clock = max(clock, value[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads the JSON file written through `onChange`; empty if missing or unreadable.
|
||||
public static func load(from url: URL) -> Data {
|
||||
guard let raw = try? Foundation.Data(contentsOf: url),
|
||||
let data = try? JSONDecoder().decode(Data.self, from: raw) else { return [:] }
|
||||
return data
|
||||
}
|
||||
|
||||
static func key(_ syllables: [Syllable]) -> String {
|
||||
syllables.map(\.toneless).joined(separator: "-")
|
||||
}
|
||||
|
||||
public func record(_ syllables: [Syllable], phrase: String) {
|
||||
guard !syllables.isEmpty, !phrase.isEmpty else { return }
|
||||
clock += 1
|
||||
let key = UserHistory.key(syllables)
|
||||
let count = data[key]?[phrase]?.first ?? 0
|
||||
data[key, default: [:]][phrase] = [min(UserHistory.maxCount, count + 1), clock]
|
||||
evict()
|
||||
onChange(data)
|
||||
}
|
||||
|
||||
private func evict() {
|
||||
var pairs: [(used: Int, reading: String, phrase: String)] = []
|
||||
for (r, phrases) in data { for (p, v) in phrases { pairs.append((v[1], r, p)) } }
|
||||
guard pairs.count > maxEntries else { return }
|
||||
pairs.sort { $0.used < $1.used }
|
||||
for pair in pairs.prefix(pairs.count - maxEntries) {
|
||||
data[pair.reading]?[pair.phrase] = nil
|
||||
if data[pair.reading]?.isEmpty == true { data[pair.reading] = nil }
|
||||
}
|
||||
}
|
||||
|
||||
public func clear() {
|
||||
data = [:]
|
||||
onChange(data)
|
||||
}
|
||||
|
||||
/// Lexicon entries with picked phrases moved to the top: just above the best entry,
|
||||
/// more so the more often they were picked.
|
||||
func adjust(_ syllables: [Syllable], _ entries: [LexiconEntry]) -> [LexiconEntry] {
|
||||
guard let picked = data[UserHistory.key(syllables)], let top = entries.first?.score else { return entries }
|
||||
var out = entries.map { e -> LexiconEntry in
|
||||
guard let count = picked[e.phrase]?.first else { return e }
|
||||
return LexiconEntry(phrase: e.phrase, score: max(e.score, top + 0.01 + 0.1 * Double(count)))
|
||||
}
|
||||
// Stable sort (keep lexicon order among equal scores), like Python's sort.
|
||||
out = out.enumerated().sorted { $0.element.score != $1.element.score
|
||||
? $0.element.score > $1.element.score : $0.offset < $1.offset }.map(\.element)
|
||||
return out
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user