Files
AutoZhuyin/windows/autozhuyin/core/lexicon.py
T
ays953andClaude Opus 5.5 2cebf0e700 Remember picked candidates (user history)
Candidates chosen with the arrow-down list are recorded on commit as
toneless reading -> phrase with a count. The converter lifts a picked phrase just
above the best lexicon entry for that reading (more for more picks), so it
comes first next time without overriding longer phrases. Up to 2000 pairs,
least recently used evicted. Stored as user-history.json on every platform; the
Mac menu's clear item clears it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 14:30:55 +08:00

145 lines
5.1 KiB
Python

"""Lexicon and Viterbi converter. Port of Lexicon.swift / Converter.swift."""
_TONE_MARKS = set("ˊˇˋ˙")
def _strip_tones(reading):
return "".join(c for c in reading if c not in _TONE_MARKS)
def _is_bopomofo(s):
return bool(s) and (0x3105 <= ord(s[0]) <= 0x312F or s[0] == "˙")
class Lexicon:
"""Maps readings ("ㄋㄧˇ-ㄏㄠˇ") to [(phrase, log10 score)], best first."""
def __init__(self):
self.exact = {}
self._toneless = None # built on first use (only untoned syllables need it)
self.max_phrase_length = 1
@classmethod
def from_file(cls, path):
lex = cls()
with open(path, encoding="utf-8") as f:
lex.load(f)
return lex
def load(self, lines):
for line in lines:
line = line.strip()
if not line or line.startswith("#"):
continue
cols = line.split()
if len(cols) < 2:
continue
reading, phrase = (cols[0], cols[1]) if _is_bopomofo(cols[0]) else (cols[1], cols[0])
if not _is_bopomofo(reading):
continue
try:
score = float(cols[2]) if len(cols) >= 3 else -6.0
except ValueError:
score = -6.0
self.add(reading, phrase, score)
for v in self.exact.values():
if len(v) > 1:
v.sort(key=lambda e: -e[1])
self._toneless = None
def add(self, reading, phrase, score):
syllables = [s[1:] + "˙" if s.startswith("˙") else s for s in reading.split("-")]
reading = "-".join(syllables)
self.max_phrase_length = max(self.max_phrase_length, len(syllables))
entries = self.exact.setdefault(reading, [])
if not any(p == phrase for p, _ in entries):
entries.append((phrase, score))
self._toneless = None
@property
def toneless(self):
"""Index ignoring tones: toneless reading -> [(phrase, best score)], best first."""
if self._toneless is None:
best = {}
for reading, entries in self.exact.items():
bucket = best.setdefault(_strip_tones(reading), {})
for phrase, score in entries:
if score > bucket.get(phrase, float("-inf")):
bucket[phrase] = score
self._toneless = {k: sorted(v.items(), key=lambda e: -e[1]) for k, v in best.items()}
return self._toneless
def entries(self, syllables):
if all(s.tone is not None for s in syllables):
return self.exact.get("-".join(s.reading for s in syllables), [])
return self.toneless.get("-".join(s.toneless for s in syllables), [])
def best_score(self, syllable):
e = self.entries([syllable])
return e[0][1] if e else None
UNKNOWN_SCORE = -30.0
TONELESS_PENALTY = -1.5
class Converter:
def __init__(self, lexicon, history=None):
self.lexicon = lexicon
self.history = history # UserHistory: picked candidates come first
def _entries(self, syllables):
entries = self.lexicon.entries(syllables)
return self.history.adjust(syllables, entries) if self.history else entries
def _options(self, syllables, i, j, fixed):
if (i, j) in fixed:
return [(fixed[(i, j)], 0.0)]
if any(a < j and i < b for (a, b) in fixed):
return []
sl = syllables[i:j]
penalty = TONELESS_PENALTY if any(s.tone is None for s in sl) else 0.0
out = [(p, s + penalty) for p, s in self._entries(sl)]
if not out and j - i == 1:
out = [(sl[0].reading, UNKNOWN_SCORE)]
return out
def convert(self, syllables, fixed=None):
"""Returns [((start, end), text)] for the best path. `fixed` maps (start, end) -> text."""
fixed = fixed or {}
n = len(syllables)
if n == 0:
return []
max_len = max(1, self.lexicon.max_phrase_length)
best = [(float("-inf"), None)] * (n + 1)
best[0] = (0.0, None)
for j in range(1, n + 1):
for length in range(1, min(max_len, j) + 1):
i = j - length
if best[i][0] == float("-inf"):
continue
for text, score in self._options(syllables, i, j, fixed):
total = best[i][0] + score
if total > best[j][0]:
best[j] = (total, ((i, j), text))
segs = []
j = n
while j > 0 and best[j][1] is not None:
seg = best[j][1]
segs.append(seg)
j = seg[0][0]
return segs[::-1]
def candidates(self, syllables, end):
"""[((start, end), text)] for phrases ending at syllable `end`, longest first."""
if end <= 0 or end > len(syllables):
return []
out, seen = [], set()
for length in range(min(self.lexicon.max_phrase_length, end), 0, -1):
i = end - length
for phrase, _ in self._entries(syllables[i:end]):
if phrase not in seen:
seen.add(phrase)
out.append(((i, end), phrase))
return out