Python port of the reranker: core/lm.py runs the same CKIP gpt2-tiny-chinese model with numpy (batched per candidate length, scores match the Swift and reference implementations), Converter.kbest, core/reranker.py (modes, time budget, pause after overruns, cache), core/system_monitor.py (Windows: CPU usage and battery saver; Linux: load average, thermal zones, low-power profile and low battery) and core/ai.py (settings.json, background model loading, status line). Windows: numpy wheels for PIME's 32-bit Python 3.8 (PIME 1.3) and 3.15 (PIME 2) are bundled under autozhuyin/vendor and picked by Python version; a settings menu on the taskbar IME icon and the language bar (left click on the icon toggles English mode). Linux: install.sh uses the system numpy or downloads the matching wheel from PyPI (tools/fetch_numpy.py, SHA-256 checked); an IBus settings menu. Without numpy or the model the input method works as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
167 lines
6.0 KiB
Python
167 lines
6.0 KiB
Python
"""Lexicon and Viterbi converter. Port of Lexicon.swift / Converter.swift."""
|
|
|
|
_TONE_MARKS = set("ˊˇˋ˙")
|
|
|
|
|
|
def _strip_tones(reading):
|
|
return "".join(c for c in reading if c not in _TONE_MARKS)
|
|
|
|
|
|
def _is_bopomofo(s):
|
|
return bool(s) and (0x3105 <= ord(s[0]) <= 0x312F or s[0] == "˙")
|
|
|
|
|
|
class Lexicon:
|
|
"""Maps readings ("ㄋㄧˇ-ㄏㄠˇ") to [(phrase, log10 score)], best first."""
|
|
|
|
def __init__(self):
|
|
self.exact = {}
|
|
self._toneless = None # built on first use (only untoned syllables need it)
|
|
self.max_phrase_length = 1
|
|
|
|
@classmethod
|
|
def from_file(cls, path):
|
|
lex = cls()
|
|
with open(path, encoding="utf-8") as f:
|
|
lex.load(f)
|
|
return lex
|
|
|
|
def load(self, lines):
|
|
for line in lines:
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
cols = line.split()
|
|
if len(cols) < 2:
|
|
continue
|
|
reading, phrase = (cols[0], cols[1]) if _is_bopomofo(cols[0]) else (cols[1], cols[0])
|
|
if not _is_bopomofo(reading):
|
|
continue
|
|
try:
|
|
score = float(cols[2]) if len(cols) >= 3 else -6.0
|
|
except ValueError:
|
|
score = -6.0
|
|
self.add(reading, phrase, score)
|
|
for v in self.exact.values():
|
|
if len(v) > 1:
|
|
v.sort(key=lambda e: -e[1])
|
|
self._toneless = None
|
|
|
|
def add(self, reading, phrase, score):
|
|
syllables = [s[1:] + "˙" if s.startswith("˙") else s for s in reading.split("-")]
|
|
reading = "-".join(syllables)
|
|
self.max_phrase_length = max(self.max_phrase_length, len(syllables))
|
|
entries = self.exact.setdefault(reading, [])
|
|
if not any(p == phrase for p, _ in entries):
|
|
entries.append((phrase, score))
|
|
self._toneless = None
|
|
|
|
@property
|
|
def toneless(self):
|
|
"""Index ignoring tones: toneless reading -> [(phrase, best score)], best first."""
|
|
if self._toneless is None:
|
|
best = {}
|
|
for reading, entries in self.exact.items():
|
|
bucket = best.setdefault(_strip_tones(reading), {})
|
|
for phrase, score in entries:
|
|
if score > bucket.get(phrase, float("-inf")):
|
|
bucket[phrase] = score
|
|
self._toneless = {k: sorted(v.items(), key=lambda e: -e[1]) for k, v in best.items()}
|
|
return self._toneless
|
|
|
|
def entries(self, syllables):
|
|
if all(s.tone is not None for s in syllables):
|
|
return self.exact.get("-".join(s.reading for s in syllables), [])
|
|
return self.toneless.get("-".join(s.toneless for s in syllables), [])
|
|
|
|
def best_score(self, syllable):
|
|
e = self.entries([syllable])
|
|
return e[0][1] if e else None
|
|
|
|
|
|
UNKNOWN_SCORE = -30.0
|
|
TONELESS_PENALTY = -1.5
|
|
|
|
|
|
class Converter:
|
|
def __init__(self, lexicon, history=None):
|
|
self.lexicon = lexicon
|
|
self.history = history # UserHistory: picked candidates come first
|
|
|
|
def _entries(self, syllables):
|
|
entries = self.lexicon.entries(syllables)
|
|
return self.history.adjust(syllables, entries) if self.history else entries
|
|
|
|
def _options(self, syllables, i, j, fixed):
|
|
if (i, j) in fixed:
|
|
return [(fixed[(i, j)], 0.0)]
|
|
if any(a < j and i < b for (a, b) in fixed):
|
|
return []
|
|
sl = syllables[i:j]
|
|
penalty = TONELESS_PENALTY if any(s.tone is None for s in sl) else 0.0
|
|
out = [(p, s + penalty) for p, s in self._entries(sl)]
|
|
if not out and j - i == 1:
|
|
out = [(sl[0].reading, UNKNOWN_SCORE)]
|
|
return out
|
|
|
|
def convert(self, syllables, fixed=None):
|
|
"""Returns [((start, end), text)] for the best path. `fixed` maps (start, end) -> text."""
|
|
fixed = fixed or {}
|
|
n = len(syllables)
|
|
if n == 0:
|
|
return []
|
|
max_len = max(1, self.lexicon.max_phrase_length)
|
|
best = [(float("-inf"), None)] * (n + 1)
|
|
best[0] = (0.0, None)
|
|
for j in range(1, n + 1):
|
|
for length in range(1, min(max_len, j) + 1):
|
|
i = j - length
|
|
if best[i][0] == float("-inf"):
|
|
continue
|
|
for text, score in self._options(syllables, i, j, fixed):
|
|
total = best[i][0] + score
|
|
if total > best[j][0]:
|
|
best[j] = (total, ((i, j), text))
|
|
segs = []
|
|
j = n
|
|
while j > 0 and best[j][1] is not None:
|
|
seg = best[j][1]
|
|
segs.append(seg)
|
|
j = seg[0][0]
|
|
return segs[::-1]
|
|
|
|
def kbest(self, syllables, k, fixed=None):
|
|
"""The k best paths [(log10 score, [((start, end), text)])], best first."""
|
|
fixed = fixed or {}
|
|
n = len(syllables)
|
|
if n == 0 or k <= 0:
|
|
return []
|
|
max_len = max(1, self.lexicon.max_phrase_length)
|
|
beams = [[] for _ in range(n + 1)]
|
|
beams[0] = [(0.0, [])]
|
|
for j in range(1, n + 1):
|
|
cand = []
|
|
for length in range(1, min(max_len, j) + 1):
|
|
i = j - length
|
|
if not beams[i]:
|
|
continue
|
|
for text, score in self._options(syllables, i, j, fixed):
|
|
for s, segs in beams[i]:
|
|
cand.append((s + score, segs + [((i, j), text)]))
|
|
cand.sort(key=lambda e: -e[0])
|
|
beams[j] = cand[:k]
|
|
return beams[n]
|
|
|
|
def candidates(self, syllables, end):
|
|
"""[((start, end), text)] for phrases ending at syllable `end`, longest first."""
|
|
if end <= 0 or end > len(syllables):
|
|
return []
|
|
out, seen = [], set()
|
|
for length in range(min(self.lexicon.max_phrase_length, end), 0, -1):
|
|
i = end - length
|
|
for phrase, _ in self._entries(syllables[i:end]):
|
|
if phrase not in seen:
|
|
seen.add(phrase)
|
|
out.append(((i, end), phrase))
|
|
return out
|