Files
AutoZhuyin/windows/autozhuyin/core/lexicon.py
T
ays953andClaude Opus 5.5 bfeed22916 Cut input method startup from ~0.95 s to ~0.15 s
After login macOS starts the input method on first use, and keys typed
while it loads its data were delayed or lost. Startup now:
- parses lexicon.txt as raw bytes (no per-line String splitting or Double
  parsing) and builds the tone-ignoring index lazily on a background queue;
- loads the precomputed English dictionary and bigram table (shared with the
  Windows build) instead of retraining the bigram model on every launch, and
  uses binary search instead of building a 230k-entry Set.
Measured (release, warm cache): lexicon 0.65 -> 0.10 s, English 0.30 -> 0.04 s.
The Windows (Python) core also builds its toneless index lazily: 0.56 -> 0.37 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 12:13:03 +08:00

140 lines
4.9 KiB
Python

"""Lexicon and Viterbi converter. Port of Lexicon.swift / Converter.swift."""
_TONE_MARKS = set("ˊˇˋ˙")
def _strip_tones(reading):
return "".join(c for c in reading if c not in _TONE_MARKS)
def _is_bopomofo(s):
return bool(s) and (0x3105 <= ord(s[0]) <= 0x312F or s[0] == "˙")
class Lexicon:
"""Maps readings ("ㄋㄧˇ-ㄏㄠˇ") to [(phrase, log10 score)], best first."""
def __init__(self):
self.exact = {}
self._toneless = None # built on first use (only untoned syllables need it)
self.max_phrase_length = 1
@classmethod
def from_file(cls, path):
lex = cls()
with open(path, encoding="utf-8") as f:
lex.load(f)
return lex
def load(self, lines):
for line in lines:
line = line.strip()
if not line or line.startswith("#"):
continue
cols = line.split()
if len(cols) < 2:
continue
reading, phrase = (cols[0], cols[1]) if _is_bopomofo(cols[0]) else (cols[1], cols[0])
if not _is_bopomofo(reading):
continue
try:
score = float(cols[2]) if len(cols) >= 3 else -6.0
except ValueError:
score = -6.0
self.add(reading, phrase, score)
for v in self.exact.values():
if len(v) > 1:
v.sort(key=lambda e: -e[1])
self._toneless = None
def add(self, reading, phrase, score):
syllables = [s[1:] + "˙" if s.startswith("˙") else s for s in reading.split("-")]
reading = "-".join(syllables)
self.max_phrase_length = max(self.max_phrase_length, len(syllables))
entries = self.exact.setdefault(reading, [])
if not any(p == phrase for p, _ in entries):
entries.append((phrase, score))
self._toneless = None
@property
def toneless(self):
"""Index ignoring tones: toneless reading -> [(phrase, best score)], best first."""
if self._toneless is None:
best = {}
for reading, entries in self.exact.items():
bucket = best.setdefault(_strip_tones(reading), {})
for phrase, score in entries:
if score > bucket.get(phrase, float("-inf")):
bucket[phrase] = score
self._toneless = {k: sorted(v.items(), key=lambda e: -e[1]) for k, v in best.items()}
return self._toneless
def entries(self, syllables):
if all(s.tone is not None for s in syllables):
return self.exact.get("-".join(s.reading for s in syllables), [])
return self.toneless.get("-".join(s.toneless for s in syllables), [])
def best_score(self, syllable):
e = self.entries([syllable])
return e[0][1] if e else None
UNKNOWN_SCORE = -30.0
TONELESS_PENALTY = -1.5
class Converter:
def __init__(self, lexicon):
self.lexicon = lexicon
def _options(self, syllables, i, j, fixed):
if (i, j) in fixed:
return [(fixed[(i, j)], 0.0)]
if any(a < j and i < b for (a, b) in fixed):
return []
sl = syllables[i:j]
penalty = TONELESS_PENALTY if any(s.tone is None for s in sl) else 0.0
out = [(p, s + penalty) for p, s in self.lexicon.entries(sl)]
if not out and j - i == 1:
out = [(sl[0].reading, UNKNOWN_SCORE)]
return out
def convert(self, syllables, fixed=None):
"""Returns [((start, end), text)] for the best path. `fixed` maps (start, end) -> text."""
fixed = fixed or {}
n = len(syllables)
if n == 0:
return []
max_len = max(1, self.lexicon.max_phrase_length)
best = [(float("-inf"), None)] * (n + 1)
best[0] = (0.0, None)
for j in range(1, n + 1):
for length in range(1, min(max_len, j) + 1):
i = j - length
if best[i][0] == float("-inf"):
continue
for text, score in self._options(syllables, i, j, fixed):
total = best[i][0] + score
if total > best[j][0]:
best[j] = (total, ((i, j), text))
segs = []
j = n
while j > 0 and best[j][1] is not None:
seg = best[j][1]
segs.append(seg)
j = seg[0][0]
return segs[::-1]
def candidates(self, syllables, end):
"""[((start, end), text)] for phrases ending at syllable `end`, longest first."""
if end <= 0 or end > len(syllables):
return []
out, seen = [], set()
for length in range(min(self.lexicon.max_phrase_length, end), 0, -1):
i = end - length
for phrase, _ in self.lexicon.entries(syllables[i:end]):
if phrase not in seen:
seen.add(phrase)
out.append(((i, end), phrase))
return out