Files
ays953andClaude Opus 5.5 c8b3c4ae6f AI candidate selection on Windows and Linux
Python port of the reranker: core/lm.py runs the same CKIP gpt2-tiny-chinese
model with numpy (batched per candidate length, scores match the Swift and
reference implementations), Converter.kbest, core/reranker.py (modes, time
budget, pause after overruns, cache), core/system_monitor.py (Windows: CPU usage
and battery saver; Linux: load average, thermal zones, low-power profile and low
battery) and core/ai.py (settings.json, background model loading, status line).

Windows: numpy wheels for PIME's 32-bit Python 3.8 (PIME 1.3) and 3.15 (PIME 2)
are bundled under autozhuyin/vendor and picked by Python version; a settings menu
on the taskbar IME icon and the language bar (left click on the icon toggles
English mode). Linux: install.sh uses the system numpy or downloads the matching
wheel from PyPI (tools/fetch_numpy.py, SHA-256 checked); an IBus settings menu.
Without numpy or the model the input method works as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 16:51:41 +08:00

599 lines
24 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Input method state machine. Port of CompositionEngine.swift.
Keys are given as one-character strings, or one of the names in SPECIAL_KEYS.
handle() returns (handled, commit_text_or_None).
The buffer is split into Zhuyin / literal segments, e.g. "ji3hello" -> 我 + hello.
"""
import time
from . import bopomofo
from .classifier import LITERAL, ZHUYIN
from .features import dot
from .lexicon import Converter
BACKSPACE, ENTER, ESCAPE, TAB = "<backspace>", "<enter>", "<escape>", "<tab>"
UP, DOWN, LEFT, RIGHT = "<up>", "<down>", "<left>", "<right>"
SHIFT_TAP = "<shift-tap>" # Shift pressed and released on its own
SPECIAL_KEYS = {BACKSPACE, ENTER, ESCAPE, TAB, UP, DOWN, LEFT, RIGHT, SHIFT_TAP}
FULLWIDTH = {
"<": ",", ">": "。", "?": "?", "!": "!", ":": ":", '"': "「", "(": "(", ")": ")",
"{": "『", "}": "』", "[": "「", "]": "」", "~": "~", "_": "——",
}
# The first word of a composition that also reads as Zhuyin is not committed as English
# on space; the next word decides ("up jo4" = 因為, "up the" = up the, "no su3" = no 你).
# Right after English text only common one-syllable readings are held (up = 因, url = 教),
# so normal English typing is not delayed.
HOLD_MIN_SCORE = -3.5
# "`" then one of these keys always gives the full-width symbol, whatever the context.
# "`" followed by anything else is just a backtick (so ``` still works).
BACKTICK = "`"
SYMBOLS = {
",": ",", "<": ",", ".": "。", ">": "。", "/": "?", "?": "?", ";": ";", ":": ":",
"'": "、", "\\": "、", '"': "「", "[": "「", "]": "」", "{": "『", "}": "』",
"(": "(", ")": ")", "!": "!", "~": "~", "-": "——", "_": "——", "=": "=",
}
PASS = (False, None)
CONSUMED = (True, None)
class CompositionEngine:
page_size = 9
context_window = 20.0
def __init__(self, classifier, now=time.monotonic):
self.classifier = classifier
self.converter = Converter(classifier.lexicon, classifier.history)
self.now = now
self.keys = []
self.forced_mode = None
# English mode, toggled by a Shift tap when nothing is being typed: keys go straight
# to the app (no composition, no underline) until Shift is tapped again.
self.english_mode = False
self.backtick = False # "`" typed; the next key may be a forced symbol
self.fixed = {} # zhuyin segment start -> {(syl_start, syl_end): text}
self.candidates = []
self.candidate_index = 0
self._candidate_segment_start = 0
# Composition cursor: a key index into self.keys (None = at the end). ← → move it
# between syllables; ↓ opens candidates for the phrase before it; typing there
# inserts a syllable (kept in self.inserting until its tone key, then spliced in).
self.cursor = None
self.inserting = []
# Index of a held space (see HOLD_MIN_SCORE), or None.
self.held = None
self.reranker = None # "AI 選字" (reranker.Reranker); None = lexicon only
self._recent_text = "" # last characters committed: context for the reranker
self._previous_mode = None
self._previous_time = float("-inf")
self._cache_key = None
self._cache_value = None
# --- derived state ---
@property
def context(self):
return self._previous_mode if self.now() - self._previous_time < self.context_window else None
def _segments(self, keys, finishing=False):
if not keys:
return []
if self.forced_mode:
return [(0, len(keys), self.forced_mode)]
ctx = self.context
held = self.held if self.held is not None and self.held < len(keys) and keys[self.held] == " " else None
cache_key = (tuple(keys), finishing, ctx, held)
if cache_key != self._cache_key:
self._cache_key = cache_key
if held is None:
self._cache_value = self.classifier.segment(keys, finishing, ctx)
else:
self._cache_value = self._held_segments(keys, held, finishing)
return self._cache_value
def _held_segments(self, keys, held, finishing):
"""The held word is Chinese if the words typed after it start with Chinese."""
rest = keys[held + 1:]
base = held + 1
rest_segs = [(a + base, b + base, m) for a, b, m in self.classifier.segment(rest, finishing, None)]
if rest_segs and rest_segs[0][2] == ZHUYIN and self._held_reads_as_chinese(keys, held, rest_segs[0][1], finishing):
return [(0, rest_segs[0][1], ZHUYIN)] + rest_segs[1:]
return [(0, base, LITERAL)] + rest_segs
def _held_reads_as_chinese(self, keys, held, end, finishing):
"""Chinese all the way to `end` vs. the held English word followed by Chinese."""
c, w, ctx = self.classifier, self.classifier.weights, self.context
is_last = end == len(keys)
zh_whole = c.segment_features(keys[:end], is_last, False, finishing, ctx)[0]
word = c.segment_features(keys[:held], False, False, True, ctx)[1]
zh_rest = c.segment_features(keys[held + 1:end], is_last, True, finishing, None)[0]
if zh_whole is None:
return False
if word is None or zh_rest is None:
return True
return dot(w, zh_whole) >= dot(w, word) + dot(w, zh_rest)
def _is_ambiguous_word(self, keys):
"""Whether keys + space also read as a common Chinese syllable (see HOLD_MIN_SCORE)."""
if not all(k.isascii() and k.isalpha() for k in keys):
return False
p = bopomofo.parse(list(keys) + [" "])
if p.pending is not None or not p.syllables or p.stray_tones or p.foreign_keys:
return False
best = []
for ps in p.syllables:
entries = self.classifier.lexicon.entries([ps.syllable])
if not entries:
return False
best.append(entries[0][1])
if self.context != LITERAL:
return True # first word: hold anything that reads as Zhuyin
return len(best) == 1 and best[0] >= HOLD_MIN_SCORE
@property
def segments(self):
return self._segments(self.keys)
@property
def mode(self):
segs = self.segments
return segs[-1][2] if segs else ZHUYIN
@property
def showing_candidates(self):
return bool(self.candidates)
@property
def page_start(self):
return (self.candidate_index // self.page_size) * self.page_size
def _convertible(self, seg, final):
"""(syllables converted to Chinese, pending Bopomofo) for a Zhuyin segment."""
start, end, _ = seg
p = bopomofo.parse(self.keys[start:end])
if final:
return [s.syllable for s in p.syllables], ""
return [s.syllable for s in p.completed], (p.pending.syllable.reading if p.pending else "")
def _conversion(self, seg, final):
syl, pending = self._convertible(seg, final)
fx = {k: v for k, v in self.fixed.get(seg[0], {}).items() if k[1] <= len(syl)}
if self.reranker is not None:
chosen = self.reranker.choose(self.converter, syl, fx, self._rerank_context())
if chosen is not None:
return chosen, pending
return self.converter.convert(syl, fx), pending
def _rerank_context(self):
return self._recent_text if self.now() - self._previous_time < self.context_window else ""
def _render(self, seg, final):
start, end, mode = seg
if mode == LITERAL:
return "".join(self.keys[start:end])
converted, pending = self._conversion(seg, final)
return "".join(t for _, t in converted) + pending
# --- cursor ---
def _syllable_ends(self, seg):
"""Absolute key index after each syllable of a Zhuyin segment."""
return [seg[0] + ps.end for ps in bopomofo.parse(self.keys[seg[0]:seg[1]]).syllables]
def _cursor_positions(self):
"""Key indices the cursor can stop at: after every Zhuyin syllable and at every
segment boundary, but never before the first Chinese character (there would be
nothing to pick candidates for)."""
out, ends = {len(self.keys)}, []
for seg in self.segments:
out.add(seg[1])
if seg[2] == ZHUYIN:
ends += self._syllable_ends(seg)
out.update(ends)
first = min(ends) if ends else len(self.keys)
return sorted(p for p in out if p >= first)
def _stop_at(self, pos):
"""(segment index, syllables before pos) for the Zhuyin segment pos is in."""
segs = self.segments
for i, seg in enumerate(segs):
if seg[2] == ZHUYIN and seg[0] < pos <= seg[1]:
n = len(self._convertible(seg, i < len(segs) - 1)[0])
local = sum(1 for e in self._syllable_ends(seg) if e <= pos)
return (i, min(local, n)) if local else None
return None
def _candidate_stop(self):
"""Where ↓ opens candidates: before the cursor, or the last syllable."""
if self.cursor is not None:
return self._stop_at(self.cursor)
segs = self.segments
for i in range(len(segs) - 1, -1, -1):
if segs[i][2] == ZHUYIN:
n = len(self._convertible(segs[i], i < len(segs) - 1)[0])
if n:
return i, n
return None
def _base_cursor_offset(self):
"""Cursor position in the rendered segments (without the syllable being inserted)."""
pieces = self._segment_pieces()
if self.cursor is None:
return sum(len(t) for t, _ in pieces)
segs = self.segments
for i, seg in enumerate(segs):
if not seg[0] < self.cursor <= seg[1]:
continue
offset = sum(len(t) for t, _ in pieces[:i])
if seg[2] == LITERAL:
return offset + self.cursor - seg[0]
stop = self._stop_at(self.cursor)
local = stop[1] if stop else 0
for (a, b), text in self._conversion(seg, i < len(segs) - 1)[0]:
if b <= local:
offset += len(text)
elif a < local:
offset += min(local - a, len(text))
return offset
return 0
@property
def cursor_offset(self):
"""Cursor position in marked_text (characters)."""
return self._base_cursor_offset() + len(self._inserting_text())
def _move_cursor(self, delta):
self.inserting = []
stops = self._cursor_positions()
here = len(self.keys) if self.cursor is None else self.cursor
if delta < 0:
before = [p for p in stops if p < here]
nxt = before[-1] if before else here
else:
after = [p for p in stops if p > here]
nxt = after[0] if after else here
self.cursor = None if nxt >= len(self.keys) else nxt
def _inserting_text(self):
if not self.inserting:
return ""
return "".join(ps.syllable.reading for ps in bopomofo.parse(self.inserting).syllables)
# --- editing in the middle ---
def _trim_fixed(self, pos):
"""Keeps picked phrases that end at or before key index pos (they don't move)."""
kept = {}
for seg in self.segments:
picks = self.fixed.get(seg[0])
if not picks or seg[2] != ZHUYIN:
continue
if seg[1] <= pos:
kept[seg[0]] = picks
elif seg[0] < pos:
ends = self._syllable_ends(seg)
here = {r: t for r, t in picks.items() if r[1] <= len(ends) and ends[r[1] - 1] <= pos}
if here:
kept[seg[0]] = here
self.fixed = kept
def _splice(self, new_keys):
"""Inserts keys at the cursor and moves the cursor after them."""
pos = self.cursor
self._trim_fixed(pos)
self.keys[pos:pos] = new_keys
if self.held is not None and self.held >= pos:
self.held += len(new_keys)
self.cursor = pos + len(new_keys)
if self.cursor >= len(self.keys):
self.cursor = None
def _delete_before_cursor(self):
"""Backspace in the middle: removes the syllable (or literal key) before the cursor."""
pos = self.cursor
start = pos - 1
for seg in self.segments:
if seg[2] == ZHUYIN and seg[0] < pos <= seg[1]:
for ps in bopomofo.parse(self.keys[seg[0]:seg[1]]).syllables:
if seg[0] + ps.end == pos:
start = seg[0] + ps.start
self._trim_fixed(start)
del self.keys[start:pos]
if self.held is not None:
if start <= self.held < pos:
self.held = None
elif self.held >= pos:
self.held -= pos - start
self.cursor = start
if not self.keys:
self.reset()
def _insert_key(self, c):
"""Typing with the cursor in the middle: compose one syllable, splice it in on its tone."""
if c == " ":
if not self.inserting:
return True, self._commit()
self._splice(self.inserting + [" "])
self.inserting = []
return CONSUMED
self.inserting.append(c)
p = bopomofo.parse(self.inserting)
if p.stray_tones or p.foreign_keys:
self.inserting.pop() # a tone with nothing to attach to
return CONSUMED
if p.syllables and p.syllables[-1].toned and len(p.syllables) == 1:
self._splice(self.inserting)
self.inserting = []
elif len(p.syllables) > 1:
# A new syllable started without a tone: splice the first one as it is.
first = p.syllables[0].end
self._splice(self.inserting[:first])
self.inserting = self.inserting[first:]
return CONSUMED
@property
def marked_pieces(self):
pieces = self._segment_pieces()
if self.inserting:
# Show the syllable being inserted at the cursor.
at, out, done = self._base_cursor_offset(), [], False
for text, mode in pieces:
if not done and at <= len(text):
out += [(text[:at], mode), (self._inserting_text(), ZHUYIN), (text[at:], mode)]
done = True
else:
out.append((text, mode))
at -= len(text)
pieces = [(t, m) for t, m in out if t]
if self.backtick:
pieces.append((BACKTICK, LITERAL))
return pieces
def _segment_pieces(self):
segs = self.segments
return [(self._render(s, i < len(segs) - 1), s[2]) for i, s in enumerate(segs)]
@property
def marked_text(self):
return "".join(t for t, _ in self.marked_pieces)
# --- key handling ---
def _flip(self):
"""Forces the whole buffer into the other mode (a correction that is learned)."""
self.forced_mode = LITERAL if self.mode == ZHUYIN else ZHUYIN
def wants(self, key):
"""Whether handle(key) would consume the key, without changing state."""
if self.candidates or key == SHIFT_TAP or self.backtick or self.inserting:
return True
if self.english_mode and not self.keys:
return False # English mode: everything goes to the app as typed
if key in SPECIAL_KEYS:
return bool(self.keys)
if key == BACKTICK:
return True
if self.keys:
return True
if key == " ":
return False
if bopomofo.is_zhuyin_key(key) or (key.isascii() and key.isalpha()):
return True
return self.context == ZHUYIN and key in FULLWIDTH
def handle(self, key):
if self.candidates:
return self._handle_candidate_key(key)
if self.backtick:
return self._after_backtick(key)
if self.inserting and key in (BACKSPACE, ESCAPE):
if key == BACKSPACE:
self.inserting.pop()
else:
self.inserting = []
return CONSUMED
if key == BACKSPACE:
if not self.keys:
return PASS
if self.cursor is not None:
if self.cursor > 0:
self._delete_before_cursor()
return CONSUMED
self.keys.pop()
if self.held is not None and len(self.keys) <= self.held:
self.held = None
self.fixed = {k: v for k, v in self.fixed.items() if k < len(self.keys)}
if not self.keys:
self.forced_mode = None
self.fixed = {}
return CONSUMED
if key == ENTER:
if not self.keys:
return PASS
if self.inserting:
self._splice(self.inserting)
self.inserting = []
return True, self._commit()
if key == ESCAPE:
if not self.keys:
return PASS
self.reset()
return CONSUMED
if key == TAB:
if not self.keys:
return PASS
self._flip()
return CONSUMED
if key == SHIFT_TAP:
if not self.keys:
# Toggle English mode; it stays on until the next Shift tap.
self.english_mode = not self.english_mode
self._previous_mode = LITERAL if self.english_mode else None
self._previous_time = self.now()
else:
self._flip()
return CONSUMED
if key == DOWN:
if not self.keys:
return PASS
if not self.inserting:
self._open_candidates()
return CONSUMED
if key in (LEFT, RIGHT):
if not self.keys:
return PASS
self._move_cursor(-1 if key == LEFT else 1)
return CONSUMED
if key == UP:
return CONSUMED if self.keys else PASS
return self._handle_character(key)
def _after_backtick(self, key):
self.backtick = False
if key in (ESCAPE, BACKSPACE):
return CONSUMED
committed = self._commit() if self.keys else ""
if key in SYMBOLS:
return True, committed + SYMBOLS[key]
# Not a symbol: the backtick was meant literally; then handle the key as usual.
handled, more = self.handle(key)
if not handled and len(key) == 1:
handled, more = True, key
return handled, committed + BACKTICK + (more or "")
def _last_parse(self, segs):
if not segs or segs[-1][2] != ZHUYIN:
return None
start, end, _ = segs[-1]
return bopomofo.parse(self.keys[start:end])
def _handle_character(self, c):
if self.cursor is not None and self.keys and (c == " " or bopomofo.is_zhuyin_key(c)):
return self._insert_key(c)
if self.inserting:
self._splice(self.inserting)
self.inserting = []
self.cursor = None # anything else continues at the end
if self.english_mode and not self.keys:
return PASS # English mode: every key goes to the app as typed
if c == BACKTICK:
self.backtick = True
return CONSUMED
if c == " ":
if not self.keys:
return PASS
segs = self._segments(self.keys, finishing=True)
if segs[-1][2] == LITERAL:
if (self.held is None and len(segs) == 1 and not self.forced_mode
and self._is_ambiguous_word(self.keys)):
self.held = len(self.keys)
self.keys.append(" ")
return CONSUMED
return True, self._commit(segs) + " "
p = self._last_parse(segs)
if p is not None and p.pending is not None:
self.keys.append(" ")
return CONSUMED
return True, self._commit(segs)
composable = bopomofo.is_zhuyin_key(c) or (c.isascii() and c.isalpha()) or (c == "'" and self.keys)
if composable:
# ", . / ; -" are Zhuyin symbols but act as punctuation after an English word.
if self.keys and c in bopomofo.SYMBOLS and not c.isalnum() and self.forced_mode != ZHUYIN:
with_key = self._segments(self.keys + [c])
if with_key[-1][2] == LITERAL and not all(k.isdigit() for k in self.keys):
return True, self._commit() + c
self.keys.append(c)
return CONSUMED
chinese = (self.context == ZHUYIN) if not self.keys else (self.mode == ZHUYIN)
committed = self._commit() if self.keys else ""
punct = FULLWIDTH.get(c, c) if chinese else c
if not committed and punct == c:
return PASS
return True, committed + punct
# --- candidates (for the last Zhuyin segment) ---
def _open_candidates(self):
"""Candidates for the phrase ending at the cursor (default: the last syllable)."""
stop = self._candidate_stop()
if stop is None:
return
seg_index, local = stop
segs = self.segments
seg = segs[seg_index]
syl = self._convertible(seg, seg_index < len(segs) - 1)[0]
self.candidates = self.converter.candidates(syl, local)
self._candidate_segment_start = seg[0]
self.candidate_index = 0
def page(self):
"""Candidates on the current page and the cursor within it."""
start = self.page_start
return [t for _, t in self.candidates[start:start + self.page_size]], self.candidate_index - start
def select_candidate(self, index):
if not 0 <= index < len(self.candidates):
return
rng, text = self.candidates[index]
here = {k: v for k, v in self.fixed.get(self._candidate_segment_start, {}).items()
if not (k[0] < rng[1] and rng[0] < k[1])}
here[rng] = text
self.fixed[self._candidate_segment_start] = here
self.candidates = []
def _handle_candidate_key(self, key):
if len(key) == 1 and key.isdigit() and key != "0":
self.select_candidate(self.page_start + int(key) - 1)
elif key in (" ", ENTER):
self.select_candidate(self.candidate_index)
elif key in (DOWN, RIGHT):
self.candidate_index = min(len(self.candidates) - 1, self.candidate_index + 1)
elif key in (UP, LEFT):
self.candidate_index = max(0, self.candidate_index - 1)
elif key in (ESCAPE, BACKSPACE):
self.candidates = []
return CONSUMED
# --- commit ---
def _commit(self, segs=None):
segs = segs if segs is not None else self.segments
text = "".join(self._render(s, True) for s in segs)
self._remember_picks(segs)
self._recent_text = (self._rerank_context() + text)[-20:]
if self.forced_mode:
self.classifier.learned.record("".join(self.keys), self.forced_mode)
self._previous_mode = segs[-1][2] if segs else None
self._previous_time = self.now()
self.reset()
return text
def _remember_picks(self, segs):
"""Candidates the user picked are remembered and come first next time."""
for i, seg in enumerate(segs):
picks = self.fixed.get(seg[0])
if seg[2] != ZHUYIN or not picks:
continue
syl = self._convertible(seg, True)[0]
for (a, b), text in picks.items():
if b <= len(syl):
self.classifier.history.record(syl[a:b], text)
def commit_all(self):
return self._commit() if self.keys else ""
def reset(self):
self.keys = []
self.inserting = []
self.held = None
self.backtick = False
self.cursor = None
self.forced_mode = None
self.fixed = {}
self.candidates = []