Files
AutoZhuyin/training/lm/eval_rerank.py
T
ays953andClaude Opus 5.5 fd87da8061 AI candidate selection: rerank with a Traditional Chinese language model (macOS)
The converter's K best paths are rescored with CKIP gpt2-tiny-chinese (GPL-3.0,
character-level, ~12M parameters, 22 MB as float16):
score = lexicon log-prob + 0.6 x LM log-prob, with the recently committed text
as context. Inference is plain Accelerate (Apple Silicon and Intel).

Modes off / saver (K=3, 15 ms) / standard (K=10, 40 ms) / quality (K=30, 120 ms)
in the input menu. A rerank over its time budget falls back to the lexicon; three
in a row pause AI for a minute. With auto-pause on, AI pauses when the Mac is hot
or the CPU is busy and drops to saver in Low Power Mode. The model loads in the
background and is not loaded at all when off.

Evaluation (training/lm/eval_rerank.py): chat 96.6% -> 97.5% chars,
Wikipedia 92.9% -> 94.7% chars (about 25-30% fewer wrong characters).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 16:33:58 +08:00

148 lines
5.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Measures how much language-model reranking improves Zhuyin → Chinese conversion.
For each test sentence: turn the text into Dachen keystrokes (via the lexicon's
readings), convert with the current Viterbi converter (baseline), take the K best
paths, and rerank them with lexicon score + λ · LM log-probability.
Usage (needs numpy and the model in build/models/ckip-gpt2-tiny):
python training/lm/eval_rerank.py build/lm-eval/chat.txt build/lm-eval/wiki.txt
"""
import heapq
import math
import os
import sys
import time
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, os.path.join(ROOT, "windows"))
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from autozhuyin.core import bopomofo # noqa: E402
from autozhuyin.core.lexicon import Converter, Lexicon # noqa: E402
from gpt2 import GPT2 # noqa: E402
KEY_OF = {sym: key for key, sym in bopomofo.SYMBOLS.items()}
TONE_KEY = {"ˊ": "6", "ˇ": "3", "ˋ": "4", "˙": "7"}
LN10 = math.log(10)
def reading_keys(reading):
out = []
for syl in reading.split("-"):
tone = " "
for ch in syl:
if ch in TONE_KEY:
tone = TONE_KEY[ch]
else:
out.append(KEY_OF[ch])
out.append(tone)
return out
def build_reverse(lex):
best = {}
for reading, entries in lex.exact.items():
for phrase, score in entries:
if phrase not in best or score > best[phrase][1]:
best[phrase] = (reading, score)
return best
def text_to_keys(text, reverse, max_len):
"""Most likely segmentation of the text into lexicon phrases → keystrokes."""
n = len(text)
best = [(-math.inf, None)] * (n + 1)
best[0] = (0.0, None)
for j in range(1, n + 1):
for i in range(max(0, j - max_len), j):
if best[i][0] == -math.inf:
continue
hit = reverse.get(text[i:j])
if hit:
s = best[i][0] + hit[1]
if s > best[j][0]:
best[j] = (s, (i, hit[0]))
if best[n][1] is None:
return None
readings, j = [], n
while j > 0:
i, r = best[j][1]
readings.append(r)
j = i
keys = []
for r in reversed(readings):
keys += reading_keys(r)
return keys
def kbest(conv, syllables, k):
"""K best (score, text) paths through the converter's lattice (scores in log10)."""
n = len(syllables)
max_len = max(1, conv.lexicon.max_phrase_length)
beams = [[] for _ in range(n + 1)]
beams[0] = [(0.0, "")]
for j in range(1, n + 1):
cand = []
for length in range(1, min(max_len, j) + 1):
i = j - length
if not beams[i]:
continue
for text, score in conv._options(syllables, i, j, {}):
for s, t in beams[i]:
cand.append((s + score, t + text))
beams[j] = heapq.nlargest(k, cand, key=lambda e: e[0])
return beams[n]
def char_errors(a, b):
return sum(x != y for x, y in zip(a, b)) + abs(len(a) - len(b))
def main(paths, k=10):
lex = Lexicon.from_file(os.path.join(ROOT, "windows/autozhuyin/data/lexicon.txt"))
conv = Converter(lex)
lm = GPT2(os.path.join(ROOT, "build/models/ckip-gpt2-tiny"))
reverse = build_reverse(lex)
cases = []
for path in paths:
for line in open(path, encoding="utf-8"):
text = line.strip().replace(",", "").replace("、", "")
if not text:
continue
keys = text_to_keys(text, reverse, lex.max_phrase_length)
if keys is None:
continue
syl = [s.syllable for s in bopomofo.parse(keys).syllables]
paths_k = kbest(conv, syl, k)
t0 = time.time()
lm_scores = [lm.logprob(t) for _, t in paths_k]
cases.append((os.path.basename(path), text, paths_k, lm_scores, time.time() - t0))
def evaluate(lam, subset):
chars = errs = sent_ok = 0
for _, text, paths_k, lm_scores, _ in subset:
best = max(range(len(paths_k)), key=lambda i: paths_k[i][0] * LN10 + lam * lm_scores[i])
out = paths_k[best][1]
e = char_errors(out, text)
chars += len(text)
errs += e
sent_ok += e == 0
return 1 - errs / chars, sent_ok / len(subset)
lams = [0, 0.1, 0.2, 0.3, 0.5, 0.7, 1.0, 1.5, 2.0]
for name in sorted({c[0] for c in cases}):
sub = [c for c in cases if c[0] == name]
oracle = sum(any(t == c[1] for _, t in c[2]) for c in sub) / len(sub)
print(f"\n== {name}: {len(sub)} sentences, top-{k} contains the answer: {oracle:.1%}")
for lam in lams:
ca, sa = evaluate(lam, sub)
print(f" λ={lam:<4} char acc {ca:.2%} sentence acc {sa:.1%}" + (" (baseline)" if lam == 0 else ""))
ms = sum(c[4] for c in cases) / len(cases) * 1000
print(f"\nLM time per sentence (top-{k}): {ms:.1f} ms")
return cases
if __name__ == "__main__":
main(sys.argv[1:])