AI candidate selection: rerank with a Traditional Chinese language model (macOS)
The converter's K best paths are rescored with CKIP gpt2-tiny-chinese (GPL-3.0, character-level, ~12M parameters, 22 MB as float16): score = lexicon log-prob + 0.6 x LM log-prob, with the recently committed text as context. Inference is plain Accelerate (Apple Silicon and Intel). Modes off / saver (K=3, 15 ms) / standard (K=10, 40 ms) / quality (K=30, 120 ms) in the input menu. A rerank over its time budget falls back to the lexicon; three in a row pause AI for a minute. With auto-pause on, AI pauses when the Mac is hot or the CPU is busy and drops to saver in Low Power Mode. The model loads in the background and is not loaded at all when off. Evaluation (training/lm/eval_rerank.py): chat 96.6% -> 97.5% chars, Wikipedia 92.9% -> 94.7% chars (about 25-30% fewer wrong characters). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
"""Measures how much language-model reranking improves Zhuyin → Chinese conversion.
|
||||
|
||||
For each test sentence: turn the text into Dachen keystrokes (via the lexicon's
|
||||
readings), convert with the current Viterbi converter (baseline), take the K best
|
||||
paths, and rerank them with lexicon score + λ · LM log-probability.
|
||||
|
||||
Usage (needs numpy and the model in build/models/ckip-gpt2-tiny):
|
||||
python training/lm/eval_rerank.py build/lm-eval/chat.txt build/lm-eval/wiki.txt
|
||||
"""
|
||||
import heapq
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.insert(0, os.path.join(ROOT, "windows"))
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from autozhuyin.core import bopomofo # noqa: E402
|
||||
from autozhuyin.core.lexicon import Converter, Lexicon # noqa: E402
|
||||
from gpt2 import GPT2 # noqa: E402
|
||||
|
||||
KEY_OF = {sym: key for key, sym in bopomofo.SYMBOLS.items()}
|
||||
TONE_KEY = {"ˊ": "6", "ˇ": "3", "ˋ": "4", "˙": "7"}
|
||||
LN10 = math.log(10)
|
||||
|
||||
|
||||
def reading_keys(reading):
|
||||
out = []
|
||||
for syl in reading.split("-"):
|
||||
tone = " "
|
||||
for ch in syl:
|
||||
if ch in TONE_KEY:
|
||||
tone = TONE_KEY[ch]
|
||||
else:
|
||||
out.append(KEY_OF[ch])
|
||||
out.append(tone)
|
||||
return out
|
||||
|
||||
|
||||
def build_reverse(lex):
|
||||
best = {}
|
||||
for reading, entries in lex.exact.items():
|
||||
for phrase, score in entries:
|
||||
if phrase not in best or score > best[phrase][1]:
|
||||
best[phrase] = (reading, score)
|
||||
return best
|
||||
|
||||
|
||||
def text_to_keys(text, reverse, max_len):
|
||||
"""Most likely segmentation of the text into lexicon phrases → keystrokes."""
|
||||
n = len(text)
|
||||
best = [(-math.inf, None)] * (n + 1)
|
||||
best[0] = (0.0, None)
|
||||
for j in range(1, n + 1):
|
||||
for i in range(max(0, j - max_len), j):
|
||||
if best[i][0] == -math.inf:
|
||||
continue
|
||||
hit = reverse.get(text[i:j])
|
||||
if hit:
|
||||
s = best[i][0] + hit[1]
|
||||
if s > best[j][0]:
|
||||
best[j] = (s, (i, hit[0]))
|
||||
if best[n][1] is None:
|
||||
return None
|
||||
readings, j = [], n
|
||||
while j > 0:
|
||||
i, r = best[j][1]
|
||||
readings.append(r)
|
||||
j = i
|
||||
keys = []
|
||||
for r in reversed(readings):
|
||||
keys += reading_keys(r)
|
||||
return keys
|
||||
|
||||
|
||||
def kbest(conv, syllables, k):
|
||||
"""K best (score, text) paths through the converter's lattice (scores in log10)."""
|
||||
n = len(syllables)
|
||||
max_len = max(1, conv.lexicon.max_phrase_length)
|
||||
beams = [[] for _ in range(n + 1)]
|
||||
beams[0] = [(0.0, "")]
|
||||
for j in range(1, n + 1):
|
||||
cand = []
|
||||
for length in range(1, min(max_len, j) + 1):
|
||||
i = j - length
|
||||
if not beams[i]:
|
||||
continue
|
||||
for text, score in conv._options(syllables, i, j, {}):
|
||||
for s, t in beams[i]:
|
||||
cand.append((s + score, t + text))
|
||||
beams[j] = heapq.nlargest(k, cand, key=lambda e: e[0])
|
||||
return beams[n]
|
||||
|
||||
|
||||
def char_errors(a, b):
|
||||
return sum(x != y for x, y in zip(a, b)) + abs(len(a) - len(b))
|
||||
|
||||
|
||||
def main(paths, k=10):
|
||||
lex = Lexicon.from_file(os.path.join(ROOT, "windows/autozhuyin/data/lexicon.txt"))
|
||||
conv = Converter(lex)
|
||||
lm = GPT2(os.path.join(ROOT, "build/models/ckip-gpt2-tiny"))
|
||||
reverse = build_reverse(lex)
|
||||
|
||||
cases = []
|
||||
for path in paths:
|
||||
for line in open(path, encoding="utf-8"):
|
||||
text = line.strip().replace(",", "").replace("、", "")
|
||||
if not text:
|
||||
continue
|
||||
keys = text_to_keys(text, reverse, lex.max_phrase_length)
|
||||
if keys is None:
|
||||
continue
|
||||
syl = [s.syllable for s in bopomofo.parse(keys).syllables]
|
||||
paths_k = kbest(conv, syl, k)
|
||||
t0 = time.time()
|
||||
lm_scores = [lm.logprob(t) for _, t in paths_k]
|
||||
cases.append((os.path.basename(path), text, paths_k, lm_scores, time.time() - t0))
|
||||
|
||||
def evaluate(lam, subset):
|
||||
chars = errs = sent_ok = 0
|
||||
for _, text, paths_k, lm_scores, _ in subset:
|
||||
best = max(range(len(paths_k)), key=lambda i: paths_k[i][0] * LN10 + lam * lm_scores[i])
|
||||
out = paths_k[best][1]
|
||||
e = char_errors(out, text)
|
||||
chars += len(text)
|
||||
errs += e
|
||||
sent_ok += e == 0
|
||||
return 1 - errs / chars, sent_ok / len(subset)
|
||||
|
||||
lams = [0, 0.1, 0.2, 0.3, 0.5, 0.7, 1.0, 1.5, 2.0]
|
||||
for name in sorted({c[0] for c in cases}):
|
||||
sub = [c for c in cases if c[0] == name]
|
||||
oracle = sum(any(t == c[1] for _, t in c[2]) for c in sub) / len(sub)
|
||||
print(f"\n== {name}: {len(sub)} sentences, top-{k} contains the answer: {oracle:.1%}")
|
||||
for lam in lams:
|
||||
ca, sa = evaluate(lam, sub)
|
||||
print(f" λ={lam:<4} char acc {ca:.2%} sentence acc {sa:.1%}" + (" (baseline)" if lam == 0 else ""))
|
||||
ms = sum(c[4] for c in cases) / len(cases) * 1000
|
||||
print(f"\nLM time per sentence (top-{k}): {ms:.1f} ms")
|
||||
return cases
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Converts the gpt2-tiny-chinese safetensors file to the input method's format.
|
||||
|
||||
Format (little-endian):
|
||||
"AZLM" magic, uint32 header length, JSON header, then float16 tensors back to back.
|
||||
Header: {"n_layer", "n_head", "n_embd", "vocab", "eps", "tensors": [[name, [shape], offset]]}
|
||||
(offset in float16 elements from the start of the data). Linear weights keep the
|
||||
GPT-2 Conv1D layout [in, out]. The attention mask buffers are dropped.
|
||||
|
||||
Usage: export_lm.py <model dir> <out.bin> (standard library only)
|
||||
"""
|
||||
import array
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
|
||||
|
||||
def main(src, out):
|
||||
cfg = json.load(open(f"{src}/config.json"))
|
||||
with open(f"{src}/model.safetensors", "rb") as f:
|
||||
n = struct.unpack("<Q", f.read(8))[0]
|
||||
header = json.loads(f.read(n))
|
||||
base = 8 + n
|
||||
names = sorted(k for k, v in header.items() if k != "__metadata__" and v["dtype"] == "F32" and v["shape"])
|
||||
tensors, blobs, offset = [], [], 0
|
||||
for name in names:
|
||||
info = header[name]
|
||||
a, b = info["data_offsets"]
|
||||
f.seek(base + a)
|
||||
floats = array.array("f")
|
||||
floats.frombytes(f.read(b - a))
|
||||
if sys.byteorder != "little":
|
||||
floats.byteswap()
|
||||
half = struct.pack(f"<{len(floats)}e", *floats)
|
||||
tensors.append([name.replace("transformer.", ""), info["shape"], offset])
|
||||
blobs.append(half)
|
||||
offset += len(floats)
|
||||
meta = json.dumps({"n_layer": cfg["n_layer"], "n_head": cfg["n_head"], "n_embd": cfg["n_embd"],
|
||||
"vocab": cfg["vocab_size"], "eps": cfg["layer_norm_epsilon"],
|
||||
"tensors": tensors}).encode()
|
||||
with open(out, "wb") as f:
|
||||
f.write(b"AZLM" + struct.pack("<I", len(meta)) + meta)
|
||||
for blob in blobs:
|
||||
f.write(blob)
|
||||
print(f"wrote {out}: {offset} weights")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1], sys.argv[2])
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Minimal GPT-2 forward pass in numpy, for the CKIP gpt2-tiny-chinese model
|
||||
(https://huggingface.co/ckiplab/gpt2-tiny-chinese, GPL-3.0). Character-level:
|
||||
each Chinese character is one token of the bert-base-chinese vocabulary.
|
||||
|
||||
Used to evaluate language-model reranking before porting it to the input methods.
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
|
||||
CLS, UNK = 101, 100
|
||||
|
||||
|
||||
def load_safetensors(path):
|
||||
with open(path, "rb") as f:
|
||||
n = struct.unpack("<Q", f.read(8))[0]
|
||||
header = json.loads(f.read(n))
|
||||
base = 8 + n
|
||||
out = {}
|
||||
for name, info in header.items():
|
||||
if name == "__metadata__" or info["dtype"] != "F32":
|
||||
continue
|
||||
a, b = info["data_offsets"]
|
||||
f.seek(base + a)
|
||||
out[name] = np.frombuffer(f.read(b - a), dtype=np.float32).reshape(info["shape"])
|
||||
return out
|
||||
|
||||
|
||||
class GPT2:
|
||||
def __init__(self, model_dir):
|
||||
self.w = load_safetensors(f"{model_dir}/model.safetensors")
|
||||
cfg = json.load(open(f"{model_dir}/config.json"))
|
||||
self.n_layer, self.n_head = cfg["n_layer"], cfg["n_head"]
|
||||
self.eps = cfg["layer_norm_epsilon"]
|
||||
self.vocab = {}
|
||||
for i, line in enumerate(open(f"{model_dir}/vocab.txt", encoding="utf-8")):
|
||||
self.vocab.setdefault(line.rstrip("\n"), i)
|
||||
|
||||
def ids(self, text):
|
||||
return [self.vocab.get(c, UNK) for c in text]
|
||||
|
||||
def _ln(self, x, name):
|
||||
mu = x.mean(-1, keepdims=True)
|
||||
var = ((x - mu) ** 2).mean(-1, keepdims=True)
|
||||
return (x - mu) / np.sqrt(var + self.eps) * self.w[name + ".weight"] + self.w[name + ".bias"]
|
||||
|
||||
def hidden(self, ids):
|
||||
"""Final hidden states [T, d] for token ids."""
|
||||
w, T = self.w, len(ids)
|
||||
x = w["transformer.wte.weight"][ids] + w["transformer.wpe.weight"][:T]
|
||||
mask = np.triu(np.full((T, T), -1e9, dtype=np.float32), 1)
|
||||
for l in range(self.n_layer):
|
||||
p = f"transformer.h.{l}."
|
||||
h = self._ln(x, p + "ln_1")
|
||||
qkv = h @ w[p + "attn.c_attn.weight"] + w[p + "attn.c_attn.bias"]
|
||||
q, k, v = np.split(qkv, 3, axis=-1)
|
||||
d = q.shape[-1] // self.n_head
|
||||
q = q.reshape(T, self.n_head, d).transpose(1, 0, 2)
|
||||
k = k.reshape(T, self.n_head, d).transpose(1, 0, 2)
|
||||
v = v.reshape(T, self.n_head, d).transpose(1, 0, 2)
|
||||
att = q @ k.transpose(0, 2, 1) / math.sqrt(d) + mask
|
||||
att = np.exp(att - att.max(-1, keepdims=True))
|
||||
att /= att.sum(-1, keepdims=True)
|
||||
a = (att @ v).transpose(1, 0, 2).reshape(T, -1)
|
||||
x = x + a @ w[p + "attn.c_proj.weight"] + w[p + "attn.c_proj.bias"]
|
||||
h = self._ln(x, p + "ln_2")
|
||||
h = h @ w[p + "mlp.c_fc.weight"] + w[p + "mlp.c_fc.bias"]
|
||||
h = 0.5 * h * (1 + np.tanh(math.sqrt(2 / math.pi) * (h + 0.044715 * h ** 3))) # gelu_new
|
||||
x = x + h @ w[p + "mlp.c_proj.weight"] + w[p + "mlp.c_proj.bias"]
|
||||
return self._ln(x, "transformer.ln_f")
|
||||
|
||||
def logprob(self, text, context=""):
|
||||
"""Natural-log probability of `text` given `context` (both strings)."""
|
||||
ctx, tgt = self.ids(context), self.ids(text)
|
||||
ids = [CLS] + ctx + tgt
|
||||
h = self.hidden(ids[:-1])
|
||||
logits = h @ self.w["transformer.wte.weight"].T
|
||||
logits -= logits.max(-1, keepdims=True)
|
||||
lse = np.log(np.exp(logits).sum(-1))
|
||||
start = len(ctx)
|
||||
return float(sum(logits[start + i, t] - lse[start + i] for i, t in enumerate(tgt)))
|
||||
Reference in New Issue
Block a user