Learn classifier weights from synthetic typing data

Scoring is now features x weights (Features.swift / features.py), with the
old hand-tuned values as defaults. training/ generates synthetic mixed
Chinese/English buffers from the lexicon and word lists and trains the
weights with an averaged structured perceptron over the same segmentation
DP the input method runs. On 10,000 test buffers commit-time accuracy goes
from 97.58% to 99.62%; typing-time (prefix) accuracy from 86.15% to 87.81%.

Both apps load Resources/classifier-weights.json. A parity test checks the
Swift segmentation matches the Python one on 400 fixtures.

Also fixes space after a toned syllable being scored as an extra stray tone,
which could turn short Chinese like "u4 " into English.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 08:58:08 +08:00
co-authored by Claude Opus 5.5
parent 0a7242ddb3
commit 476858d841
19 changed files with 5712 additions and 225 deletions
+30
View File
@@ -0,0 +1,30 @@
"""Shared setup: imports the Python core (windows/autozhuyin/core) and its data."""
import os
import sys
import types
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
WIN = os.path.join(ROOT, "windows")
DATA = os.path.join(WIN, "autozhuyin", "data")
RESOURCES = os.path.join(ROOT, "Resources")
WEIGHTS_PATH = os.path.join(RESOURCES, "classifier-weights.json")
sys.path.insert(0, WIN)
from autozhuyin.core import bopomofo, features # noqa: E402
from autozhuyin.core.classifier import LITERAL, ZHUYIN, Classifier # noqa: E402
from autozhuyin.core.english import EnglishModel # noqa: E402
from autozhuyin.core.lexicon import Lexicon # noqa: E402
core = types.SimpleNamespace(bopomofo=bopomofo, features=features, Classifier=Classifier,
ZHUYIN=ZHUYIN, LITERAL=LITERAL)
def load_models():
if not os.path.exists(os.path.join(DATA, "english-dict.txt")):
sys.exit("Run windows/tools/build_data.py first.")
lexicon = Lexicon.from_file(os.path.join(DATA, "lexicon.txt"))
english = EnglishModel.from_files(os.path.join(DATA, "english-common.txt"),
os.path.join(DATA, "english-dict.txt"),
os.path.join(DATA, "english-bigram.json"))
return lexicon, english
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Writes training/parity-fixtures.json: segmentations produced by the Python core
with the current weights. The Swift tests (ParityTests) must produce the same ones,
which keeps the two implementations in sync. Re-run after changing features or weights.
"""
import json
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from common import DATA, WEIGHTS_PATH, core, load_models # noqa: E402
from synth import Generator # noqa: E402
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parity-fixtures.json")
HAND_PICKED = ["ji3hello", "hellosu3cl3", "su3cl3github", "ji3hel", "2;3", "2.5", "uvl3", "so", "up",
"u4", "183", "2024", "Hello", "don't", "mp3", "ji3emailsu3cl3", "rup wu0 wu0 fu4cp3cl3"]
def main():
lexicon, english = load_models()
clf = core.Classifier(lexicon, english, weights=core.Classifier.load_weights(WEIGHTS_PATH))
gen = Generator(os.path.join(DATA, "lexicon.txt"), os.path.join(DATA, "english-common.txt"),
os.path.join(DATA, "english-dict.txt"), seed=7)
cases = [(k, f) for k in HAND_PICKED for f in (False, True)]
while len(cases) < 400:
ex = gen.example()
if len(ex["keys"]) <= 28:
cases.append((ex["keys"], ex["finishing"]))
cases.append((gen.prefix_of(ex)["keys"], False))
fixtures = [{"keys": k, "finishing": f,
"segments": [[s, e, m] for s, e, m in clf.segment(list(k), f)]} for k, f in cases]
with open(OUT, "w", encoding="utf-8") as fh:
json.dump(fixtures, fh, ensure_ascii=False, indent=0)
print("wrote %d fixtures to %s" % (len(fixtures), OUT))
if __name__ == "__main__":
main()
File diff suppressed because it is too large Load Diff
+132
View File
@@ -0,0 +1,132 @@
"""Synthetic typing data for the Zhuyin / literal classifier.
Each example is what sits in the input buffer when the user commits it, with the
intended mode of every key:
{"keys": "ji3hello", "labels": "zzzlllll", "finishing": True, "kind": "zh+en"}
labels: "z" = Zhuyin, "l" = literal. finishing: committed with space (True) or Enter.
Chinese comes from lexicon phrases sampled by frequency and converted to Dachen keys;
English from the common-word list and the dictionary. Real typing commits English at
a space, so a buffer holds at most one English word next to Chinese.
"""
import random
from common import core
bopomofo = core.bopomofo
KEY_OF = {sym: key for key, sym in bopomofo.SYMBOLS.items()}
TONE_KEY = {"ˊ": "6", "ˇ": "3", "ˋ": "4", "˙": "7"}
def reading_to_keys(reading):
"""'ㄋㄧˇ-ㄏㄠˇ' -> 'su3cl3'; first tone -> space. None if not typeable."""
out = []
for syl in reading.split("-"):
tone = " "
if syl and syl[-1] in TONE_KEY:
tone = TONE_KEY[syl[-1]]
syl = syl[:-1]
if not syl or syl not in bopomofo.VALID:
return None
out.append("".join(KEY_OF[c] for c in syl) + tone)
return "".join(out)
def _is_cjk(s):
return all(0x4E00 <= ord(c) <= 0x9FFF for c in s)
class Generator:
def __init__(self, lexicon_path, common_path, dict_path, seed=0):
self.rng = random.Random(seed)
phrases, weights = [], []
with open(lexicon_path, encoding="utf-8") as f:
for line in f:
cols = line.split()
if len(cols) != 3 or line.startswith("#") or not _is_cjk(cols[1]):
continue
n = cols[0].count("-") + 1
if n > 4:
continue
keys = reading_to_keys(cols[0])
if keys is None:
continue
phrases.append((cols[1], keys))
weights.append(10 ** (float(cols[2]) * 0.6)) # tempered frequency
self.phrases = phrases
self.phrase_cum = _cumulative(weights)
self.common = [w.strip() for w in open(common_path, encoding="utf-8")
if w.strip() and not w.startswith("#")]
self.dict_words = [w.strip() for w in open(dict_path, encoding="utf-8")
if 3 <= len(w.strip()) <= 9]
# --- pieces ---
def chinese(self):
n = self.rng.choice([1, 1, 2, 2, 3])
picked = self.rng.choices(self.phrases, cum_weights=self.phrase_cum, k=n)
return "".join(keys for _, keys in picked)
def english(self):
w = self.rng.choice(self.common) if self.rng.random() < 0.75 else self.rng.choice(self.dict_words)
if self.rng.random() < 0.1:
w = w[:1].upper() + w[1:]
return w
def number(self):
r = self.rng.random()
if r < 0.4:
return str(self.rng.randint(0, 9999))
if r < 0.7:
return str(self.rng.choice([2023, 2024, 2025, 2026, 100, 50, 10, 30]))
return "%d.%d" % (self.rng.randint(0, 99), self.rng.randint(0, 9))
# --- examples ---
def example(self):
r = self.rng.random()
if r < 0.40:
parts, kind = [("z", self.chinese())], "zh"
elif r < 0.65:
parts, kind = [("l", self.english())], "en"
elif r < 0.80:
parts, kind = [("z", self.chinese()), ("l", self.english())], "zh+en"
elif r < 0.93:
parts, kind = [("l", self.english()), ("z", self.chinese())], "en+zh"
elif r < 0.96:
parts, kind = [("z", self.chinese()), ("l", self.english()), ("z", self.chinese())], "zh+en+zh"
else:
parts, kind = [("l", self.number())], "num"
keys = "".join(p for _, p in parts)
labels = "".join(m * len(p) for m, p in parts)
if parts[-1][0] == "l":
finishing = True # English / numbers end with a space
else:
# Chinese ends with Enter, or space after a finished (toned) syllable.
finishing = keys[-1] != " " and self.rng.random() < 0.5
return {"keys": keys, "labels": labels, "finishing": finishing, "kind": kind}
def prefix_of(self, ex):
"""The buffer while still typing: a random cut, not finishing."""
cut = self.rng.randint(1, len(ex["keys"]) - 1) if len(ex["keys"]) > 1 else 1
return {"keys": ex["keys"][:cut], "labels": ex["labels"][:cut], "finishing": False,
"kind": ex["kind"] + "/prefix"}
def _cumulative(ws):
out, t = [], 0.0
for w in ws:
t += w
out.append(t)
return out
def gold_segments(labels):
"""'zzzlll' -> [(0, 3, 'zhuyin'), (3, 6, 'literal')]"""
segs, start = [], 0
for i in range(1, len(labels) + 1):
if i == len(labels) or labels[i] != labels[start]:
segs.append((start, i, core.ZHUYIN if labels[start] == "z" else core.LITERAL))
start = i
return segs
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Learns the classifier weights from synthetic typing data (averaged structured
perceptron over the same segmentation DP the input method runs) and reports
accuracy against the hand-tuned defaults.
Usage (from the repo root, after windows/tools/build_data.py):
python3 training/train.py # train, evaluate, write Resources/classifier-weights.json
python3 training/train.py --eval-only # just evaluate the current weights file
"""
import argparse
import datetime
import json
import os
import random
import sys
import time
from collections import defaultdict
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from common import DATA, WEIGHTS_PATH, core, load_models # noqa: E402
from synth import Generator, gold_segments # noqa: E402
MAX_KEYS = 28
FIXED = {"lit.upper"} # hard rule (Shift = English), not learned
def make_set(gen, n):
out = []
while len(out) < 2 * n:
ex = gen.example()
if len(ex["keys"]) > MAX_KEYS:
continue
out.append(ex)
if len(ex["keys"]) > 1:
out.append(gen.prefix_of(ex))
return out
def labels_of(segs, n):
lab = [""] * n
for s, e, m in segs:
for i in range(s, e):
lab[i] = "z" if m == core.ZHUYIN else "l"
return "".join(lab)
def seg_features(clf, keys, segs, finishing):
"""Summed features of a segmentation, or None if any segment is impossible."""
total = defaultdict(float)
n = len(keys)
for idx, (s, e, m) in enumerate(segs):
fz, fl = clf.segment_features(keys[s:e], e == n, idx > 0, finishing)
f = fz if m == core.ZHUYIN else fl
if f is None:
return None
for k, v in f.items():
total[k] += v
return total
def evaluate(clf, weights, data):
by_kind = defaultdict(lambda: [0, 0])
for ex in data:
segs = clf.segment(list(ex["keys"]), ex["finishing"], None, weights)
ok = labels_of(segs, len(ex["keys"])) == ex["labels"]
bucket = by_kind["prefix" if ex["kind"].endswith("/prefix") else ex["kind"]]
bucket[0] += ok
bucket[1] += 1
commit = [v for k, v in by_kind.items() if k != "prefix"]
summary = {
"commit": sum(c for c, _ in commit) / max(1, sum(t for _, t in commit)),
"prefix": by_kind["prefix"][0] / max(1, by_kind["prefix"][1]),
"by_kind": {k: round(c / t, 4) for k, (c, t) in sorted(by_kind.items())},
}
return summary
def show(name, s):
kinds = " ".join("%s %.1f%%" % (k, 100 * v) for k, v in s["by_kind"].items())
print("%-9s commit %.2f%% typing %.2f%% [%s]" % (name, 100 * s["commit"], 100 * s["prefix"], kinds))
def train(clf, data, epochs, lr, seed):
rng = random.Random(seed)
w = dict(clf.weights)
total = defaultdict(float)
steps = 0
for epoch in range(epochs):
rng.shuffle(data)
mistakes = 0
t0 = time.time()
for ex in data:
keys = list(ex["keys"])
pred = clf.segment(keys, ex["finishing"], None, w)
if labels_of(pred, len(keys)) != ex["labels"]:
gold = gold_segments(ex["labels"])
fg = seg_features(clf, keys, gold, ex["finishing"])
fp = seg_features(clf, keys, pred, ex["finishing"])
if fg is not None and fp is not None:
mistakes += 1
scale = lr * (0.5 if ex["kind"].endswith("/prefix") else 1.0)
for k in set(fg) | set(fp):
if k not in FIXED:
w[k] = w.get(k, 0.0) + scale * (fg.get(k, 0.0) - fp.get(k, 0.0))
steps += 1
for k, v in w.items():
total[k] += v
print("epoch %d: %d updates (%.0fs)" % (epoch + 1, mistakes, time.time() - t0), flush=True)
return {k: round(v / steps, 4) for k, v in total.items()}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--train", type=int, default=3000, help="full examples (plus as many prefixes)")
ap.add_argument("--test", type=int, default=1500)
ap.add_argument("--epochs", type=int, default=6)
ap.add_argument("--lr", type=float, default=0.1)
ap.add_argument("--eval-only", action="store_true")
args = ap.parse_args()
lexicon, english = load_models()
paths = (os.path.join(DATA, "lexicon.txt"), os.path.join(DATA, "english-common.txt"),
os.path.join(DATA, "english-dict.txt"))
test = make_set(Generator(*paths, seed=2), args.test)
clf = core.Classifier(lexicon, english) # default (hand-tuned) weights
show("default", evaluate(clf, clf.weights, test))
if os.path.exists(WEIGHTS_PATH):
current = dict(clf.weights)
current.update(core.Classifier.load_weights(WEIGHTS_PATH))
show("current", evaluate(clf, current, test))
if args.eval_only:
return
data = make_set(Generator(*paths, seed=1), args.train)
print("training on %d buffers (%d test)" % (len(data), len(test)), flush=True)
learned = train(clf, data, args.epochs, args.lr, seed=3)
result = evaluate(clf, learned, test)
show("learned", result)
with open(WEIGHTS_PATH, "w", encoding="utf-8") as f:
json.dump({
"about": "AutoZhuyin classifier weights, learned by training/train.py",
"trained": datetime.date.today().isoformat(),
"examples": len(data),
"test_accuracy": {"commit": round(result["commit"], 4), "typing": round(result["prefix"], 4)},
"weights": dict(sorted(learned.items())),
}, f, ensure_ascii=False, indent=2)
f.write("\n")
print("wrote", WEIGHTS_PATH)
if __name__ == "__main__":
main()