Learn classifier weights from synthetic typing data
Scoring is now features x weights (Features.swift / features.py), with the old hand-tuned values as defaults. training/ generates synthetic mixed Chinese/English buffers from the lexicon and word lists and trains the weights with an averaged structured perceptron over the same segmentation DP the input method runs. On 10,000 test buffers commit-time accuracy goes from 97.58% to 99.62%; typing-time (prefix) accuracy from 86.15% to 87.81%. Both apps load Resources/classifier-weights.json. A parity test checks the Swift segmentation matches the Python one on 400 fixtures. Also fixes space after a toned syllable being scored as an extra stray tone, which could turn short Chinese like "u4 " into English. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
"""Shared setup: imports the Python core (windows/autozhuyin/core) and its data."""
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
WIN = os.path.join(ROOT, "windows")
|
||||
DATA = os.path.join(WIN, "autozhuyin", "data")
|
||||
RESOURCES = os.path.join(ROOT, "Resources")
|
||||
WEIGHTS_PATH = os.path.join(RESOURCES, "classifier-weights.json")
|
||||
|
||||
sys.path.insert(0, WIN)
|
||||
|
||||
from autozhuyin.core import bopomofo, features # noqa: E402
|
||||
from autozhuyin.core.classifier import LITERAL, ZHUYIN, Classifier # noqa: E402
|
||||
from autozhuyin.core.english import EnglishModel # noqa: E402
|
||||
from autozhuyin.core.lexicon import Lexicon # noqa: E402
|
||||
|
||||
core = types.SimpleNamespace(bopomofo=bopomofo, features=features, Classifier=Classifier,
|
||||
ZHUYIN=ZHUYIN, LITERAL=LITERAL)
|
||||
|
||||
|
||||
def load_models():
|
||||
if not os.path.exists(os.path.join(DATA, "english-dict.txt")):
|
||||
sys.exit("Run windows/tools/build_data.py first.")
|
||||
lexicon = Lexicon.from_file(os.path.join(DATA, "lexicon.txt"))
|
||||
english = EnglishModel.from_files(os.path.join(DATA, "english-common.txt"),
|
||||
os.path.join(DATA, "english-dict.txt"),
|
||||
os.path.join(DATA, "english-bigram.json"))
|
||||
return lexicon, english
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Writes training/parity-fixtures.json: segmentations produced by the Python core
|
||||
with the current weights. The Swift tests (ParityTests) must produce the same ones,
|
||||
which keeps the two implementations in sync. Re-run after changing features or weights.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from common import DATA, WEIGHTS_PATH, core, load_models # noqa: E402
|
||||
from synth import Generator # noqa: E402
|
||||
|
||||
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parity-fixtures.json")
|
||||
|
||||
HAND_PICKED = ["ji3hello", "hellosu3cl3", "su3cl3github", "ji3hel", "2;3", "2.5", "uvl3", "so", "up",
|
||||
"u4", "183", "2024", "Hello", "don't", "mp3", "ji3emailsu3cl3", "rup wu0 wu0 fu4cp3cl3"]
|
||||
|
||||
|
||||
def main():
|
||||
lexicon, english = load_models()
|
||||
clf = core.Classifier(lexicon, english, weights=core.Classifier.load_weights(WEIGHTS_PATH))
|
||||
gen = Generator(os.path.join(DATA, "lexicon.txt"), os.path.join(DATA, "english-common.txt"),
|
||||
os.path.join(DATA, "english-dict.txt"), seed=7)
|
||||
cases = [(k, f) for k in HAND_PICKED for f in (False, True)]
|
||||
while len(cases) < 400:
|
||||
ex = gen.example()
|
||||
if len(ex["keys"]) <= 28:
|
||||
cases.append((ex["keys"], ex["finishing"]))
|
||||
cases.append((gen.prefix_of(ex)["keys"], False))
|
||||
fixtures = [{"keys": k, "finishing": f,
|
||||
"segments": [[s, e, m] for s, e, m in clf.segment(list(k), f)]} for k, f in cases]
|
||||
with open(OUT, "w", encoding="utf-8") as fh:
|
||||
json.dump(fixtures, fh, ensure_ascii=False, indent=0)
|
||||
print("wrote %d fixtures to %s" % (len(fixtures), OUT))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,132 @@
|
||||
"""Synthetic typing data for the Zhuyin / literal classifier.
|
||||
|
||||
Each example is what sits in the input buffer when the user commits it, with the
|
||||
intended mode of every key:
|
||||
{"keys": "ji3hello", "labels": "zzzlllll", "finishing": True, "kind": "zh+en"}
|
||||
labels: "z" = Zhuyin, "l" = literal. finishing: committed with space (True) or Enter.
|
||||
|
||||
Chinese comes from lexicon phrases sampled by frequency and converted to Dachen keys;
|
||||
English from the common-word list and the dictionary. Real typing commits English at
|
||||
a space, so a buffer holds at most one English word next to Chinese.
|
||||
"""
|
||||
import random
|
||||
|
||||
from common import core
|
||||
|
||||
bopomofo = core.bopomofo
|
||||
|
||||
KEY_OF = {sym: key for key, sym in bopomofo.SYMBOLS.items()}
|
||||
TONE_KEY = {"ˊ": "6", "ˇ": "3", "ˋ": "4", "˙": "7"}
|
||||
|
||||
|
||||
def reading_to_keys(reading):
|
||||
"""'ㄋㄧˇ-ㄏㄠˇ' -> 'su3cl3'; first tone -> space. None if not typeable."""
|
||||
out = []
|
||||
for syl in reading.split("-"):
|
||||
tone = " "
|
||||
if syl and syl[-1] in TONE_KEY:
|
||||
tone = TONE_KEY[syl[-1]]
|
||||
syl = syl[:-1]
|
||||
if not syl or syl not in bopomofo.VALID:
|
||||
return None
|
||||
out.append("".join(KEY_OF[c] for c in syl) + tone)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _is_cjk(s):
|
||||
return all(0x4E00 <= ord(c) <= 0x9FFF for c in s)
|
||||
|
||||
|
||||
class Generator:
|
||||
def __init__(self, lexicon_path, common_path, dict_path, seed=0):
|
||||
self.rng = random.Random(seed)
|
||||
phrases, weights = [], []
|
||||
with open(lexicon_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
cols = line.split()
|
||||
if len(cols) != 3 or line.startswith("#") or not _is_cjk(cols[1]):
|
||||
continue
|
||||
n = cols[0].count("-") + 1
|
||||
if n > 4:
|
||||
continue
|
||||
keys = reading_to_keys(cols[0])
|
||||
if keys is None:
|
||||
continue
|
||||
phrases.append((cols[1], keys))
|
||||
weights.append(10 ** (float(cols[2]) * 0.6)) # tempered frequency
|
||||
self.phrases = phrases
|
||||
self.phrase_cum = _cumulative(weights)
|
||||
self.common = [w.strip() for w in open(common_path, encoding="utf-8")
|
||||
if w.strip() and not w.startswith("#")]
|
||||
self.dict_words = [w.strip() for w in open(dict_path, encoding="utf-8")
|
||||
if 3 <= len(w.strip()) <= 9]
|
||||
|
||||
# --- pieces ---
|
||||
|
||||
def chinese(self):
|
||||
n = self.rng.choice([1, 1, 2, 2, 3])
|
||||
picked = self.rng.choices(self.phrases, cum_weights=self.phrase_cum, k=n)
|
||||
return "".join(keys for _, keys in picked)
|
||||
|
||||
def english(self):
|
||||
w = self.rng.choice(self.common) if self.rng.random() < 0.75 else self.rng.choice(self.dict_words)
|
||||
if self.rng.random() < 0.1:
|
||||
w = w[:1].upper() + w[1:]
|
||||
return w
|
||||
|
||||
def number(self):
|
||||
r = self.rng.random()
|
||||
if r < 0.4:
|
||||
return str(self.rng.randint(0, 9999))
|
||||
if r < 0.7:
|
||||
return str(self.rng.choice([2023, 2024, 2025, 2026, 100, 50, 10, 30]))
|
||||
return "%d.%d" % (self.rng.randint(0, 99), self.rng.randint(0, 9))
|
||||
|
||||
# --- examples ---
|
||||
|
||||
def example(self):
|
||||
r = self.rng.random()
|
||||
if r < 0.40:
|
||||
parts, kind = [("z", self.chinese())], "zh"
|
||||
elif r < 0.65:
|
||||
parts, kind = [("l", self.english())], "en"
|
||||
elif r < 0.80:
|
||||
parts, kind = [("z", self.chinese()), ("l", self.english())], "zh+en"
|
||||
elif r < 0.93:
|
||||
parts, kind = [("l", self.english()), ("z", self.chinese())], "en+zh"
|
||||
elif r < 0.96:
|
||||
parts, kind = [("z", self.chinese()), ("l", self.english()), ("z", self.chinese())], "zh+en+zh"
|
||||
else:
|
||||
parts, kind = [("l", self.number())], "num"
|
||||
keys = "".join(p for _, p in parts)
|
||||
labels = "".join(m * len(p) for m, p in parts)
|
||||
if parts[-1][0] == "l":
|
||||
finishing = True # English / numbers end with a space
|
||||
else:
|
||||
# Chinese ends with Enter, or space after a finished (toned) syllable.
|
||||
finishing = keys[-1] != " " and self.rng.random() < 0.5
|
||||
return {"keys": keys, "labels": labels, "finishing": finishing, "kind": kind}
|
||||
|
||||
def prefix_of(self, ex):
|
||||
"""The buffer while still typing: a random cut, not finishing."""
|
||||
cut = self.rng.randint(1, len(ex["keys"]) - 1) if len(ex["keys"]) > 1 else 1
|
||||
return {"keys": ex["keys"][:cut], "labels": ex["labels"][:cut], "finishing": False,
|
||||
"kind": ex["kind"] + "/prefix"}
|
||||
|
||||
|
||||
def _cumulative(ws):
|
||||
out, t = [], 0.0
|
||||
for w in ws:
|
||||
t += w
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def gold_segments(labels):
|
||||
"""'zzzlll' -> [(0, 3, 'zhuyin'), (3, 6, 'literal')]"""
|
||||
segs, start = [], 0
|
||||
for i in range(1, len(labels) + 1):
|
||||
if i == len(labels) or labels[i] != labels[start]:
|
||||
segs.append((start, i, core.ZHUYIN if labels[start] == "z" else core.LITERAL))
|
||||
start = i
|
||||
return segs
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Learns the classifier weights from synthetic typing data (averaged structured
|
||||
perceptron over the same segmentation DP the input method runs) and reports
|
||||
accuracy against the hand-tuned defaults.
|
||||
|
||||
Usage (from the repo root, after windows/tools/build_data.py):
|
||||
python3 training/train.py # train, evaluate, write Resources/classifier-weights.json
|
||||
python3 training/train.py --eval-only # just evaluate the current weights file
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from common import DATA, WEIGHTS_PATH, core, load_models # noqa: E402
|
||||
from synth import Generator, gold_segments # noqa: E402
|
||||
|
||||
MAX_KEYS = 28
|
||||
FIXED = {"lit.upper"} # hard rule (Shift = English), not learned
|
||||
|
||||
|
||||
def make_set(gen, n):
|
||||
out = []
|
||||
while len(out) < 2 * n:
|
||||
ex = gen.example()
|
||||
if len(ex["keys"]) > MAX_KEYS:
|
||||
continue
|
||||
out.append(ex)
|
||||
if len(ex["keys"]) > 1:
|
||||
out.append(gen.prefix_of(ex))
|
||||
return out
|
||||
|
||||
|
||||
def labels_of(segs, n):
|
||||
lab = [""] * n
|
||||
for s, e, m in segs:
|
||||
for i in range(s, e):
|
||||
lab[i] = "z" if m == core.ZHUYIN else "l"
|
||||
return "".join(lab)
|
||||
|
||||
|
||||
def seg_features(clf, keys, segs, finishing):
|
||||
"""Summed features of a segmentation, or None if any segment is impossible."""
|
||||
total = defaultdict(float)
|
||||
n = len(keys)
|
||||
for idx, (s, e, m) in enumerate(segs):
|
||||
fz, fl = clf.segment_features(keys[s:e], e == n, idx > 0, finishing)
|
||||
f = fz if m == core.ZHUYIN else fl
|
||||
if f is None:
|
||||
return None
|
||||
for k, v in f.items():
|
||||
total[k] += v
|
||||
return total
|
||||
|
||||
|
||||
def evaluate(clf, weights, data):
|
||||
by_kind = defaultdict(lambda: [0, 0])
|
||||
for ex in data:
|
||||
segs = clf.segment(list(ex["keys"]), ex["finishing"], None, weights)
|
||||
ok = labels_of(segs, len(ex["keys"])) == ex["labels"]
|
||||
bucket = by_kind["prefix" if ex["kind"].endswith("/prefix") else ex["kind"]]
|
||||
bucket[0] += ok
|
||||
bucket[1] += 1
|
||||
commit = [v for k, v in by_kind.items() if k != "prefix"]
|
||||
summary = {
|
||||
"commit": sum(c for c, _ in commit) / max(1, sum(t for _, t in commit)),
|
||||
"prefix": by_kind["prefix"][0] / max(1, by_kind["prefix"][1]),
|
||||
"by_kind": {k: round(c / t, 4) for k, (c, t) in sorted(by_kind.items())},
|
||||
}
|
||||
return summary
|
||||
|
||||
|
||||
def show(name, s):
|
||||
kinds = " ".join("%s %.1f%%" % (k, 100 * v) for k, v in s["by_kind"].items())
|
||||
print("%-9s commit %.2f%% typing %.2f%% [%s]" % (name, 100 * s["commit"], 100 * s["prefix"], kinds))
|
||||
|
||||
|
||||
def train(clf, data, epochs, lr, seed):
|
||||
rng = random.Random(seed)
|
||||
w = dict(clf.weights)
|
||||
total = defaultdict(float)
|
||||
steps = 0
|
||||
for epoch in range(epochs):
|
||||
rng.shuffle(data)
|
||||
mistakes = 0
|
||||
t0 = time.time()
|
||||
for ex in data:
|
||||
keys = list(ex["keys"])
|
||||
pred = clf.segment(keys, ex["finishing"], None, w)
|
||||
if labels_of(pred, len(keys)) != ex["labels"]:
|
||||
gold = gold_segments(ex["labels"])
|
||||
fg = seg_features(clf, keys, gold, ex["finishing"])
|
||||
fp = seg_features(clf, keys, pred, ex["finishing"])
|
||||
if fg is not None and fp is not None:
|
||||
mistakes += 1
|
||||
scale = lr * (0.5 if ex["kind"].endswith("/prefix") else 1.0)
|
||||
for k in set(fg) | set(fp):
|
||||
if k not in FIXED:
|
||||
w[k] = w.get(k, 0.0) + scale * (fg.get(k, 0.0) - fp.get(k, 0.0))
|
||||
steps += 1
|
||||
for k, v in w.items():
|
||||
total[k] += v
|
||||
print("epoch %d: %d updates (%.0fs)" % (epoch + 1, mistakes, time.time() - t0), flush=True)
|
||||
return {k: round(v / steps, 4) for k, v in total.items()}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--train", type=int, default=3000, help="full examples (plus as many prefixes)")
|
||||
ap.add_argument("--test", type=int, default=1500)
|
||||
ap.add_argument("--epochs", type=int, default=6)
|
||||
ap.add_argument("--lr", type=float, default=0.1)
|
||||
ap.add_argument("--eval-only", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
lexicon, english = load_models()
|
||||
paths = (os.path.join(DATA, "lexicon.txt"), os.path.join(DATA, "english-common.txt"),
|
||||
os.path.join(DATA, "english-dict.txt"))
|
||||
test = make_set(Generator(*paths, seed=2), args.test)
|
||||
|
||||
clf = core.Classifier(lexicon, english) # default (hand-tuned) weights
|
||||
show("default", evaluate(clf, clf.weights, test))
|
||||
if os.path.exists(WEIGHTS_PATH):
|
||||
current = dict(clf.weights)
|
||||
current.update(core.Classifier.load_weights(WEIGHTS_PATH))
|
||||
show("current", evaluate(clf, current, test))
|
||||
if args.eval_only:
|
||||
return
|
||||
|
||||
data = make_set(Generator(*paths, seed=1), args.train)
|
||||
print("training on %d buffers (%d test)" % (len(data), len(test)), flush=True)
|
||||
learned = train(clf, data, args.epochs, args.lr, seed=3)
|
||||
result = evaluate(clf, learned, test)
|
||||
show("learned", result)
|
||||
|
||||
with open(WEIGHTS_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump({
|
||||
"about": "AutoZhuyin classifier weights, learned by training/train.py",
|
||||
"trained": datetime.date.today().isoformat(),
|
||||
"examples": len(data),
|
||||
"test_accuracy": {"commit": round(result["commit"], 4), "typing": round(result["prefix"], 4)},
|
||||
"weights": dict(sorted(learned.items())),
|
||||
}, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
print("wrote", WEIGHTS_PATH)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user