Files
AutoZhuyin/training/synth.py
T
ays953andClaude Opus 5.5 2c5c70c066 Learn the previous-word context; fix first-tone characters after English
After an English word, 23 of 50 common first-tone characters (機, 吃, 書…)
were read as English when space was pressed: the fixed +1.5 context bonus
was far too strong for the trained weight scale, and the training data never
contained the "space just pressed on a first-tone syllable" decision.

Context is now the ctx.zh / ctx.lit features, trained with the rest; the
synthetic data includes the previous mode and the first-tone space decision.
Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the
language differs from the previous word 92.0% -> 99.3%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 11:59:57 +08:00

148 lines
5.7 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Synthetic typing data for the Zhuyin / literal classifier.
Each example is what sits in the input buffer when the user commits it, with the
intended mode of every key:
{"keys": "ji3hello", "labels": "zzzlllll", "finishing": True, "kind": "zh+en"}
labels: "z" = Zhuyin, "l" = literal. finishing: committed with space (True) or Enter.
Chinese comes from lexicon phrases sampled by frequency and converted to Dachen keys;
English from the common-word list and the dictionary. Real typing commits English at
a space, so a buffer holds at most one English word next to Chinese.
"""
import random
from common import core
bopomofo = core.bopomofo
KEY_OF = {sym: key for key, sym in bopomofo.SYMBOLS.items()}
TONE_KEY = {"ˊ": "6", "ˇ": "3", "ˋ": "4", "˙": "7"}
def reading_to_keys(reading):
"""'ㄋㄧˇ-ㄏㄠˇ' -> 'su3cl3'; first tone -> space. None if not typeable."""
out = []
for syl in reading.split("-"):
tone = " "
if syl and syl[-1] in TONE_KEY:
tone = TONE_KEY[syl[-1]]
syl = syl[:-1]
if not syl or syl not in bopomofo.VALID:
return None
out.append("".join(KEY_OF[c] for c in syl) + tone)
return "".join(out)
def _is_cjk(s):
return all(0x4E00 <= ord(c) <= 0x9FFF for c in s)
class Generator:
def __init__(self, lexicon_path, common_path, dict_path, seed=0):
self.rng = random.Random(seed)
phrases, weights = [], []
with open(lexicon_path, encoding="utf-8") as f:
for line in f:
cols = line.split()
if len(cols) != 3 or line.startswith("#") or not _is_cjk(cols[1]):
continue
n = cols[0].count("-") + 1
if n > 4:
continue
keys = reading_to_keys(cols[0])
if keys is None:
continue
phrases.append((cols[1], keys))
weights.append(10 ** (float(cols[2]) * 0.6)) # tempered frequency
self.phrases = phrases
self.phrase_cum = _cumulative(weights)
self.common = [w.strip() for w in open(common_path, encoding="utf-8")
if w.strip() and not w.startswith("#")]
self.dict_words = [w.strip() for w in open(dict_path, encoding="utf-8")
if 3 <= len(w.strip()) <= 9]
# --- pieces ---
def chinese(self):
n = self.rng.choice([1, 1, 2, 2, 3])
picked = self.rng.choices(self.phrases, cum_weights=self.phrase_cum, k=n)
return "".join(keys for _, keys in picked)
def english(self):
w = self.rng.choice(self.common) if self.rng.random() < 0.75 else self.rng.choice(self.dict_words)
if self.rng.random() < 0.1:
w = w[:1].upper() + w[1:]
return w
def number(self):
r = self.rng.random()
if r < 0.4:
return str(self.rng.randint(0, 9999))
if r < 0.7:
return str(self.rng.choice([2023, 2024, 2025, 2026, 100, 50, 10, 30]))
return "%d.%d" % (self.rng.randint(0, 99), self.rng.randint(0, 9))
# --- examples ---
def example(self):
r = self.rng.random()
if r < 0.40:
parts, kind = [("z", self.chinese())], "zh"
elif r < 0.65:
parts, kind = [("l", self.english())], "en"
elif r < 0.80:
parts, kind = [("z", self.chinese()), ("l", self.english())], "zh+en"
elif r < 0.93:
parts, kind = [("l", self.english()), ("z", self.chinese())], "en+zh"
elif r < 0.96:
parts, kind = [("z", self.chinese()), ("l", self.english()), ("z", self.chinese())], "zh+en+zh"
else:
parts, kind = [("l", self.number())], "num"
keys = "".join(p for _, p in parts)
labels = "".join(m * len(p) for m, p in parts)
if parts[-1][0] == "l":
finishing = True # English / numbers end with a space
elif keys[-1] == " " and self.rng.random() < 0.6:
# The moment the space (first tone) is pressed: the buffer has no space yet and
# the input method decides "first tone" vs "English word + space".
keys, labels, finishing = keys[:-1], labels[:-1], True
kind += "/tone1"
else:
# Chinese ends with Enter, or space after a finished (toned) syllable.
finishing = keys[-1] != " " and self.rng.random() < 0.5
return {"keys": keys, "labels": labels, "finishing": finishing, "kind": kind,
"previous": self.previous_mode(labels[0])}
def previous_mode(self, first_label):
"""Mode of the text committed just before: none, mostly the same, sometimes the other."""
r = self.rng.random()
if r < 0.35:
return None
same = core.ZHUYIN if first_label == "z" else core.LITERAL
other = core.LITERAL if first_label == "z" else core.ZHUYIN
return same if r < 0.80 else other
def prefix_of(self, ex):
"""The buffer while still typing: a random cut, not finishing."""
cut = self.rng.randint(1, len(ex["keys"]) - 1) if len(ex["keys"]) > 1 else 1
return {"keys": ex["keys"][:cut], "labels": ex["labels"][:cut], "finishing": False,
"kind": ex["kind"] + "/prefix", "previous": ex["previous"]}
def _cumulative(ws):
out, t = [], 0.0
for w in ws:
t += w
out.append(t)
return out
def gold_segments(labels):
"""'zzzlll' -> [(0, 3, 'zhuyin'), (3, 6, 'literal')]"""
segs, start = [], 0
for i in range(1, len(labels) + 1):
if i == len(labels) or labels[i] != labels[start]:
segs.append((start, i, core.ZHUYIN if labels[start] == "z" else core.LITERAL))
start = i
return segs