After an English word, 23 of 50 common first-tone characters (機, 吃, 書…) were read as English when space was pressed: the fixed +1.5 context bonus was far too strong for the trained weight scale, and the training data never contained the "space just pressed on a first-tone syllable" decision. Context is now the ctx.zh / ctx.lit features, trained with the rest; the synthetic data includes the previous mode and the first-tone space decision. Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the language differs from the previous word 92.0% -> 99.3%. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
148 lines
5.7 KiB
Python
148 lines
5.7 KiB
Python
"""Synthetic typing data for the Zhuyin / literal classifier.
|
||
|
||
Each example is what sits in the input buffer when the user commits it, with the
|
||
intended mode of every key:
|
||
{"keys": "ji3hello", "labels": "zzzlllll", "finishing": True, "kind": "zh+en"}
|
||
labels: "z" = Zhuyin, "l" = literal. finishing: committed with space (True) or Enter.
|
||
|
||
Chinese comes from lexicon phrases sampled by frequency and converted to Dachen keys;
|
||
English from the common-word list and the dictionary. Real typing commits English at
|
||
a space, so a buffer holds at most one English word next to Chinese.
|
||
"""
|
||
import random
|
||
|
||
from common import core
|
||
|
||
bopomofo = core.bopomofo
|
||
|
||
KEY_OF = {sym: key for key, sym in bopomofo.SYMBOLS.items()}
|
||
TONE_KEY = {"ˊ": "6", "ˇ": "3", "ˋ": "4", "˙": "7"}
|
||
|
||
|
||
def reading_to_keys(reading):
|
||
"""'ㄋㄧˇ-ㄏㄠˇ' -> 'su3cl3'; first tone -> space. None if not typeable."""
|
||
out = []
|
||
for syl in reading.split("-"):
|
||
tone = " "
|
||
if syl and syl[-1] in TONE_KEY:
|
||
tone = TONE_KEY[syl[-1]]
|
||
syl = syl[:-1]
|
||
if not syl or syl not in bopomofo.VALID:
|
||
return None
|
||
out.append("".join(KEY_OF[c] for c in syl) + tone)
|
||
return "".join(out)
|
||
|
||
|
||
def _is_cjk(s):
|
||
return all(0x4E00 <= ord(c) <= 0x9FFF for c in s)
|
||
|
||
|
||
class Generator:
|
||
def __init__(self, lexicon_path, common_path, dict_path, seed=0):
|
||
self.rng = random.Random(seed)
|
||
phrases, weights = [], []
|
||
with open(lexicon_path, encoding="utf-8") as f:
|
||
for line in f:
|
||
cols = line.split()
|
||
if len(cols) != 3 or line.startswith("#") or not _is_cjk(cols[1]):
|
||
continue
|
||
n = cols[0].count("-") + 1
|
||
if n > 4:
|
||
continue
|
||
keys = reading_to_keys(cols[0])
|
||
if keys is None:
|
||
continue
|
||
phrases.append((cols[1], keys))
|
||
weights.append(10 ** (float(cols[2]) * 0.6)) # tempered frequency
|
||
self.phrases = phrases
|
||
self.phrase_cum = _cumulative(weights)
|
||
self.common = [w.strip() for w in open(common_path, encoding="utf-8")
|
||
if w.strip() and not w.startswith("#")]
|
||
self.dict_words = [w.strip() for w in open(dict_path, encoding="utf-8")
|
||
if 3 <= len(w.strip()) <= 9]
|
||
|
||
# --- pieces ---
|
||
|
||
def chinese(self):
|
||
n = self.rng.choice([1, 1, 2, 2, 3])
|
||
picked = self.rng.choices(self.phrases, cum_weights=self.phrase_cum, k=n)
|
||
return "".join(keys for _, keys in picked)
|
||
|
||
def english(self):
|
||
w = self.rng.choice(self.common) if self.rng.random() < 0.75 else self.rng.choice(self.dict_words)
|
||
if self.rng.random() < 0.1:
|
||
w = w[:1].upper() + w[1:]
|
||
return w
|
||
|
||
def number(self):
|
||
r = self.rng.random()
|
||
if r < 0.4:
|
||
return str(self.rng.randint(0, 9999))
|
||
if r < 0.7:
|
||
return str(self.rng.choice([2023, 2024, 2025, 2026, 100, 50, 10, 30]))
|
||
return "%d.%d" % (self.rng.randint(0, 99), self.rng.randint(0, 9))
|
||
|
||
# --- examples ---
|
||
|
||
def example(self):
|
||
r = self.rng.random()
|
||
if r < 0.40:
|
||
parts, kind = [("z", self.chinese())], "zh"
|
||
elif r < 0.65:
|
||
parts, kind = [("l", self.english())], "en"
|
||
elif r < 0.80:
|
||
parts, kind = [("z", self.chinese()), ("l", self.english())], "zh+en"
|
||
elif r < 0.93:
|
||
parts, kind = [("l", self.english()), ("z", self.chinese())], "en+zh"
|
||
elif r < 0.96:
|
||
parts, kind = [("z", self.chinese()), ("l", self.english()), ("z", self.chinese())], "zh+en+zh"
|
||
else:
|
||
parts, kind = [("l", self.number())], "num"
|
||
keys = "".join(p for _, p in parts)
|
||
labels = "".join(m * len(p) for m, p in parts)
|
||
if parts[-1][0] == "l":
|
||
finishing = True # English / numbers end with a space
|
||
elif keys[-1] == " " and self.rng.random() < 0.6:
|
||
# The moment the space (first tone) is pressed: the buffer has no space yet and
|
||
# the input method decides "first tone" vs "English word + space".
|
||
keys, labels, finishing = keys[:-1], labels[:-1], True
|
||
kind += "/tone1"
|
||
else:
|
||
# Chinese ends with Enter, or space after a finished (toned) syllable.
|
||
finishing = keys[-1] != " " and self.rng.random() < 0.5
|
||
return {"keys": keys, "labels": labels, "finishing": finishing, "kind": kind,
|
||
"previous": self.previous_mode(labels[0])}
|
||
|
||
def previous_mode(self, first_label):
|
||
"""Mode of the text committed just before: none, mostly the same, sometimes the other."""
|
||
r = self.rng.random()
|
||
if r < 0.35:
|
||
return None
|
||
same = core.ZHUYIN if first_label == "z" else core.LITERAL
|
||
other = core.LITERAL if first_label == "z" else core.ZHUYIN
|
||
return same if r < 0.80 else other
|
||
|
||
def prefix_of(self, ex):
|
||
"""The buffer while still typing: a random cut, not finishing."""
|
||
cut = self.rng.randint(1, len(ex["keys"]) - 1) if len(ex["keys"]) > 1 else 1
|
||
return {"keys": ex["keys"][:cut], "labels": ex["labels"][:cut], "finishing": False,
|
||
"kind": ex["kind"] + "/prefix", "previous": ex["previous"]}
|
||
|
||
|
||
def _cumulative(ws):
|
||
out, t = [], 0.0
|
||
for w in ws:
|
||
t += w
|
||
out.append(t)
|
||
return out
|
||
|
||
|
||
def gold_segments(labels):
|
||
"""'zzzlll' -> [(0, 3, 'zhuyin'), (3, 6, 'literal')]"""
|
||
segs, start = [], 0
|
||
for i in range(1, len(labels) + 1):
|
||
if i == len(labels) or labels[i] != labels[start]:
|
||
segs.append((start, i, core.ZHUYIN if labels[start] == "z" else core.LITERAL))
|
||
start = i
|
||
return segs
|