After an English word, 23 of 50 common first-tone characters (機, 吃, 書…) were read as English when space was pressed: the fixed +1.5 context bonus was far too strong for the trained weight scale, and the training data never contained the "space just pressed on a first-tone syllable" decision. Context is now the ctx.zh / ctx.lit features, trained with the rest; the synthetic data includes the previous mode and the first-tone space decision. Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the language differs from the previous word 92.0% -> 99.3%. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
41 lines
1.8 KiB
Python
41 lines
1.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Writes training/parity-fixtures.json: segmentations produced by the Python core
|
|
with the current weights. The Swift tests (ParityTests) must produce the same ones,
|
|
which keeps the two implementations in sync. Re-run after changing features or weights.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
from common import DATA, WEIGHTS_PATH, core, load_models # noqa: E402
|
|
from synth import Generator # noqa: E402
|
|
|
|
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parity-fixtures.json")
|
|
|
|
HAND_PICKED = ["ji3hello", "hellosu3cl3", "su3cl3github", "ji3hel", "2;3", "2.5", "uvl3", "so", "up",
|
|
"u4", "183", "2024", "Hello", "don't", "mp3", "ji3emailsu3cl3", "rup wu0 wu0 fu4cp3cl3"]
|
|
|
|
|
|
def main():
|
|
lexicon, english = load_models()
|
|
clf = core.Classifier(lexicon, english, weights=core.Classifier.load_weights(WEIGHTS_PATH))
|
|
gen = Generator(os.path.join(DATA, "lexicon.txt"), os.path.join(DATA, "english-common.txt"),
|
|
os.path.join(DATA, "english-dict.txt"), seed=7)
|
|
cases = [(k, f, p) for k in HAND_PICKED for f in (False, True) for p in (None, "literal")]
|
|
while len(cases) < 500:
|
|
ex = gen.example()
|
|
if len(ex["keys"]) <= 28:
|
|
cases.append((ex["keys"], ex["finishing"], ex["previous"]))
|
|
cases.append((gen.prefix_of(ex)["keys"], False, ex["previous"]))
|
|
fixtures = [{"keys": k, "finishing": f, "previous": p,
|
|
"segments": [[s, e, m] for s, e, m in clf.segment(list(k), f, p)]} for k, f, p in cases]
|
|
with open(OUT, "w", encoding="utf-8") as fh:
|
|
json.dump(fixtures, fh, ensure_ascii=False, indent=0)
|
|
print("wrote %d fixtures to %s" % (len(fixtures), OUT))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|