Files
ays953andClaude Opus 5.5 2c5c70c066 Learn the previous-word context; fix first-tone characters after English
After an English word, 23 of 50 common first-tone characters (機, 吃, 書…)
were read as English when space was pressed: the fixed +1.5 context bonus
was far too strong for the trained weight scale, and the training data never
contained the "space just pressed on a first-tone syllable" decision.

Context is now the ctx.zh / ctx.lit features, trained with the rest; the
synthetic data includes the previous mode and the first-tone space decision.
Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the
language differs from the previous word 92.0% -> 99.3%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 11:59:57 +08:00

41 lines
1.8 KiB
Python

#!/usr/bin/env python3
"""Writes training/parity-fixtures.json: segmentations produced by the Python core
with the current weights. The Swift tests (ParityTests) must produce the same ones,
which keeps the two implementations in sync. Re-run after changing features or weights.
"""
import json
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from common import DATA, WEIGHTS_PATH, core, load_models # noqa: E402
from synth import Generator # noqa: E402
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parity-fixtures.json")
HAND_PICKED = ["ji3hello", "hellosu3cl3", "su3cl3github", "ji3hel", "2;3", "2.5", "uvl3", "so", "up",
"u4", "183", "2024", "Hello", "don't", "mp3", "ji3emailsu3cl3", "rup wu0 wu0 fu4cp3cl3"]
def main():
lexicon, english = load_models()
clf = core.Classifier(lexicon, english, weights=core.Classifier.load_weights(WEIGHTS_PATH))
gen = Generator(os.path.join(DATA, "lexicon.txt"), os.path.join(DATA, "english-common.txt"),
os.path.join(DATA, "english-dict.txt"), seed=7)
cases = [(k, f, p) for k in HAND_PICKED for f in (False, True) for p in (None, "literal")]
while len(cases) < 500:
ex = gen.example()
if len(ex["keys"]) <= 28:
cases.append((ex["keys"], ex["finishing"], ex["previous"]))
cases.append((gen.prefix_of(ex)["keys"], False, ex["previous"]))
fixtures = [{"keys": k, "finishing": f, "previous": p,
"segments": [[s, e, m] for s, e, m in clf.segment(list(k), f, p)]} for k, f, p in cases]
with open(OUT, "w", encoding="utf-8") as fh:
json.dump(fixtures, fh, ensure_ascii=False, indent=0)
print("wrote %d fixtures to %s" % (len(fixtures), OUT))
if __name__ == "__main__":
main()