Learn the previous-word context; fix first-tone characters after English

After an English word, 23 of 50 common first-tone characters (機, 吃, 書…)
were read as English when space was pressed: the fixed +1.5 context bonus
was far too strong for the trained weight scale, and the training data never
contained the "space just pressed on a first-tone syllable" decision.

Context is now the ctx.zh / ctx.lit features, trained with the rest; the
synthetic data includes the previous mode and the first-tone space decision.
Test accuracy: commit 98.22% -> 99.52%, typing 88.73% -> 91.49%, and when the
language differs from the previous word 92.0% -> 99.3%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 11:59:57 +08:00
co-authored by Claude Opus 5.5
parent 17f7d6f50d
commit 2c5c70c066
12 changed files with 5257 additions and 3455 deletions
+5 -5
View File
@@ -144,12 +144,12 @@ python3 training/make_parity.py # 權重或特徵改了之後都要重
目前的結果(10,000 筆測試資料): 目前的結果(10,000 筆測試資料):
| | 送出時整段判斷正確 | 打字過程中顯示正確 | | | 送出時整段判斷正確 | 打字過程中顯示正確 | 跟前一個字語言不同時 |
|---|---|---| |---|---|---|---|
| 手調權重 | 97.58% | 86.15% | | 手調權重 | 97.84% | 88.19% | 96.2% |
| 訓練後 | **99.62%** | 87.81% | | 訓練後 | **99.52%** | **91.49%** | **99.3%** |
送出時的錯誤率從 2.4% 降到 0.4%。「打字過程中」本來就有歧義(例如 `ji3h` 的 `h` 可能是英文的開頭,也可能是注音 ㄘ),所以準確率比較低,但最後送出的結果才是重點。 訓練資料也模擬了「前一個字是中文或英文」和「按下空白(一聲)的那一刻」,所以「前一個字」的影響(`ctx.zh` / `ctx.lit`)也是訓練出來的,不是手調的。「打字過程中」本來就有歧義(例如 `ji3h` 的 `h` 可能是英文的開頭,也可能是注音 ㄘ),所以準確率比較低,但最後送出的結果才是重點。
新增特徵時,Swift 和 Python 兩邊都要加,預設權重設成 0,然後重新訓練。 新增特徵時,Swift 和 Python 兩邊都要加,預設權重設成 0,然後重新訓練。
+28 -26
View File
@@ -1,39 +1,41 @@
{ {
"about": "AutoZhuyin classifier weights, learned by training/train.py", "about": "AutoZhuyin classifier weights, learned by training/train.py",
"trained": "2026-11-11", "trained": "2026-11-11",
"examples": 40001, "examples": 40000,
"test_accuracy": { "test_accuracy": {
"commit": 0.9962, "commit": 0.9952,
"typing": 0.8781 "typing": 0.9149
}, },
"weights": { "weights": {
"lit.bias": 0.4276, "ctx.lit": 0.0916,
"lit.common": -0.3465, "ctx.zh": 0.1349,
"lit.common_len": 0.81, "lit.bias": 0.3982,
"lit.dict": -1.7902, "lit.common": -0.2906,
"lit.dict_len": 0.6765, "lit.common_len": 0.8046,
"lit.digits": -2.7403, "lit.dict": -1.6662,
"lit.num_digits": -0.0751, "lit.dict_len": 0.6441,
"lit.num_others": -0.7433, "lit.digits": -2.9689,
"lit.others": -1.5685, "lit.num_digits": -0.018,
"lit.prefix_common_len": 0.2082, "lit.num_others": -0.5554,
"lit.prefix_dict_len": 0.0003, "lit.others": -2.0002,
"lit.short": -0.2397, "lit.prefix_common_len": 0.2831,
"lit.unknown_score": 0.213, "lit.prefix_dict_len": 0.0299,
"lit.short": -0.2482,
"lit.unknown_score": 0.2479,
"lit.upper": 20.0, "lit.upper": 20.0,
"seg.switch": -1.7203, "seg.switch": -1.7596,
"zh.bias": 1.3522, "zh.bias": 1.3423,
"zh.lexscore": 0.26, "zh.lexscore": 0.2917,
"zh.pending_invalid": -5.0, "zh.pending_invalid": -5.0,
"zh.pending_prefix": -0.9295, "zh.pending_prefix": -0.9162,
"zh.phrase": 0.4595, "zh.phrase": 0.4499,
"zh.reordered": -1.4348, "zh.reordered": -1.3079,
"zh.stray": -5.0, "zh.stray": -5.0,
"zh.toned_invalid": -5.0, "zh.toned_invalid": -5.0,
"zh.toned_valid_lex": -0.9823, "zh.toned_valid_lex": -0.9018,
"zh.toned_valid_nolex": -1.8742, "zh.toned_valid_nolex": -1.6341,
"zh.unconfirmed": -0.7891, "zh.unconfirmed": -0.7952,
"zh.untoned_mid_invalid": -5.0, "zh.untoned_mid_invalid": -5.0,
"zh.untoned_mid_valid": -2.8009 "zh.untoned_mid_valid": -2.7479
} }
} }
+10 -14
View File
@@ -58,9 +58,6 @@ public final class Classifier {
/// Feature weights: the hand-tuned defaults overlaid with trained ones. /// Feature weights: the hand-tuned defaults overlaid with trained ones.
public private(set) var weights: FeatureVector public private(set) var weights: FeatureVector
/// How much the previous segment's mode pulls the current one.
public var contextWeight = 1.5
public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences(), public init(lexicon: Lexicon, english: EnglishModel, learned: LearnedPreferences = LearnedPreferences(),
weights: FeatureVector = [:]) { weights: FeatureVector = [:]) {
self.lexicon = lexicon self.lexicon = lexicon
@@ -69,23 +66,22 @@ public final class Classifier {
self.weights = Features.defaultWeights.merging(weights) { _, new in new } self.weights = Features.defaultWeights.merging(weights) { _, new in new }
} }
/// (Zhuyin features or nil, literal features or nil) for one segment. /// (Zhuyin features or nil, literal features or nil) for one segment. `previous` (mode of
public func segmentFeatures(_ seg: [Character], isLast: Bool, afterOther: Bool, /// the text committed just before) only affects the first segment.
finishing: Bool) -> (FeatureVector?, FeatureVector?) { public func segmentFeatures(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
(Features.zhuyin(seg, lexicon: lexicon, isLast: isLast, afterOther: afterOther, finishing: finishing), previous: InputMode? = nil) -> (FeatureVector?, FeatureVector?) {
Features.literal(seg, english: english, isLast: isLast, afterOther: afterOther, finishing: finishing)) (Features.zhuyin(seg, lexicon: lexicon, isLast: isLast, afterOther: afterOther,
finishing: finishing, previous: previous),
Features.literal(seg, english: english, isLast: isLast, afterOther: afterOther,
finishing: finishing, previous: previous))
} }
private func scores(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool, private func scores(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
previous: InputMode?, whole: Bool) -> [Double] { previous: InputMode?, whole: Bool) -> [Double] {
let (fz, fl) = segmentFeatures(seg, isLast: isLast, afterOther: afterOther, finishing: finishing) let (fz, fl) = segmentFeatures(seg, isLast: isLast, afterOther: afterOther,
finishing: finishing, previous: previous)
var zh = fz.map { Features.dot(weights, $0) } ?? -.infinity var zh = fz.map { Features.dot(weights, $0) } ?? -.infinity
var lit = fl.map { Features.dot(weights, $0) } ?? -.infinity var lit = fl.map { Features.dot(weights, $0) } ?? -.infinity
switch previous {
case .zhuyin?: zh += contextWeight
case .literal?: lit += contextWeight
case nil: break
}
if whole { if whole {
let b = learned.bias(for: String(seg)) let b = learned.bias(for: String(seg))
if b > 0 { zh += b } else { lit -= b } if b > 0 { zh += b } else { lit -= b }
+17 -4
View File
@@ -42,6 +42,9 @@ public enum Features {
"lit.others": -1.5, "lit.others": -1.5,
// Segmentation // Segmentation
"seg.switch": -3.0, "seg.switch": -3.0,
// Context: the first segment has the same mode as the previously committed text
"ctx.zh": 1.5,
"ctx.lit": 1.5,
] ]
/// Reads {"weights": {...}} from a JSON file; empty if missing or invalid. /// Reads {"weights": {...}} from a JSON file; empty if missing or invalid.
@@ -58,7 +61,8 @@ public enum Features {
/// Features of `keys` read as Zhuyin, or nil if impossible. /// Features of `keys` read as Zhuyin, or nil if impossible.
public static func zhuyin(_ keys: [Character], lexicon: Lexicon, isLast: Bool = true, public static func zhuyin(_ keys: [Character], lexicon: Lexicon, isLast: Bool = true,
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? { afterOther: Bool = false, finishing: Bool = false,
previous: InputMode? = nil) -> FeatureVector? {
let parse = ZhuyinParser.parse(keys) let parse = ZhuyinParser.parse(keys)
// A Zhuyin segment followed by another must end with a tone. // A Zhuyin segment followed by another must end with a tone.
if !isLast && parse.pending != nil { return nil } if !isLast && parse.pending != nil { return nil }
@@ -107,18 +111,27 @@ public enum Features {
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) { if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
add("zh.unconfirmed") add("zh.unconfirmed")
} }
if afterOther { add("seg.switch") } if afterOther {
add("seg.switch")
} else if previous == .zhuyin {
add("ctx.zh")
}
return f return f
} }
/// Features of `keys` output as typed, or nil if impossible. /// Features of `keys` output as typed, or nil if impossible.
public static func literal(_ keys: [Character], english: EnglishModel, isLast: Bool = true, public static func literal(_ keys: [Character], english: EnglishModel, isLast: Bool = true,
afterOther: Bool = false, finishing: Bool = false) -> FeatureVector? { afterOther: Bool = false, finishing: Bool = false,
previous: InputMode? = nil) -> FeatureVector? {
if keys.isEmpty || keys.contains(" ") { return nil } if keys.isEmpty || keys.contains(" ") { return nil }
let complete = !isLast || finishing let complete = !isLast || finishing
var f: FeatureVector = ["lit.bias": 1] var f: FeatureVector = ["lit.bias": 1]
func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v } func add(_ k: String, _ v: Double = 1) { f[k, default: 0] += v }
if afterOther { add("seg.switch") } if afterOther {
add("seg.switch")
} else if previous == .literal {
add("ctx.lit")
}
if keys.contains(where: { $0.isUppercase }) { add("lit.upper") } if keys.contains(where: { $0.isUppercase }) { add("lit.upper") }
let letters = String(keys.filter { $0.isLetter }) let letters = String(keys.filter { $0.isLetter })
@@ -265,10 +265,13 @@ final class ParityTests: XCTestCase {
for fx in fixtures { for fx in fixtures {
let keys = Array(fx["keys"] as! String) let keys = Array(fx["keys"] as! String)
let expected = (fx["segments"] as! [[Any]]).map { "\($0[0])-\($0[1]):\($0[2])" } let expected = (fx["segments"] as! [[Any]]).map { "\($0[0])-\($0[1]):\($0[2])" }
let got = c.segment(keys, finishing: fx["finishing"] as! Bool).map { let previous = (fx["previous"] as? String).flatMap(InputMode.init(rawValue:))
let got = c.segment(keys, finishing: fx["finishing"] as! Bool, previous: previous).map {
"\($0.range.lowerBound)-\($0.range.upperBound):\($0.mode.rawValue)" "\($0.range.lowerBound)-\($0.range.upperBound):\($0.mode.rawValue)"
} }
if got != expected { mismatches.append("\(String(keys)) finishing=\(fx["finishing"]!): \(got) vs \(expected)") } if got != expected {
mismatches.append("\(String(keys)) finishing=\(fx["finishing"]!) previous=\(String(describing: previous)): \(got) vs \(expected)")
}
} }
XCTAssertGreaterThan(fixtures.count, 300) XCTAssertGreaterThan(fixtures.count, 300)
XCTAssertTrue(mismatches.isEmpty, "\(mismatches.count) mismatches:\n" + mismatches.prefix(10).joined(separator: "\n")) XCTAssertTrue(mismatches.isEmpty, "\(mismatches.count) mismatches:\n" + mismatches.prefix(10).joined(separator: "\n"))
@@ -408,3 +411,16 @@ final class CursorTests: XCTestCase {
XCTAssertTrue(e.isShowingCandidates) XCTAssertTrue(e.isShowingCandidates)
} }
} }
final class FirstToneAfterEnglishTests: XCTestCase {
/// After an English word, a first-tone syllable + space must stay Chinese (機, 吃, 書…).
func testFirstToneAfterEnglish() {
for keys in ["ru", "t", "gj", "vup", "tj", "el"] {
let e = CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights))
var out = ""
for c in "hello " + keys + " " { if let t = e.handle(.character(c)).commit { out += t } }
XCTAssertEqual(out, "hello ", keys)
XCTAssertEqual(e.markedText.count, 1, keys) // still composing one Chinese character
}
}
}
+6 -6
View File
@@ -23,14 +23,14 @@ def main():
clf = core.Classifier(lexicon, english, weights=core.Classifier.load_weights(WEIGHTS_PATH)) clf = core.Classifier(lexicon, english, weights=core.Classifier.load_weights(WEIGHTS_PATH))
gen = Generator(os.path.join(DATA, "lexicon.txt"), os.path.join(DATA, "english-common.txt"), gen = Generator(os.path.join(DATA, "lexicon.txt"), os.path.join(DATA, "english-common.txt"),
os.path.join(DATA, "english-dict.txt"), seed=7) os.path.join(DATA, "english-dict.txt"), seed=7)
cases = [(k, f) for k in HAND_PICKED for f in (False, True)] cases = [(k, f, p) for k in HAND_PICKED for f in (False, True) for p in (None, "literal")]
while len(cases) < 400: while len(cases) < 500:
ex = gen.example() ex = gen.example()
if len(ex["keys"]) <= 28: if len(ex["keys"]) <= 28:
cases.append((ex["keys"], ex["finishing"])) cases.append((ex["keys"], ex["finishing"], ex["previous"]))
cases.append((gen.prefix_of(ex)["keys"], False)) cases.append((gen.prefix_of(ex)["keys"], False, ex["previous"]))
fixtures = [{"keys": k, "finishing": f, fixtures = [{"keys": k, "finishing": f, "previous": p,
"segments": [[s, e, m] for s, e, m in clf.segment(list(k), f)]} for k, f in cases] "segments": [[s, e, m] for s, e, m in clf.segment(list(k), f, p)]} for k, f, p in cases]
with open(OUT, "w", encoding="utf-8") as fh: with open(OUT, "w", encoding="utf-8") as fh:
json.dump(fixtures, fh, ensure_ascii=False, indent=0) json.dump(fixtures, fh, ensure_ascii=False, indent=0)
print("wrote %d fixtures to %s" % (len(fixtures), OUT)) print("wrote %d fixtures to %s" % (len(fixtures), OUT))
+5114 -3374
View File
File diff suppressed because it is too large Load Diff
+17 -2
View File
@@ -102,16 +102,31 @@ class Generator:
labels = "".join(m * len(p) for m, p in parts) labels = "".join(m * len(p) for m, p in parts)
if parts[-1][0] == "l": if parts[-1][0] == "l":
finishing = True # English / numbers end with a space finishing = True # English / numbers end with a space
elif keys[-1] == " " and self.rng.random() < 0.6:
# The moment the space (first tone) is pressed: the buffer has no space yet and
# the input method decides "first tone" vs "English word + space".
keys, labels, finishing = keys[:-1], labels[:-1], True
kind += "/tone1"
else: else:
# Chinese ends with Enter, or space after a finished (toned) syllable. # Chinese ends with Enter, or space after a finished (toned) syllable.
finishing = keys[-1] != " " and self.rng.random() < 0.5 finishing = keys[-1] != " " and self.rng.random() < 0.5
return {"keys": keys, "labels": labels, "finishing": finishing, "kind": kind} return {"keys": keys, "labels": labels, "finishing": finishing, "kind": kind,
"previous": self.previous_mode(labels[0])}
def previous_mode(self, first_label):
"""Mode of the text committed just before: none, mostly the same, sometimes the other."""
r = self.rng.random()
if r < 0.35:
return None
same = core.ZHUYIN if first_label == "z" else core.LITERAL
other = core.LITERAL if first_label == "z" else core.ZHUYIN
return same if r < 0.80 else other
def prefix_of(self, ex): def prefix_of(self, ex):
"""The buffer while still typing: a random cut, not finishing.""" """The buffer while still typing: a random cut, not finishing."""
cut = self.rng.randint(1, len(ex["keys"]) - 1) if len(ex["keys"]) > 1 else 1 cut = self.rng.randint(1, len(ex["keys"]) - 1) if len(ex["keys"]) > 1 else 1
return {"keys": ex["keys"][:cut], "labels": ex["labels"][:cut], "finishing": False, return {"keys": ex["keys"][:cut], "labels": ex["labels"][:cut], "finishing": False,
"kind": ex["kind"] + "/prefix"} "kind": ex["kind"] + "/prefix", "previous": ex["previous"]}
def _cumulative(ws): def _cumulative(ws):
+19 -10
View File
@@ -45,12 +45,12 @@ def labels_of(segs, n):
return "".join(lab) return "".join(lab)
def seg_features(clf, keys, segs, finishing): def seg_features(clf, keys, segs, finishing, previous):
"""Summed features of a segmentation, or None if any segment is impossible.""" """Summed features of a segmentation, or None if any segment is impossible."""
total = defaultdict(float) total = defaultdict(float)
n = len(keys) n = len(keys)
for idx, (s, e, m) in enumerate(segs): for idx, (s, e, m) in enumerate(segs):
fz, fl = clf.segment_features(keys[s:e], e == n, idx > 0, finishing) fz, fl = clf.segment_features(keys[s:e], e == n, idx > 0, finishing, previous if idx == 0 else None)
f = fz if m == core.ZHUYIN else fl f = fz if m == core.ZHUYIN else fl
if f is None: if f is None:
return None return None
@@ -62,12 +62,13 @@ def seg_features(clf, keys, segs, finishing):
def evaluate(clf, weights, data): def evaluate(clf, weights, data):
by_kind = defaultdict(lambda: [0, 0]) by_kind = defaultdict(lambda: [0, 0])
for ex in data: for ex in data:
segs = clf.segment(list(ex["keys"]), ex["finishing"], None, weights) segs = clf.segment(list(ex["keys"]), ex["finishing"], ex["previous"], weights)
ok = labels_of(segs, len(ex["keys"])) == ex["labels"] ok = labels_of(segs, len(ex["keys"])) == ex["labels"]
bucket = by_kind["prefix" if ex["kind"].endswith("/prefix") else ex["kind"]] kind = "prefix" if ex["kind"].endswith("/prefix") else ex["kind"]
bucket[0] += ok for k in (kind, "after-other") if _after_other(ex) else (kind,):
bucket[1] += 1 by_kind[k][0] += ok
commit = [v for k, v in by_kind.items() if k != "prefix"] by_kind[k][1] += 1
commit = [v for k, v in by_kind.items() if k not in ("prefix", "after-other")]
summary = { summary = {
"commit": sum(c for c, _ in commit) / max(1, sum(t for _, t in commit)), "commit": sum(c for c, _ in commit) / max(1, sum(t for _, t in commit)),
"prefix": by_kind["prefix"][0] / max(1, by_kind["prefix"][1]), "prefix": by_kind["prefix"][0] / max(1, by_kind["prefix"][1]),
@@ -76,6 +77,14 @@ def evaluate(clf, weights, data):
return summary return summary
def _after_other(ex):
"""Committed buffer whose first segment differs from the text typed just before."""
if ex["kind"].endswith("/prefix") or ex["previous"] is None:
return False
first = core.ZHUYIN if ex["labels"][0] == "z" else core.LITERAL
return ex["previous"] != first
def show(name, s): def show(name, s):
kinds = " ".join("%s %.1f%%" % (k, 100 * v) for k, v in s["by_kind"].items()) kinds = " ".join("%s %.1f%%" % (k, 100 * v) for k, v in s["by_kind"].items())
print("%-9s commit %.2f%% typing %.2f%% [%s]" % (name, 100 * s["commit"], 100 * s["prefix"], kinds)) print("%-9s commit %.2f%% typing %.2f%% [%s]" % (name, 100 * s["commit"], 100 * s["prefix"], kinds))
@@ -92,11 +101,11 @@ def train(clf, data, epochs, lr, seed):
t0 = time.time() t0 = time.time()
for ex in data: for ex in data:
keys = list(ex["keys"]) keys = list(ex["keys"])
pred = clf.segment(keys, ex["finishing"], None, w) pred = clf.segment(keys, ex["finishing"], ex["previous"], w)
if labels_of(pred, len(keys)) != ex["labels"]: if labels_of(pred, len(keys)) != ex["labels"]:
gold = gold_segments(ex["labels"]) gold = gold_segments(ex["labels"])
fg = seg_features(clf, keys, gold, ex["finishing"]) fg = seg_features(clf, keys, gold, ex["finishing"], ex["previous"])
fp = seg_features(clf, keys, pred, ex["finishing"]) fp = seg_features(clf, keys, pred, ex["finishing"], ex["previous"])
if fg is not None and fp is not None: if fg is not None and fp is not None:
mistakes += 1 mistakes += 1
scale = lr * (0.5 if ex["kind"].endswith("/prefix") else 1.0) scale = lr * (0.5 if ex["kind"].endswith("/prefix") else 1.0)
+6 -10
View File
@@ -30,7 +30,6 @@ class LearnedPreferences:
class Classifier: class Classifier:
context_weight = 1.5
def __init__(self, lexicon, english, learned=None, weights=None): def __init__(self, lexicon, english, learned=None, weights=None):
self.lexicon = lexicon self.lexicon = lexicon
@@ -49,20 +48,17 @@ class Classifier:
except (OSError, ValueError): except (OSError, ValueError):
return {} return {}
def segment_features(self, seg, is_last, after_other, finishing): def segment_features(self, seg, is_last, after_other, finishing, previous=None):
"""(zhuyin features or None, literal features or None) for one segment.""" """(zhuyin features or None, literal features or None) for one segment.
`previous` (mode of the text committed just before) only affects the first segment."""
seg = list(seg) seg = list(seg)
return (zhuyin_features(seg, self.lexicon, is_last, after_other, finishing), return (zhuyin_features(seg, self.lexicon, is_last, after_other, finishing, previous),
literal_features(seg, self.english, is_last, after_other, finishing)) literal_features(seg, self.english, is_last, after_other, finishing, previous))
def _scores(self, seg, is_last, after_other, finishing, previous, whole, weights): def _scores(self, seg, is_last, after_other, finishing, previous, whole, weights):
fz, fl = self.segment_features(seg, is_last, after_other, finishing) fz, fl = self.segment_features(seg, is_last, after_other, finishing, previous)
zh = dot(weights, fz) if fz is not None else NEG_INF zh = dot(weights, fz) if fz is not None else NEG_INF
lit = dot(weights, fl) if fl is not None else NEG_INF lit = dot(weights, fl) if fl is not None else NEG_INF
if previous == ZHUYIN:
zh += self.context_weight
elif previous == LITERAL:
lit += self.context_weight
if whole: if whole:
b = self.learned.bias("".join(seg)) b = self.learned.bias("".join(seg))
if b > 0: if b > 0:
+9 -2
View File
@@ -41,6 +41,9 @@ DEFAULT_WEIGHTS = {
"lit.others": -1.5, "lit.others": -1.5,
# Segmentation # Segmentation
"seg.switch": -3.0, "seg.switch": -3.0,
# Context: the first segment has the same mode as the previously committed text
"ctx.zh": 1.5,
"ctx.lit": 1.5,
} }
@@ -52,7 +55,7 @@ def _add(f, k, v=1.0):
f[k] = f.get(k, 0.0) + v f[k] = f.get(k, 0.0) + v
def zhuyin_features(keys, lexicon, is_last=True, after_other=False, finishing=False): def zhuyin_features(keys, lexicon, is_last=True, after_other=False, finishing=False, previous=None):
"""Features of `keys` read as Zhuyin, or None if impossible.""" """Features of `keys` read as Zhuyin, or None if impossible."""
parse = bopomofo.parse(keys) parse = bopomofo.parse(keys)
if not is_last and parse.pending is not None: if not is_last and parse.pending is not None:
@@ -99,10 +102,12 @@ def zhuyin_features(keys, lexicon, is_last=True, after_other=False, finishing=Fa
_add(f, "zh.unconfirmed") _add(f, "zh.unconfirmed")
if after_other: if after_other:
_add(f, "seg.switch") _add(f, "seg.switch")
elif previous == "zhuyin":
_add(f, "ctx.zh")
return f return f
def literal_features(keys, english, is_last=True, after_other=False, finishing=False): def literal_features(keys, english, is_last=True, after_other=False, finishing=False, previous=None):
"""Features of `keys` output as typed, or None if impossible.""" """Features of `keys` output as typed, or None if impossible."""
if not keys or " " in keys: if not keys or " " in keys:
return None return None
@@ -110,6 +115,8 @@ def literal_features(keys, english, is_last=True, after_other=False, finishing=F
f = {"lit.bias": 1.0} f = {"lit.bias": 1.0}
if after_other: if after_other:
_add(f, "seg.switch") _add(f, "seg.switch")
elif previous == "literal":
_add(f, "ctx.lit")
if any(c.isupper() for c in keys): if any(c.isupper() for c in keys):
_add(f, "lit.upper") _add(f, "lit.upper")
letters = "".join(c for c in keys if c.isalpha()) letters = "".join(c for c in keys if c.isalpha())
+8
View File
@@ -271,5 +271,13 @@ class CursorTests(unittest.TestCase):
self.assertTrue(e.showing_candidates) self.assertTrue(e.showing_candidates)
class FirstToneAfterEnglishTests(unittest.TestCase):
def test_first_tone_after_english(self):
for keys in ["ru", "t", "gj", "vup", "tj", "el"]:
e = make_engine()
self.assertEqual(type_keys("hello " + keys + " ", e), "hello ", keys)
self.assertEqual(len(e.marked_text), 1, keys)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()