Files
AutoZhuyin/windows/autozhuyin/core/english.py
T
ays953andClaude Opus 5.5 e0eecc1eb5 Add Windows version on PIME
Port the core (Zhuyin parser, classifier, converter, composition engine) to
Python as a PIME input method, with the same test cases as the Swift version
plus wrapper tests against stand-in PIME modules. scripts/package-windows.sh
builds a zip with install/uninstall scripts and a README for testers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 14:31:42 +08:00

87 lines
3.0 KiB
Python

"""English-likeness scoring. Port of EnglishModel.swift.
Windows has no /usr/share/dict/words, so the dictionary ships in data/english-dict.txt,
and the letter-bigram table is precomputed into data/english-bigram.json (training it
in Python at every start would take a couple of seconds).
"""
import bisect
import json
import math
import os
def _read_words(path):
if not path or not os.path.exists(path):
return []
with open(path, encoding="utf-8") as f:
return [w.strip() for w in f if w.strip() and not w.startswith("#")]
class EnglishModel:
def __init__(self, common_words, dictionary_words=(), bigram=None):
self.common = {w.lower() for w in common_words if w}
self.dictionary = {w.lower() for w in dictionary_words if len(w) > 2}
self.sorted_common = sorted(self.common)
self.sorted_words = sorted(self.common | self.dictionary)
if bigram is None:
bigram = train_bigram(self.sorted_words, self.common)
self.bigram = bigram["table"]
self.unseen = bigram["unseen"]
@classmethod
def from_files(cls, common_path, dict_path=None, bigram_path=None):
bigram = None
if bigram_path and os.path.exists(bigram_path):
with open(bigram_path, encoding="utf-8") as f:
bigram = json.load(f)
return cls(_read_words(common_path), _read_words(dict_path), bigram)
def is_common(self, w):
return w.lower() in self.common
def score(self, word, complete):
w = word.lower()
n = float(len(w))
if not w:
return 0.0
if w in self.common:
return 3 + n
if w in self.dictionary:
return 1 + 0.6 * n
if not complete:
if _has_prefix(w, self.sorted_common):
return 0.4 * n
if _has_prefix(w, self.sorted_words):
return 0.2 * n
plaus = self.plausibility(w)
return min(0.5 * n, max(-2 * n, n * (plaus + 4.2)))
def plausibility(self, w):
chars = ["^"] + list(w) + ["$"]
total = sum(self.bigram.get(chars[i - 1] + chars[i], self.unseen) for i in range(1, len(chars)))
return total / (len(chars) - 1)
def _has_prefix(p, words):
"""True if some word strictly longer than p starts with p."""
lo = bisect.bisect_left(words, p)
if lo < len(words) and words[lo] == p:
lo += 1
return lo < len(words) and words[lo].startswith(p)
def train_bigram(words, common):
pair, first = {}, {}
for w in words:
if not (w.isascii() and w.isalpha()):
continue
weight = 5.0 if w in common else 1.0
chars = ["^"] + list(w) + ["$"]
for i in range(1, len(chars)):
k = chars[i - 1] + chars[i]
pair[k] = pair.get(k, 0.0) + weight
first[chars[i - 1]] = first.get(chars[i - 1], 0.0) + weight
table = {k: math.log2((c + 1) / (first.get(k[0], 0.0) + 27)) for k, c in pair.items()}
unseen = math.log2(1.0 / (max(first.values(), default=1000) + 27))
return {"table": table, "unseen": unseen}