Python port of the reranker: core/lm.py runs the same CKIP gpt2-tiny-chinese model with numpy (batched per candidate length, scores match the Swift and reference implementations), Converter.kbest, core/reranker.py (modes, time budget, pause after overruns, cache), core/system_monitor.py (Windows: CPU usage and battery saver; Linux: load average, thermal zones, low-power profile and low battery) and core/ai.py (settings.json, background model loading, status line). Windows: numpy wheels for PIME's 32-bit Python 3.8 (PIME 1.3) and 3.15 (PIME 2) are bundled under autozhuyin/vendor and picked by Python version; a settings menu on the taskbar IME icon and the language bar (left click on the icon toggles English mode). Linux: install.sh uses the system numpy or downloads the matching wheel from PyPI (tools/fetch_numpy.py, SHA-256 checked); an IBus settings menu. Without numpy or the model the input method works as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
45 lines
1.9 KiB
Python
Executable File
45 lines
1.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Fills windows/autozhuyin/data/ from the shared Mac resources.
|
|
|
|
- lexicon.txt, english-common.txt, classifier-weights.json: copied from Resources/
|
|
- english-dict.txt: filtered from a system word list (default /usr/share/dict/words,
|
|
Webster's 2nd edition, public domain), since Windows has no such file
|
|
- english-bigram.json: precomputed letter-bigram table
|
|
- lm.bin, lm-vocab.txt: the "AI 選字" language model, copied from build/lm/ if present
|
|
|
|
Usage: build_data.py [word-list]
|
|
"""
|
|
import json
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
WIN = ROOT / "windows"
|
|
sys.path.insert(0, str(WIN))
|
|
|
|
from autozhuyin.core.english import train_bigram # noqa: E402
|
|
|
|
data = WIN / "autozhuyin" / "data"
|
|
data.mkdir(parents=True, exist_ok=True)
|
|
for name in ("lexicon.txt", "english-common.txt", "classifier-weights.json"):
|
|
shutil.copy(ROOT / "Resources" / name, data / name)
|
|
|
|
word_list = Path(sys.argv[1] if len(sys.argv) > 1 else "/usr/share/dict/words")
|
|
words = sorted({w.strip().lower() for w in word_list.read_text(encoding="utf-8").split()
|
|
if len(w.strip()) > 2 and w.strip().isascii() and w.strip().isalpha()})
|
|
(data / "english-dict.txt").write_text("\n".join(words) + "\n", encoding="utf-8")
|
|
|
|
common = {w.strip().lower() for w in (data / "english-common.txt").read_text(encoding="utf-8").splitlines()
|
|
if w.strip() and not w.startswith("#")}
|
|
bigram = train_bigram(sorted(common | set(words)), common)
|
|
(data / "english-bigram.json").write_text(json.dumps(bigram), encoding="utf-8")
|
|
|
|
# "AI 選字" language model (scripts/fetch-lm.sh makes build/lm/); optional.
|
|
lm_dir = ROOT / "build" / "lm"
|
|
for name in ("lm.bin", "lm-vocab.txt"):
|
|
if (lm_dir / name).exists():
|
|
shutil.copy(lm_dir / name, data / name)
|
|
|
|
print("data ready: %d dictionary words, %d bigrams" % (len(words), len(bigram["table"])))
|