diff --git a/.gitignore b/.gitignore index 243e1ad..bcf76b7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ build/ .swiftpm/ *.xcodeproj/xcuserdata/ .DS_Store +windows/autozhuyin/data/ +__pycache__/ +*.pyc diff --git a/README.md b/README.md index e75fce0..f8a98c2 100644 --- a/README.md +++ b/README.md @@ -83,3 +83,15 @@ Tests/ 單元測試 - [ ] 用 Core ML 小模型取代手調分數,參考整句語境 - [ ] 使用者詞頻學習(選過的候選字優先) - [ ] 偏好設定視窗 + +## Windows 版 + +`windows/` 是給 Windows 用的版本,建立在開源輸入法框架 [PIME](https://github.com/EasyIME/PIME) 上。 +核心邏輯用 Python 重寫(`windows/autozhuyin/core/`),行為和測試案例都跟 Mac 版一樣;詞庫共用 `Resources/`。 + +```bash +scripts/package-windows.sh # 產生 build/AutoZhuyin-Windows-<版本>.zip(會先跑測試) +``` + +zip 裡附有 `install.bat`、`uninstall.bat` 和給測試者看的 `README.txt`。使用者需要先安裝 PIME。 +修改核心邏輯時,Swift(`Sources/AutoZhuyinCore/`)和 Python(`windows/autozhuyin/core/`)兩邊都要改。 diff --git a/scripts/package-windows.sh b/scripts/package-windows.sh new file mode 100755 index 0000000..93fdb71 --- /dev/null +++ b/scripts/package-windows.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Builds build/AutoZhuyin-Windows-.zip: the PIME input method folder plus +# install/uninstall scripts and a README for testers. +set -euo pipefail +cd "$(dirname "$0")/.." + +VERSION="$(python3 -I -c 'import json,sys; print(json.load(open(sys.argv[1]))["version"])' windows/autozhuyin/ime.json)" +NAME="AutoZhuyin-Windows-$VERSION" +OUT="build/$NAME" + +python3 -I windows/tools/build_data.py +(cd windows && python3 -m unittest discover -q tests) + +# Icon: render the Mac icon to PNG and wrap it in an .ico (PNG-in-ICO, Vista+). +if [ ! -f build/icon.tiff ]; then + DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" xcrun swift scripts/make-icon.swift build/icon.tiff +fi +sips -s format png -z 32 32 build/icon.tiff --out build/icon-32.png >/dev/null +python3 -I windows/tools/png_to_ico.py build/icon-32.png windows/autozhuyin/icon.ico + +rm -rf "$OUT" "$OUT.zip" +mkdir -p "$OUT" +rsync -a --exclude __pycache__ --exclude '*.pyc' windows/autozhuyin "$OUT/" + +# Windows tools expect CRLF line endings in .bat files. +for f in install.bat uninstall.bat README.txt; do + sed 's/$/\r/' "windows/installer/$f" > "$OUT/$f" +done + +(cd build && zip -qr -X "$NAME.zip" "$NAME") +echo "Built build/$NAME.zip" diff --git a/windows/autozhuyin/__init__.py b/windows/autozhuyin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/windows/autozhuyin/autozhuyin_ime.py b/windows/autozhuyin/autozhuyin_ime.py new file mode 100644 index 0000000..ede1619 --- /dev/null +++ b/windows/autozhuyin/autozhuyin_ime.py @@ -0,0 +1,137 @@ +"""PIME text service for AutoZhuyin (自動注音). + +PIME imports this as `input_methods.autozhuyin.autozhuyin_ime` and creates one +AutoZhuyinTextService per client. All the logic lives in .core (shared with the +tests); this file only translates PIME key events and replies. +""" +import json +import os + +from keycodes import * # noqa: F401,F403 (VK_* constants, provided by PIME) +from textService import TextService + +from .core.classifier import Classifier, LearnedPreferences +from .core.engine import BACKSPACE, DOWN, ENTER, ESCAPE, LEFT, RIGHT, TAB, UP, CompositionEngine +from .core.english import EnglishModel +from .core.lexicon import Lexicon + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_DATA = os.path.join(_HERE, "data") +_USER_DIR = os.path.join(os.environ.get("APPDATA", _HERE), "PIME", "autozhuyin") +_LEARNED_FILE = os.path.join(_USER_DIR, "learned.json") +_USER_PHRASES = os.path.join(_USER_DIR, "user-phrases.txt") + +_classifier = None + + +def _save_learned(weights): + try: + os.makedirs(_USER_DIR, exist_ok=True) + with open(_LEARNED_FILE, "w", encoding="utf-8") as f: + json.dump(weights, f, ensure_ascii=False) + except OSError: + pass + + +def _shared_classifier(): + """Loads the lexicon and English model once per PIME server process.""" + global _classifier + if _classifier is None: + lexicon = Lexicon.from_file(os.path.join(_DATA, "lexicon.txt")) + if os.path.exists(_USER_PHRASES): + with open(_USER_PHRASES, encoding="utf-8") as f: + lexicon.load(f) + english = EnglishModel.from_files( + os.path.join(_DATA, "english-common.txt"), + os.path.join(_DATA, "english-dict.txt"), + os.path.join(_DATA, "english-bigram.json"), + ) + weights = {} + try: + with open(_LEARNED_FILE, encoding="utf-8") as f: + weights = json.load(f) + except (OSError, ValueError): + pass + _classifier = Classifier(lexicon, english, LearnedPreferences(weights, _save_learned)) + return _classifier + + +_SPECIAL = { + VK_BACK: BACKSPACE, VK_RETURN: ENTER, VK_ESCAPE: ESCAPE, VK_TAB: TAB, # noqa: F405 + VK_UP: UP, VK_DOWN: DOWN, VK_LEFT: LEFT, VK_RIGHT: RIGHT, # noqa: F405 +} + + +class AutoZhuyinTextService(TextService): + def __init__(self, client): + TextService.__init__(self, client) + self.engine = CompositionEngine(_shared_classifier()) + + # --- lifecycle --- + + def onActivate(self): + TextService.onActivate(self) + self.setSelKeys("123456789") + self.customizeUI(candFontSize=16, candPerRow=1, candUseCursor=True) + + def onDeactivate(self): + self.engine.reset() + TextService.onDeactivate(self) + + def onCompositionTerminated(self, forced): + # The app ended the composition (focus change, mouse click): the shown text stays. + self.engine.reset() + if self.showCandidates: + self.setShowCandidates(False) + TextService.onCompositionTerminated(self, forced) + + # --- keys --- + + def _key(self, ev): + if ev.keyCode in _SPECIAL: + return _SPECIAL[ev.keyCode] + if ev.keyCode == VK_SPACE: # noqa: F405 + return " " + if not ev.isPrintableChar(): + return None + c = chr(ev.charCode) + # Caps Lock without Shift: keep Zhuyin working by using lowercase. + if ev.isKeyToggled(VK_CAPITAL) and not ev.isKeyDown(VK_SHIFT): # noqa: F405 + c = c.lower() + return c + + def _busy(self): + return bool(self.engine.keys or self.engine.candidates) + + def filterKeyDown(self, ev): + # Shortcuts (Ctrl/Alt) go to the app. + if ev.isKeyDown(VK_CONTROL) or ev.isKeyDown(VK_MENU): # noqa: F405 + return False + key = self._key(ev) + if key is None: + return self._busy() # swallow other keys (Delete, Home…) while composing + return self.engine.wants(key) + + def onKeyDown(self, ev): + key = self._key(ev) + if key is None: + return self._busy() + handled, commit = self.engine.handle(key) + self._update(commit) + return handled + + # --- replies --- + + def _update(self, commit): + if commit: + self.setCommitString(commit) + marked = self.engine.marked_text + self.setCompositionString(marked) + self.setCompositionCursor(len(marked)) + if self.engine.showing_candidates: + page, cursor = self.engine.page() + self.setCandidateList(page) + self.setCandidateCursor(cursor) + self.setShowCandidates(True) + elif self.showCandidates: + self.setShowCandidates(False) diff --git a/windows/autozhuyin/core/__init__.py b/windows/autozhuyin/core/__init__.py new file mode 100644 index 0000000..c9459c4 --- /dev/null +++ b/windows/autozhuyin/core/__init__.py @@ -0,0 +1 @@ +"""AutoZhuyin core logic (no PIME / Windows dependencies).""" diff --git a/windows/autozhuyin/core/bopomofo.py b/windows/autozhuyin/core/bopomofo.py new file mode 100644 index 0000000..8accf53 --- /dev/null +++ b/windows/autozhuyin/core/bopomofo.py @@ -0,0 +1,208 @@ +"""Dachen (大千) Zhuyin layout, syllable table and keystroke parser. + +Python port of Sources/AutoZhuyinCore/{Bopomofo,SyllableTable,ZhuyinParser}.swift. +""" + +INITIAL, MEDIAL, FINAL = 0, 1, 2 + +SYMBOLS = { + "1": "ㄅ", "q": "ㄆ", "a": "ㄇ", "z": "ㄈ", + "2": "ㄉ", "w": "ㄊ", "s": "ㄋ", "x": "ㄌ", + "e": "ㄍ", "d": "ㄎ", "c": "ㄏ", + "r": "ㄐ", "f": "ㄑ", "v": "ㄒ", + "5": "ㄓ", "t": "ㄔ", "g": "ㄕ", "b": "ㄖ", + "y": "ㄗ", "h": "ㄘ", "n": "ㄙ", + "u": "ㄧ", "j": "ㄨ", "m": "ㄩ", + "8": "ㄚ", "i": "ㄛ", "k": "ㄜ", ",": "ㄝ", + "9": "ㄞ", "o": "ㄟ", "l": "ㄠ", ".": "ㄡ", + "0": "ㄢ", "p": "ㄣ", ";": "ㄤ", "/": "ㄥ", "-": "ㄦ", +} + +# Tone number by key; first tone is the space bar. +TONES = {" ": 1, "6": 2, "3": 3, "4": 4, "7": 5} +TONE_MARKS = {1: "", 2: "ˊ", 3: "ˇ", 4: "ˋ", 5: "˙"} + +_INITIALS = set("ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙ") +_MEDIALS = set("ㄧㄨㄩ") + + +def is_zhuyin_key(c): + return c in SYMBOLS or c in TONES + + +def slot_of(symbol): + if symbol in _INITIALS: + return INITIAL + if symbol in _MEDIALS: + return MEDIAL + return FINAL + + +class Syllable: + __slots__ = ("slots", "tone") + + def __init__(self, slots=None, tone=None): + self.slots = list(slots) if slots else [None, None, None] + self.tone = tone + + def copy(self): + return Syllable(self.slots, self.tone) + + @property + def is_empty(self): + return not any(self.slots) + + @property + def toneless(self): + return "".join(s for s in self.slots if s) + + @property + def reading(self): + return self.toneless + (TONE_MARKS[self.tone] if self.tone else "") + + def __eq__(self, other): + return isinstance(other, Syllable) and self.slots == other.slots and self.tone == other.tone + + def __repr__(self): + return "Syllable(%s)" % self.reading + + +# --- valid syllables --------------------------------------------------------- + +_RAW = """ +ㄚ ㄛ ㄜ ㄝ ㄞ ㄟ ㄠ ㄡ ㄢ ㄣ ㄤ ㄥ ㄦ +ㄧ ㄧㄚ ㄧㄛ ㄧㄝ ㄧㄞ ㄧㄠ ㄧㄡ ㄧㄢ ㄧㄣ ㄧㄤ ㄧㄥ +ㄨ ㄨㄚ ㄨㄛ ㄨㄞ ㄨㄟ ㄨㄢ ㄨㄣ ㄨㄤ ㄨㄥ +ㄩ ㄩㄝ ㄩㄢ ㄩㄣ ㄩㄥ +ㄅㄚ ㄅㄛ ㄅㄞ ㄅㄟ ㄅㄠ ㄅㄢ ㄅㄣ ㄅㄤ ㄅㄥ ㄅㄧ ㄅㄧㄝ ㄅㄧㄠ ㄅㄧㄢ ㄅㄧㄣ ㄅㄧㄥ ㄅㄨ +ㄆㄚ ㄆㄛ ㄆㄞ ㄆㄟ ㄆㄠ ㄆㄡ ㄆㄢ ㄆㄣ ㄆㄤ ㄆㄥ ㄆㄧ ㄆㄧㄝ ㄆㄧㄠ ㄆㄧㄢ ㄆㄧㄣ ㄆㄧㄥ ㄆㄨ +ㄇㄚ ㄇㄛ ㄇㄜ ㄇㄞ ㄇㄟ ㄇㄠ ㄇㄡ ㄇㄢ ㄇㄣ ㄇㄤ ㄇㄥ ㄇㄧ ㄇㄧㄝ ㄇㄧㄠ ㄇㄧㄡ ㄇㄧㄢ ㄇㄧㄣ ㄇㄧㄥ ㄇㄨ +ㄈㄚ ㄈㄛ ㄈㄟ ㄈㄡ ㄈㄢ ㄈㄣ ㄈㄤ ㄈㄥ ㄈㄨ +ㄉㄚ ㄉㄜ ㄉㄞ ㄉㄟ ㄉㄠ ㄉㄡ ㄉㄢ ㄉㄣ ㄉㄤ ㄉㄥ ㄉㄧ ㄉㄧㄚ ㄉㄧㄝ ㄉㄧㄠ ㄉㄧㄡ ㄉㄧㄢ ㄉㄧㄥ ㄉㄨ ㄉㄨㄛ ㄉㄨㄟ ㄉㄨㄢ ㄉㄨㄣ ㄉㄨㄥ +ㄊㄚ ㄊㄜ ㄊㄞ ㄊㄠ ㄊㄡ ㄊㄢ ㄊㄤ ㄊㄥ ㄊㄧ ㄊㄧㄝ ㄊㄧㄠ ㄊㄧㄢ ㄊㄧㄥ ㄊㄨ ㄊㄨㄛ ㄊㄨㄟ ㄊㄨㄢ ㄊㄨㄣ ㄊㄨㄥ +ㄋㄚ ㄋㄜ ㄋㄞ ㄋㄟ ㄋㄠ ㄋㄡ ㄋㄢ ㄋㄣ ㄋㄤ ㄋㄥ ㄋㄧ ㄋㄧㄝ ㄋㄧㄠ ㄋㄧㄡ ㄋㄧㄢ ㄋㄧㄣ ㄋㄧㄤ ㄋㄧㄥ ㄋㄨ ㄋㄨㄛ ㄋㄨㄢ ㄋㄨㄥ ㄋㄩ ㄋㄩㄝ +ㄌㄚ ㄌㄛ ㄌㄜ ㄌㄞ ㄌㄟ ㄌㄠ ㄌㄡ ㄌㄢ ㄌㄤ ㄌㄥ ㄌㄧ ㄌㄧㄚ ㄌㄧㄝ ㄌㄧㄠ ㄌㄧㄡ ㄌㄧㄢ ㄌㄧㄣ ㄌㄧㄤ ㄌㄧㄥ ㄌㄨ ㄌㄨㄛ ㄌㄨㄢ ㄌㄨㄣ ㄌㄨㄥ ㄌㄩ ㄌㄩㄝ ㄌㄩㄢ +ㄍㄚ ㄍㄜ ㄍㄞ ㄍㄟ ㄍㄠ ㄍㄡ ㄍㄢ ㄍㄣ ㄍㄤ ㄍㄥ ㄍㄨ ㄍㄨㄚ ㄍㄨㄛ ㄍㄨㄞ ㄍㄨㄟ ㄍㄨㄢ ㄍㄨㄣ ㄍㄨㄤ ㄍㄨㄥ +ㄎㄚ ㄎㄜ ㄎㄞ ㄎㄟ ㄎㄠ ㄎㄡ ㄎㄢ ㄎㄣ ㄎㄤ ㄎㄥ ㄎㄨ ㄎㄨㄚ ㄎㄨㄛ ㄎㄨㄞ ㄎㄨㄟ ㄎㄨㄢ ㄎㄨㄣ ㄎㄨㄤ ㄎㄨㄥ +ㄏㄚ ㄏㄜ ㄏㄞ ㄏㄟ ㄏㄠ ㄏㄡ ㄏㄢ ㄏㄣ ㄏㄤ ㄏㄥ ㄏㄨ ㄏㄨㄚ ㄏㄨㄛ ㄏㄨㄞ ㄏㄨㄟ ㄏㄨㄢ ㄏㄨㄣ ㄏㄨㄤ ㄏㄨㄥ +ㄐㄧ ㄐㄧㄚ ㄐㄧㄝ ㄐㄧㄠ ㄐㄧㄡ ㄐㄧㄢ ㄐㄧㄣ ㄐㄧㄤ ㄐㄧㄥ ㄐㄩ ㄐㄩㄝ ㄐㄩㄢ ㄐㄩㄣ ㄐㄩㄥ +ㄑㄧ ㄑㄧㄚ ㄑㄧㄝ ㄑㄧㄠ ㄑㄧㄡ ㄑㄧㄢ ㄑㄧㄣ ㄑㄧㄤ ㄑㄧㄥ ㄑㄩ ㄑㄩㄝ ㄑㄩㄢ ㄑㄩㄣ ㄑㄩㄥ +ㄒㄧ ㄒㄧㄚ ㄒㄧㄝ ㄒㄧㄠ ㄒㄧㄡ ㄒㄧㄢ ㄒㄧㄣ ㄒㄧㄤ ㄒㄧㄥ ㄒㄩ ㄒㄩㄝ ㄒㄩㄢ ㄒㄩㄣ ㄒㄩㄥ +ㄓ ㄓㄚ ㄓㄜ ㄓㄞ ㄓㄟ ㄓㄠ ㄓㄡ ㄓㄢ ㄓㄣ ㄓㄤ ㄓㄥ ㄓㄨ ㄓㄨㄚ ㄓㄨㄛ ㄓㄨㄞ ㄓㄨㄟ ㄓㄨㄢ ㄓㄨㄣ ㄓㄨㄤ ㄓㄨㄥ +ㄔ ㄔㄚ ㄔㄜ ㄔㄞ ㄔㄠ ㄔㄡ ㄔㄢ ㄔㄣ ㄔㄤ ㄔㄥ ㄔㄨ ㄔㄨㄚ ㄔㄨㄛ ㄔㄨㄞ ㄔㄨㄟ ㄔㄨㄢ ㄔㄨㄣ ㄔㄨㄤ ㄔㄨㄥ +ㄕ ㄕㄚ ㄕㄜ ㄕㄞ ㄕㄟ ㄕㄠ ㄕㄡ ㄕㄢ ㄕㄣ ㄕㄤ ㄕㄥ ㄕㄨ ㄕㄨㄚ ㄕㄨㄛ ㄕㄨㄞ ㄕㄨㄟ ㄕㄨㄢ ㄕㄨㄣ ㄕㄨㄤ +ㄖ ㄖㄜ ㄖㄠ ㄖㄡ ㄖㄢ ㄖㄣ ㄖㄤ ㄖㄥ ㄖㄨ ㄖㄨㄛ ㄖㄨㄟ ㄖㄨㄢ ㄖㄨㄣ ㄖㄨㄥ +ㄗ ㄗㄚ ㄗㄜ ㄗㄞ ㄗㄟ ㄗㄠ ㄗㄡ ㄗㄢ ㄗㄣ ㄗㄤ ㄗㄥ ㄗㄨ ㄗㄨㄛ ㄗㄨㄟ ㄗㄨㄢ ㄗㄨㄣ ㄗㄨㄥ +ㄘ ㄘㄚ ㄘㄜ ㄘㄞ ㄘㄠ ㄘㄡ ㄘㄢ ㄘㄣ ㄘㄤ ㄘㄥ ㄘㄨ ㄘㄨㄛ ㄘㄨㄟ ㄘㄨㄢ ㄘㄨㄣ ㄘㄨㄥ +ㄙ ㄙㄚ ㄙㄜ ㄙㄞ ㄙㄠ ㄙㄡ ㄙㄢ ㄙㄣ ㄙㄤ ㄙㄥ ㄙㄨ ㄙㄨㄛ ㄙㄨㄟ ㄙㄨㄢ ㄙㄨㄣ ㄙㄨㄥ +""" + +VALID = set(_RAW.split()) +VALID_PREFIXES = {syl[:n] for syl in VALID for n in range(1, len(syl) + 1)} + + +def _patterns(): + out = set() + for syl in VALID: + slots = [None, None, None] + for ch in syl: + slots[slot_of(ch)] = ch + for mask in range(8): + out.add("".join((slots[i] or "_") if mask & (1 << i) else "_" for i in range(3))) + return out + + +_COMPATIBLE = _patterns() + + +def is_valid(syl): + return syl.toneless in VALID + + +def is_valid_prefix(syl): + return syl.is_empty or syl.toneless in VALID_PREFIXES + + +def is_compatible(syl): + """True if some valid syllable has these slots (used for out-of-order typing).""" + return "".join(s or "_" for s in syl.slots) in _COMPATIBLE + + +# --- parser ------------------------------------------------------------------ + +class ParsedSyllable: + __slots__ = ("syllable", "start", "end", "reordered") + + def __init__(self, syllable, start, end, reordered=False): + self.syllable = syllable + self.start = start + self.end = end + self.reordered = reordered + + @property + def toned(self): + return self.syllable.tone is not None + + +class ZhuyinParse: + def __init__(self): + self.syllables = [] + self.stray_tones = 0 + self.foreign_keys = 0 + + @property + def pending(self): + if self.syllables and not self.syllables[-1].toned: + return self.syllables[-1] + return None + + @property + def completed(self): + return self.syllables if self.pending is None else self.syllables[:-1] + + +def parse(keys): + """Splits keystrokes into syllables (Dachen rules, out-of-order symbols merged).""" + result = ZhuyinParse() + current = Syllable() + start = 0 + reordered = False + + def close(end): + nonlocal current, start, reordered + if not current.is_empty: + result.syllables.append(ParsedSyllable(current, start, end, reordered)) + current = Syllable() + start = end + reordered = False + + for i, key in enumerate(keys): + if key in TONES: + if current.is_empty: + result.stray_tones += 1 + start = i + 1 + else: + current.tone = TONES[key] + close(i + 1) + elif key in SYMBOLS: + symbol = SYMBOLS[key] + slot = slot_of(symbol) + later_filled = any(current.slots[s] for s in range(slot + 1, 3)) + if current.slots[slot]: + close(i) + elif later_filled: + merged = current.copy() + merged.slots[slot] = symbol + if is_compatible(merged): + reordered = True + else: + close(i) + if current.is_empty: + start = i + current.slots[slot] = symbol + else: + close(i) + result.foreign_keys += 1 + start = i + 1 + close(len(keys)) + return result diff --git a/windows/autozhuyin/core/classifier.py b/windows/autozhuyin/core/classifier.py new file mode 100644 index 0000000..1ff5d93 --- /dev/null +++ b/windows/autozhuyin/core/classifier.py @@ -0,0 +1,94 @@ +"""Zhuyin-vs-literal classifier with learned corrections. Port of Classifier.swift.""" +from . import bopomofo + +ZHUYIN = "zhuyin" +LITERAL = "literal" + +NEG_INF = float("-inf") + + +class LearnedPreferences: + """Per key-sequence corrections. Positive weight = prefers Zhuyin.""" + + def __init__(self, weights=None, on_change=None): + self.weights = dict(weights or {}) + self._on_change = on_change or (lambda w: None) + + def record(self, keys, mode): + delta = 1 if mode == ZHUYIN else -1 + self.weights[keys] = max(-3, min(3, self.weights.get(keys, 0) + delta)) + self._on_change(self.weights) + + def clear(self): + self.weights = {} + self._on_change(self.weights) + + def bias(self, keys): + return self.weights.get(keys, 0) * 4.0 + + +class Classifier: + zhuyin_prior = 0.5 + context_weight = 1.5 + + def __init__(self, lexicon, english, learned=None): + self.lexicon = lexicon + self.english = english + self.learned = learned or LearnedPreferences() + + def classify(self, keys, finishing=False, previous=None): + """Returns (mode, zhuyin_score, literal_score).""" + keys = list(keys) + zh = self.zhuyin_score(keys + [" "] if finishing else keys) + self.zhuyin_prior + lit = self.literal_score(keys, complete=finishing) + if previous == ZHUYIN: + zh += self.context_weight + elif previous == LITERAL: + lit += self.context_weight + b = self.learned.bias("".join(keys)) + if b > 0: + zh += b + else: + lit -= b + return (ZHUYIN if zh >= lit else LITERAL), zh, lit + + def mode(self, keys, finishing=False, previous=None): + return self.classify(keys, finishing, previous)[0] + + def zhuyin_score(self, keys): + p = bopomofo.parse(keys) + if p.foreign_keys > 0 or not p.syllables: + return NEG_INF + s = -5.0 * p.stray_tones + last = len(p.syllables) - 1 + for i, ps in enumerate(p.syllables): + if ps.reordered: + s -= 0.5 + valid = bopomofo.is_valid(ps.syllable) + if ps.toned: + if valid: + s += 2 + (0.8 if self.lexicon.best_score(ps.syllable) is not None else -1.5) + else: + s -= 5 + elif i == last: + s += 0 if bopomofo.is_valid_prefix(ps.syllable) else -5 + else: + s += -2.5 if valid else -5 + return s + + def literal_score(self, keys, complete): + if not keys or " " in keys: + return NEG_INF + score = 0.0 + if any(c.isupper() for c in keys): + score += 20 + letters = "".join(c for c in keys if c.isalpha()) + digits = sum(1 for c in keys if c.isdigit()) + others = len(keys) - len(letters) - digits + if not letters: + return score + 0.8 * digits - 1.0 * others + score += self.english.score(letters, complete or digits > 0) + score -= 1.5 * digits + apostrophes = keys.count("'") + score -= 1.5 * (others - apostrophes) + return score diff --git a/windows/autozhuyin/core/engine.py b/windows/autozhuyin/core/engine.py new file mode 100644 index 0000000..8163427 --- /dev/null +++ b/windows/autozhuyin/core/engine.py @@ -0,0 +1,215 @@ +"""Input method state machine. Port of CompositionEngine.swift. + +Keys are given as one-character strings, or one of the names in SPECIAL_KEYS. +handle() returns (handled, commit_text_or_None). +""" +import time + +from . import bopomofo +from .classifier import LITERAL, ZHUYIN +from .lexicon import Converter + +BACKSPACE, ENTER, ESCAPE, TAB = "", "", "", "" +UP, DOWN, LEFT, RIGHT = "", "", "", "" +SPECIAL_KEYS = {BACKSPACE, ENTER, ESCAPE, TAB, UP, DOWN, LEFT, RIGHT} + +FULLWIDTH = { + "<": ",", ">": "。", "?": "?", "!": "!", ":": ":", '"': "「", "(": "(", ")": ")", + "{": "『", "}": "』", "[": "「", "]": "」", "~": "~", "_": "——", +} + +PASS = (False, None) +CONSUMED = (True, None) + + +class CompositionEngine: + page_size = 9 + context_window = 20.0 + + def __init__(self, classifier, now=time.monotonic): + self.classifier = classifier + self.converter = Converter(classifier.lexicon) + self.now = now + self.keys = [] + self.forced_mode = None + self.fixed = {} + self.candidates = [] + self.candidate_index = 0 + self._previous_mode = None + self._previous_time = float("-inf") + + # --- derived state --- + + @property + def context(self): + return self._previous_mode if self.now() - self._previous_time < self.context_window else None + + @property + def mode(self): + return self.forced_mode or self.classifier.mode(self.keys, previous=self.context) + + @property + def showing_candidates(self): + return bool(self.candidates) + + @property + def page_start(self): + return (self.candidate_index // self.page_size) * self.page_size + + def _parse(self): + return bopomofo.parse(self.keys) + + def _valid_fixed(self, p): + n = len(p.completed) + return {k: v for k, v in self.fixed.items() if k[1] <= n} + + @property + def marked_text(self): + if not self.keys: + return "" + if self.mode == LITERAL: + return "".join(self.keys) + p = self._parse() + done = self.converter.convert([s.syllable for s in p.completed], self._valid_fixed(p)) + pending = p.pending.syllable.reading if p.pending else "" + return "".join(t for _, t in done) + pending + + def _zhuyin_commit_text(self): + p = self._parse() + segs = self.converter.convert([s.syllable for s in p.syllables], self._valid_fixed(p)) + return "".join(t for _, t in segs) + + # --- key handling --- + + def wants(self, key): + """Whether handle(key) would consume the key, without changing state.""" + if self.candidates: + return True + if key in SPECIAL_KEYS: + return bool(self.keys) + if self.keys: + return True + if key == " ": + return False + if bopomofo.is_zhuyin_key(key) or (key.isascii() and key.isalpha()): + return True + return self.context == ZHUYIN and key in FULLWIDTH + + def handle(self, key): + if self.candidates: + return self._handle_candidate_key(key) + if key == BACKSPACE: + if not self.keys: + return PASS + self.keys.pop() + self.fixed = self._valid_fixed(self._parse()) + if not self.keys: + self.forced_mode = None + self.fixed = {} + return CONSUMED + if key == ENTER: + return (True, self._commit()) if self.keys else PASS + if key == ESCAPE: + if not self.keys: + return PASS + self.reset() + return CONSUMED + if key == TAB: + if not self.keys: + return PASS + self.forced_mode = LITERAL if self.mode == ZHUYIN else ZHUYIN + return CONSUMED + if key == DOWN: + if not self.keys: + return PASS + self._open_candidates() + return CONSUMED + if key in (UP, LEFT, RIGHT): + return CONSUMED if self.keys else PASS + return self._handle_character(key) + + def _handle_character(self, c): + if c == " ": + if not self.keys: + return PASS + decided = self.forced_mode or self.classifier.mode(self.keys, finishing=True, previous=self.context) + if decided == LITERAL: + return True, self._commit(LITERAL) + " " + if self._parse().pending is not None: + self.keys.append(" ") + return CONSUMED + return True, self._commit(ZHUYIN) + + composable = bopomofo.is_zhuyin_key(c) or (c.isascii() and c.isalpha()) or (c == "'" and self.keys) + if composable: + # ", . / ; -" are Zhuyin symbols but act as punctuation after an English word. + if self.keys and c in bopomofo.SYMBOLS and not c.isalnum() and self.forced_mode != ZHUYIN: + with_key = self.forced_mode or self.classifier.mode(self.keys + [c], previous=self.context) + if with_key == LITERAL and not all(k.isdigit() for k in self.keys): + return True, self._commit(LITERAL) + c + self.keys.append(c) + return CONSUMED + + chinese = (self.context == ZHUYIN) if not self.keys else (self.mode == ZHUYIN) + committed = self._commit() if self.keys else "" + punct = FULLWIDTH.get(c, c) if chinese else c + if not committed and punct == c: + return PASS + return True, committed + punct + + # --- candidates --- + + def _open_candidates(self): + p = self._parse() + if self.mode != ZHUYIN or not p.completed: + return + syl = [s.syllable for s in p.completed] + self.candidates = self.converter.candidates(syl, len(syl)) + self.candidate_index = 0 + + def page(self): + """Candidates on the current page and the cursor within it.""" + start = self.page_start + return [t for _, t in self.candidates[start:start + self.page_size]], self.candidate_index - start + + def select_candidate(self, index): + if not 0 <= index < len(self.candidates): + return + rng, text = self.candidates[index] + self.fixed = {k: v for k, v in self.fixed.items() if not (k[0] < rng[1] and rng[0] < k[1])} + self.fixed[rng] = text + self.candidates = [] + + def _handle_candidate_key(self, key): + if len(key) == 1 and key.isdigit() and key != "0": + self.select_candidate(self.page_start + int(key) - 1) + elif key in (" ", ENTER): + self.select_candidate(self.candidate_index) + elif key in (DOWN, RIGHT): + self.candidate_index = min(len(self.candidates) - 1, self.candidate_index + 1) + elif key in (UP, LEFT): + self.candidate_index = max(0, self.candidate_index - 1) + elif key in (ESCAPE, BACKSPACE): + self.candidates = [] + return CONSUMED + + # --- commit --- + + def _commit(self, decided=None): + m = decided or self.mode + text = "".join(self.keys) if m == LITERAL else self._zhuyin_commit_text() + if self.forced_mode: + self.classifier.learned.record("".join(self.keys), self.forced_mode) + self._previous_mode = m + self._previous_time = self.now() + self.reset() + return text + + def commit_all(self): + return self._commit() if self.keys else "" + + def reset(self): + self.keys = [] + self.forced_mode = None + self.fixed = {} + self.candidates = [] diff --git a/windows/autozhuyin/core/english.py b/windows/autozhuyin/core/english.py new file mode 100644 index 0000000..ebdeeee --- /dev/null +++ b/windows/autozhuyin/core/english.py @@ -0,0 +1,86 @@ +"""English-likeness scoring. Port of EnglishModel.swift. + +Windows has no /usr/share/dict/words, so the dictionary ships in data/english-dict.txt, +and the letter-bigram table is precomputed into data/english-bigram.json (training it +in Python at every start would take a couple of seconds). +""" +import bisect +import json +import math +import os + + +def _read_words(path): + if not path or not os.path.exists(path): + return [] + with open(path, encoding="utf-8") as f: + return [w.strip() for w in f if w.strip() and not w.startswith("#")] + + +class EnglishModel: + def __init__(self, common_words, dictionary_words=(), bigram=None): + self.common = {w.lower() for w in common_words if w} + self.dictionary = {w.lower() for w in dictionary_words if len(w) > 2} + self.sorted_common = sorted(self.common) + self.sorted_words = sorted(self.common | self.dictionary) + if bigram is None: + bigram = train_bigram(self.sorted_words, self.common) + self.bigram = bigram["table"] + self.unseen = bigram["unseen"] + + @classmethod + def from_files(cls, common_path, dict_path=None, bigram_path=None): + bigram = None + if bigram_path and os.path.exists(bigram_path): + with open(bigram_path, encoding="utf-8") as f: + bigram = json.load(f) + return cls(_read_words(common_path), _read_words(dict_path), bigram) + + def is_common(self, w): + return w.lower() in self.common + + def score(self, word, complete): + w = word.lower() + n = float(len(w)) + if not w: + return 0.0 + if w in self.common: + return 3 + n + if w in self.dictionary: + return 1 + 0.6 * n + if not complete: + if _has_prefix(w, self.sorted_common): + return 0.4 * n + if _has_prefix(w, self.sorted_words): + return 0.2 * n + plaus = self.plausibility(w) + return min(0.5 * n, max(-2 * n, n * (plaus + 4.2))) + + def plausibility(self, w): + chars = ["^"] + list(w) + ["$"] + total = sum(self.bigram.get(chars[i - 1] + chars[i], self.unseen) for i in range(1, len(chars))) + return total / (len(chars) - 1) + + +def _has_prefix(p, words): + """True if some word strictly longer than p starts with p.""" + lo = bisect.bisect_left(words, p) + if lo < len(words) and words[lo] == p: + lo += 1 + return lo < len(words) and words[lo].startswith(p) + + +def train_bigram(words, common): + pair, first = {}, {} + for w in words: + if not (w.isascii() and w.isalpha()): + continue + weight = 5.0 if w in common else 1.0 + chars = ["^"] + list(w) + ["$"] + for i in range(1, len(chars)): + k = chars[i - 1] + chars[i] + pair[k] = pair.get(k, 0.0) + weight + first[chars[i - 1]] = first.get(chars[i - 1], 0.0) + weight + table = {k: math.log2((c + 1) / (first.get(k[0], 0.0) + 27)) for k, c in pair.items()} + unseen = math.log2(1.0 / (max(first.values(), default=1000) + 27)) + return {"table": table, "unseen": unseen} diff --git a/windows/autozhuyin/core/lexicon.py b/windows/autozhuyin/core/lexicon.py new file mode 100644 index 0000000..c685d9d --- /dev/null +++ b/windows/autozhuyin/core/lexicon.py @@ -0,0 +1,133 @@ +"""Lexicon and Viterbi converter. Port of Lexicon.swift / Converter.swift.""" + +_TONE_MARKS = set("ˊˇˋ˙") + + +def _strip_tones(reading): + return "".join(c for c in reading if c not in _TONE_MARKS) + + +def _is_bopomofo(s): + return bool(s) and (0x3105 <= ord(s[0]) <= 0x312F or s[0] == "˙") + + +class Lexicon: + """Maps readings ("ㄋㄧˇ-ㄏㄠˇ") to [(phrase, log10 score)], best first.""" + + def __init__(self): + self.exact = {} + self.toneless = {} + self.max_phrase_length = 1 + + @classmethod + def from_file(cls, path): + lex = cls() + with open(path, encoding="utf-8") as f: + lex.load(f) + return lex + + def load(self, lines): + for line in lines: + line = line.strip() + if not line or line.startswith("#"): + continue + cols = line.split() + if len(cols) < 2: + continue + reading, phrase = (cols[0], cols[1]) if _is_bopomofo(cols[0]) else (cols[1], cols[0]) + if not _is_bopomofo(reading): + continue + try: + score = float(cols[2]) if len(cols) >= 3 else -6.0 + except ValueError: + score = -6.0 + self.add(reading, phrase, score) + for table in (self.exact, self.toneless): + for v in table.values(): + v.sort(key=lambda e: -e[1]) + + def add(self, reading, phrase, score): + syllables = [s[1:] + "˙" if s.startswith("˙") else s for s in reading.split("-")] + reading = "-".join(syllables) + self.max_phrase_length = max(self.max_phrase_length, len(syllables)) + entries = self.exact.setdefault(reading, []) + if not any(p == phrase for p, _ in entries): + entries.append((phrase, score)) + key = _strip_tones(reading) + tl = self.toneless.setdefault(key, []) + for i, (p, s) in enumerate(tl): + if p == phrase: + if score > s: + tl[i] = (p, score) + break + else: + tl.append((phrase, score)) + + def entries(self, syllables): + if all(s.tone is not None for s in syllables): + return self.exact.get("-".join(s.reading for s in syllables), []) + return self.toneless.get("-".join(s.toneless for s in syllables), []) + + def best_score(self, syllable): + e = self.entries([syllable]) + return e[0][1] if e else None + + +UNKNOWN_SCORE = -30.0 +TONELESS_PENALTY = -1.5 + + +class Converter: + def __init__(self, lexicon): + self.lexicon = lexicon + + def _options(self, syllables, i, j, fixed): + if (i, j) in fixed: + return [(fixed[(i, j)], 0.0)] + if any(a < j and i < b for (a, b) in fixed): + return [] + sl = syllables[i:j] + penalty = TONELESS_PENALTY if any(s.tone is None for s in sl) else 0.0 + out = [(p, s + penalty) for p, s in self.lexicon.entries(sl)] + if not out and j - i == 1: + out = [(sl[0].reading, UNKNOWN_SCORE)] + return out + + def convert(self, syllables, fixed=None): + """Returns [((start, end), text)] for the best path. `fixed` maps (start, end) -> text.""" + fixed = fixed or {} + n = len(syllables) + if n == 0: + return [] + max_len = max(1, self.lexicon.max_phrase_length) + best = [(float("-inf"), None)] * (n + 1) + best[0] = (0.0, None) + for j in range(1, n + 1): + for length in range(1, min(max_len, j) + 1): + i = j - length + if best[i][0] == float("-inf"): + continue + for text, score in self._options(syllables, i, j, fixed): + total = best[i][0] + score + if total > best[j][0]: + best[j] = (total, ((i, j), text)) + segs = [] + j = n + while j > 0 and best[j][1] is not None: + seg = best[j][1] + segs.append(seg) + j = seg[0][0] + return segs[::-1] + + def candidates(self, syllables, end): + """[((start, end), text)] for phrases ending at syllable `end`, longest first.""" + if end <= 0 or end > len(syllables): + return [] + out, seen = [], set() + for length in range(min(self.lexicon.max_phrase_length, end), 0, -1): + i = end - length + for phrase, _ in self.lexicon.entries(syllables[i:end]): + if phrase not in seen: + seen.add(phrase) + out.append(((i, end), phrase)) + return out diff --git a/windows/autozhuyin/icon.ico b/windows/autozhuyin/icon.ico new file mode 100644 index 0000000..405de58 Binary files /dev/null and b/windows/autozhuyin/icon.ico differ diff --git a/windows/autozhuyin/ime.json b/windows/autozhuyin/ime.json new file mode 100644 index 0000000..dccb47a --- /dev/null +++ b/windows/autozhuyin/ime.json @@ -0,0 +1,11 @@ +{ + "name": "自動注音 AutoZhuyin", + "version": "0.1.0", + "guid": "{3895C6E3-6D40-4B4B-B4EA-CEEC75382D23}", + "locale": "zh-Hant-TW", + "fallbackLocale": "zh-TW", + "icon": "icon.ico", + "win8_icon": "", + "moduleName": "autozhuyin_ime", + "serviceName": "AutoZhuyinTextService" +} diff --git a/windows/installer/README.txt b/windows/installer/README.txt new file mode 100644 index 0000000..f7752ae --- /dev/null +++ b/windows/installer/README.txt @@ -0,0 +1,49 @@ +自動注音 AutoZhuyin(Windows 測試版) +===================================== + +用大千注音鍵盤直接打字,不用切換中英文,輸入法會自動判斷你打的是注音還是英文。 + + 按鍵: ji3 hello world su3cl3 (最後按空白) + 輸出: 我hello world 你好 + + +安裝 +---- +1. 先安裝 PIME 輸入法框架(免費、開源): + https://github.com/EasyIME/PIME/releases + 下載 PIME-x.x.x-setup.exe 安裝(預設選項即可)。 + +2. 解壓縮這個 zip,在 install.bat 上按右鍵 →「以系統管理員身分執行」。 + +3. 設定 → 時間與語言 → 語言與地區 → 中文(台灣) → 語言選項 → 新增鍵盤, + 加入「自動注音 AutoZhuyin」。如果清單裡沒有,重新開機一次。 + +4. 用 Win + 空白鍵 切換到自動注音。 + + +使用方式 +-------- + 空白 注音:一聲;音節打完時送出中文。英文:送出單字加空白 + Enter 送出目前組字 + Tab 判斷錯了?按 Tab 在中/英之間翻轉,輸入法會記住這次修正 + ↓ 開啟候選字,按數字鍵 1–9 選字 + Esc 取消組字 + Shift+字母 強制英文 + < > ? ! : 中文語境下自動變成全形 ,。?!: + +注音符號的順序打錯也沒關係,例如 ㄧㄒㄠˇ 會自動變成 ㄒㄧㄠˇ(小)。 + + +回報問題 +-------- +請告訴我: + - 按了哪些鍵(例如 2;3) + - 預期打出什麼、實際出現什麼 + - 在哪個程式裡打字(Word、LINE、Chrome…) + +自訂詞:%APPDATA%\PIME\autozhuyin\user-phrases.txt + 每行一個詞,格式:ㄗˋ-ㄉㄨㄥˋ-ㄓㄨˋ-ㄧㄣ 自動注音 -4 + +解除安裝:在 uninstall.bat 上按右鍵 →「以系統管理員身分執行」。 + +詞庫來源:小麥注音 McBopomofo(MIT 授權)。 diff --git a/windows/installer/install.bat b/windows/installer/install.bat new file mode 100644 index 0000000..e458da6 --- /dev/null +++ b/windows/installer/install.bat @@ -0,0 +1,45 @@ +@echo off +chcp 65001 >nul +setlocal + +net session >nul 2>&1 +if errorlevel 1 ( + echo 請在 install.bat 上按右鍵,選「以系統管理員身分執行」。 + pause + exit /b 1 +) + +set "PIME=%ProgramFiles(x86)%\PIME" +if not exist "%PIME%\python\input_methods" ( + echo 找不到 PIME。請先安裝 PIME: + echo https://github.com/EasyIME/PIME/releases + pause + exit /b 1 +) + +echo 正在安裝自動注音... +taskkill /f /im PIMELauncher.exe >nul 2>&1 + +set "DEST=%PIME%\python\input_methods\autozhuyin" +if exist "%DEST%" rmdir /s /q "%DEST%" +robocopy "%~dp0autozhuyin" "%DEST%" /E /NFL /NDL /NJH /NJS /NP >nul +if errorlevel 8 ( + echo 複製檔案失敗。 + pause + exit /b 1 +) + +rem Registering PIME scans input_methods\*\ime.json and adds AutoZhuyin to Windows. +if exist "%PIME%\x86\PIMETextService.dll" regsvr32 /s "%PIME%\x86\PIMETextService.dll" +if exist "%PIME%\x64\PIMETextService.dll" regsvr32 /s "%PIME%\x64\PIMETextService.dll" +if exist "%PIME%\arm64\PIMETextService.dll" regsvr32 /s "%PIME%\arm64\PIMETextService.dll" + +rem Start PIMELauncher as the normal (non-admin) user. +explorer.exe "%PIME%\PIMELauncher.exe" + +echo. +echo 安裝完成! +echo 到「設定 > 時間與語言 > 語言與地區 > 中文(台灣) > 語言選項 > 新增鍵盤」 +echo 加入「自動注音 AutoZhuyin」,再用 Win+空白鍵 切換過去。 +echo 如果清單裡沒有,請重新開機一次。 +pause diff --git a/windows/installer/uninstall.bat b/windows/installer/uninstall.bat new file mode 100644 index 0000000..16136f3 --- /dev/null +++ b/windows/installer/uninstall.bat @@ -0,0 +1,26 @@ +@echo off +chcp 65001 >nul +setlocal + +net session >nul 2>&1 +if errorlevel 1 ( + echo 請在 uninstall.bat 上按右鍵,選「以系統管理員身分執行」。 + pause + exit /b 1 +) + +set "PIME=%ProgramFiles(x86)%\PIME" +taskkill /f /im PIMELauncher.exe >nul 2>&1 +if exist "%PIME%\python\input_methods\autozhuyin" rmdir /s /q "%PIME%\python\input_methods\autozhuyin" + +rem Unregister all PIME input methods, then register the remaining ones again. +for %%A in (x86 x64 arm64) do ( + if exist "%PIME%\%%A\PIMETextService.dll" ( + regsvr32 /u /s "%PIME%\%%A\PIMETextService.dll" + regsvr32 /s "%PIME%\%%A\PIMETextService.dll" + ) +) +explorer.exe "%PIME%\PIMELauncher.exe" + +echo 已移除自動注音。學習紀錄在 %APPDATA%\PIME\autozhuyin,可以手動刪除。 +pause diff --git a/windows/tests/test_core.py b/windows/tests/test_core.py new file mode 100644 index 0000000..ce01abe --- /dev/null +++ b/windows/tests/test_core.py @@ -0,0 +1,158 @@ +"""Tests for the Python core (mirror of the Swift tests). Run from windows/: + python3 tools/build_data.py && python3 -m unittest discover tests +""" +import os +import sys +import time +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) + +from autozhuyin.core import bopomofo # noqa: E402 +from autozhuyin.core.classifier import LITERAL, ZHUYIN, Classifier, LearnedPreferences # noqa: E402 +from autozhuyin.core.engine import DOWN, ENTER, TAB, CompositionEngine # noqa: E402 +from autozhuyin.core.english import EnglishModel # noqa: E402 +from autozhuyin.core.lexicon import Converter, Lexicon # noqa: E402 + +DATA = os.path.join(os.path.dirname(HERE), "autozhuyin", "data") +_start = time.time() +LEX = Lexicon.from_file(os.path.join(DATA, "lexicon.txt")) +ENG = EnglishModel.from_files(os.path.join(DATA, "english-common.txt"), + os.path.join(DATA, "english-dict.txt"), + os.path.join(DATA, "english-bigram.json")) +LOAD_TIME = time.time() - _start + + +def readings(keys): + return [s.syllable.reading for s in bopomofo.parse(list(keys)).syllables] + + +def convert(keys): + syl = [s.syllable for s in bopomofo.parse(list(keys)).syllables] + return "".join(t for _, t in Converter(LEX).convert(syl)) + + +def make_engine(): + return CompositionEngine(Classifier(LEX, ENG)) + + +def type_keys(s, e): + out = "" + for c in s: + key = ENTER if c == "\n" else TAB if c == "\t" else c + handled, commit = e.handle(key) + if commit is not None: + out += commit + elif not handled: + out += c + return out + + +class ParserTests(unittest.TestCase): + def test_ni_hao(self): + self.assertEqual(readings("su3cl3"), ["ㄋㄧˇ", "ㄏㄠˇ"]) + + def test_space_is_first_tone(self): + self.assertEqual(readings("wu0 "), ["ㄊㄧㄢ"]) + + def test_pending(self): + p = bopomofo.parse(list("su3c")) + self.assertEqual(len(p.completed), 1) + self.assertEqual(p.pending.syllable.reading, "ㄏ") + + def test_out_of_order(self): + for keys in ("uvl3", "vlu3", "ulv3"): + p = bopomofo.parse(list(keys)) + self.assertEqual([s.syllable.reading for s in p.syllables], ["ㄒㄧㄠˇ"], keys) + self.assertTrue(p.syllables[0].reordered) + self.assertFalse(bopomofo.parse(list("vul3")).syllables[0].reordered) + self.assertEqual(len(bopomofo.parse(list("me")).syllables), 2) + + +class ConverterTests(unittest.TestCase): + def test_phrase(self): + self.assertEqual(convert("su3cl3"), "你好") + + def test_sentence(self): + self.assertEqual(convert("rup wu0 wu0 fu4cp3cl3"), "今天天氣很好") + + def test_tone_omission(self): + self.assertEqual(convert("jiap7"), "我們") + + def test_load_time(self): + print("\nlexicon + english load: %.2fs" % LOAD_TIME) + self.assertLess(LOAD_TIME, 5) + + +class ClassifierTests(unittest.TestCase): + c = Classifier(LEX, ENG) + + def test_zhuyin(self): + for s in ["su3cl3", "ji3", "cl3", "ru ", "u ", "fu ", "183", "2k7", "5j/ "]: + self.assertEqual(self.c.mode(list(s)), ZHUYIN, s) + for s in ["ru", "ji", "fu"]: + self.assertEqual(self.c.mode(list(s), finishing=True), ZHUYIN, s) + + def test_english(self): + for s in ["hello", "the", "is", "ok", "github", "Swift", "2024", "so", "go", "me", + "no", "a", "an", "up", "i", "meeting", "email", "push"]: + self.assertEqual(self.c.mode(list(s), finishing=True), LITERAL, s) + + def test_learning(self): + learned = LearnedPreferences() + c = Classifier(LEX, ENG, learned) + self.assertEqual(c.mode(list("so"), finishing=True), LITERAL) + learned.record("so", ZHUYIN) + self.assertEqual(c.mode(list("so"), finishing=True), ZHUYIN) + + +class EngineTests(unittest.TestCase): + def test_mixed_sentence(self): + self.assertEqual(type_keys("ji3 hello world su3cl3 ", make_engine()), "我hello world 你好") + + def test_marked_text(self): + e = make_engine() + type_keys("su3c", e) + self.assertEqual(e.marked_text, "你ㄏ") + type_keys("l3", e) + self.assertEqual(e.marked_text, "你好") + + def test_enter_commits_english(self): + self.assertEqual(type_keys("github\n", make_engine()), "github") + + def test_tab_learns(self): + e = make_engine() + type_keys("so", e) + self.assertEqual(e.mode, LITERAL) + e.handle(TAB) + self.assertEqual(e.mode, ZHUYIN) + type_keys("\n", e) + self.assertEqual(e.classifier.learned.weights["so"], 1) + + def test_punctuation(self): + self.assertEqual(type_keys("hello, ", make_engine()), "hello, ") + self.assertEqual(type_keys("su3cl3?", make_engine()), "你好?") + + def test_candidates(self): + e = make_engine() + type_keys("su3", e) + e.handle(DOWN) + self.assertTrue(e.showing_candidates) + idx = [t for _, t in e.candidates].index("妳") + e.select_candidate(idx) + self.assertEqual(type_keys("\n", e), "妳") + + def test_digit_initials(self): + e = make_engine() + type_keys("2;3", e) + self.assertEqual(len(e.marked_text), 1) + self.assertEqual(type_keys("2j/ \n", make_engine()), "東") + self.assertEqual(type_keys("2.5\n", make_engine()), "2.5") + + def test_out_of_order_converts(self): + self.assertEqual(type_keys("uvl3\n", make_engine()), "小") + + +if __name__ == "__main__": + unittest.main() diff --git a/windows/tests/test_pime_wrapper.py b/windows/tests/test_pime_wrapper.py new file mode 100644 index 0000000..e7aeefc --- /dev/null +++ b/windows/tests/test_pime_wrapper.py @@ -0,0 +1,127 @@ +"""Drives autozhuyin_ime.AutoZhuyinTextService through minimal stand-ins for PIME's +`keycodes` and `textService` modules, to catch wrapper bugs without Windows.""" +import os +import sys +import types +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) + +# --- stand-ins for PIME modules (only what the wrapper uses) --- +keycodes = types.ModuleType("keycodes") +for name, code in dict(VK_BACK=0x08, VK_TAB=0x09, VK_RETURN=0x0D, VK_SHIFT=0x10, VK_CONTROL=0x11, + VK_MENU=0x12, VK_CAPITAL=0x14, VK_ESCAPE=0x1B, VK_SPACE=0x20, VK_LEFT=0x25, + VK_UP=0x26, VK_RIGHT=0x27, VK_DOWN=0x28).items(): + setattr(keycodes, name, code) +sys.modules["keycodes"] = keycodes + +textService = types.ModuleType("textService") + + +class FakeTextService: + def __init__(self, client): + self.client = client + self.showCandidates = False + self.reply = {} + + def onActivate(self): pass + def onDeactivate(self): pass + def onCompositionTerminated(self, forced): pass + def setSelKeys(self, k): self.reply["setSelKeys"] = k + def customizeUI(self, **kw): self.reply["customizeUI"] = kw + def setCommitString(self, s): self.reply["commitString"] = s + def setCompositionString(self, s): self.reply["compositionString"] = s + def setCompositionCursor(self, p): self.reply["compositionCursor"] = p + def setCandidateList(self, c): self.reply["candidateList"] = c + def setCandidateCursor(self, p): self.reply["candidateCursor"] = p + + def setShowCandidates(self, show): + self.showCandidates = show + self.reply["showCandidates"] = show + + +textService.TextService = FakeTextService +sys.modules["textService"] = textService + +from autozhuyin.autozhuyin_ime import AutoZhuyinTextService # noqa: E402 + +VK = {"\n": 0x0D, "\t": 0x09, " ": 0x20, "\b": 0x08, "↓": 0x28} + + +class FakeKeyEvent: + def __init__(self, ch, shift=False, ctrl=False): + self.keyCode = VK.get(ch, ord(ch.upper()) if ch.isalnum() else 0xBA) + self.charCode = 0 if ch in ("\n", "\t", "\b", "↓") else ord(ch) + self.keyStates = [0] * 256 + if shift: + self.keyStates[0x10] = 0x80 + if ctrl: + self.keyStates[0x11] = 0x80 + + def isKeyDown(self, code): return bool(self.keyStates[code] & 0x80) + def isKeyToggled(self, code): return bool(self.keyStates[code] & 1) + def isPrintableChar(self): return self.charCode > 0x1F and self.charCode != 0x7F + + +def press(svc, ch, **kw): + """Simulates PIME: filterKeyDown, then onKeyDown if filtered. Returns (eaten, reply).""" + svc.reply = {} + ev = FakeKeyEvent(ch, **kw) + if not svc.filterKeyDown(ev): + return False, {} + svc.onKeyDown(ev) + return True, svc.reply + + +def type_text(svc, text): + out = "" + for ch in text: + eaten, reply = press(svc, ch) + if not eaten: + out += ch if ch not in VK or ch == " " else "" + out += reply.get("commitString", "") + return out + + +class WrapperTests(unittest.TestCase): + def setUp(self): + self.svc = AutoZhuyinTextService(client=None) + self.svc.onActivate() + + def test_mixed_sentence(self): + self.assertEqual(type_text(self.svc, "ji3 hello world su3cl3 "), "我hello world 你好") + + def test_composition_shown(self): + type_text(self.svc, "su3c") + self.assertEqual(self.svc.reply["compositionString"], "你ㄏ") + self.assertEqual(self.svc.reply["compositionCursor"], 2) + + def test_idle_keys_pass_through(self): + self.assertFalse(press(self.svc, " ")[0]) + self.assertFalse(press(self.svc, "\n")[0]) + self.assertFalse(press(self.svc, "c", ctrl=True)[0]) + + def test_candidates_reply(self): + type_text(self.svc, "su3") + eaten, reply = press(self.svc, "↓") + self.assertTrue(eaten) + self.assertTrue(reply["showCandidates"]) + self.assertIn("妳", reply["candidateList"]) + self.assertLessEqual(len(reply["candidateList"]), 9) + n = reply["candidateList"].index("妳") + 1 + eaten, reply = press(self.svc, str(n)) + self.assertFalse(reply["showCandidates"]) + self.assertEqual(type_text(self.svc, "\n"), "妳") + + def test_shift_letter_is_english(self): + out = "" + for ch in "Hi": + eaten, reply = press(self.svc, ch, shift=ch.isupper()) + out += reply.get("commitString", "") + eaten, reply = press(self.svc, " ") + self.assertEqual(out + reply.get("commitString", ""), "Hi ") + + +if __name__ == "__main__": + unittest.main() diff --git a/windows/tools/build_data.py b/windows/tools/build_data.py new file mode 100755 index 0000000..41c5e6c --- /dev/null +++ b/windows/tools/build_data.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Fills windows/autozhuyin/data/ from the shared Mac resources. + +- lexicon.txt, english-common.txt: copied from Resources/ +- english-dict.txt: filtered from a system word list (default /usr/share/dict/words, + Webster's 2nd edition, public domain), since Windows has no such file +- english-bigram.json: precomputed letter-bigram table + +Usage: build_data.py [word-list] +""" +import json +import shutil +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WIN = ROOT / "windows" +sys.path.insert(0, str(WIN)) + +from autozhuyin.core.english import train_bigram # noqa: E402 + +data = WIN / "autozhuyin" / "data" +data.mkdir(parents=True, exist_ok=True) +for name in ("lexicon.txt", "english-common.txt"): + shutil.copy(ROOT / "Resources" / name, data / name) + +word_list = Path(sys.argv[1] if len(sys.argv) > 1 else "/usr/share/dict/words") +words = sorted({w.strip().lower() for w in word_list.read_text(encoding="utf-8").split() + if len(w.strip()) > 2 and w.strip().isascii() and w.strip().isalpha()}) +(data / "english-dict.txt").write_text("\n".join(words) + "\n", encoding="utf-8") + +common = {w.strip().lower() for w in (data / "english-common.txt").read_text(encoding="utf-8").splitlines() + if w.strip() and not w.startswith("#")} +bigram = train_bigram(sorted(common | set(words)), common) +(data / "english-bigram.json").write_text(json.dumps(bigram), encoding="utf-8") + +print("data ready: %d dictionary words, %d bigrams" % (len(words), len(bigram["table"]))) diff --git a/windows/tools/png_to_ico.py b/windows/tools/png_to_ico.py new file mode 100755 index 0000000..e7aba29 --- /dev/null +++ b/windows/tools/png_to_ico.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +"""Wraps a PNG into a single-image .ico file. Usage: png_to_ico.py in.png out.ico""" +import struct +import sys + +png = open(sys.argv[1], "rb").read() +width, height = struct.unpack(">II", png[16:24]) +header = struct.pack("