diff --git a/README.md b/README.md index 086c9e7..9e5c6e7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ ## 功能 - **中英自動判斷**:每按一個鍵就重新判斷,組字區即時顯示結果(Mac 版英文以藍字顯示) +- **中英混打不用空白**:`ji3hello` → 我hello、`hellosu3cl3` → hello你好,同一段內會自動切分 - **判斷錯了按 Tab**:在中/英之間翻轉,輸入法會記住這組按鍵,下次自動判斷對 - **注音順序打錯也能修正**:`ㄧㄒㄠˇ`、`ㄒㄠㄧˇ` 都會變成 `ㄒㄧㄠˇ`(小) - **可省略聲調**:`jiap7`(ㄨㄛ ㄇㄣ˙)一樣能打出「我們」 @@ -48,7 +49,7 @@ SmartScreen 出現警告時,按「其他資訊 → 仍要執行」。解除安 |---|---| | 空白 | 注音:一聲;音節已完成時送出中文。英文:送出單字加空白 | | Enter | 送出目前組字 | -| **Tab** | 判斷錯了?按 Tab 在中/英之間翻轉,輸入法會**記住**這次修正 | +| **Tab** | 判斷錯了?按 Tab 把整段翻轉成另一種語言,輸入法會**記住**這次修正 | | ↓ | 開啟候選字(選最後一個字或詞),數字鍵 1–9 選字,Esc 關閉 | | Esc | 取消組字 | | Shift + 字母 | 強制英文 | @@ -70,6 +71,11 @@ SmartScreen 出現警告時,按「其他資訊 → 仍要執行」。解除安 分數高的一方勝出。按空白時,注音那邊會把空白當成一聲來評分,英文那邊當作完整單字評分。 +中英混打時,`Classifier.segment` 會用動態規劃試遍所有切分點,找出總分最高的切法(例如 `ji3hello` → `ji3` 注音+`hello` 英文)。為了避免把正常的單字或句子切碎: +- 每多切一刀扣 3 分(`segmentPenalty`) +- 後面還接著其他段的注音段,最後一個音節必須打了聲調 +- 接在英文後面、還沒打聲調的注音段再扣 2 分(`unconfirmedSwitchPenalty`),所以打到 `ji3hel` 時會先顯示「我hel」,不會變成「我heㄠ」 + 音節解析照大千規則:符號依「聲母 → 介音 → 韻母」填入,聲調鍵結束音節。順序打錯時,如果合併後仍可能是合法音節,就併進同一個音節,否則才切成新音節。 轉換用 Viterbi 演算法,在詞庫中找出整串音節機率最高的詞組組合。 @@ -129,10 +135,24 @@ scripts/import-mcbopomofo.sh ### 發布新版本 -1. 更新 `Resources/Info.plist` 和 `windows/autozhuyin/ime.json` 的版本號。 -2. 執行 `scripts/package.sh` 和 `scripts/package-windows.sh`。 -3. commit,打 tag(例如 `git tag -a v0.2.0 -m "AutoZhuyin v0.2.0"`),push commit 和 tag。 -4. 在 Gitea 用這個 tag 建立 Release,附上 `build/` 裡的 `.pkg` 和 `.zip`。 +先 commit 所有修改,再執行: + +```bash +scripts/release.sh # 修正版:0.1.0 → 0.1.1 +scripts/release.sh minor # 新功能:0.1.1 → 0.2.0 +scripts/release.sh major # 大改版:0.2.0 → 1.0.0 +scripts/release.sh minor --push # 同時 push commit 和 tag +``` + +`release.sh` 會自動: +1. 遞增版本號(`Resources/Info.plist` 的版本與 build 號、`windows/autozhuyin/ime.json`) +2. 跑 Swift 和 Python 全部測試 +3. 產生 `build/AutoZhuyin-<版本>.pkg` 和 `build/AutoZhuyin-Windows-<版本>.zip` +4. commit「Release v<版本>」並打上 `v<版本>` tag + +任何一步失敗都會把版本號改回去。只想改版本號不打包:`scripts/bump-version.sh [patch|minor|major]`。 + +最後在 Gitea 用這個 tag 建立 Release,附上 `build/` 裡的 `.pkg` 和 `.zip`。 ## 專案結構 diff --git a/Sources/AutoZhuyin/InputController.swift b/Sources/AutoZhuyin/InputController.swift index 8bb351d..1341465 100644 --- a/Sources/AutoZhuyin/InputController.swift +++ b/Sources/AutoZhuyin/InputController.swift @@ -90,13 +90,15 @@ final class InputController: IMKInputController { } private func updateComposition(_ client: IMKTextInput) { - let text = engine.markedText - let attrs: [NSAttributedString.Key: Any] = [ - .underlineStyle: NSUnderlineStyle.single.rawValue, - // Literal (English) composition is shown in a different colour as a hint. - .foregroundColor: engine.mode == .literal ? NSColor.systemBlue : NSColor.textColor, - ] - let marked = NSAttributedString(string: text, attributes: text.isEmpty ? [:] : attrs) + // Each segment is underlined; literal (English) segments are blue as a hint. + let marked = NSMutableAttributedString() + for piece in engine.markedPieces { + marked.append(NSAttributedString(string: piece.text, attributes: [ + .underlineStyle: NSUnderlineStyle.single.rawValue, + .foregroundColor: piece.mode == .literal ? NSColor.systemBlue : NSColor.textColor, + ])) + } + let text = marked.string client.setMarkedText(marked, selectionRange: NSRange(location: (text as NSString).length, length: 0), replacementRange: NSRange(location: NSNotFound, length: 0)) diff --git a/Sources/AutoZhuyinCore/Classifier.swift b/Sources/AutoZhuyinCore/Classifier.swift index 06ecbf2..4221625 100644 --- a/Sources/AutoZhuyinCore/Classifier.swift +++ b/Sources/AutoZhuyinCore/Classifier.swift @@ -7,6 +7,17 @@ public enum InputMode: String, Codable { case literal } +/// A run of keys (indices into the buffer) interpreted in one mode. +public struct Segment: Equatable { + public var range: Range + public var mode: InputMode + + public init(range: Range, mode: InputMode) { + self.range = range + self.mode = mode + } +} + public struct Classification { public var mode: InputMode public var zhuyinScore: Double @@ -72,6 +83,83 @@ public final class Classifier { return Classification(mode: zh >= lit ? .zhuyin : .literal, zhuyinScore: zh, literalScore: lit) } + // MARK: - Segmentation + + /// Cost of each switch between Zhuyin and literal inside one buffer, so that a normal + /// word or sentence is not chopped up. + public var segmentPenalty = 3.0 + + /// Splits a buffer into alternating Zhuyin / literal segments, e.g. "ji3hello" → + /// [ji3: zhuyin, hello: literal]. Dynamic programming over every split point. + /// A Zhuyin segment followed by another segment must end with a tone key. + public func segment(_ keys: [Character], finishing: Bool = false, previous: InputMode? = nil) -> [Segment] { + let n = keys.count + guard n > 0 else { return [] } + let modes: [InputMode] = [.zhuyin, .literal] + // best[i][m]: best score for keys[0.. 0, + finishing: finishing, previous: j == 0 ? previous : nil, + whole: j == 0 && i == n) + for m in 0..<2 where scores[m] > -.infinity { + // Adjacent segments must alternate modes; same-mode neighbours are one segment. + let total = j == 0 ? scores[m] : best[j][1 - m] + scores[m] - segmentPenalty + if total > best[i][m] { + best[i][m] = total + back[i][m] = j + } + } + } + } + + guard best[n][0] > -.infinity || best[n][1] > -.infinity else { + return [Segment(range: 0..= best[n][1] ? 0 : 1 + var i = n + var out: [Segment] = [] + while i > 0 { + let j = back[i][m] + out.append(Segment(range: j.. [Double] { + var zh: Double + let parse = ZhuyinParser.parse(seg) + if !isLast && parse.pending != nil { + zh = -.infinity + } else { + zh = zhuyinScore(finishing && isLast ? seg + [" "] : seg) + zhuyinPrior + if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) { + zh -= unconfirmedSwitchPenalty + } + } + var lit = literalScore(seg, complete: !isLast || finishing) + switch previous { + case .zhuyin?: zh += contextWeight + case .literal?: lit += contextWeight + case nil: break + } + if whole { + let b = learned.bias(for: String(seg)) + if b > 0 { zh += b } else { lit -= b } + } + return [zh, lit] + } + public func zhuyinScore(_ keys: [Character]) -> Double { let parse = ZhuyinParser.parse(keys) if parse.foreignKeys > 0 || parse.syllables.isEmpty { return -.infinity } diff --git a/Sources/AutoZhuyinCore/CompositionEngine.swift b/Sources/AutoZhuyinCore/CompositionEngine.swift index ab1b669..d4a5dda 100644 --- a/Sources/AutoZhuyinCore/CompositionEngine.swift +++ b/Sources/AutoZhuyinCore/CompositionEngine.swift @@ -16,22 +16,29 @@ public struct KeyResult: Equatable { public static let consumed = KeyResult(handled: true, commit: nil) } +/// A piece of the composition as shown to the user. +public struct MarkedPiece: Equatable { + public var text: String + public var mode: InputMode +} + /// The whole input-method state machine, independent of InputMethodKit. /// -/// Keystrokes accumulate in a buffer. On every key the classifier decides whether the -/// buffer is Zhuyin or literal text; the composition (marked text) shows the result -/// live. Space / Enter / punctuation commit it. Tab flips the decision and the -/// correction is learned. +/// Keystrokes accumulate in a buffer. On every key the classifier splits the buffer +/// into Zhuyin and literal segments (e.g. "ji3hello" → 我 + hello); the composition +/// (marked text) shows the result live. Space / Enter / punctuation commit it. Tab +/// forces the whole buffer into the other mode and the correction is learned. public final class CompositionEngine { public let classifier: Classifier public let converter: Converter public private(set) var keys: [Character] = [] - /// Set by Tab: overrides the classifier for the current buffer. + /// Set by Tab: overrides the classifier for the whole buffer. public private(set) var forcedMode: InputMode? - /// User-picked phrases, by syllable range. - public private(set) var fixed: [Range: String] = [:] + /// User-picked phrases: Zhuyin segment start (key index) → syllable range → text. + private var fixed: [Int: [Range: String]] = [:] public private(set) var candidates: [Candidate] = [] + private var candidateSegmentStart = 0 public var candidateIndex = 0 public var isShowingCandidates: Bool { !candidates.isEmpty } @@ -52,33 +59,41 @@ public final class CompositionEngine { now().timeIntervalSince(previousCommitTime) < contextWindow ? previousMode : nil } - public var mode: InputMode { - forcedMode ?? classifier.classify(keys, previous: context).mode + private func segments(of keys: [Character], finishing: Bool = false) -> [Segment] { + guard !keys.isEmpty else { return [] } + if let forced = forcedMode { return [Segment(range: 0.. [Range: String] { - fixed.filter { $0.key.upperBound <= p.completed.count } - } - - private func zhuyinCommitText() -> String { - let p = parse - // An unfinished trailing syllable is converted as untoned. - return converter.convert(p.syllables.map(\.syllable), fixed: validFixed(p)).map(\.text).joined() + /// Text for one segment. A non-final Zhuyin segment shows its unfinished syllable as Bopomofo. + private func render(_ seg: Segment, final: Bool) -> String { + let segKeys = Array(keys[seg.range]) + guard seg.mode == .zhuyin else { return String(segKeys) } + let p = ZhuyinParser.parse(segKeys) + let fixedHere = (fixed[seg.range.lowerBound] ?? [:]) + if final { + let all = p.syllables.map(\.syllable) + return converter.convert(all, fixed: fixedHere.filter { $0.key.upperBound <= all.count }).map(\.text).joined() + } + let done = p.completed.map(\.syllable) + let text = converter.convert(done, fixed: fixedHere.filter { $0.key.upperBound <= done.count }).map(\.text).joined() + return text + (p.pending?.syllable.reading ?? "") } // MARK: - Key handling @@ -92,7 +107,7 @@ public final class CompositionEngine { case .backspace: guard !keys.isEmpty else { return .passThrough } keys.removeLast() - fixed = validFixed(parse) + fixed = fixed.filter { $0.key < keys.count } if keys.isEmpty { forcedMode = nil; fixed = [:] } return .consumed case .enter: @@ -118,15 +133,15 @@ public final class CompositionEngine { private func handleCharacter(_ c: Character) -> KeyResult { if c == " " { guard !keys.isEmpty else { return .passThrough } - let decided = forcedMode ?? classifier.classify(keys, finishing: true, previous: context).mode - if decided == .literal { - return KeyResult(handled: true, commit: commitCurrent(as: .literal) + " ") + let segs = segments(of: keys, finishing: true) + if segs.last?.mode == .literal { + return KeyResult(handled: true, commit: commitCurrent(segs) + " ") } - if parse.pending != nil { + if parse(ofLast: segs)?.pending != nil { keys.append(" ") // first tone return .consumed } - return KeyResult(handled: true, commit: commitCurrent(as: .zhuyin)) + return KeyResult(handled: true, commit: commitCurrent(segs)) } let composable = DachenLayout.isZhuyinKey(c) || (c.isASCII && c.isLetter) || (c == "'" && !keys.isEmpty) @@ -134,9 +149,9 @@ public final class CompositionEngine { // ", . / ; -" are Zhuyin symbols (ㄝ ㄡ ㄥ ㄤ ㄦ) but act as punctuation after an // English word. Decide with the key included: "2;" is ㄉㄤ (檔…), "hello," is English. if !keys.isEmpty, DachenLayout.symbols[c] != nil, !c.isLetter, !c.isNumber, forcedMode != .zhuyin { - let withKey = forcedMode ?? classifier.classify(keys + [c], previous: context).mode - if withKey == .literal && !keys.allSatisfy(\.isNumber) { - return KeyResult(handled: true, commit: commitCurrent(as: .literal) + String(c)) + let withKey = segments(of: keys + [c]) + if withKey.last?.mode == .literal && !keys.allSatisfy(\.isNumber) { + return KeyResult(handled: true, commit: commitCurrent() + String(c)) } } keys.append(c) @@ -151,25 +166,34 @@ public final class CompositionEngine { return KeyResult(handled: true, commit: committed + punct) } + private func parse(ofLast segs: [Segment]) -> ZhuyinParse? { + guard let last = segs.last, last.mode == .zhuyin else { return nil } + return ZhuyinParser.parse(Array(keys[last.range])) + } + static let fullwidth: [Character: String] = [ "<": ",", ">": "。", "?": "?", "!": "!", ":": ":", "\"": "「", "(": "(", ")": ")", "{": "『", "}": "』", "[": "「", "]": "」", "~": "~", "_": "——", ] - // MARK: - Candidates + // MARK: - Candidates (for the last Zhuyin segment) private func openCandidates() { - let p = parse - guard mode == .zhuyin, !p.completed.isEmpty else { return } - candidates = converter.candidates(p.completed.map(\.syllable), endingAt: p.completed.count) + let segs = segments + guard let last = segs.last, last.mode == .zhuyin else { return } + let done = ZhuyinParser.parse(Array(keys[last.range])).completed.map(\.syllable) + guard !done.isEmpty else { return } + candidates = converter.candidates(done, endingAt: done.count) + candidateSegmentStart = last.range.lowerBound candidateIndex = 0 } public func selectCandidate(_ index: Int) { guard candidates.indices.contains(index) else { return } let c = candidates[index] - fixed = fixed.filter { !$0.key.overlaps(c.range) } - fixed[c.range] = c.text + var here = (fixed[candidateSegmentStart] ?? [:]).filter { !$0.key.overlaps(c.range) } + here[c.range] = c.text + fixed[candidateSegmentStart] = here candidates = [] } @@ -197,13 +221,13 @@ public final class CompositionEngine { // MARK: - Commit @discardableResult - private func commitCurrent(as decided: InputMode? = nil) -> String { - let m = decided ?? mode - let text = m == .literal ? String(keys) : zhuyinCommitText() + private func commitCurrent(_ given: [Segment]? = nil) -> String { + let segs = given ?? segments + let text = segs.map { render($0, final: true) }.joined() if let forced = forcedMode { classifier.learned.record(String(keys), as: forced) } - previousMode = m + previousMode = segs.last?.mode previousCommitTime = now() reset() return text diff --git a/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift b/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift index 65e7eee..100fd1d 100644 --- a/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift +++ b/Tests/AutoZhuyinCoreTests/AutoZhuyinCoreTests.swift @@ -219,3 +219,37 @@ final class DigitInitialTests: XCTestCase { XCTAssertEqual(run("hello, ").out, "hello, ") } } + +final class MixedSegmentTests: XCTestCase { + func run(_ s: String) -> (out: String, marked: String) { + let e = CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish)) + var out = "" + for c in s { + let r = e.handle(c == "\n" ? .enter : .character(c)) + if let t = r.commit { out += t } else if !r.handled { out.append(c) } + } + return (out, e.markedText) + } + + func testChineseThenEnglish() { + XCTAssertEqual(run("ji3hello ").out, "我hello ") + XCTAssertEqual(run("su3cl3github\n").out, "你好github") + XCTAssertEqual(run("ji3hel").marked, "我hel") + } + + func testEnglishThenChinese() { + XCTAssertEqual(run("hellosu3cl3 ").out, "hello你好") + XCTAssertEqual(run("oksu3cl3\n").out, "ok你好") + } + + func testThreeSegments() { + XCTAssertEqual(run("ji3emailsu3cl3\n").out, "我email你好") + } + + func testSingleLanguageUnchanged() { + XCTAssertEqual(run("rup wu0 wu0 fu4cp3cl3\n").out, "今天天氣很好") + XCTAssertEqual(run("helloworld\n").out, "helloworld") + XCTAssertEqual(run("meeting ").out, "meeting ") + XCTAssertEqual(run("183\n").out.count, 1) + } +} diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh new file mode 100755 index 0000000..d4197a4 --- /dev/null +++ b/scripts/bump-version.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Increments the version in every place it lives: +# Resources/Info.plist CFBundleShortVersionString (x.y.z) and CFBundleVersion (build number) +# windows/autozhuyin/ime.json "version" +# +# Usage: scripts/bump-version.sh [patch|minor|major] (default: patch) +# Prints the new version. +set -euo pipefail +cd "$(dirname "$0")/.." + +LEVEL="${1:-patch}" +PLIST=Resources/Info.plist +IME=windows/autozhuyin/ime.json + +CURRENT="$(plutil -extract CFBundleShortVersionString raw "$PLIST")" +IFS=. read -r MAJOR MINOR PATCH <<< "$CURRENT" +case "$LEVEL" in + major) MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 ;; + minor) MINOR=$((MINOR + 1)); PATCH=0 ;; + patch) PATCH=$((PATCH + 1)) ;; + *) echo "usage: $0 [patch|minor|major]" >&2; exit 2 ;; +esac +NEW="$MAJOR.$MINOR.$PATCH" +BUILD=$(( $(plutil -extract CFBundleVersion raw "$PLIST") + 1 )) + +plutil -replace CFBundleShortVersionString -string "$NEW" "$PLIST" +plutil -replace CFBundleVersion -string "$BUILD" "$PLIST" +# Edit only the version line so the rest of ime.json keeps its formatting. +sed -i '' -E "s/(\"version\"[[:space:]]*:[[:space:]]*\")[^\"]*\"/\1$NEW\"/" "$IME" + +grep -q "\"version\": \"$NEW\"" "$IME" || { echo "failed to update $IME" >&2; exit 1; } +echo "$NEW" diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..66b8486 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# One-step release: bump the version, run all tests, build both installers, then +# commit the version change and tag it. +# +# Usage: scripts/release.sh [patch|minor|major] [--push] (default: patch) +# --push also pushes the commit and tag to origin +set -euo pipefail +cd "$(dirname "$0")/.." + +LEVEL=patch +PUSH=0 +for arg in "$@"; do + case "$arg" in + patch|minor|major) LEVEL="$arg" ;; + --push) PUSH=1 ;; + *) echo "usage: $0 [patch|minor|major] [--push]" >&2; exit 2 ;; + esac +done + +if [ -n "$(git status --porcelain)" ]; then + echo "Working tree has uncommitted changes; commit them first." >&2 + exit 1 +fi + +export DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" + +VERSION="$(scripts/bump-version.sh "$LEVEL")" +echo "==> Version $VERSION" + +restore() { + echo "Release failed; reverting the version bump." >&2 + git checkout -- Resources/Info.plist windows/autozhuyin/ime.json +} +trap restore ERR + +echo "==> Swift tests" +xcrun swift test -q 2>&1 | tail -3 +echo "==> Packaging (runs the Python tests too)" +scripts/package.sh >/dev/null +scripts/package-windows.sh >/dev/null + +trap - ERR +git add Resources/Info.plist windows/autozhuyin/ime.json +git commit -q -m "Release v$VERSION" +git tag -a "v$VERSION" -m "AutoZhuyin v$VERSION" +echo "==> Tagged v$VERSION" +ls -lh "build/AutoZhuyin-$VERSION.pkg" "build/AutoZhuyin-Windows-$VERSION.zip" +shasum -a 256 "build/AutoZhuyin-$VERSION.pkg" "build/AutoZhuyin-Windows-$VERSION.zip" + +if [ "$PUSH" = 1 ]; then + git push origin HEAD "v$VERSION" +else + echo "Push with: git push origin HEAD v$VERSION" +fi diff --git a/windows/autozhuyin/core/classifier.py b/windows/autozhuyin/core/classifier.py index 1ff5d93..caafbd8 100644 --- a/windows/autozhuyin/core/classifier.py +++ b/windows/autozhuyin/core/classifier.py @@ -55,6 +55,69 @@ class Classifier: def mode(self, keys, finishing=False, previous=None): return self.classify(keys, finishing, previous)[0] + # --- segmentation --- + + segment_penalty = 3.0 + unconfirmed_switch_penalty = 2.0 + + def segment(self, keys, finishing=False, previous=None): + """Splits keys into alternating segments: [(start, end, mode)], e.g. + "ji3hello" -> [(0, 3, zhuyin), (3, 8, literal)]. Dynamic programming over + split points; a Zhuyin segment followed by another must end with a tone.""" + keys = list(keys) + n = len(keys) + if n == 0: + return [] + modes = (ZHUYIN, LITERAL) + best = [[NEG_INF, NEG_INF] for _ in range(n + 1)] + back = [[-1, -1] for _ in range(n + 1)] + for i in range(1, n + 1): + for j in range(i): + scores = self._segment_scores(keys[j:i], i == n, j > 0, finishing, + previous if j == 0 else None, j == 0 and i == n) + for m in (0, 1): + if scores[m] == NEG_INF: + continue + if j == 0: + total = scores[m] + else: + if best[j][1 - m] == NEG_INF: + continue + total = best[j][1 - m] + scores[m] - self.segment_penalty + if total > best[i][m]: + best[i][m] = total + back[i][m] = j + if best[n][0] == NEG_INF and best[n][1] == NEG_INF: + return [(0, n, self.mode(keys, finishing, previous))] + m = 0 if best[n][0] >= best[n][1] else 1 + out, i = [], n + while i > 0: + j = back[i][m] + out.append((j, i, modes[m])) + i, m = j, 1 - m + return out[::-1] + + def _segment_scores(self, seg, is_last, after_other, finishing, previous, whole): + parse = bopomofo.parse(seg) + if not is_last and parse.pending is not None: + zh = NEG_INF + else: + zh = self.zhuyin_score(seg + [" "] if finishing and is_last else seg) + self.zhuyin_prior + if is_last and after_other and not finishing and not any(s.toned for s in parse.syllables): + zh -= self.unconfirmed_switch_penalty + lit = self.literal_score(seg, complete=(not is_last) or finishing) + if previous == ZHUYIN: + zh += self.context_weight + elif previous == LITERAL: + lit += self.context_weight + if whole: + b = self.learned.bias("".join(seg)) + if b > 0: + zh += b + else: + lit -= b + return zh, lit + def zhuyin_score(self, keys): p = bopomofo.parse(keys) if p.foreign_keys > 0 or not p.syllables: diff --git a/windows/autozhuyin/core/engine.py b/windows/autozhuyin/core/engine.py index 8163427..f4930bc 100644 --- a/windows/autozhuyin/core/engine.py +++ b/windows/autozhuyin/core/engine.py @@ -2,6 +2,7 @@ Keys are given as one-character strings, or one of the names in SPECIAL_KEYS. handle() returns (handled, commit_text_or_None). +The buffer is split into Zhuyin / literal segments, e.g. "ji3hello" -> 我 + hello. """ import time @@ -32,11 +33,14 @@ class CompositionEngine: self.now = now self.keys = [] self.forced_mode = None - self.fixed = {} + self.fixed = {} # zhuyin segment start -> {(syl_start, syl_end): text} self.candidates = [] self.candidate_index = 0 + self._candidate_segment_start = 0 self._previous_mode = None self._previous_time = float("-inf") + self._cache_key = None + self._cache_value = None # --- derived state --- @@ -44,9 +48,26 @@ class CompositionEngine: def context(self): return self._previous_mode if self.now() - self._previous_time < self.context_window else None + def _segments(self, keys, finishing=False): + if not keys: + return [] + if self.forced_mode: + return [(0, len(keys), self.forced_mode)] + ctx = self.context + cache_key = (tuple(keys), finishing, ctx) + if cache_key != self._cache_key: + self._cache_key = cache_key + self._cache_value = self.classifier.segment(keys, finishing, ctx) + return self._cache_value + + @property + def segments(self): + return self._segments(self.keys) + @property def mode(self): - return self.forced_mode or self.classifier.mode(self.keys, previous=self.context) + segs = self.segments + return segs[-1][2] if segs else ZHUYIN @property def showing_candidates(self): @@ -56,28 +77,28 @@ class CompositionEngine: def page_start(self): return (self.candidate_index // self.page_size) * self.page_size - def _parse(self): - return bopomofo.parse(self.keys) + def _render(self, seg, final): + start, end, mode = seg + seg_keys = self.keys[start:end] + if mode == LITERAL: + return "".join(seg_keys) + p = bopomofo.parse(seg_keys) + fixed_here = self.fixed.get(start, {}) + syl = [s.syllable for s in (p.syllables if final else p.completed)] + fx = {k: v for k, v in fixed_here.items() if k[1] <= len(syl)} + text = "".join(t for _, t in self.converter.convert(syl, fx)) + if not final and p.pending: + text += p.pending.syllable.reading + return text - def _valid_fixed(self, p): - n = len(p.completed) - return {k: v for k, v in self.fixed.items() if k[1] <= n} + @property + def marked_pieces(self): + segs = self.segments + return [(self._render(s, i < len(segs) - 1), s[2]) for i, s in enumerate(segs)] @property def marked_text(self): - if not self.keys: - return "" - if self.mode == LITERAL: - return "".join(self.keys) - p = self._parse() - done = self.converter.convert([s.syllable for s in p.completed], self._valid_fixed(p)) - pending = p.pending.syllable.reading if p.pending else "" - return "".join(t for _, t in done) + pending - - def _zhuyin_commit_text(self): - p = self._parse() - segs = self.converter.convert([s.syllable for s in p.syllables], self._valid_fixed(p)) - return "".join(t for _, t in segs) + return "".join(t for t, _ in self.marked_pieces) # --- key handling --- @@ -102,7 +123,7 @@ class CompositionEngine: if not self.keys: return PASS self.keys.pop() - self.fixed = self._valid_fixed(self._parse()) + self.fixed = {k: v for k, v in self.fixed.items() if k < len(self.keys)} if not self.keys: self.forced_mode = None self.fixed = {} @@ -128,25 +149,32 @@ class CompositionEngine: return CONSUMED if self.keys else PASS return self._handle_character(key) + def _last_parse(self, segs): + if not segs or segs[-1][2] != ZHUYIN: + return None + start, end, _ = segs[-1] + return bopomofo.parse(self.keys[start:end]) + def _handle_character(self, c): if c == " ": if not self.keys: return PASS - decided = self.forced_mode or self.classifier.mode(self.keys, finishing=True, previous=self.context) - if decided == LITERAL: - return True, self._commit(LITERAL) + " " - if self._parse().pending is not None: + segs = self._segments(self.keys, finishing=True) + if segs[-1][2] == LITERAL: + return True, self._commit(segs) + " " + p = self._last_parse(segs) + if p is not None and p.pending is not None: self.keys.append(" ") return CONSUMED - return True, self._commit(ZHUYIN) + return True, self._commit(segs) composable = bopomofo.is_zhuyin_key(c) or (c.isascii() and c.isalpha()) or (c == "'" and self.keys) if composable: # ", . / ; -" are Zhuyin symbols but act as punctuation after an English word. if self.keys and c in bopomofo.SYMBOLS and not c.isalnum() and self.forced_mode != ZHUYIN: - with_key = self.forced_mode or self.classifier.mode(self.keys + [c], previous=self.context) - if with_key == LITERAL and not all(k.isdigit() for k in self.keys): - return True, self._commit(LITERAL) + c + with_key = self._segments(self.keys + [c]) + if with_key[-1][2] == LITERAL and not all(k.isdigit() for k in self.keys): + return True, self._commit() + c self.keys.append(c) return CONSUMED @@ -157,14 +185,18 @@ class CompositionEngine: return PASS return True, committed + punct - # --- candidates --- + # --- candidates (for the last Zhuyin segment) --- def _open_candidates(self): - p = self._parse() - if self.mode != ZHUYIN or not p.completed: + segs = self.segments + if not segs or segs[-1][2] != ZHUYIN: + return + start, end, _ = segs[-1] + syl = [s.syllable for s in bopomofo.parse(self.keys[start:end]).completed] + if not syl: return - syl = [s.syllable for s in p.completed] self.candidates = self.converter.candidates(syl, len(syl)) + self._candidate_segment_start = start self.candidate_index = 0 def page(self): @@ -176,8 +208,10 @@ class CompositionEngine: if not 0 <= index < len(self.candidates): return rng, text = self.candidates[index] - self.fixed = {k: v for k, v in self.fixed.items() if not (k[0] < rng[1] and rng[0] < k[1])} - self.fixed[rng] = text + here = {k: v for k, v in self.fixed.get(self._candidate_segment_start, {}).items() + if not (k[0] < rng[1] and rng[0] < k[1])} + here[rng] = text + self.fixed[self._candidate_segment_start] = here self.candidates = [] def _handle_candidate_key(self, key): @@ -195,12 +229,12 @@ class CompositionEngine: # --- commit --- - def _commit(self, decided=None): - m = decided or self.mode - text = "".join(self.keys) if m == LITERAL else self._zhuyin_commit_text() + def _commit(self, segs=None): + segs = segs if segs is not None else self.segments + text = "".join(self._render(s, True) for s in segs) if self.forced_mode: self.classifier.learned.record("".join(self.keys), self.forced_mode) - self._previous_mode = m + self._previous_mode = segs[-1][2] if segs else None self._previous_time = self.now() self.reset() return text diff --git a/windows/tests/test_core.py b/windows/tests/test_core.py index ce01abe..fec03ee 100644 --- a/windows/tests/test_core.py +++ b/windows/tests/test_core.py @@ -154,5 +154,39 @@ class EngineTests(unittest.TestCase): self.assertEqual(type_keys("uvl3\n", make_engine()), "小") +class MixedSegmentTests(unittest.TestCase): + def run_keys(self, s): + e = make_engine() + return type_keys(s, e), e.marked_text + + def test_chinese_then_english(self): + self.assertEqual(self.run_keys("ji3hello ")[0], "我hello ") + self.assertEqual(self.run_keys("su3cl3github\n")[0], "你好github") + self.assertEqual(self.run_keys("ji3hel")[1], "我hel") + + def test_english_then_chinese(self): + self.assertEqual(self.run_keys("hellosu3cl3 ")[0], "hello你好") + self.assertEqual(self.run_keys("oksu3cl3\n")[0], "ok你好") + + def test_three_segments(self): + self.assertEqual(self.run_keys("ji3emailsu3cl3\n")[0], "我email你好") + + def test_single_language_unchanged(self): + self.assertEqual(self.run_keys("rup wu0 wu0 fu4cp3cl3\n")[0], "今天天氣很好") + self.assertEqual(self.run_keys("helloworld\n")[0], "helloworld") + self.assertEqual(self.run_keys("meeting ")[0], "meeting ") + self.assertEqual(len(self.run_keys("183\n")[0]), 1) + + def test_speed(self): + e = make_engine() + start = time.time() + for c in "ji3emailsu3cl3rup wu0 wu0 fu4": + e.handle(c) + e.marked_text + per_key = (time.time() - start) / 30 + print("\nper key: %.1f ms" % (per_key * 1000)) + self.assertLess(per_key, 0.05) + + if __name__ == "__main__": unittest.main()