Split mixed Chinese/English input without spaces; add release script

The classifier now segments a buffer into alternating Zhuyin and literal runs
by dynamic programming over split points, so "ji3hello" becomes 我hello and
"hellosu3cl3" becomes hello你好. Each switch costs a penalty, a Zhuyin run
followed by another must end with a tone, and an untoned trailing Zhuyin run
after English is penalized so partial words don't flicker into Bopomofo.
Implemented in both Swift and Python with the same tests; the Mac app colours
each segment separately.

scripts/bump-version.sh bumps the version in Info.plist and ime.json;
scripts/release.sh bumps, tests, packages, commits and tags in one step.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-10 16:21:13 +08:00
co-authored by Claude Opus 5.5
parent dbe08e00a7
commit d1f1112c65
10 changed files with 482 additions and 97 deletions
+9 -7
View File
@@ -90,13 +90,15 @@ final class InputController: IMKInputController {
}
private func updateComposition(_ client: IMKTextInput) {
let text = engine.markedText
let attrs: [NSAttributedString.Key: Any] = [
.underlineStyle: NSUnderlineStyle.single.rawValue,
// Literal (English) composition is shown in a different colour as a hint.
.foregroundColor: engine.mode == .literal ? NSColor.systemBlue : NSColor.textColor,
]
let marked = NSAttributedString(string: text, attributes: text.isEmpty ? [:] : attrs)
// Each segment is underlined; literal (English) segments are blue as a hint.
let marked = NSMutableAttributedString()
for piece in engine.markedPieces {
marked.append(NSAttributedString(string: piece.text, attributes: [
.underlineStyle: NSUnderlineStyle.single.rawValue,
.foregroundColor: piece.mode == .literal ? NSColor.systemBlue : NSColor.textColor,
]))
}
let text = marked.string
client.setMarkedText(marked,
selectionRange: NSRange(location: (text as NSString).length, length: 0),
replacementRange: NSRange(location: NSNotFound, length: 0))
+88
View File
@@ -7,6 +7,17 @@ public enum InputMode: String, Codable {
case literal
}
/// A run of keys (indices into the buffer) interpreted in one mode.
public struct Segment: Equatable {
public var range: Range<Int>
public var mode: InputMode
public init(range: Range<Int>, mode: InputMode) {
self.range = range
self.mode = mode
}
}
public struct Classification {
public var mode: InputMode
public var zhuyinScore: Double
@@ -72,6 +83,83 @@ public final class Classifier {
return Classification(mode: zh >= lit ? .zhuyin : .literal, zhuyinScore: zh, literalScore: lit)
}
// MARK: - Segmentation
/// Cost of each switch between Zhuyin and literal inside one buffer, so that a normal
/// word or sentence is not chopped up.
public var segmentPenalty = 3.0
/// Splits a buffer into alternating Zhuyin / literal segments, e.g. "ji3hello" →
/// [ji3: zhuyin, hello: literal]. Dynamic programming over every split point.
/// A Zhuyin segment followed by another segment must end with a tone key.
public func segment(_ keys: [Character], finishing: Bool = false, previous: InputMode? = nil) -> [Segment] {
let n = keys.count
guard n > 0 else { return [] }
let modes: [InputMode] = [.zhuyin, .literal]
// best[i][m]: best score for keys[0..<i] whose last segment has mode m.
var best = Array(repeating: [Double](repeating: -.infinity, count: 2), count: n + 1)
var back = Array(repeating: [Int](repeating: -1, count: 2), count: n + 1) // segment start j
for i in 1...n {
for j in 0..<i {
let scores = segmentScores(Array(keys[j..<i]), isLast: i == n, afterOther: j > 0,
finishing: finishing, previous: j == 0 ? previous : nil,
whole: j == 0 && i == n)
for m in 0..<2 where scores[m] > -.infinity {
// Adjacent segments must alternate modes; same-mode neighbours are one segment.
let total = j == 0 ? scores[m] : best[j][1 - m] + scores[m] - segmentPenalty
if total > best[i][m] {
best[i][m] = total
back[i][m] = j
}
}
}
}
guard best[n][0] > -.infinity || best[n][1] > -.infinity else {
return [Segment(range: 0..<n, mode: classify(keys, finishing: finishing, previous: previous).mode)]
}
var m = best[n][0] >= best[n][1] ? 0 : 1
var i = n
var out: [Segment] = []
while i > 0 {
let j = back[i][m]
out.append(Segment(range: j..<i, mode: modes[m]))
i = j
m = 1 - m
}
return out.reversed()
}
/// Extra cost for a trailing Zhuyin segment after English that has no tone key yet
/// ("ji3hel" should stay 我hel, not become 我heㄠ, until a tone confirms the switch).
public var unconfirmedSwitchPenalty = 2.0
private func segmentScores(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
previous: InputMode?, whole: Bool) -> [Double] {
var zh: Double
let parse = ZhuyinParser.parse(seg)
if !isLast && parse.pending != nil {
zh = -.infinity
} else {
zh = zhuyinScore(finishing && isLast ? seg + [" "] : seg) + zhuyinPrior
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
zh -= unconfirmedSwitchPenalty
}
}
var lit = literalScore(seg, complete: !isLast || finishing)
switch previous {
case .zhuyin?: zh += contextWeight
case .literal?: lit += contextWeight
case nil: break
}
if whole {
let b = learned.bias(for: String(seg))
if b > 0 { zh += b } else { lit -= b }
}
return [zh, lit]
}
public func zhuyinScore(_ keys: [Character]) -> Double {
let parse = ZhuyinParser.parse(keys)
if parse.foreignKeys > 0 || parse.syllables.isEmpty { return -.infinity }
+70 -46
View File
@@ -16,22 +16,29 @@ public struct KeyResult: Equatable {
public static let consumed = KeyResult(handled: true, commit: nil)
}
/// A piece of the composition as shown to the user.
public struct MarkedPiece: Equatable {
public var text: String
public var mode: InputMode
}
/// The whole input-method state machine, independent of InputMethodKit.
///
/// Keystrokes accumulate in a buffer. On every key the classifier decides whether the
/// buffer is Zhuyin or literal text; the composition (marked text) shows the result
/// live. Space / Enter / punctuation commit it. Tab flips the decision and the
/// correction is learned.
/// Keystrokes accumulate in a buffer. On every key the classifier splits the buffer
/// into Zhuyin and literal segments (e.g. "ji3hello" → 我 + hello); the composition
/// (marked text) shows the result live. Space / Enter / punctuation commit it. Tab
/// forces the whole buffer into the other mode and the correction is learned.
public final class CompositionEngine {
public let classifier: Classifier
public let converter: Converter
public private(set) var keys: [Character] = []
/// Set by Tab: overrides the classifier for the current buffer.
/// Set by Tab: overrides the classifier for the whole buffer.
public private(set) var forcedMode: InputMode?
/// User-picked phrases, by syllable range.
public private(set) var fixed: [Range<Int>: String] = [:]
/// User-picked phrases: Zhuyin segment start (key index) → syllable range → text.
private var fixed: [Int: [Range<Int>: String]] = [:]
public private(set) var candidates: [Candidate] = []
private var candidateSegmentStart = 0
public var candidateIndex = 0
public var isShowingCandidates: Bool { !candidates.isEmpty }
@@ -52,33 +59,41 @@ public final class CompositionEngine {
now().timeIntervalSince(previousCommitTime) < contextWindow ? previousMode : nil
}
public var mode: InputMode {
forcedMode ?? classifier.classify(keys, previous: context).mode
private func segments(of keys: [Character], finishing: Bool = false) -> [Segment] {
guard !keys.isEmpty else { return [] }
if let forced = forcedMode { return [Segment(range: 0..<keys.count, mode: forced)] }
return classifier.segment(keys, finishing: finishing, previous: context)
}
public var parse: ZhuyinParse { ZhuyinParser.parse(keys) }
/// Current split of the buffer into Zhuyin / literal runs.
public var segments: [Segment] { segments(of: keys) }
/// Mode of the last segment (what the user is typing right now).
public var mode: InputMode { segments.last?.mode ?? .zhuyin }
/// What to show underlined in the app while composing.
public var markedText: String {
guard !keys.isEmpty else { return "" }
switch mode {
case .literal:
return String(keys)
case .zhuyin:
let p = parse
let done = converter.convert(p.completed.map(\.syllable), fixed: validFixed(p)).map(\.text).joined()
return done + (p.pending?.syllable.reading ?? "")
public var markedText: String { markedPieces.map(\.text).joined() }
public var markedPieces: [MarkedPiece] {
let segs = segments
return segs.enumerated().map { i, seg in
MarkedPiece(text: render(seg, final: i < segs.count - 1), mode: seg.mode)
}
}
private func validFixed(_ p: ZhuyinParse) -> [Range<Int>: String] {
fixed.filter { $0.key.upperBound <= p.completed.count }
}
private func zhuyinCommitText() -> String {
let p = parse
// An unfinished trailing syllable is converted as untoned.
return converter.convert(p.syllables.map(\.syllable), fixed: validFixed(p)).map(\.text).joined()
/// Text for one segment. A non-final Zhuyin segment shows its unfinished syllable as Bopomofo.
private func render(_ seg: Segment, final: Bool) -> String {
let segKeys = Array(keys[seg.range])
guard seg.mode == .zhuyin else { return String(segKeys) }
let p = ZhuyinParser.parse(segKeys)
let fixedHere = (fixed[seg.range.lowerBound] ?? [:])
if final {
let all = p.syllables.map(\.syllable)
return converter.convert(all, fixed: fixedHere.filter { $0.key.upperBound <= all.count }).map(\.text).joined()
}
let done = p.completed.map(\.syllable)
let text = converter.convert(done, fixed: fixedHere.filter { $0.key.upperBound <= done.count }).map(\.text).joined()
return text + (p.pending?.syllable.reading ?? "")
}
// MARK: - Key handling
@@ -92,7 +107,7 @@ public final class CompositionEngine {
case .backspace:
guard !keys.isEmpty else { return .passThrough }
keys.removeLast()
fixed = validFixed(parse)
fixed = fixed.filter { $0.key < keys.count }
if keys.isEmpty { forcedMode = nil; fixed = [:] }
return .consumed
case .enter:
@@ -118,15 +133,15 @@ public final class CompositionEngine {
private func handleCharacter(_ c: Character) -> KeyResult {
if c == " " {
guard !keys.isEmpty else { return .passThrough }
let decided = forcedMode ?? classifier.classify(keys, finishing: true, previous: context).mode
if decided == .literal {
return KeyResult(handled: true, commit: commitCurrent(as: .literal) + " ")
let segs = segments(of: keys, finishing: true)
if segs.last?.mode == .literal {
return KeyResult(handled: true, commit: commitCurrent(segs) + " ")
}
if parse.pending != nil {
if parse(ofLast: segs)?.pending != nil {
keys.append(" ") // first tone
return .consumed
}
return KeyResult(handled: true, commit: commitCurrent(as: .zhuyin))
return KeyResult(handled: true, commit: commitCurrent(segs))
}
let composable = DachenLayout.isZhuyinKey(c) || (c.isASCII && c.isLetter) || (c == "'" && !keys.isEmpty)
@@ -134,9 +149,9 @@ public final class CompositionEngine {
// ", . / ; -" are Zhuyin symbols (ㄝ ㄡ ㄥ ㄤ ㄦ) but act as punctuation after an
// English word. Decide with the key included: "2;" is ㄉㄤ (檔…), "hello," is English.
if !keys.isEmpty, DachenLayout.symbols[c] != nil, !c.isLetter, !c.isNumber, forcedMode != .zhuyin {
let withKey = forcedMode ?? classifier.classify(keys + [c], previous: context).mode
if withKey == .literal && !keys.allSatisfy(\.isNumber) {
return KeyResult(handled: true, commit: commitCurrent(as: .literal) + String(c))
let withKey = segments(of: keys + [c])
if withKey.last?.mode == .literal && !keys.allSatisfy(\.isNumber) {
return KeyResult(handled: true, commit: commitCurrent() + String(c))
}
}
keys.append(c)
@@ -151,25 +166,34 @@ public final class CompositionEngine {
return KeyResult(handled: true, commit: committed + punct)
}
private func parse(ofLast segs: [Segment]) -> ZhuyinParse? {
guard let last = segs.last, last.mode == .zhuyin else { return nil }
return ZhuyinParser.parse(Array(keys[last.range]))
}
static let fullwidth: [Character: String] = [
"<": ",", ">": "。", "?": "?", "!": "!", ":": ":", "\"": "「", "(": "(", ")": ")",
"{": "『", "}": "』", "[": "「", "]": "」", "~": "~", "_": "——",
]
// MARK: - Candidates
// MARK: - Candidates (for the last Zhuyin segment)
private func openCandidates() {
let p = parse
guard mode == .zhuyin, !p.completed.isEmpty else { return }
candidates = converter.candidates(p.completed.map(\.syllable), endingAt: p.completed.count)
let segs = segments
guard let last = segs.last, last.mode == .zhuyin else { return }
let done = ZhuyinParser.parse(Array(keys[last.range])).completed.map(\.syllable)
guard !done.isEmpty else { return }
candidates = converter.candidates(done, endingAt: done.count)
candidateSegmentStart = last.range.lowerBound
candidateIndex = 0
}
public func selectCandidate(_ index: Int) {
guard candidates.indices.contains(index) else { return }
let c = candidates[index]
fixed = fixed.filter { !$0.key.overlaps(c.range) }
fixed[c.range] = c.text
var here = (fixed[candidateSegmentStart] ?? [:]).filter { !$0.key.overlaps(c.range) }
here[c.range] = c.text
fixed[candidateSegmentStart] = here
candidates = []
}
@@ -197,13 +221,13 @@ public final class CompositionEngine {
// MARK: - Commit
@discardableResult
private func commitCurrent(as decided: InputMode? = nil) -> String {
let m = decided ?? mode
let text = m == .literal ? String(keys) : zhuyinCommitText()
private func commitCurrent(_ given: [Segment]? = nil) -> String {
let segs = given ?? segments
let text = segs.map { render($0, final: true) }.joined()
if let forced = forcedMode {
classifier.learned.record(String(keys), as: forced)
}
previousMode = m
previousMode = segs.last?.mode
previousCommitTime = now()
reset()
return text