Split mixed Chinese/English input without spaces; add release script
The classifier now segments a buffer into alternating Zhuyin and literal runs by dynamic programming over split points, so "ji3hello" becomes 我hello and "hellosu3cl3" becomes hello你好. Each switch costs a penalty, a Zhuyin run followed by another must end with a tone, and an untoned trailing Zhuyin run after English is penalized so partial words don't flicker into Bopomofo. Implemented in both Swift and Python with the same tests; the Mac app colours each segment separately. scripts/bump-version.sh bumps the version in Info.plist and ime.json; scripts/release.sh bumps, tests, packages, commits and tags in one step. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,17 @@ public enum InputMode: String, Codable {
|
||||
case literal
|
||||
}
|
||||
|
||||
/// A run of keys (indices into the buffer) interpreted in one mode.
|
||||
public struct Segment: Equatable {
|
||||
public var range: Range<Int>
|
||||
public var mode: InputMode
|
||||
|
||||
public init(range: Range<Int>, mode: InputMode) {
|
||||
self.range = range
|
||||
self.mode = mode
|
||||
}
|
||||
}
|
||||
|
||||
public struct Classification {
|
||||
public var mode: InputMode
|
||||
public var zhuyinScore: Double
|
||||
@@ -72,6 +83,83 @@ public final class Classifier {
|
||||
return Classification(mode: zh >= lit ? .zhuyin : .literal, zhuyinScore: zh, literalScore: lit)
|
||||
}
|
||||
|
||||
// MARK: - Segmentation
|
||||
|
||||
/// Cost of each switch between Zhuyin and literal inside one buffer, so that a normal
|
||||
/// word or sentence is not chopped up.
|
||||
public var segmentPenalty = 3.0
|
||||
|
||||
/// Splits a buffer into alternating Zhuyin / literal segments, e.g. "ji3hello" →
|
||||
/// [ji3: zhuyin, hello: literal]. Dynamic programming over every split point.
|
||||
/// A Zhuyin segment followed by another segment must end with a tone key.
|
||||
public func segment(_ keys: [Character], finishing: Bool = false, previous: InputMode? = nil) -> [Segment] {
|
||||
let n = keys.count
|
||||
guard n > 0 else { return [] }
|
||||
let modes: [InputMode] = [.zhuyin, .literal]
|
||||
// best[i][m]: best score for keys[0..<i] whose last segment has mode m.
|
||||
var best = Array(repeating: [Double](repeating: -.infinity, count: 2), count: n + 1)
|
||||
var back = Array(repeating: [Int](repeating: -1, count: 2), count: n + 1) // segment start j
|
||||
|
||||
for i in 1...n {
|
||||
for j in 0..<i {
|
||||
let scores = segmentScores(Array(keys[j..<i]), isLast: i == n, afterOther: j > 0,
|
||||
finishing: finishing, previous: j == 0 ? previous : nil,
|
||||
whole: j == 0 && i == n)
|
||||
for m in 0..<2 where scores[m] > -.infinity {
|
||||
// Adjacent segments must alternate modes; same-mode neighbours are one segment.
|
||||
let total = j == 0 ? scores[m] : best[j][1 - m] + scores[m] - segmentPenalty
|
||||
if total > best[i][m] {
|
||||
best[i][m] = total
|
||||
back[i][m] = j
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
guard best[n][0] > -.infinity || best[n][1] > -.infinity else {
|
||||
return [Segment(range: 0..<n, mode: classify(keys, finishing: finishing, previous: previous).mode)]
|
||||
}
|
||||
var m = best[n][0] >= best[n][1] ? 0 : 1
|
||||
var i = n
|
||||
var out: [Segment] = []
|
||||
while i > 0 {
|
||||
let j = back[i][m]
|
||||
out.append(Segment(range: j..<i, mode: modes[m]))
|
||||
i = j
|
||||
m = 1 - m
|
||||
}
|
||||
return out.reversed()
|
||||
}
|
||||
|
||||
/// Extra cost for a trailing Zhuyin segment after English that has no tone key yet
|
||||
/// ("ji3hel" should stay 我hel, not become 我heㄠ, until a tone confirms the switch).
|
||||
public var unconfirmedSwitchPenalty = 2.0
|
||||
|
||||
private func segmentScores(_ seg: [Character], isLast: Bool, afterOther: Bool, finishing: Bool,
|
||||
previous: InputMode?, whole: Bool) -> [Double] {
|
||||
var zh: Double
|
||||
let parse = ZhuyinParser.parse(seg)
|
||||
if !isLast && parse.pending != nil {
|
||||
zh = -.infinity
|
||||
} else {
|
||||
zh = zhuyinScore(finishing && isLast ? seg + [" "] : seg) + zhuyinPrior
|
||||
if isLast && afterOther && !finishing && !parse.syllables.contains(where: \.toned) {
|
||||
zh -= unconfirmedSwitchPenalty
|
||||
}
|
||||
}
|
||||
var lit = literalScore(seg, complete: !isLast || finishing)
|
||||
switch previous {
|
||||
case .zhuyin?: zh += contextWeight
|
||||
case .literal?: lit += contextWeight
|
||||
case nil: break
|
||||
}
|
||||
if whole {
|
||||
let b = learned.bias(for: String(seg))
|
||||
if b > 0 { zh += b } else { lit -= b }
|
||||
}
|
||||
return [zh, lit]
|
||||
}
|
||||
|
||||
public func zhuyinScore(_ keys: [Character]) -> Double {
|
||||
let parse = ZhuyinParser.parse(keys)
|
||||
if parse.foreignKeys > 0 || parse.syllables.isEmpty { return -.infinity }
|
||||
|
||||
Reference in New Issue
Block a user