import XCTest @testable import AutoZhuyinCore private let resources = URL(fileURLWithPath: #filePath) .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() .appendingPathComponent("Resources") private let sharedLexicon = try! Lexicon(contentsOf: resources.appendingPathComponent("lexicon.txt")) private let sharedEnglish = EnglishModel(commonWordsURL: resources.appendingPathComponent("english-common.txt")) private let sharedWeights = Features.loadWeights(from: resources.appendingPathComponent("classifier-weights.json")) final class ParserTests: XCTestCase { func testParsesNiHao() { let p = ZhuyinParser.parse(Array("su3cl3")) XCTAssertEqual(p.syllables.map(\.syllable.reading), ["ㄋㄧˇ", "ㄏㄠˇ"]) XCTAssertNil(p.pending) } func testSpaceIsFirstTone() { let p = ZhuyinParser.parse(Array("wu0 ")) XCTAssertEqual(p.syllables.map(\.syllable.reading), ["ㄊㄧㄢ"]) XCTAssertEqual(p.syllables.first?.syllable.tone, .first) } func testPendingSyllable() { let p = ZhuyinParser.parse(Array("su3c")) XCTAssertEqual(p.completed.count, 1) XCTAssertEqual(p.pending?.syllable.reading, "ㄏ") } func testOutOfOrderSymbols() { // ㄧ ㄒ ㄠ ˇ → ㄒㄧㄠˇ (小); ㄒ ㄠ ㄧ ˇ → ㄒㄧㄠˇ for keys in ["uvl3", "vlu3", "ulv3"] { let p = ZhuyinParser.parse(Array(keys)) XCTAssertEqual(p.syllables.map(\.syllable.reading), ["ㄒㄧㄠˇ"], keys) XCTAssertTrue(p.syllables[0].reordered, keys) } // In-order typing is unchanged and not flagged. XCTAssertFalse(ZhuyinParser.parse(Array("vul3")).syllables[0].reordered) // Symbols that no valid syllable combines (ㄩ then ㄍ) still start a new syllable. XCTAssertEqual(ZhuyinParser.parse(Array("me")).syllables.count, 2) } func testSyllableTable() { XCTAssertTrue(SyllableTable.valid.contains("ㄋㄧ")) XCTAssertTrue(SyllableTable.valid.contains("ㄓ")) XCTAssertFalse(SyllableTable.valid.contains("ㄐㄚ")) XCTAssertTrue(SyllableTable.validPrefixes.contains("ㄐㄧ")) } } final class ConverterTests: XCTestCase { func testConvertsPhrase() { let c = Converter(lexicon: sharedLexicon) let syl = ZhuyinParser.parse(Array("su3cl3")).syllables.map(\.syllable) XCTAssertEqual(c.convert(syl).map(\.text).joined(), "你好") } func testSentence() { let c = Converter(lexicon: sharedLexicon) let syl = ZhuyinParser.parse(Array("rup wu0 wu0 fu4cp3cl3")).syllables.map(\.syllable) XCTAssertEqual(c.convert(syl).map(\.text).joined(), "今天天氣很好") } func testLexiconLoadTime() throws { let start = Date() _ = try Lexicon(contentsOf: resources.appendingPathComponent("lexicon.txt")) let elapsed = Date().timeIntervalSince(start) print("lexicon load: \(elapsed)s") XCTAssertLessThan(elapsed, 3) } func testFixedCandidateWins() { let c = Converter(lexicon: sharedLexicon) let syl = ZhuyinParser.parse(Array("su3")).syllables.map(\.syllable) XCTAssertEqual(c.convert(syl, fixed: [0..<1: "妳"]).map(\.text).joined(), "妳") } func testToneOmissionStillConverts() { let c = Converter(lexicon: sharedLexicon) // ㄨㄛ (untoned) + ㄇㄣ˙ → 我們 let syl = ZhuyinParser.parse(Array("jiap7")).syllables.map(\.syllable) XCTAssertEqual(c.convert(syl).map(\.text).joined(), "我們") } } final class ClassifierTests: XCTestCase { let classifier = Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights) func mode(_ s: String, finishing: Bool = false) -> InputMode { classifier.classify(Array(s), finishing: finishing).mode } func testZhuyin() { for s in ["su3cl3", "ji3", "cl3", "ru ", "u ", "fu ", "183", "2k7", "5j/ "] { XCTAssertEqual(mode(s), .zhuyin, s) } for s in ["ru", "ji", "fu"] { XCTAssertEqual(mode(s, finishing: true), .zhuyin, s) } } func testEnglish() { for s in ["hello", "the", "is", "ok", "github", "Swift", "2024", "so", "go", "me", "no", "a", "an", "up", "i", "meeting", "email", "push"] { XCTAssertEqual(mode(s, finishing: true), .literal, s) } XCTAssertEqual(mode("hello"), .literal) } func testLearningFlipsDecision() { let learned = LearnedPreferences() let c = Classifier(lexicon: sharedLexicon, english: sharedEnglish, learned: learned, weights: sharedWeights) XCTAssertEqual(c.classify(Array("so"), finishing: true).mode, .literal) learned.record("so", as: .zhuyin) XCTAssertEqual(c.classify(Array("so"), finishing: true).mode, .zhuyin) } } final class EngineTests: XCTestCase { func makeEngine() -> CompositionEngine { CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights)) } /// Types a string and returns everything committed. func type(_ s: String, into e: CompositionEngine) -> String { var out = "" for c in s { let key: KeyInput = c == "\n" ? .enter : c == "\t" ? .tab : .character(c) let r = e.handle(key) if let t = r.commit { out += t } else if !r.handled { out.append(c) } } return out } func testMixedSentence() { let e = makeEngine() // 我 + "hello world " + 你好 XCTAssertEqual(type("ji3 hello world su3cl3 ", into: e), "我hello world 你好") } func testOutOfOrderTypingConverts() { let e = makeEngine() XCTAssertEqual(type("uvl3\n", into: e), "小") // A phrase typed with one syllable out of order converts like the in-order version. XCTAssertEqual(type("ulv3cl3\n", into: makeEngine()), type("vul3cl3\n", into: makeEngine())) } func testMarkedTextWhileTyping() { let e = makeEngine() _ = type("su3c", into: e) XCTAssertEqual(e.markedText, "你ㄏ") _ = type("l3", into: e) XCTAssertEqual(e.markedText, "你好") } func testEnterCommitsEnglish() { let e = makeEngine() XCTAssertEqual(type("github\n", into: e), "github") } func testTabTogglesAndLearns() { let e = makeEngine() _ = type("so", into: e) XCTAssertEqual(e.mode, .literal) _ = e.handle(.tab) XCTAssertEqual(e.mode, .zhuyin) _ = type("\n", into: e) XCTAssertEqual(e.classifier.learned.weights["so"], 1) } func testEnglishPunctuation() { let e = makeEngine() XCTAssertEqual(type("hello, ", into: e), "hello, ") } func testFullwidthPunctuationAfterChinese() { let e = makeEngine() XCTAssertEqual(type("su3cl3?", into: e), "你好?") } func testCandidates() { let e = makeEngine() _ = type("su3", into: e) _ = e.handle(.down) XCTAssertTrue(e.isShowingCandidates) let idx = e.candidates.firstIndex { $0.text == "妳" }! e.selectCandidate(idx) XCTAssertEqual(type("\n", into: e), "妳") } } final class DigitInitialTests: XCTestCase { func engine() -> CompositionEngine { CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights)) } func run(_ s: String) -> (out: String, marked: String) { let e = engine() var out = "" for c in s { let r = e.handle(c == "\n" ? .enter : .character(c)) if let t = r.commit { out += t } else if !r.handled { out.append(c) } } return (out, e.markedText) } func testDang3() { // 檔 = ㄉㄤˇ = "2;3": ";" (ㄤ) after a digit initial must not be treated as punctuation. let r = run("2;3") XCTAssertEqual(r.out, "") XCTAssertTrue(r.marked.hasPrefix("檔") || r.marked.hasPrefix("黨") || r.marked.hasPrefix("擋"), r.marked) } func testOtherDigitInitialSyllables() { XCTAssertEqual(run("2;3\n").out.count, 1) // ㄉㄤˇ XCTAssertEqual(run("2j/ \n").out, "東") // ㄉㄨㄥ XCTAssertEqual(run("1/4\n").out.count, 1) // ㄅㄥˋ } func testNumbersStillWork() { XCTAssertEqual(run("2.5\n").out, "2.5") XCTAssertEqual(run("hello, ").out, "hello, ") } } final class MixedSegmentTests: XCTestCase { func run(_ s: String) -> (out: String, marked: String) { let e = CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights)) var out = "" for c in s { let r = e.handle(c == "\n" ? .enter : .character(c)) if let t = r.commit { out += t } else if !r.handled { out.append(c) } } return (out, e.markedText) } func testChineseThenEnglish() { XCTAssertEqual(run("ji3hello ").out, "我hello ") XCTAssertEqual(run("su3cl3github\n").out, "你好github") XCTAssertEqual(run("ji3hel").marked, "我hel") } func testEnglishThenChinese() { XCTAssertEqual(run("hellosu3cl3 ").out, "hello你好") XCTAssertEqual(run("oksu3cl3\n").out, "ok你好") } func testThreeSegments() { XCTAssertEqual(run("ji3emailsu3cl3\n").out, "我email你好") } func testSingleLanguageUnchanged() { XCTAssertEqual(run("rup wu0 wu0 fu4cp3cl3\n").out, "今天天氣很好") XCTAssertEqual(run("helloworld\n").out, "helloworld") XCTAssertEqual(run("meeting ").out, "meeting ") XCTAssertEqual(run("183\n").out.count, 1) } } /// The Swift and Python cores must segment identically (fixtures from training/make_parity.py). final class ParityTests: XCTestCase { func testMatchesPythonSegmentation() throws { let url = resources.deletingLastPathComponent().appendingPathComponent("training/parity-fixtures.json") let fixtures = try JSONSerialization.jsonObject(with: Data(contentsOf: url)) as! [[String: Any]] let c = Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights) var mismatches: [String] = [] for fx in fixtures { let keys = Array(fx["keys"] as! String) let expected = (fx["segments"] as! [[Any]]).map { "\($0[0])-\($0[1]):\($0[2])" } let got = c.segment(keys, finishing: fx["finishing"] as! Bool).map { "\($0.range.lowerBound)-\($0.range.upperBound):\($0.mode.rawValue)" } if got != expected { mismatches.append("\(String(keys)) finishing=\(fx["finishing"]!): \(got) vs \(expected)") } } XCTAssertGreaterThan(fixtures.count, 300) XCTAssertTrue(mismatches.isEmpty, "\(mismatches.count) mismatches:\n" + mismatches.prefix(10).joined(separator: "\n")) } func testToneThenSpaceStaysChinese() { let e = CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights)) var out = "" for c in "u4 " { if let t = e.handle(.character(c)).commit { out += t } } XCTAssertEqual(out.count, 1, out) } } final class ShiftTapTests: XCTestCase { func engine() -> CompositionEngine { CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights)) } func type(_ s: String, into e: CompositionEngine) -> String { var out = "" for c in s { let key: KeyInput = c == "⇧" ? .shiftTap : c == "\n" ? .enter : .character(c) let r = e.handle(key) if let t = r.commit { out += t } else if !r.handled { out.append(c) } } return out } func testShiftTapForcesNextWordEnglishLowercase() { let e = engine() // "ru" alone would be ㄐㄧ (機); after a Shift tap it stays English, lowercase. XCTAssertEqual(type("⇧ru ", into: e), "ru ") // Automatic detection resumes after the word. XCTAssertFalse(e.isEnglishArmed) XCTAssertEqual(type("su3cl3\n", into: e), "你好") } func testShiftTapTwiceCancels() { XCTAssertEqual(type("⇧⇧su3cl3\n", into: engine()), "你好") } func testShiftTapMidWordFlips() { let e = engine() _ = type("so", into: e) XCTAssertEqual(e.mode, .literal) _ = e.handle(.shiftTap) XCTAssertEqual(e.mode, .zhuyin) _ = type("\n", into: e) XCTAssertEqual(e.classifier.learned.weights["so"], 1) // a correction is learned } func testForcingBeforeTypingIsNotLearned() { let e = engine() _ = type("⇧hello ", into: e) XCTAssertNil(e.classifier.learned.weights["hello"]) } func testArmingSurvivesBackspace() { let e = engine() _ = type("⇧r", into: e) _ = e.handle(.backspace) XCTAssertTrue(e.isEnglishArmed) XCTAssertEqual(type("ru ", into: e), "ru ") } func testDetector() { var d = ShiftTapDetector() XCTAssertFalse(d.modifiersChanged(shift: true, otherModifiers: false, time: 0)) XCTAssertTrue(d.modifiersChanged(shift: false, otherModifiers: false, time: 0.1)) // Shift used for a capital letter: not a tap. _ = d.modifiersChanged(shift: true, otherModifiers: false, time: 1) d.keyPressed() XCTAssertFalse(d.modifiersChanged(shift: false, otherModifiers: false, time: 1.1)) // Held too long: not a tap. _ = d.modifiersChanged(shift: true, otherModifiers: false, time: 2) XCTAssertFalse(d.modifiersChanged(shift: false, otherModifiers: false, time: 3)) // Shift + Command: not a tap. _ = d.modifiersChanged(shift: true, otherModifiers: false, time: 4) _ = d.modifiersChanged(shift: true, otherModifiers: true, time: 4.1) XCTAssertFalse(d.modifiersChanged(shift: false, otherModifiers: false, time: 4.2)) } } final class CursorTests: XCTestCase { func engine() -> CompositionEngine { CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights)) } func type(_ s: String, into e: CompositionEngine) -> String { var out = "" for c in s { let key: KeyInput = c == "←" ? .left : c == "→" ? .right : c == "↓" ? .down : c == "\n" ? .enter : .character(c) let r = e.handle(key) if let t = r.commit { out += t } else if !r.handled { out.append(c) } } return out } func testCursorMovesAndClamps() { let e = engine() _ = type("rup wu0 wu0 fu4", into: e) // 今天天氣 XCTAssertEqual(e.cursorOffset, 4) _ = type("←", into: e) XCTAssertEqual(e.cursorOffset, 3) _ = type("←←←←", into: e) XCTAssertEqual(e.cursorOffset, 1) // stops after the first character _ = type("→→→→", into: e) XCTAssertEqual(e.cursorOffset, 4) } func testChooseCandidateInTheMiddle() { let e = engine() _ = type("rup wu0 wu0 fu4", into: e) _ = type("←←↓", into: e) // cursor after the 2nd syllable (天) XCTAssertTrue(e.isShowingCandidates) XCTAssertTrue(e.candidates.allSatisfy { $0.range.upperBound == 2 }) let i = e.candidates.firstIndex { $0.text == "添" }! e.selectCandidate(i) XCTAssertEqual(type("\n", into: e), "今添天氣") } func testTypingReturnsCursorToEnd() { let e = engine() _ = type("rup wu0 ←", into: e) XCTAssertEqual(e.cursorOffset, 1) _ = type("wu0 ", into: e) XCTAssertEqual(e.cursorOffset, e.markedText.count) } func testCandidatesForChineseBeforeEnglish() { let e = engine() _ = type("su3cl3hello", into: e) // 你好hello _ = type("↓", into: e) // last Chinese syllable, not the English word XCTAssertTrue(e.isShowingCandidates) } } final class DirectEnglishTests: XCTestCase { func testForcedEnglishHasNoComposition() { let e = CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights)) _ = e.handle(.shiftTap) let r = e.handle(.character("r")) XCTAssertEqual(r.commit, "r") // inserted right away XCTAssertEqual(e.markedText, "") // nothing underlined XCTAssertEqual(e.handle(.character("u")).commit, "u") XCTAssertEqual(e.handle(.character("'")).commit, "'") XCTAssertFalse(e.handle(.backspace).handled) // the app deletes it XCTAssertTrue(e.isEnglishArmed) XCTAssertEqual(e.handle(.character(" ")).commit, " ") XCTAssertFalse(e.isEnglishArmed) // back to automatic } func testEnterEndsForcedEnglish() { let e = CompositionEngine(classifier: Classifier(lexicon: sharedLexicon, english: sharedEnglish, weights: sharedWeights)) _ = e.handle(.shiftTap) _ = e.handle(.character("a")) XCTAssertFalse(e.handle(.enter).handled) XCTAssertFalse(e.isEnglishArmed) } }