#!/usr/bin/env python3 """Builds Resources/lexicon.txt from McBopomofo's raw data files (MIT License). Only reads data files; it does not run any McBopomofo code. Inputs (from McBopomofo's Source/Data): BPMFBase.txt, BPMFMappings.txt, phrase.occ, heterophony{1,2,3}.list. Score = log10(count / total), with multi-reading characters split by how common each reading is (heterophony1 = main reading, 2 and 3 = secondary). Usage: convert-mcbopomofo.py """ import math import sys from collections import defaultdict from pathlib import Path data, seed, out = Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3]) def rows(name): with open(data / name, encoding="utf-8") as f: for line in f: line = line.strip() if line and not line.startswith("#"): yield line.split() counts = defaultdict(int) for cols in rows("phrase.occ"): if len(cols) == 2 and cols[1].isdigit(): counts[cols[0]] += int(cols[1]) total = sum(counts.values()) # Rank of each (char, reading): 1 = main reading. rank = {} for r in (3, 2, 1): for cols in rows(f"heterophony{r}.list"): if len(cols) >= 2: rank[(cols[0], cols[1])] = r has_main = {ch for (ch, _), r in rank.items() if r == 1} RANK_WEIGHT = {1: 1.0, 2: 0.25, 3: 0.08} entries = {} # (reading, phrase) -> score def add(reading, phrase, count): score = round(math.log10((count + 0.3) / total), 4) key = (reading, phrase) if key not in entries or entries[key] < score: entries[key] = score # Single characters. char_readings = defaultdict(list) for cols in rows("BPMFBase.txt"): if len(cols) >= 2: char_readings[cols[0]].append(cols[1]) for ch, readings in char_readings.items(): readings = list(dict.fromkeys(readings)) c = counts.get(ch, 0) for rd in readings: if len(readings) == 1: w = 1.0 elif (ch, rd) in rank: w = RANK_WEIGHT[rank[(ch, rd)]] elif ch in has_main: w = 0.02 else: w = 1.0 / len(readings) add(rd, ch, c * w) # Multi-character phrases. for cols in rows("BPMFMappings.txt"): if len(cols) >= 3 and len(cols[0]) == len(cols) - 1: add("-".join(cols[1:]), cols[0], counts.get(cols[0], 0)) with open(out, "w", encoding="utf-8") as f: f.write("# Generated by scripts/convert-mcbopomofo.py from McBopomofo data (MIT License,\n") f.write("# https://github.com/openvanilla/McBopomofo). Format: \n") for (reading, phrase), score in sorted(entries.items()): f.write(f"{reading} {phrase} {score}\n") # Starter entries come last: the loader keeps the first copy of a duplicate. f.write("\n") f.write(seed.read_text(encoding="utf-8")) print(f"{len(entries)} entries from McBopomofo, total occurrences {total}")