Files
AutoZhuyin/scripts/convert-mcbopomofo.py
T
2026-11-10 10:50:04 +08:00

88 lines
2.8 KiB
Python

#!/usr/bin/env python3
"""Builds Resources/lexicon.txt from McBopomofo's raw data files (MIT License).
Only reads data files; it does not run any McBopomofo code. Inputs (from
McBopomofo's Source/Data): BPMFBase.txt, BPMFMappings.txt, phrase.occ,
heterophony{1,2,3}.list.
Score = log10(count / total), with multi-reading characters split by how
common each reading is (heterophony1 = main reading, 2 and 3 = secondary).
Usage: convert-mcbopomofo.py <McBopomofo/Source/Data> <seed lexicon> <output>
"""
import math
import sys
from collections import defaultdict
from pathlib import Path
data, seed, out = Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3])
def rows(name):
with open(data / name, encoding="utf-8") as f:
for line in f:
line = line.strip()
if line and not line.startswith("#"):
yield line.split()
counts = defaultdict(int)
for cols in rows("phrase.occ"):
if len(cols) == 2 and cols[1].isdigit():
counts[cols[0]] += int(cols[1])
total = sum(counts.values())
# Rank of each (char, reading): 1 = main reading.
rank = {}
for r in (3, 2, 1):
for cols in rows(f"heterophony{r}.list"):
if len(cols) >= 2:
rank[(cols[0], cols[1])] = r
has_main = {ch for (ch, _), r in rank.items() if r == 1}
RANK_WEIGHT = {1: 1.0, 2: 0.25, 3: 0.08}
entries = {} # (reading, phrase) -> score
def add(reading, phrase, count):
score = round(math.log10((count + 0.3) / total), 4)
key = (reading, phrase)
if key not in entries or entries[key] < score:
entries[key] = score
# Single characters.
char_readings = defaultdict(list)
for cols in rows("BPMFBase.txt"):
if len(cols) >= 2:
char_readings[cols[0]].append(cols[1])
for ch, readings in char_readings.items():
readings = list(dict.fromkeys(readings))
c = counts.get(ch, 0)
for rd in readings:
if len(readings) == 1:
w = 1.0
elif (ch, rd) in rank:
w = RANK_WEIGHT[rank[(ch, rd)]]
elif ch in has_main:
w = 0.02
else:
w = 1.0 / len(readings)
add(rd, ch, c * w)
# Multi-character phrases.
for cols in rows("BPMFMappings.txt"):
if len(cols) >= 3 and len(cols[0]) == len(cols) - 1:
add("-".join(cols[1:]), cols[0], counts.get(cols[0], 0))
with open(out, "w", encoding="utf-8") as f:
f.write("# Generated by scripts/convert-mcbopomofo.py from McBopomofo data (MIT License,\n")
f.write("# https://github.com/openvanilla/McBopomofo). Format: <reading> <phrase> <log10 score>\n")
for (reading, phrase), score in sorted(entries.items()):
f.write(f"{reading} {phrase} {score}\n")
# Starter entries come last: the loader keeps the first copy of a duplicate.
f.write("\n")
f.write(seed.read_text(encoding="utf-8"))
print(f"{len(entries)} entries from McBopomofo, total occurrences {total}")