88 lines
2.8 KiB
Python
88 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Builds Resources/lexicon.txt from McBopomofo's raw data files (MIT License).
|
|
|
|
Only reads data files; it does not run any McBopomofo code. Inputs (from
|
|
McBopomofo's Source/Data): BPMFBase.txt, BPMFMappings.txt, phrase.occ,
|
|
heterophony{1,2,3}.list.
|
|
|
|
Score = log10(count / total), with multi-reading characters split by how
|
|
common each reading is (heterophony1 = main reading, 2 and 3 = secondary).
|
|
|
|
Usage: convert-mcbopomofo.py <McBopomofo/Source/Data> <seed lexicon> <output>
|
|
"""
|
|
import math
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
data, seed, out = Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3])
|
|
|
|
|
|
def rows(name):
|
|
with open(data / name, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line and not line.startswith("#"):
|
|
yield line.split()
|
|
|
|
|
|
counts = defaultdict(int)
|
|
for cols in rows("phrase.occ"):
|
|
if len(cols) == 2 and cols[1].isdigit():
|
|
counts[cols[0]] += int(cols[1])
|
|
total = sum(counts.values())
|
|
|
|
# Rank of each (char, reading): 1 = main reading.
|
|
rank = {}
|
|
for r in (3, 2, 1):
|
|
for cols in rows(f"heterophony{r}.list"):
|
|
if len(cols) >= 2:
|
|
rank[(cols[0], cols[1])] = r
|
|
has_main = {ch for (ch, _), r in rank.items() if r == 1}
|
|
RANK_WEIGHT = {1: 1.0, 2: 0.25, 3: 0.08}
|
|
|
|
entries = {} # (reading, phrase) -> score
|
|
|
|
|
|
def add(reading, phrase, count):
|
|
score = round(math.log10((count + 0.3) / total), 4)
|
|
key = (reading, phrase)
|
|
if key not in entries or entries[key] < score:
|
|
entries[key] = score
|
|
|
|
|
|
# Single characters.
|
|
char_readings = defaultdict(list)
|
|
for cols in rows("BPMFBase.txt"):
|
|
if len(cols) >= 2:
|
|
char_readings[cols[0]].append(cols[1])
|
|
for ch, readings in char_readings.items():
|
|
readings = list(dict.fromkeys(readings))
|
|
c = counts.get(ch, 0)
|
|
for rd in readings:
|
|
if len(readings) == 1:
|
|
w = 1.0
|
|
elif (ch, rd) in rank:
|
|
w = RANK_WEIGHT[rank[(ch, rd)]]
|
|
elif ch in has_main:
|
|
w = 0.02
|
|
else:
|
|
w = 1.0 / len(readings)
|
|
add(rd, ch, c * w)
|
|
|
|
# Multi-character phrases.
|
|
for cols in rows("BPMFMappings.txt"):
|
|
if len(cols) >= 3 and len(cols[0]) == len(cols) - 1:
|
|
add("-".join(cols[1:]), cols[0], counts.get(cols[0], 0))
|
|
|
|
with open(out, "w", encoding="utf-8") as f:
|
|
f.write("# Generated by scripts/convert-mcbopomofo.py from McBopomofo data (MIT License,\n")
|
|
f.write("# https://github.com/openvanilla/McBopomofo). Format: <reading> <phrase> <log10 score>\n")
|
|
for (reading, phrase), score in sorted(entries.items()):
|
|
f.write(f"{reading} {phrase} {score}\n")
|
|
# Starter entries come last: the loader keeps the first copy of a duplicate.
|
|
f.write("\n")
|
|
f.write(seed.read_text(encoding="utf-8"))
|
|
|
|
print(f"{len(entries)} entries from McBopomofo, total occurrences {total}")
|