first
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Builds Resources/lexicon.txt from McBopomofo's raw data files (MIT License).
|
||||
|
||||
Only reads data files; it does not run any McBopomofo code. Inputs (from
|
||||
McBopomofo's Source/Data): BPMFBase.txt, BPMFMappings.txt, phrase.occ,
|
||||
heterophony{1,2,3}.list.
|
||||
|
||||
Score = log10(count / total), with multi-reading characters split by how
|
||||
common each reading is (heterophony1 = main reading, 2 and 3 = secondary).
|
||||
|
||||
Usage: convert-mcbopomofo.py <McBopomofo/Source/Data> <seed lexicon> <output>
|
||||
"""
|
||||
import math
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
data, seed, out = Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3])
|
||||
|
||||
|
||||
def rows(name):
|
||||
with open(data / name, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
yield line.split()
|
||||
|
||||
|
||||
counts = defaultdict(int)
|
||||
for cols in rows("phrase.occ"):
|
||||
if len(cols) == 2 and cols[1].isdigit():
|
||||
counts[cols[0]] += int(cols[1])
|
||||
total = sum(counts.values())
|
||||
|
||||
# Rank of each (char, reading): 1 = main reading.
|
||||
rank = {}
|
||||
for r in (3, 2, 1):
|
||||
for cols in rows(f"heterophony{r}.list"):
|
||||
if len(cols) >= 2:
|
||||
rank[(cols[0], cols[1])] = r
|
||||
has_main = {ch for (ch, _), r in rank.items() if r == 1}
|
||||
RANK_WEIGHT = {1: 1.0, 2: 0.25, 3: 0.08}
|
||||
|
||||
entries = {} # (reading, phrase) -> score
|
||||
|
||||
|
||||
def add(reading, phrase, count):
|
||||
score = round(math.log10((count + 0.3) / total), 4)
|
||||
key = (reading, phrase)
|
||||
if key not in entries or entries[key] < score:
|
||||
entries[key] = score
|
||||
|
||||
|
||||
# Single characters.
|
||||
char_readings = defaultdict(list)
|
||||
for cols in rows("BPMFBase.txt"):
|
||||
if len(cols) >= 2:
|
||||
char_readings[cols[0]].append(cols[1])
|
||||
for ch, readings in char_readings.items():
|
||||
readings = list(dict.fromkeys(readings))
|
||||
c = counts.get(ch, 0)
|
||||
for rd in readings:
|
||||
if len(readings) == 1:
|
||||
w = 1.0
|
||||
elif (ch, rd) in rank:
|
||||
w = RANK_WEIGHT[rank[(ch, rd)]]
|
||||
elif ch in has_main:
|
||||
w = 0.02
|
||||
else:
|
||||
w = 1.0 / len(readings)
|
||||
add(rd, ch, c * w)
|
||||
|
||||
# Multi-character phrases.
|
||||
for cols in rows("BPMFMappings.txt"):
|
||||
if len(cols) >= 3 and len(cols[0]) == len(cols) - 1:
|
||||
add("-".join(cols[1:]), cols[0], counts.get(cols[0], 0))
|
||||
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
f.write("# Generated by scripts/convert-mcbopomofo.py from McBopomofo data (MIT License,\n")
|
||||
f.write("# https://github.com/openvanilla/McBopomofo). Format: <reading> <phrase> <log10 score>\n")
|
||||
for (reading, phrase), score in sorted(entries.items()):
|
||||
f.write(f"{reading} {phrase} {score}\n")
|
||||
# Starter entries come last: the loader keeps the first copy of a duplicate.
|
||||
f.write("\n")
|
||||
f.write(seed.read_text(encoding="utf-8"))
|
||||
|
||||
print(f"{len(entries)} entries from McBopomofo, total occurrences {total}")
|
||||
Reference in New Issue
Block a user