| import json |
|
|
|
|
| |
| class HomophonesReplacer: |
| """ |
| Homophones Replacer |
| |
| Replace the mispronounced characters with correctly pronounced ones. |
| |
| Creation process of homophones_map.json: |
| |
| 1. Establish a word corpus using the [Tencent AI Lab Embedding Corpora v0.2.0 large] with 12 million entries. After cleaning, approximately 1.8 million entries remain. Use ChatTTS to infer the text. |
| 2. Record discrepancies between the inferred and input text, identifying about 180,000 misread words. |
| 3. Create a pinyin to common characters mapping using correctly read characters by ChatTTS. |
| 4. For each discrepancy, extract the correct pinyin using [python-pinyin] and find homophones with the correct pronunciation from the mapping. |
| |
| Thanks to: |
| [Tencent AI Lab Embedding Corpora for Chinese and English Words and Phrases](https://ai.tencent.com/ailab/nlp/en/embedding.html) |
| [python-pinyin](https://github.com/mozillazg/python-pinyin) |
| |
| """ |
|
|
| def __init__(self, map_file_path): |
| self.homophones_map = self.load_homophones_map(map_file_path) |
|
|
| def load_homophones_map(self, map_file_path): |
| with open(map_file_path, "r", encoding="utf-8") as f: |
| homophones_map = json.load(f) |
| return homophones_map |
|
|
| def replace(self, text): |
| result = [] |
| for char in text: |
| if char in self.homophones_map: |
| result.append(self.homophones_map[char]) |
| else: |
| result.append(char) |
| return "".join(result) |
|
|
