| """Translate english to chinese via a dict.""" |
| from typing import List, Union |
|
|
| from gradiobee.en2zh import en2zh |
| from gradiobee.insert_spaces import insert_spaces |
|
|
|
|
| |
| def en2zh_tokens( |
| |
| text: Union[str, List[str]], |
| dedup: bool = True, |
| ) -> List[List[str]]: |
| |
| """Translate english to chinese tokens via a dict. |
| |
| Args |
| text: to translate, list of str |
| dedup: if True, remove all duplicates |
| Returns |
| res: list of list of str/token/char |
| """ |
| res = en2zh(text) |
|
|
| if dedup: |
| return [list(set(insert_spaces(elm).split())) for elm in res] |
|
|
| return [insert_spaces(elm).split() for elm in res] |
|
|
