| import json
|
| import os
|
| from typing import List, Optional
|
| from transformers import PreTrainedTokenizer
|
|
|
|
|
| class CenturioTokenizer(PreTrainedTokenizer):
|
| vocab_files_names = {"vocab_file": "centurio_vocab.json"}
|
| model_input_names = ["input_ids", "attention_mask"]
|
|
|
| def __init__(
|
| self,
|
| vocab_file=None,
|
| unk_token="<unk>",
|
| bos_token="<s>",
|
| eos_token="</s>",
|
| pad_token="<pad>",
|
| sep_token="<sep>",
|
| cls_token="<cls>",
|
| mask_token="<mask>",
|
| space_token="▁",
|
| **kwargs
|
| ):
|
| self.space_token = space_token
|
| self._vocab = {}
|
| self._inv_vocab = {}
|
|
|
| super().__init__(
|
| unk_token=unk_token,
|
| bos_token=bos_token,
|
| eos_token=eos_token,
|
| pad_token=pad_token,
|
| sep_token=sep_token,
|
| cls_token=cls_token,
|
| mask_token=mask_token,
|
| **kwargs
|
| )
|
|
|
| if vocab_file is not None:
|
| self._load_vocab(vocab_file)
|
| else:
|
| self._build_default_vocab()
|
|
|
| def _build_default_vocab(self):
|
| special_tokens = [
|
| self.unk_token, self.bos_token, self.eos_token,
|
| self.pad_token, self.sep_token, self.cls_token,
|
| self.mask_token, self.space_token
|
| ]
|
| self._vocab = {token: i for i, token in enumerate(special_tokens)}
|
| self._inv_vocab = {i: token for token, i in self._vocab.items()}
|
|
|
| def _load_vocab(self, vocab_file):
|
| with open(vocab_file, "r", encoding="utf-8") as f:
|
| self._vocab = json.load(f)
|
| self._inv_vocab = {v: k for k, v in self._vocab.items()}
|
|
|
| def get_vocab(self):
|
| return self._vocab.copy()
|
|
|
| @property
|
| def vocab_size(self):
|
| return len(self._vocab)
|
|
|
| def _tokenize(self, text: str) -> List[str]:
|
| text = text.replace(" ", self.space_token)
|
| tokens = []
|
| current = ""
|
| for ch in text:
|
| if ch.isalnum() or ch in "абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ":
|
| current += ch
|
| else:
|
| if current:
|
| tokens.append(current)
|
| current = ""
|
| tokens.append(ch if ch != self.space_token else self.space_token)
|
| if current:
|
| tokens.append(current)
|
| return tokens
|
|
|
| def _convert_token_to_id(self, token: str) -> int:
|
| return self._vocab.get(token, self._vocab.get(self.unk_token, 0))
|
|
|
| def _convert_id_to_token(self, index: int) -> str:
|
| return self._inv_vocab.get(index, self.unk_token)
|
|
|
| def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple:
|
| if not os.path.isdir(save_directory):
|
| os.makedirs(save_directory)
|
| vocab_file = os.path.join(
|
| save_directory,
|
| (filename_prefix + "-" if filename_prefix else "") + "centurio_vocab.json"
|
| )
|
| with open(vocab_file, "w", encoding="utf-8") as f:
|
| json.dump(self._vocab, f, ensure_ascii=False, indent=2)
|
| return (vocab_file,)
|
|
|
| def build_vocab_from_corpus(self, corpus: List[str], min_freq: int = 2):
|
| from collections import Counter
|
| token_counter = Counter()
|
| for text in corpus:
|
| tokens = self._tokenize(text)
|
| token_counter.update(tokens)
|
|
|
| special_tokens = [
|
| self.unk_token, self.bos_token, self.eos_token,
|
| self.pad_token, self.sep_token, self.cls_token,
|
| self.mask_token, self.space_token
|
| ]
|
| new_vocab = {token: i for i, token in enumerate(special_tokens)}
|
| idx = len(new_vocab)
|
|
|
| for token, freq in token_counter.items():
|
| if freq >= min_freq and token not in new_vocab:
|
| new_vocab[token] = idx
|
| idx += 1
|
|
|
| self._vocab = new_vocab
|
| self._inv_vocab = {v: k for k, v in self._vocab.items()}
|
|
|
| if __name__ == "__main__":
|
| corpus = [
|
| "Привет, как дела!",
|
| "Я учу немецкий язык.",
|
| "Морфемы помогают понять структуру слов."
|
| ]
|
|
|
| tokenizer = CenturioTokenizer()
|
| tokenizer.build_vocab_from_corpus(corpus, min_freq=1)
|
| tokenizer.save_pretrained("./centurio_model")
|
|
|
| for text in corpus:
|
| tokens = tokenizer.tokenize(text)
|
| ids = tokenizer.encode(text)
|
| back = tokenizer.decode(ids)
|
| print(f"\nTEXT : {text}")
|
| print(f"TOKENS : {tokens}")
|
| print(f"IDS : {ids}")
|
| print(f"BACK : {back}")
|
| print(f"VOCAB : {tokenizer.vocab_size}")
|
|
|
