Upload tokenization_splinter.py
Browse files- tokenization_splinter.py +4 -1
tokenization_splinter.py
CHANGED
|
@@ -215,5 +215,8 @@ class SplinterBertTokenizerFast(BertTokenizerFast):
|
|
| 215 |
])
|
| 216 |
self._tokenizer.decoder = Decoder.custom(SplinterDecoder(self.splinter))
|
| 217 |
|
| 218 |
-
def
|
|
|
|
|
|
|
|
|
|
| 219 |
print('Cannot save SplinterBertTokenizerFast, please copy the files directly from the repository')
|
|
|
|
| 215 |
])
|
| 216 |
self._tokenizer.decoder = Decoder.custom(SplinterDecoder(self.splinter))
|
| 217 |
|
| 218 |
+
def save_pretrained(self, *args, **kwargs):
|
| 219 |
+
self._save_pretrained(*args, **kwargs)
|
| 220 |
+
|
| 221 |
+
def _save_pretrained(self, *args, **kwargs):
|
| 222 |
print('Cannot save SplinterBertTokenizerFast, please copy the files directly from the repository')
|