File size: 184 Bytes
dc7b215 | 1 2 3 4 5 6 7 8 9 | from transformers import AutoTokenizer, PreTrainedTokenizerFast
class MtgGloVeTokenizer(PreTrainedTokenizerFast):
pass
MtgGloVeTokenizer.register_for_auto_class(AutoTokenizer)
|
dc7b215 | 1 2 3 4 5 6 7 8 9 | from transformers import AutoTokenizer, PreTrainedTokenizerFast
class MtgGloVeTokenizer(PreTrainedTokenizerFast):
pass
MtgGloVeTokenizer.register_for_auto_class(AutoTokenizer)
|