Upload tokenizer

tokenizer_script.py

@@ -109,6 +109,18 @@ class CharacterTokenizer(PreTrainedTokenizer):
 
     def convert_tokens_to_string(self, tokens):
         return "".join(tokens)
+    
+
+      @classmethod
+    def from_json(cls, vocab_file, **kwargs):
+        with open(vocab_file, 'r', encoding='utf-8') as f:
+            vocab = json.load(f)
+        
+        return cls(vocab=vocab, **kwargs)
+    
+    @classmethod
+    def from_vocab(cls, vocab, **kwargs):
+        return cls(vocab=vocab, **kwargs)
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):