Adapt tokenization_interns1.py to transformers>=5.0.0 (#15)

- Adapt tokenization_interns1.py to transformers>=5.0.0 (65efc111527b17792b13f09fbe7ddbb9d9df263e)
- Update: fix bug (7260346cf2e4d0d8faf80b45ff37a06a14e4b74a)
- Update: fix bug (7b33977d07711c291f31851e21f44f995dc74c3f)


Co-authored-by: Kevin Zhang <Zhangyc02@users.noreply.huggingface.co>

tokenization_interns1.py

@@ -24,11 +24,14 @@ from functools import lru_cache
 import regex as re
 import sentencepiece as spm
 
+import transformers
 from transformers.tokenization_utils_base import AddedToken, TextInput
-from transformers.tokenization_utils import PreTrainedTokenizer
 from transformers.utils import logging
-# from transformers.utils.import_utils import requires
-
+from packaging import version
+if version.parse(transformers.__version__) >= version.parse("5.0.0"):
+    from transformers.tokenization_python import PreTrainedTokenizer
+else:
+    from transformers.tokenization_utils import PreTrainedTokenizer
 
 logger = logging.get_logger(__name__)
 
@@ -506,6 +509,7 @@ class InternS1Tokenizer(PreTrainedTokenizer):
         pad_token="<|endoftext|>",
         clean_up_tokenization_spaces=False,
         split_special_tokens=False,
+        special_tokens_pattern="none",
         **kwargs,
     ):
         bos_token = (
@@ -566,6 +570,7 @@ class InternS1Tokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
             split_special_tokens=split_special_tokens,
+            special_tokens_pattern="none",
             **kwargs,
         )
 
@@ -715,9 +720,6 @@ class InternS1Tokenizer(PreTrainedTokenizer):
 
         text, kwargs = self.prepare_for_tokenization(text, **kwargs)
 
-        if kwargs:
-            logger.warning(f"Keyword arguments {kwargs} not recognized.")
-
         if hasattr(self, "do_lower_case") and self.do_lower_case:
             # convert non-special tokens to lowercase. Might be super slow as well?
             escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]