Update tokenization_nandi.py

tokenization_nandi.py

@@ -91,34 +91,40 @@ class NandiTokenizer(TokenizersBackend):
             **kwargs,
         )
 
-    def encode(
-        self,
-        text,
-        text_pair=None,
-        add_special_tokens: bool = True,
-        padding=False,
-        truncation=None,
-        max_length=None,
-        stride: int = 0,
-        padding_side=None,
-        return_tensors=None,
-        **kwargs,
-    ):
+    def prepare_for_tokenization(self, text, **kwargs):
+        
         if isinstance(text, str):
-            # This is a temporary fix to match the behaviour of the training pipeline
-            text = "<|im_start|>" + " " + text
-        return super().encode(
-            text,
-            text_pair=text_pair,
-            add_special_tokens=add_special_tokens,
-            padding=padding,
-            truncation=truncation,
-            max_length=max_length,
-            stride=stride,
-            padding_side=padding_side,
-            return_tensors=return_tensors,
-            **kwargs,
-        )
+            text = "<|im_start|> " + text
+        return (text, kwargs)
+
+    # def encode(
+    #     self,
+    #     text,
+    #     text_pair=None,
+    #     add_special_tokens: bool = True,
+    #     padding=False,
+    #     truncation=None,
+    #     max_length=None,
+    #     stride: int = 0,
+    #     padding_side=None,
+    #     return_tensors=None,
+    #     **kwargs,
+    # ):
+    #     if isinstance(text, str):
+    #         # This is a temporary fix to match the behaviour of the training pipeline
+    #         text = "<|im_start|>" + " " + text
+    #     return super().encode(
+    #         text,
+    #         text_pair=text_pair,
+    #         add_special_tokens=add_special_tokens,
+    #         padding=padding,
+    #         truncation=truncation,
+    #         max_length=max_length,
+    #         stride=stride,
+    #         padding_side=padding_side,
+    #         return_tensors=return_tensors,
+    #         **kwargs,
+    #     )
 
 
 __all__ = ["NandiTokenizer"]