Update tokenization_nandi.py

tokenization_nandi.py

@@ -91,11 +91,24 @@ class NandiTokenizer(TokenizersBackend):
             **kwargs,
         )
 
-    def prepare_for_tokenization(self, text, **kwargs):
-        
-        if isinstance(text, str):
-            text = "<|im_start|> " + text
-        return (text, kwargs)
+    def __call__(self, text, *args, **kwargs):
+        add_special_tokens = kwargs.get("add_special_tokens", False)
+    
+        def add_prefix(t):
+            if isinstance(t, str):
+                return "<|im_start|> " + t
+            return t
+    
+        # Only inject when special tokens are disabled
+        if not add_special_tokens:
+            if isinstance(text, list):
+                text = [add_prefix(t) for t in text]
+            else:
+                text = add_prefix(text)
+    
+        return super().__call__(text, *args, **kwargs)
+
+
 
     # def encode(
     #     self,