Update tokenizer.py

tokenizer.py

@@ -16,6 +16,7 @@ Supports token_mask for Fine-grained Nucleotide Supervision (FNS):
 
 import os
 import json
+import warnings
 import itertools
 from typing import List, Optional, Tuple, Dict, Union, Any
 
@@ -322,7 +323,7 @@ class HybridDNATokenizer(PreTrainedTokenizer):
             else:
                 base_ids = self._base_tokenizer.encode(
                     segment_content,
-                    add_special_tokens=False
+                    add_special_tokens=add_special_tokens
                 )
                 token_ids.extend(base_ids)
                 if return_token_mask:
@@ -430,6 +431,13 @@ class HybridDNATokenizer(PreTrainedTokenizer):
         auto_dna_tags: Optional[bool] = None,
         **kwargs
     ) -> Dict[str, Any]:
+        if add_special_tokens:
+            warnings.warn(
+                "HybridTokenizer does not support add_special_tokens=True, ignoring.",
+                UserWarning
+            )
+            add_special_tokens = False
+            
         is_batch = isinstance(text, list)
         texts = text if is_batch else [text]