tokenizer: sync with Carbon-500M/3B (warnings, add_special_tokens pass-through, tolist in decode)

tokenizer.py

@@ -16,6 +16,7 @@ Supports token_mask for Fine-grained Nucleotide Supervision (FNS):
 
 import os
 import json
+import warnings
 import itertools
 from typing import List, Optional, Tuple, Dict, Union, Any
 
@@ -322,7 +323,7 @@ class HybridDNATokenizer(PreTrainedTokenizer):
             else:
                 base_ids = self._base_tokenizer.encode(
                     segment_content,
-                    add_special_tokens=False
+                    add_special_tokens=add_special_tokens
                 )
                 token_ids.extend(base_ids)
                 if return_token_mask:
@@ -344,6 +345,8 @@ class HybridDNATokenizer(PreTrainedTokenizer):
         skip_special_tokens: bool = False,
         **kwargs
     ) -> str:
+        if hasattr(token_ids, 'tolist'):
+            token_ids = token_ids.tolist()
         if isinstance(token_ids, int):
             token_ids = [token_ids]
 
@@ -430,6 +433,13 @@ class HybridDNATokenizer(PreTrainedTokenizer):
         auto_dna_tags: Optional[bool] = None,
         **kwargs
     ) -> Dict[str, Any]:
+        if add_special_tokens:
+            warnings.warn(
+                "HybridTokenizer does not support add_special_tokens=True, ignoring.",
+                UserWarning
+            )
+            add_special_tokens = False
+            
         is_batch = isinstance(text, list)
         texts = text if is_batch else [text]