kashif HF Staff commited on
Commit
ac0ca80
·
verified ·
1 Parent(s): e66f05b

tokenizer: fix decode() to handle torch tensor input via .tolist()

Browse files
Files changed (1) hide show
  1. tokenizer.py +2 -0
tokenizer.py CHANGED
@@ -345,6 +345,8 @@ class HybridDNATokenizer(PreTrainedTokenizer):
345
  skip_special_tokens: bool = False,
346
  **kwargs
347
  ) -> str:
 
 
348
  if isinstance(token_ids, int):
349
  token_ids = [token_ids]
350
 
 
345
  skip_special_tokens: bool = False,
346
  **kwargs
347
  ) -> str:
348
+ if hasattr(token_ids, 'tolist'):
349
+ token_ids = token_ids.tolist()
350
  if isinstance(token_ids, int):
351
  token_ids = [token_ids]
352