Upload folder using huggingface_hub

special_tokens_map.json

@@ -13,13 +13,6 @@
     "rstrip": false,
     "single_word": false
   },
-  "mask_token": {
-    "content": "#",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
   "pad_token": {
     "content": "!",
     "lstrip": false,

tokenizer_config.json

@@ -1,39 +1,31 @@
 {
   "added_tokens_decoder": {
-    "27": {
-      "content": "*",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "28": {
-      "content": "#",
+    "6": {
+      "content": "!",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "29": {
-      "content": "@",
+    "7": {
+      "content": "*",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "30": {
-      "content": "!",
+    "8": {
+      "content": "/",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "31": {
-      "content": "/",
+    "9": {
+      "content": "@",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -50,7 +42,6 @@
   "bos_token": "@",
   "clean_up_tokenization_spaces": true,
   "eos_token": "*",
-  "mask_token": "#",
   "model_max_length": 2048,
   "pad_token": "!",
   "sep_token": "/",

tokenizers.py

@@ -2,9 +2,9 @@ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
 from typing import List, Optional, Union
 import os
 
-MASK = "#"
 MSA_PAD = "!"
-UL_ALPHABET_PLUS = "GATCN-*#@!/[]{}"
+#UL_ALPHABET_PLUS = "GATCN-*#@!/[]{}"
+UL_ALPHABET_PLUS = 'GATCN-!*/@[]{}'
 MSA_AAS = "GATCN-"
 GAP = "-"
 START = "@"
@@ -22,7 +22,6 @@ class DNATokenizer(PreTrainedTokenizer):
         dna_alphabet: str = UL_ALPHABET_PLUS,
         model_max_length: int = 2048,
         pad_token=MSA_PAD,
-        mask_token=MASK,
         all_aas=MSA_AAS,
         gap_token=GAP,
         bos_token=START,
@@ -44,13 +43,11 @@ class DNATokenizer(PreTrainedTokenizer):
         bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
         eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
         sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token
-        mask_token = AddedToken(mask_token, lstrip=False, rstrip=False) if isinstance(mask_token, str) else mask_token 
         pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
         gap_token = AddedToken(gap_token, lstrip=False, rstrip=False) if isinstance(gap_token, str) else gap_token
 
         super().__init__(
             pad_token=pad_token,
-            mask_token=mask_token,
             eos_token=eos_token,
             bos_token=bos_token,
             sep_token=sep_token,