Upload folder using huggingface_hub

README.md

@@ -1,3 +1,7 @@
----
-license: mit
----
+---
+{}
+---
+
+    # EnhancAR Sorted
+    EnhancAR Sorted is an autoregressive generative model of enhancer homology families, trained on 233,158,475 enhancers extracted from 241 vertebrate genomes. By "unrolling" homology families (enhancer sequences are sorted into sets of homology sequences, and input data is sequences concatenated to each other with a separator token delimiting different sequences), enhancAR Sorted learns to generate new sequences that conserve the function of prompt sequences. We demonstrate that this can be used to design new enhancers "by example", which is particularly useful when the function of enhancers is not known a priori.
+    

config.json

@@ -0,0 +1,41 @@
+{
+  "architectures": [
+    "JambaForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_layer_offset": 4,
+  "attn_layer_period": 8,
+  "bos_token_id": 9,
+  "eos_token_id": 7,
+  "expert_layer_offset": 1,
+  "expert_layer_period": 2,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "mamba_conv_bias": true,
+  "mamba_d_conv": 4,
+  "mamba_d_state": 16,
+  "mamba_dt_rank": 16,
+  "mamba_expand": 2,
+  "mamba_proj_bias": false,
+  "max_position_embeddings": 262144,
+  "model_type": "jamba",
+  "num_attention_heads": 16,
+  "num_experts": 16,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 8,
+  "num_logits_to_keep": 1,
+  "output_router_logits": true,
+  "pad_token_id": 6,
+  "rms_norm_eps": 1e-06,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.2",
+  "use_cache": false,
+  "use_mamba_kernels": true,
+  "vocab_size": 16
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 9,
+  "eos_token_id": 7,
+  "pad_token_id": 6,
+  "transformers_version": "4.48.2",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c2fa527a693668611bfeede25de64f014e12bc8d49f96a04b0bc982d94bf5ca
+size 681256576

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "bos_token": {
+    "content": "@",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "*",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "#",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "!",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "/",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "added_tokens_decoder": {
+    "27": {
+      "content": "*",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "#",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "@",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "!",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "/",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenizers.DNATokenizer",
+      null
+    ]
+  },
+  "bos_token": "@",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "*",
+  "mask_token": "#",
+  "model_max_length": 2048,
+  "pad_token": "!",
+  "sep_token": "/",
+  "tokenizer_class": "DNATokenizer"
+}

tokenizers.py

@@ -0,0 +1,126 @@
+from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
+from typing import List, Optional, Union
+import os
+
+MASK = "#"
+MSA_PAD = "!"
+UL_ALPHABET_PLUS = "GATCN-*#@!/[]{}"
+MSA_AAS = "GATCN-"
+GAP = "-"
+START = "@"
+STOP = "*"
+SEP = "/"
+END_AL = "]"
+END_UL = "}"
+START_AL = "["
+START_UL = "{"
+
+class DNATokenizer(PreTrainedTokenizer):
+
+    def __init__(
+        self,
+        dna_alphabet: str = UL_ALPHABET_PLUS,
+        model_max_length: int = 2048,
+        pad_token=MSA_PAD,
+        mask_token=MASK,
+        all_aas=MSA_AAS,
+        gap_token=GAP,
+        bos_token=START,
+        eos_token=STOP,
+        sep_token=SEP,
+        **kwargs
+    ):
+        """Character tokenizer for Hugging Face transformers.
+
+        model_max_length (int): Model maximum sequence length.
+        """
+        self.alphabet = list("".join(dna_alphabet))
+        self.all_aas = list("".join(all_aas))
+        self.a_to_i = {u: i for i, u in enumerate(self.alphabet)}
+        self.i_to_a = {i: u for i, u in enumerate(self.alphabet)}
+        self.gap_token = gap_token
+
+        
+        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
+        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
+        sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token
+        mask_token = AddedToken(mask_token, lstrip=False, rstrip=False) if isinstance(mask_token, str) else mask_token 
+        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
+        gap_token = AddedToken(gap_token, lstrip=False, rstrip=False) if isinstance(gap_token, str) else gap_token
+
+        super().__init__(
+            pad_token=pad_token,
+            mask_token=mask_token,
+            eos_token=eos_token,
+            bos_token=bos_token,
+            sep_token=sep_token,
+            model_max_length=model_max_length,
+            **kwargs
+        )
+
+    @property
+    def vocab_size(self):
+        return len(self.alphabet)
+    
+    @property
+    def gap_token_id(self):
+        return self.convert_tokens_to_ids(self.gap_token)
+
+    def get_vocab(self):
+        return self.a_to_i
+
+    def _tokenize(self, text: str) -> List[str]:
+        return list(text)
+
+    def _convert_token_to_id(self, token) -> int:
+        return self.a_to_i[token]
+
+    def _convert_id_to_token(self, index) -> str:
+        return self.i_to_a[index]
+
+    def convert_tokens_to_string(self, tokens):
+        return "".join(tokens)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        result = token_ids_0
+        if token_ids_1 is not None:
+            raise NotImplementedError("This tokenizer does not support two sequences")
+        return result
+
+    def get_special_tokens_mask(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+        already_has_special_tokens: bool = False,
+    ) -> List[int]:
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0,
+                token_ids_1=token_ids_1,
+                already_has_special_tokens=True,
+            )
+
+        result = [0] * len(token_ids_0)
+        if token_ids_1 is not None:
+            raise NotImplementedError("This tokenizer does not support two sequences")
+
+        return result
+
+    def create_token_type_ids_from_sequences(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Identifies the type of token. 0 for the first sentence, 1 for the second sentence if it exists
+        """
+
+        result = len(token_ids_0) * [0]
+
+        if token_ids_1 is not None:
+            raise NotImplementedError("This tokenizer does not support two sequences")
+        return result
+
+    def save_pretrained(self, save_directory: Union[str, os.PathLike], **kwargs):
+        super().save_pretrained(save_directory, **kwargs)
+    
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
+        return ()