| from transformers import PretrainedConfig |
| from typing import Literal, Optional |
|
|
| class DNAEncoderConfig(PretrainedConfig): |
| model_type = "dna_encoder" |
| |
| def __init__( |
| self, |
| vocab_size: int = 4, |
| embedding_dim: int = 384, |
| dim_feedforward: int = 1536, |
| num_heads: int = 12, |
| num_layers: int = 6, |
| dropout: float = 0.1, |
| activation: Literal["relu", "gelu"] = "gelu", |
| pos_embedding: Optional[str] = "SinusoidalPositionalEncoding", |
| max_position_embeddings: int = 1024, |
| **kwargs |
| ): |
| self.vocab_size = vocab_size |
| self.embedding_dim = embedding_dim |
| self.dim_feedforward = dim_feedforward |
| self.num_heads = num_heads |
| self.num_layers = num_layers |
| self.dropout = dropout |
| self.activation = activation |
| self.pos_embedding = pos_embedding |
| self.max_position_embeddings = max_position_embeddings |
| |
| super().__init__(**kwargs) |
