Upload 12 files

claspp_forward.py

@@ -0,0 +1,412 @@
+import os
+import sys
+import warnings
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+from transformers import DataCollatorWithPadding
+from transformers import EsmTokenizer
+from datasets import (
+    load_dataset,
+    Dataset,
+)
+
+from modeling_esm import EsmForSequenceClassificationCustomWidehead
+
+
+tokenizer = EsmTokenizer.from_pretrained("finalCheckpoint_25_05_11/")
+model = EsmForSequenceClassificationCustomWidehead.from_pretrained("finalCheckpoint_25_05_11/", num_labels=54).cuda()
+
+
+###############################################################################
+#helper code to make the model run smooth
+###############################################################################
+    # labs=['ST-Phosphorylation_nc0_tot5', 
+    #       'ST-Phosphorylation_nc1_tot5', 
+    #       'ST-Phosphorylation_nc2_tot5', 
+    #       'ST-Phosphorylation_nc3_tot5', 
+    #       'ST-Phosphorylation_nc4_tot5', 
+    #       'K-Ubiquitination_nc0_tot20', 
+    #       'K-Ubiquitination_nc1_tot20', 
+    #       'K-Ubiquitination_nc2_tot20', 
+    #       'K-Ubiquitination_nc3_tot20', 
+    #       'K-Ubiquitination_nc4_tot20', 
+    #       'K-Ubiquitination_nc5_tot20', 
+    #       'K-Ubiquitination_nc6_tot20', 
+    #       'K-Ubiquitination_nc7_tot20', 
+    #       'K-Ubiquitination_nc8_tot20', 
+    #       'K-Ubiquitination_nc9_tot20', 
+    #       'K-Ubiquitination_nc10_tot20', 
+    #       'K-Ubiquitination_nc11_tot20', 
+    #       'K-Ubiquitination_nc12_tot20', 
+    #       'K-Ubiquitination_nc13_tot20', 
+    #       'K-Ubiquitination_nc14_tot20', 
+    #       'K-Ubiquitination_nc15_tot20', 
+    #       'K-Ubiquitination_nc16_tot20', 
+    #       'K-Ubiquitination_nc17_tot20', 
+    #       'K-Ubiquitination_nc18_tot20', 
+    #       'K-Ubiquitination_nc19_tot20', 
+    #       'Y-Phosphorylation_nc0_tot1', 
+    #       'K-Acetylation_nc0_tot10', 
+    #       'K-Acetylation_nc1_tot10', 
+    #       'K-Acetylation_nc2_tot10', 
+    #       'K-Acetylation_nc3_tot10', 
+    #       'K-Acetylation_nc4_tot10', 
+    #       'K-Acetylation_nc5_tot10', 
+    #       'K-Acetylation_nc6_tot10', 
+    #       'K-Acetylation_nc7_tot10', 
+    #       'K-Acetylation_nc8_tot10', 
+    #       'K-Acetylation_nc9_tot10', 
+    #       'N-N-linked-Glycosylation_nc0_tot1', 
+    #       'ST-O-linked-Glycosylation_nc0_tot5', 
+    #       'ST-O-linked-Glycosylation_nc1_tot5', 
+    #       'ST-O-linked-Glycosylation_nc2_tot5', 
+    #       'ST-O-linked-Glycosylation_nc3_tot5', 
+    #       'ST-O-linked-Glycosylation_nc4_tot5', 
+    #       'RK-Methylation_nc0_tot4', 
+    #       'RK-Methylation_nc1_tot4', 
+    #       'RK-Methylation_nc2_tot4', 
+    #       'RK-Methylation_nc3_tot4', 
+    #       'K-Sumoylation_nc0_tot1', 
+    #       'K-Malonylation_nc0_tot1', 
+    #       'M-Sulfoxidation_nc0_tot1', 
+    #       'AM-Acetylation_nc0_tot1', 
+    #       'C-Glutathionylation_nc0_tot1', 
+    #       'C-S-palmitoylation_nc0_tot1', 
+    #       'PK-Hydroxylation_nc0_tot1', 
+    #       'NegLab']
+
+labsoi=set()
+lab2map={}
+labsoi.add("S_Phosphorylation")
+lab2map["S_Phosphorylation"]=0
+labsoi.add("T_Phosphorylation")
+lab2map["T_Phosphorylation"]=1
+labsoi.add("Y_Phosphorylation")
+lab2map["Y_Phosphorylation"]=3
+labsoi.add("A_Acetylation")
+lab2map["A_Acetylation"]=13
+labsoi.add("M_Acetylation")
+lab2map["M_Acetylation"]=14
+labsoi.add("K_Acetylation")
+lab2map["K_Acetylation"]=4
+labsoi.add("K_Ubiquitination")
+lab2map["K_Ubiquitination"]=2
+labsoi.add("S_O-linked-Glycosylation")
+lab2map["S_O-linked-Glycosylation"]=6
+labsoi.add("T_O-linked-Glycosylation")
+lab2map["T_O-linked-Glycosylation"]=7
+labsoi.add("N_N-linked-Glycosylation")
+lab2map["N_N-linked-Glycosylation"]=5
+labsoi.add("K_Methylation")
+lab2map["K_Methylation"]=9
+labsoi.add("R_Methylation")
+lab2map["R_Methylation"]=8
+labsoi.add("K_Malonylation")
+lab2map["K_Malonylation"]=11
+labsoi.add("K_Sumoylation")
+lab2map["K_Sumoylation"]=10
+labsoi.add("C_Glutathionylation")
+lab2map["C_Glutathionylation"]=15
+labsoi.add("P_Hydroxylation")
+lab2map["P_Hydroxylation"]=17
+labsoi.add("K_Hydroxylation")
+lab2map["K_Hydroxylation"]=18
+labsoi.add("C_S-palmitoylation")
+lab2map["C_S-palmitoylation"]=16
+lab2map['M_Sulfoxidation']=12
+pos2lab={}
+for lab in lab2map.keys():
+    pos=lab2map[lab]
+    pos2lab[pos]=lab
+# labsoi.add("K-Succinylation")
+# lab2map["K-Succinylation"]=14
+
+
+def preprocess_function(examples):
+    toks={}
+    toks['input_ids']=[]
+    toks['attention_mask']=[]
+    
+    for info in examples["pep"]:
+        info=info.replace(".", "<mask>")
+        t=tokenizer(info.replace("-", "<pad>"))
+        toks['input_ids'].append(t['input_ids'])
+        toks['attention_mask'].append(t['attention_mask'])
+    
+    
+    return toks
+
+
+def getlab(elab,res):
+    output=np.zeros((20))
+    if res=='S':
+        output[0]=max(elab[:5])
+        output[1]=0
+    elif res=='T':
+        output[0]=0
+        output[1]=max(elab[:5])
+    else:
+         output[0]=0
+         output[1]=0
+    #print(labs[:5])['ST-Phosphorylation_nc0_tot5', 'ST-Phosphorylation_nc1_tot5', 'ST-Phosphorylation_nc2_tot5', 'ST-Phosphorylation_nc3_tot5', 'ST-Phosphorylation_nc4_tot5']
+    output[2]=max(elab[5:25])
+    #print(labs[5:25])['K-Ubiquitination_nc0_tot20', 'K-Ubiquitination_nc1_tot20', 'K-Ubiquitination_nc2_tot20', 'K-Ubiquitination_nc3_tot20', 'K-Ubiquitination_nc4_tot20', 'K-Ubiquitination_nc5_tot20', 'K-Ubiquitination_nc6_tot20', 'K-Ubiquitination_nc7_tot20', 'K-Ubiquitination_nc8_tot20', 'K-Ubiquitination_nc9_tot20', 'K-Ubiquitination_nc10_tot20', 'K-Ubiquitination_nc11_tot20', 'K-Ubiquitination_nc12_tot20', 'K-Ubiquitination_nc13_tot20', 'K-Ubiquitination_nc14_tot20', 'K-Ubiquitination_nc15_tot20', 'K-Ubiquitination_nc16_tot20', 'K-Ubiquitination_nc17_tot20', 'K-Ubiquitination_nc18_tot20', 'K-Ubiquitination_nc19_tot20'] 
+    output[3]=max(elab[25:26])
+    #print(labs[25:30])['Y-Phosphorylation_nc0_tot5', 'Y-Phosphorylation_nc1_tot5', 'Y-Phosphorylation_nc2_tot5', 'Y-Phosphorylation_nc3_tot5', 'Y-Phosphorylation_nc4_tot5']
+    output[4]=max(elab[26:36])
+    #print(labs[30:40])['K-Acetylation_nc0_tot10', 'K-Acetylation_nc1_tot10', 'K-Acetylation_nc2_tot10', 'K-Acetylation_nc3_tot10', 'K-Acetylation_nc4_tot10', 'K-Acetylation_nc5_tot10', 'K-Acetylation_nc6_tot10', 'K-Acetylation_nc7_tot10', 'K-Acetylation_nc8_tot10', 'K-Acetylation_nc9_tot10']
+    output[5]=max(elab[36:37])
+    #print(labs[40:41])['N-N-linked-Glycosylation_nc0_tot1']
+    if res=='S':
+        output[6]=max(elab[37:42])
+        output[7]=0
+    elif res=='T':
+        output[6]=0
+        output[7]=max(elab[37:42])
+    else:
+         output[6]=0
+         output[7]=0
+    #print(labs[41:46])['ST-O-linked-Glycosylation_nc0_tot5', 'ST-O-linked-Glycosylation_nc1_tot5', 'ST-O-linked-Glycosylation_nc2_tot5', 'ST-O-linked-Glycosylation_nc3_tot5', 'ST-O-linked-Glycosylation_nc4_tot5']
+    if res=="R":
+        output[8]=max(elab[42:46])
+        output[9]=0
+    elif res=="K":
+        output[8]=0
+        output[9]=max(elab[42:46])
+    else:
+        output[8]=0
+        output[9]=0
+    #print(labs[46:50])['RK-Methylation_nc0_tot4', 'RK-Methylation_nc1_tot4', 'RK-Methylation_nc2_tot4', 'RK-Methylation_nc3_tot4']
+    output[10]=max(elab[46:47])
+    #print(labs[50:52])['K-Sumoylation_nc0_tot2', 'K-Sumoylation_nc1_tot2']
+    output[11]=max(elab[47:48])
+        #'K-Malonylation_nc0_tot1'
+    output[12]=max(elab[48:49])
+        #"M-Sulfoxidation_nc0_tot1'
+    if res=="A":
+        output[13]=max(elab[49:50])
+        output[14]=0
+    elif res=="M":
+        output[13]=0
+        output[14]=max(elab[49:50])
+    else:
+        output[13]=0
+        output[14]=0
+    #print(elab[50:51])
+    output[15]=max(elab[50:51])
+    #print(labs[57:58])['C-Glutathionylation_nc0_tot1']
+    output[16]=max(elab[51:52])
+    #print(labs[58:59])['C-S-palmitoylation_nc0_tot1']
+    if res=="P":
+        output[17]=max(elab[52:53])
+        output[18]=0
+    elif res=="K":
+        output[17]=0
+        output[18]=max(elab[52:53])
+    else:
+        output[17]=0
+        output[18]=0
+    #print(labs[52:54])['K-Malonylation_nc0_tot2', 'K-Malonylation_nc1_tot2']
+    output[19]=max(elab[53:54])
+    return(output)
+    #print(labs[59:60])['NegLab']
+
+
+
+
+###############################################################################
+#prediction code
+###############################################################################
+
+
+def predict(input_batches):
+    sig=nn.Sigmoid()
+    outputpreds=[]
+    r='\r'
+    for i,batches in enumerate(input_batches):
+        print(f"{i} / {len(input_batches)} batches done",end=r)
+        # tok_input_ids=tokenizer(batches)['input_ids']
+        # tensor_input_ids=torch.tensor(tok_input_ids)
+        # print(tensor_input_ids)
+        # print(torch.tensor([tokenizer(batches)['input_ids']]).cuda().shape)
+        # print(torch.tensor([tokenizer(batches)['attention_mask']]).cuda()["logits"][0].shape)
+        #print(torch.tensor([tokenizer(batches)['input_ids']]).cuda().squeeze().shape)
+
+        pred=(sig(model(torch.tensor([tokenizer(batches)['input_ids']]).squeeze().cuda(),torch.tensor([tokenizer(batches)['attention_mask']]).squeeze().cuda())["logits"]).tolist())
+        #print(len(pred[0]))
+        for p in pred:
+            outputpreds.append(p)
+    return outputpreds
+
+
+def write_output(pred,listofpeps):
+    hf=open("output_predictions.csv",'w+')
+    n="\n"
+    writethisline="pep,"
+    for i in range(len(labsoi)):
+        writethisline+=pos2lab[i]
+    hf.write(writethisline+n)
+    for p,ip in zip(pred,listofpeps):
+        writethisline=f"{ip}"
+        r=ip[10]
+        #print(p)
+        easyreadlab=getlab(p,r)
+        for sp in easyreadlab:
+            writethisline+=f"{sp},"
+        
+        writethisline=writethisline[:-1]+n
+        hf.write(writethisline)
+    hf.close()
+        
+
+DOC_HELP='''
+Usage: python3 claspp_forward.py [OPTION]... --input INPUT [FASTA_FILE or TXT_FILE]...
+predict PTM events on peptides or full sequences
+
+Example 1: python3 claspp_forward.py -B 100 -S 0 -i random.txt
+Example 2: python3 claspp_forward.py -B 50 -S 1 -i random.fasta
+
+FASTA_FILE contain protein sequences in proper fasta or a2m format
+TXT_FILE cointain protien peptides 21 in length with the center
+residue being the PTM modification site
+
+
+Pattern selection and interpretation:
+  -B, --batch_size          (int) that describes how many predictions
+                            can be predicted at a time on the GPU
+                            (reduce if you get run out of GPU space)
+
+  -S  --scrape_fasta        (int) should be a 1 or a 0 
+                            1 = read a fasta and scrape posible 21 peptides
+                            that can be modified by a PTM 
+                            0 = read a txt file that has the 21mer already 
+                            sperated and all peptides should be sperated by 
+                            a '\\n' (can be faster) than fasta option
+  
+  -h  --help                your reading it right now
+
+  -i  --input               location of the input fasta or txt
+
+  -o  --output              location of the output csv
+
+
+Report bugs to: 
+
+
+'''
+WARNING_MESSAGE="""
+        #################################
+        PLEASE READ HELP MESSAGE TO ENSURE
+        YOU KNOW HOW TO FORMAT/USE THE
+        MODEL 
+        #################################
+              """
+
+
+
+
+def main():
+    batch_size=50
+    scrape=0
+    file_output="output_predictions.csv"
+    input_file="N/A"
+    for i in range(len(sys.argv)-1):
+        if sys.argv[i]=='--scrape_fasta' or sys.argv[i]=='-S':
+            scrape = int(sys.argv[i+1])
+        if sys.argv[i]=='--batch_size' or sys.argv[i]=='-B':
+             batch_size = int(sys.argv[i+1])
+        if sys.argv[i]=='--input' or sys.argv[i]=='-i':
+             input_file = sys.argv[i+1]
+        if sys.argv[i]=='--output' or sys.argv[i]=='-o':
+             file_output = sys.argv[i+1]
+        if sys.argv[i]=='-h' or sys.argv[i]=='--h' or sys.argv[i]=='-help' or sys.argv[i]=='--help' :
+            print(DOC_HELP)
+    if input_file=='N/A':
+        print(WARNING_MESSAGE)
+        print(DOC_HELP)
+        return
+    
+    if scrape==0:
+        #todo make readerfuc
+        listofpeps=[]
+        rf=open(input_file,"r")
+        lines=rf.readlines()
+        for line in lines:
+            pep=line[:-1]
+            listofpeps.append(pep)
+            
+
+
+    else:
+        #todo make readerfuc
+        listofpeps=[]
+        acc2seq={}
+        #seq2acc={}
+        rf=open(input_file,"r")
+        lines=rf.readlines()
+        seq=""
+        acc=""
+        for line in lines:
+            if line[0]=='>':
+                if seq!='':
+                    acc2seq[acc]=seq
+                    #seq2acc[seq]=acc
+                    seq=""
+                acc=line[1:-1]
+            else:
+                seq+=line.replace('\n','')
+        acc2seq[acc]=seq
+        #seq2acc[seq]=acc 
+        for acc in acc2seq.keys():
+            seq=acc2seq[acc]
+            paddedseq='----------'+seq+'----------'
+            for i,c in enumerate(seq):
+                pep=paddedseq[i:i+21]
+                listofpeps.append(pep)
+        setofpeps=set(listofpeps)
+        listofpeps=list(setofpeps)
+    
+            
+            
+            
+
+        
+
+
+        
+    
+    input_batches=[]
+    temp=[]
+    for i,pep in enumerate(listofpeps):
+        if i%batch_size==0 and i!=0:
+            input_batches.append(temp)
+            temp=[]
+        temp.append(pep)
+    input_batches.append(temp)
+    
+    pred=predict(input_batches=input_batches)
+    write_output(pred,listofpeps)
+
+    
+
+    
+
+                
+
+
+        
+
+
+
+
+
+if __name__ == "__main__":
+    main()
+    #df=pd.read_csv("output_predictions.csv")
+    #print(df)
+       
+
+    

configuration_esm.py

@@ -0,0 +1,370 @@
+# coding=utf-8
+# Copyright 2022 Meta and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" ESM model configuration"""
+
+from dataclasses import asdict, dataclass
+from typing import Optional
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+# TODO Update this
+ESM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "facebook/esm-1b": "https://huggingface.co/facebook/esm-1b/resolve/main/config.json",
+    "facebook/esm2_t6_8M_UR50D": "https://huggingface.co/facebook/esm2_t6_8M_UR50D/blob/main/config.json"
+    # See all ESM models at https://huggingface.co/models?filter=esm
+}
+
+
+class EsmConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`ESMModel`]. It is used to instantiate a ESM model
+    according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the ESM
+    [facebook/esm-1b](https://huggingface.co/facebook/esm-1b) architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*):
+            Vocabulary size of the ESM model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`ESMModel`].
+        mask_token_id (`int`, *optional*):
+            The index of the mask token in the vocabulary. This must be included in the config because of the
+            "mask-dropout" scaling trick, which will scale the inputs depending on the number of masked tokens.
+        pad_token_id (`int`, *optional*):
+            The index of the padding token in the vocabulary. This must be included in the config because certain parts
+            of the ESM code use this instead of the attention mask.
+        hidden_size (`int`, *optional*, defaults to 768):
+            Dimensionality of the encoder layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 12):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        intermediate_size (`int`, *optional*, defaults to 3072):
+            Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
+        hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"silu"` and `"gelu_new"` are supported.
+        hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
+            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+        attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
+            The dropout ratio for the attention probabilities.
+        max_position_embeddings (`int`, *optional*, defaults to 1026):
+            The maximum sequence length that this model might ever be used with. Typically set this to something large
+            just in case (e.g., 512 or 1024 or 2048).
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-12):
+            The epsilon used by the layer normalization layers.
+        position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
+            Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query", "rotary"`.
+            For positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
+            [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
+            For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
+            with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        classifier_dropout (`float`, *optional*):
+            The dropout ratio for the classification head.
+        emb_layer_norm_before (`bool`, *optional*):
+            Whether to apply layer normalization after embeddings but before the main stem of the network.
+        token_dropout (`bool`, defaults to `False`):
+            When this is enabled, masked tokens are treated as if they had been dropped out by input dropout.
+
+    Examples:
+
+    ```python
+    >>> from transformers import EsmModel, EsmConfig
+
+    >>> # Initializing a ESM facebook/esm-1b style configuration >>> configuration = EsmConfig()
+
+    >>> # Initializing a model from the configuration >>> model = ESMModel(configuration)
+
+    >>> # Accessing the model configuration >>> configuration = model.config
+    ```"""
+    model_type = "esm"
+
+    def __init__(
+        self,
+        vocab_size=None,
+        mask_token_id=None,
+        pad_token_id=None,
+        hidden_size=768,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        intermediate_size=3072,
+        hidden_act="gelu",
+        hidden_dropout_prob=0.1,
+        attention_probs_dropout_prob=0.1,
+        max_position_embeddings=1026,
+        initializer_range=0.02,
+        layer_norm_eps=1e-12,
+        position_embedding_type="absolute",
+        use_cache=True,
+        classifier_dropout=None,
+        emb_layer_norm_before=None,
+        token_dropout=False,
+        is_folding_model=False,
+        esmfold_config=None,
+        vocab_list=None,
+        **kwargs
+    ):
+        super().__init__(pad_token_id=pad_token_id, mask_token_id=mask_token_id, **kwargs)
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.position_embedding_type = position_embedding_type
+        self.use_cache = use_cache
+        self.classifier_dropout = classifier_dropout
+        self.emb_layer_norm_before = emb_layer_norm_before
+        self.token_dropout = token_dropout
+        self.is_folding_model = is_folding_model
+        if is_folding_model:
+            if esmfold_config is None:
+                logger.info("No esmfold_config supplied for folding model, using default values.")
+                esmfold_config = EsmFoldConfig()
+            elif isinstance(esmfold_config, dict):
+                esmfold_config = EsmFoldConfig(**esmfold_config)
+            self.esmfold_config = esmfold_config
+            if vocab_list is None:
+                logger.warning("No vocab_list supplied for folding model, assuming the ESM-2 vocabulary!")
+                self.vocab_list = get_default_vocab_list()
+            else:
+                self.vocab_list = vocab_list
+        else:
+            self.esmfold_config = None
+            self.vocab_list = None
+        if self.esmfold_config is not None and getattr(self.esmfold_config, "use_esm_attn_map", False):
+            raise ValueError("The HuggingFace port of ESMFold does not support use_esm_attn_map at this time!")
+
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+        Returns:
+            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+        """
+        output = super().to_dict()
+        if isinstance(self.esmfold_config, EsmFoldConfig):
+            output["esmfold_config"] = self.esmfold_config.to_dict()
+        return output
+
+
+@dataclass
+class EsmFoldConfig:
+    esm_type: str = None
+    fp16_esm: bool = True
+    use_esm_attn_map: bool = False
+    esm_ablate_pairwise: bool = False
+    esm_ablate_sequence: bool = False
+    esm_input_dropout: float = 0
+
+    embed_aa: bool = True
+    bypass_lm: bool = False
+
+    lddt_head_hid_dim: int = 128
+    trunk: "TrunkConfig" = None
+
+    def __post_init__(self):
+        if self.trunk is None:
+            self.trunk = TrunkConfig()
+        elif isinstance(self.trunk, dict):
+            self.trunk = TrunkConfig(**self.trunk)
+
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+        Returns:
+            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+        """
+        output = asdict(self)
+        output["trunk"] = self.trunk.to_dict()
+        return output
+
+
+@dataclass
+class TrunkConfig:
+    num_blocks: int = 48
+    sequence_state_dim: int = 1024
+    pairwise_state_dim: int = 128
+    sequence_head_width: int = 32
+    pairwise_head_width: int = 32
+    position_bins: int = 32
+    dropout: float = 0
+    layer_drop: float = 0
+    cpu_grad_checkpoint: bool = False
+    max_recycles: int = 4
+    chunk_size: Optional[int] = 128
+    structure_module: "StructureModuleConfig" = None
+
+    def __post_init__(self):
+        if self.structure_module is None:
+            self.structure_module = StructureModuleConfig()
+        elif isinstance(self.structure_module, dict):
+            self.structure_module = StructureModuleConfig(**self.structure_module)
+
+        if self.max_recycles <= 0:
+            raise ValueError(f"`max_recycles` should be positive, got {self.max_recycles}.")
+        if self.sequence_state_dim % self.sequence_state_dim != 0:
+            raise ValueError(
+                "`sequence_state_dim` should be a round multiple of `sequence_state_dim`, got"
+                f" {self.sequence_state_dim} and {self.sequence_state_dim}."
+            )
+        if self.pairwise_state_dim % self.pairwise_state_dim != 0:
+            raise ValueError(
+                "`pairwise_state_dim` should be a round multiple of `pairwise_state_dim`, got"
+                f" {self.pairwise_state_dim} and {self.pairwise_state_dim}."
+            )
+
+        sequence_num_heads = self.sequence_state_dim // self.sequence_head_width
+        pairwise_num_heads = self.pairwise_state_dim // self.pairwise_head_width
+
+        if self.sequence_state_dim != sequence_num_heads * self.sequence_head_width:
+            raise ValueError(
+                "`sequence_state_dim` should be equal to `sequence_num_heads * sequence_head_width, got"
+                f" {self.sequence_state_dim} != {sequence_num_heads} * {self.sequence_head_width}."
+            )
+        if self.pairwise_state_dim != pairwise_num_heads * self.pairwise_head_width:
+            raise ValueError(
+                "`pairwise_state_dim` should be equal to `pairwise_num_heads * pairwise_head_width, got"
+                f" {self.pairwise_state_dim} != {pairwise_num_heads} * {self.pairwise_head_width}."
+            )
+        if self.pairwise_state_dim % 2 != 0:
+            raise ValueError(f"`pairwise_state_dim` should be even, got {self.pairwise_state_dim}.")
+
+        if self.dropout >= 0.4:
+            raise ValueError(f"`dropout` should not be greater than 0.4, got {self.dropout}.")
+
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+        Returns:
+            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+        """
+        output = asdict(self)
+        output["structure_module"] = self.structure_module.to_dict()
+        return output
+
+
+@dataclass
+class StructureModuleConfig:
+    """
+    Args:
+        sequence_dim:
+            Single representation channel dimension
+        pairwise_dim:
+            Pair representation channel dimension
+        ipa_dim:
+            IPA hidden channel dimension
+        resnet_dim:
+            Angle resnet (Alg. 23 lines 11-14) hidden channel dimension
+        num_heads_ipa:
+            Number of IPA heads
+        num_qk_points:
+            Number of query/key points to generate during IPA
+        num_v_points:
+            Number of value points to generate during IPA
+        dropout_rate:
+            Dropout rate used throughout the layer
+        num_blocks:
+            Number of structure module blocks
+        num_transition_layers:
+            Number of layers in the single representation transition (Alg. 23 lines 8-9)
+        num_resnet_blocks:
+            Number of blocks in the angle resnet
+        num_angles:
+            Number of angles to generate in the angle resnet
+        trans_scale_factor:
+            Scale of single representation transition hidden dimension
+        epsilon:
+            Small number used in angle resnet normalization
+        inf:
+            Large number used for attention masking
+    """
+
+    sequence_dim: int = 384
+    pairwise_dim: int = 128
+    ipa_dim: int = 16
+    resnet_dim: int = 128
+    num_heads_ipa: int = 12
+    num_qk_points: int = 4
+    num_v_points: int = 8
+    dropout_rate: float = 0.1
+    num_blocks: int = 8
+    num_transition_layers: int = 1
+    num_resnet_blocks: int = 2
+    num_angles: int = 7
+    trans_scale_factor: int = 10
+    epsilon: float = 1e-8
+    inf: float = 1e5
+
+    def to_dict(self):
+        return asdict(self)
+
+
+def get_default_vocab_list():
+    return (
+        "<cls>",
+        "<pad>",
+        "<eos>",
+        "<unk>",
+        "L",
+        "A",
+        "G",
+        "V",
+        "S",
+        "E",
+        "R",
+        "T",
+        "I",
+        "D",
+        "P",
+        "K",
+        "Q",
+        "N",
+        "F",
+        "Y",
+        "M",
+        "H",
+        "W",
+        "C",
+        "X",
+        "B",
+        "U",
+        "Z",
+        "O",
+        ".",
+        "-",
+        "<null_1>",
+        "<mask>",
+    )

finalCheckpoint_25_05_11/config.json

@@ -0,0 +1,143 @@
+{
+  "_name_or_path": "contrastive_checkpoint/checkpoint-7800",
+  "architectures": [
+    "EsmForSequenceClassificationCustomWidehead"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "classifier_dropout": null,
+  "emb_layer_norm_before": false,
+  "esmfold_config": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 640,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2",
+    "3": "LABEL_3",
+    "4": "LABEL_4",
+    "5": "LABEL_5",
+    "6": "LABEL_6",
+    "7": "LABEL_7",
+    "8": "LABEL_8",
+    "9": "LABEL_9",
+    "10": "LABEL_10",
+    "11": "LABEL_11",
+    "12": "LABEL_12",
+    "13": "LABEL_13",
+    "14": "LABEL_14",
+    "15": "LABEL_15",
+    "16": "LABEL_16",
+    "17": "LABEL_17",
+    "18": "LABEL_18",
+    "19": "LABEL_19",
+    "20": "LABEL_20",
+    "21": "LABEL_21",
+    "22": "LABEL_22",
+    "23": "LABEL_23",
+    "24": "LABEL_24",
+    "25": "LABEL_25",
+    "26": "LABEL_26",
+    "27": "LABEL_27",
+    "28": "LABEL_28",
+    "29": "LABEL_29",
+    "30": "LABEL_30",
+    "31": "LABEL_31",
+    "32": "LABEL_32",
+    "33": "LABEL_33",
+    "34": "LABEL_34",
+    "35": "LABEL_35",
+    "36": "LABEL_36",
+    "37": "LABEL_37",
+    "38": "LABEL_38",
+    "39": "LABEL_39",
+    "40": "LABEL_40",
+    "41": "LABEL_41",
+    "42": "LABEL_42",
+    "43": "LABEL_43",
+    "44": "LABEL_44",
+    "45": "LABEL_45",
+    "46": "LABEL_46",
+    "47": "LABEL_47",
+    "48": "LABEL_48",
+    "49": "LABEL_49",
+    "50": "LABEL_50",
+    "51": "LABEL_51",
+    "52": "LABEL_52",
+    "53": "LABEL_53"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 2560,
+  "is_folding_model": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_10": 10,
+    "LABEL_11": 11,
+    "LABEL_12": 12,
+    "LABEL_13": 13,
+    "LABEL_14": 14,
+    "LABEL_15": 15,
+    "LABEL_16": 16,
+    "LABEL_17": 17,
+    "LABEL_18": 18,
+    "LABEL_19": 19,
+    "LABEL_2": 2,
+    "LABEL_20": 20,
+    "LABEL_21": 21,
+    "LABEL_22": 22,
+    "LABEL_23": 23,
+    "LABEL_24": 24,
+    "LABEL_25": 25,
+    "LABEL_26": 26,
+    "LABEL_27": 27,
+    "LABEL_28": 28,
+    "LABEL_29": 29,
+    "LABEL_3": 3,
+    "LABEL_30": 30,
+    "LABEL_31": 31,
+    "LABEL_32": 32,
+    "LABEL_33": 33,
+    "LABEL_34": 34,
+    "LABEL_35": 35,
+    "LABEL_36": 36,
+    "LABEL_37": 37,
+    "LABEL_38": 38,
+    "LABEL_39": 39,
+    "LABEL_4": 4,
+    "LABEL_40": 40,
+    "LABEL_41": 41,
+    "LABEL_42": 42,
+    "LABEL_43": 43,
+    "LABEL_44": 44,
+    "LABEL_45": 45,
+    "LABEL_46": 46,
+    "LABEL_47": 47,
+    "LABEL_48": 48,
+    "LABEL_49": 49,
+    "LABEL_5": 5,
+    "LABEL_50": 50,
+    "LABEL_51": 51,
+    "LABEL_52": 52,
+    "LABEL_53": 53,
+    "LABEL_6": 6,
+    "LABEL_7": 7,
+    "LABEL_8": 8,
+    "LABEL_9": 9
+  },
+  "layer_norm_eps": 1e-05,
+  "mask_token_id": 32,
+  "max_position_embeddings": 1026,
+  "model_type": "esm",
+  "num_attention_heads": 20,
+  "num_hidden_layers": 30,
+  "pad_token_id": 1,
+  "position_embedding_type": "rotary",
+  "problem_type": "multi_label_classification",
+  "token_dropout": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "vocab_list": null,
+  "vocab_size": 33
+}

finalCheckpoint_25_05_11/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8500bf15856824253050c35105b187cf7b6b099759093573f52e8d3795a8c43a
+size 593608456

finalCheckpoint_25_05_11/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0457e64925706ef97ee05e0a2a17f1974ac4c02ab4539e4c6853af37ed02ba73
+size 4842128200

finalCheckpoint_25_05_11/model.safetensors.index.json

@@ -0,0 +1,528 @@
+{
+  "metadata": {
+    "total_size": 5435676264
+  },
+  "weight_map": {
+    "classifier.dense1.bias": "model-00002-of-00002.safetensors",
+    "classifier.dense1.weight": "model-00002-of-00002.safetensors",
+    "classifier.layer_norm.bias": "model-00002-of-00002.safetensors",
+    "classifier.layer_norm.weight": "model-00002-of-00002.safetensors",
+    "classifier.out_proj.bias": "model-00002-of-00002.safetensors",
+    "classifier.out_proj.weight": "model-00002-of-00002.safetensors",
+    "esm.embeddings.position_embeddings.weight": "model-00001-of-00002.safetensors",
+    "esm.embeddings.position_ids": "model-00001-of-00002.safetensors",
+    "esm.embeddings.word_embeddings.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.emb_layer_norm_after.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.emb_layer_norm_after.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.0.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.1.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.10.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.11.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.12.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.13.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.14.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.15.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.16.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.17.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.18.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.19.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.2.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.20.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.21.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.22.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.23.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.24.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.25.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.26.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.27.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.28.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.29.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.3.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.4.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.5.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.6.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.7.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.8.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.self.rotary_embeddings.inv_freq": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.output.dense.bias": "model-00001-of-00002.safetensors",
+    "esm.encoder.layer.9.output.dense.weight": "model-00001-of-00002.safetensors"
+  }
+}

finalCheckpoint_25_05_11/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "<cls>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

finalCheckpoint_25_05_11/tokenizer_config.json

@@ -0,0 +1,52 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<cls>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<cls>",
+  "eos_token": "<eos>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "EsmTokenizer",
+  "unk_token": "<unk>"
+}

finalCheckpoint_25_05_11/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be1edc20238f883c88163b261b9326bcb206ea88ea8b1303463ddc3d6684549e
+size 5496

finalCheckpoint_25_05_11/vocab.txt

@@ -0,0 +1,33 @@
+<cls>
+<pad>
+<eos>
+<unk>
+L
+A
+G
+V
+S
+E
+R
+T
+I
+D
+P
+K
+Q
+N
+F
+Y
+M
+H
+W
+C
+X
+B
+U
+Z
+O
+.
+-
+<null_1>
+<mask>