| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| """ EncT5 model configuration""" |
|
|
| from transformers.configuration_utils import PretrainedConfig |
|
|
|
|
| class EncT5Config(PretrainedConfig): |
| r""" |
| This is the configuration class to store the configuration of a [`EncT5`]. It is used to instantiate a EncT5 model |
| according to the specified arguments, defining the model architecture. Instantiating a configuration with the |
| defaults will yield a similar configuration to that of the T5 [t5-small](https://huggingface.co/t5-small) |
| architecture. |
| |
| Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the |
| documentation from [`PretrainedConfig`] for more information. |
| |
| Arguments: |
| vocab_size (`int`, *optional*, defaults to 32128): |
| Vocabulary size of the EncT5 model. Defines the number of different tokens that can be represented by the |
| `inputs_ids` passed when calling [`T5Model`] or [`TFT5Model`]. |
| decoder_vocab_size (`int`, *optional*, defaults to 1): |
| Decoder vocabulary size of the EncT5 model. For regression and single-label classification, this should just |
| be 1 (the default). For multi-label classification, this should be the number of labels. |
| d_model (`int`, *optional*, defaults to 512): |
| Size of the encoder layers and the pooler layer. |
| d_kv (`int`, *optional*, defaults to 64): |
| Size of the key, query, value projections per attention head. The `inner_dim` of the projection layer will |
| be defined as `num_heads * d_kv`. |
| d_ff (`int`, *optional*, defaults to 2048): |
| Size of the intermediate feed forward layer in each `T5Block`. |
| num_layers (`int`, *optional*, defaults to 6): |
| Number of hidden layers in the Transformer encoder. |
| num_decoder_layers (`int`, *optional*, defaults to 1): |
| Number of hidden layers in the Transformer decoder. |
| num_heads (`int`, *optional*, defaults to 8): |
| Number of attention heads for each attention layer in the Transformer encoder. |
| relative_attention_num_buckets (`int`, *optional*, defaults to 32): |
| The number of buckets to use for each attention layer. |
| relative_attention_max_distance (`int`, *optional*, defaults to 128): |
| The maximum distance of the longer sequences for the bucket separation. |
| dropout_rate (`float`, *optional*, defaults to 0.1): |
| The ratio for all dropout layers. |
| classifier_dropout (`float`, *optional*, defaults to 0.0): |
| The dropout ratio for classifier. |
| layer_norm_eps (`float`, *optional*, defaults to 1e-6): |
| The epsilon used by the layer normalization layers. |
| initializer_factor (`float`, *optional*, defaults to 1): |
| A factor for initializing all weight matrices (should be kept to 1, used internally for initialization |
| testing). |
| feed_forward_proj (`string`, *optional*, defaults to `"relu"`): |
| Type of feed forward layer to be used. Should be one of `"relu"` or `"gated-gelu"`. T5v1.1 uses the |
| `"gated-gelu"` feed forward projection. Original T5 uses `"relu"`. |
| use_cache (`bool`, *optional*, defaults to `True`): |
| Whether or not the model should return the last key/values attentions (not used by all models). |
| """ |
|
|
| model_type = "enct5" |
| keys_to_ignore_at_inference = ["past_key_values"] |
| attribute_map = {"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"} |
|
|
| def __init__( |
| self, |
| vocab_size=32128, |
| decoder_vocab_size=1, |
| d_model=512, |
| d_kv=64, |
| d_ff=2048, |
| num_layers=6, |
| num_decoder_layers=1, |
| num_heads=8, |
| relative_attention_num_buckets=32, |
| relative_attention_max_distance=128, |
| dropout_rate=0.1, |
| layer_norm_epsilon=1e-6, |
| initializer_factor=1.0, |
| feed_forward_proj="relu", |
| is_encoder_decoder=True, |
| use_cache=True, |
| pad_token_id=0, |
| eos_token_id=1, |
| classifier_dropout=0.0, |
| **kwargs, |
| ): |
| self.vocab_size = vocab_size |
| self.decoder_vocab_size = decoder_vocab_size |
| self.d_model = d_model |
| self.d_kv = d_kv |
| self.d_ff = d_ff |
| self.num_layers = num_layers |
| self.num_decoder_layers = num_decoder_layers |
| self.num_heads = num_heads |
| self.relative_attention_num_buckets = relative_attention_num_buckets |
| self.relative_attention_max_distance = relative_attention_max_distance |
| self.dropout_rate = dropout_rate |
| self.classifier_dropout = classifier_dropout |
| self.layer_norm_epsilon = layer_norm_epsilon |
| self.initializer_factor = initializer_factor |
| self.feed_forward_proj = feed_forward_proj |
| self.use_cache = use_cache |
|
|
| act_info = self.feed_forward_proj.split("-") |
| self.dense_act_fn = act_info[-1] |
| self.is_gated_act = act_info[0] == "gated" |
|
|
| if len(act_info) > 1 and act_info[0] != "gated" or len(act_info) > 2: |
| raise ValueError( |
| f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer. " |
| "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. " |
| "'gated-gelu' or 'relu'" |
| ) |
|
|
| |
| if feed_forward_proj == "gated-gelu": |
| self.dense_act_fn = "gelu_new" |
|
|
| super().__init__( |
| pad_token_id=pad_token_id, |
| eos_token_id=eos_token_id, |
| is_encoder_decoder=is_encoder_decoder, |
| **kwargs, |
| ) |
|
|
| |
| self.tie_word_embeddings = False |
|
|
