rita_modeling.py

import math
import os
from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import CrossEntropyLoss

from transformers.modeling_outputs import (
    BaseModelOutputWithPast,
    BaseModelOutputWithPastAndCrossAttentions,
    CausalLMOutputWithCrossAttentions,
    CausalLMOutputWithPast,
)

from transformers.modeling_utils import PreTrainedModel
from transformers.utils import logging

from .rita_configuration import RITAConfig
import torch.nn.functional as F
logger = logging.get_logger(__name__)

@torch.jit.script
def RITA_gelu(hidden_states):
    return hidden_states * 0.5 * (1.0 + torch.tanh(0.79788456 * hidden_states * (1 + 0.044715 * hidden_states * hidden_states)))

class RITAGELU(nn.Module):
    def __init__(self):
        super().__init__()
    
    def forward(self, hidden_states):
        return RITA_gelu(hidden_states)

def rotate_half(x):
    x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
    return torch.cat((-x2, x1), dim=x1.ndim - 1)

class RotaryEmbedding(nn.Module):
    def __init__(self, config):
        super().__init__()
        assert config.d_model % config.num_heads == 0
        
        self.d_model = config.d_model
        self.num_heads = config.num_heads
        self.max_seq_len = config.max_seq_len
        
        head_dim = self.d_model // self.num_heads
        inv_freq = 1.0 / (10000 ** (torch.arange(0, head_dim, 2).float() / head_dim))
        self.register_buffer('inv_freq', inv_freq)
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None
    
    def forward(self, x: torch.FloatTensor, seq_dim=1) -> torch.FloatTensor:
        seq_len = x.shape[seq_dim]
        if seq_len != self.seq_len_cached:
            self.seq_len_cached = seq_len
            t = torch.arange(x.shape[seq_dim], device=x.device).type_as(self.inv_freq)
            freqs = torch.einsum("i,j->ij", t, self.inv_freq)
            emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
            self.cos_cached = emb.cos()[None, None, :, :]
            self.sin_cached = emb.sin()[None, None, :, :]
        return self.cos_cached, self.sin_cached
    
    def apply_rotary_pos_emb(self, q, k, cos, sin):
        return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)

    
class SelfAttention(nn.Module):
    """Implementation of MultiHeadAttention following `Karpathy's MinGPT <https://github.com/karpathy/minGPT>`_.
    modified to use rotary embeddings.
    
    Parameters
    ----------
    d_model: int,
         total dimension of the model.
    num_heads: int,
        number of parallel attention heads.
    num_layers: int,
        number of layers in the model, used for the Megatron-like init.
    rotaty_embedding: Optional[Block], default None,
        a RotaryEmbedding Block to add positionnal information in Queries and Keys
    dropout: float, default 0.1,
        amount of dropout on the attention weights.
    sigma: float, default 0.02,
        standard deviation used for the init.
    trainable: bool, default True,
        if False, the Module parameters will be hidden from the optimizer.
    """

    def __init__(
        self,
        d_model: int,
        num_heads: int,
        num_layers: int,
        rotary_embedding= None,
        dropout: float = 0.1,
        sigma=0.02,
        use_cache: bool = False,
        bias=True,
    ):
        super().__init__()
        assert d_model % num_heads == 0
        self.d_model = d_model
        self.num_heads = num_heads
        self.head_dim = self.d_model // self.num_heads
        self.num_layers = num_layers
        self.dropout = dropout
        self.sigma = sigma
        self.bias = bias

        # key, query, value projections for all heads
        self.key = nn.Linear(d_model, d_model, bias=bias)
        self.query = nn.Linear(d_model, d_model, bias=bias)
        self.value = nn.Linear(d_model, d_model, bias=bias)
        # regularization
        self.attn_drop = nn.Dropout(dropout)
        self.resid_drop = nn.Dropout(dropout)
        # output projection
        self.proj = nn.Linear(d_model, d_model, bias=bias)

        self.rotary_embedding = rotary_embedding
        self.layer_id = None  # will be set by the Transformer itself
        self.use_cache = use_cache
        self.qkv = None
        self.bias = bias

    def forward(
        self,
        x,
        attn_mask: Optional[torch.BoolTensor] = None,
        padding_mask: Optional[torch.BoolTensor] = None,
    ) -> Tuple[torch.FloatTensor, torch.FloatTensor]:

        N, L, D = x.size()  # Batch_size, Context_size, d_model

        # calculate query, key, values for all heads in batch and move head forward to be the batch dim
        k = (
            self.key(x).view(N, L, self.num_heads, D // self.num_heads).transpose(1, 2)
        )  # (N, nh, L, hs)
        q = (
            self.query(x).view(N, L, self.num_heads, D // self.num_heads).transpose(1, 2)
        )  # (N, nh, L, hs)
        v = (
            self.value(x).view(N, L, self.num_heads, D // self.num_heads).transpose(1, 2)
        )  # (N, nh, L, hs)
        
        if self.rotary_embedding is not None:
            cos, sin = self.rotary_embedding(x)
            q, k = self.rotary_embedding.apply_rotary_pos_emb(q, k, cos, sin)

        # causal self-attention; Self-attend: (N, nh, L, hs) x (N, nh, hs, L) -> (N, nh, L, L)
        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
        
        if attn_mask is not None:
            att[:,:,-L:, -L: ].masked_fill_(attn_mask.view(1, 1, L, L), float("-inf"))
            
        att = (
            att.transpose(0, 2)
            .masked_fill(padding_mask.view(1, 1, N, L), float("-inf"))
            .transpose(0, 2)
            if padding_mask is not None
            else att
        )
        
        att = F.softmax(att, dim=-1)
        att = self.attn_drop(att)
        y = att @ v  # (N, nh, L, L) x (N, nh, L, hs) -> (N, nh, L, hs)
        y = (
            y.transpose(1, 2).contiguous().view(N, L, D)
        )  # re-assemble all head outputs side by side

        # output projection
        y = self.resid_drop(self.proj(y))
        return y

class DecoderLayer(nn.Module):
    """Transformer block containing the self-attention module and the feedfoward module."""

    def __init__(
        self, config
    ):
        super().__init__()
        self.self_attention = SelfAttention(config.d_model, config.num_heads, config.dropout, rotary_embedding=RotaryEmbedding(config))
        self.attn_norm = nn.LayerNorm(config.d_model)
        self.attn_dropout = nn.Dropout(config.dropout)

        self.mlp = nn.Sequential(
            nn.Linear(config.d_model, config.d_feedforward, bias=True),
            RITAGELU(),
            nn.Linear(config.d_feedforward, config.d_model, bias=True),
        )
        self.mlp_norm = nn.LayerNorm(config.d_model)
        self.mlp_dropout = nn.Dropout(config.dropout)
        
    def forward(
        self,
        x: torch.FloatTensor,
        attn_mask: torch.BoolTensor,
        padding_mask: Optional[torch.BoolTensor] = None,
    ) -> torch.FloatTensor:
        y = self.attn_norm(x)
        y = self.self_attention(y, attn_mask=attn_mask, padding_mask=padding_mask)
        x = x + self.attn_dropout(y)

        y = self.mlp_norm(x)
        y = self.mlp(y)
        x = x + self.mlp_dropout(y)
        return x
    
class RITAModel(PreTrainedModel):
    config_class = RITAConfig
    def __init__(
        self,
        config
    ):
        super().__init__(config)
        self.embedding = nn.Embedding(config.vocab_size, config.d_model)
        self.layers = nn.ModuleList([DecoderLayer(config) for _ in range(config.num_layers)])
        self.final_norm = nn.LayerNorm(config.d_model)
        self.projector = nn.Linear(config.d_model, config.vocab_size, bias = False)

    def forward(self, input_ids, attn_mask=None, padding_mask=None, return_hidden=False) -> torch.FloatTensor:
        x = self.embedding(input_ids)  # N x L x D
        if attn_mask == None:
            attn_mask = (torch.triu(torch.ones(input_ids.size(1), input_ids.size(1))) == 0).transpose(0, 1).contiguous().to(input_ids.device)
        for layer in self.layers:
            x = layer(x, attn_mask=attn_mask, padding_mask=padding_mask)
        x = self.final_norm(x)  # N x L x D

        if return_hidden:
            return x
        else:
            return self.projector(x)

    #Some common HF functions.
    def get_input_embeddings(self):
        return self.embedding

    def set_input_embeddings(self, new_embeddings):
        self.embedding = new_embeddings

    def get_output_embeddings(self):
        return self.projector

    def set_output_embeddings(self, new_projector):
        self.projector = new_projector