Spaces:

gijl
/

hj

Sleeping

App Files Files Community

gijl commited on 15 days ago

Commit

cde5e54

verified ·

1 Parent(s): 6c811ca

Upload 2 files

Browse files

Files changed (2) hide show

model.py +55 -0
tokenizer_config.json +233 -0

model.py ADDED Viewed

	@@ -0,0 +1,55 @@

+import torch
+import torch.nn as nn
+import math
+from torch.nn import functional as F
+class SelfAttention(nn.Module):
+    def __init__(self, n_embd=768, n_head=8):
+        super().__init__()
+        self.qkv = nn.Linear(n_embd, n_embd * 3, bias=False)
+        self.proj = nn.Linear(n_embd, n_embd)
+        self.n_head = n_head
+    def forward(self, x):
+        B, T, C = x.shape
+        q, k, v = self.qkv(x).split(C, dim=2)
+        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.shape[-1]))
+        att = torch.softmax(att, dim=-1)
+        y = att @ v
+        y = y.transpose(1, 2).contiguous().view(B, T, C)
+        return self.proj(y)
+class Block(nn.Module):
+    def __init__(self, n_embd=768, n_head=8):
+        super().__init__()
+        self.ln1 = nn.LayerNorm(n_embd)
+        self.attn = SelfAttention(n_embd, n_head)
+        self.ln2 = nn.LayerNorm(n_embd)
+        self.mlp = nn.Sequential(
+            nn.Linear(n_embd, 4 * n_embd),
+            nn.GELU(),
+            nn.Linear(4 * n_embd, n_embd),
+        )
+    def forward(self, x):
+        x = x + self.attn(self.ln1(x))
+        x = x + self.mlp(self.ln2(x))
+        return x
+class MedicalMasterAI(nn.Module):
+    def __init__(self, vocab_size=115, n_layer=48, n_head=8, n_embd=768):
+        super().__init__()
+        self.token_embedding = nn.Embedding(vocab_size, n_embd)
+        self.position_embedding = nn.Parameter(torch.zeros(1, 1024, n_embd))
+        self.blocks = nn.Sequential(*[Block(n_embd, n_head) for _ in range(n_layer)])
+        self.ln_f = nn.LayerNorm(n_embd)
+        self.lm_head = nn.Linear(n_embd, vocab_size)
+    def forward(self, idx):
+        b, t = idx.shape
+        x = self.token_embedding(idx) + self.position_embedding[:, :t, :]
+        x = self.blocks(x)
+        return self.lm_head(self.ln_f(x))

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,233 @@

+{
+    "stoi": {
+        " ": 0,
+        "!": 1,
+        "(": 2,
+        ")": 3,
+        "*": 4,
+        "+": 5,
+        "-": 6,
+        ".": 7,
+        "/": 8,
+        "0": 9,
+        "1": 10,
+        "2": 11,
+        "3": 12,
+        "4": 13,
+        "5": 14,
+        "6": 15,
+        "7": 16,
+        "8": 17,
+        "9": 18,
+        "=": 19,
+        "A": 20,
+        "B": 21,
+        "C": 22,
+        "D": 23,
+        "E": 24,
+        "F": 25,
+        "G": 26,
+        "H": 27,
+        "I": 28,
+        "J": 29,
+        "K": 30,
+        "L": 31,
+        "M": 32,
+        "N": 33,
+        "O": 34,
+        "P": 35,
+        "Q": 36,
+        "R": 37,
+        "S": 38,
+        "T": 39,
+        "U": 40,
+        "V": 41,
+        "W": 42,
+        "X": 43,
+        "Y": 44,
+        "Z": 45,
+        "[": 46,
+        "]": 47,
+        "a": 48,
+        "b": 49,
+        "c": 50,
+        "d": 51,
+        "e": 52,
+        "f": 53,
+        "g": 54,
+        "h": 55,
+        "i": 56,
+        "j": 57,
+        "k": 58,
+        "l": 59,
+        "m": 60,
+        "n": 61,
+        "o": 62,
+        "p": 63,
+        "q": 64,
+        "r": 65,
+        "s": 66,
+        "t": 67,
+        "u": 68,
+        "v": 69,
+        "w": 70,
+        "x": 71,
+        "y": 72,
+        "z": 73,
+        "{": 74,
+        "}": 75,
+        "،": 76,
+        "؟": 77,
+        "ء": 78,
+        "آ": 79,
+        "أ": 80,
+        "ؤ": 81,
+        "إ": 82,
+        "ئ": 83,
+        "ب": 84,
+        "ة": 85,
+        "ت": 86,
+        "ث": 87,
+        "ج": 88,
+        "ح": 89,
+        "خ": 90,
+        "د": 91,
+        "ذ": 92,
+        "ر": 93,
+        "ز": 94,
+        "س": 95,
+        "ش": 96,
+        "ص": 97,
+        "ض": 98,
+        "ط": 99,
+        "ظ": 100,
+        "ع": 101,
+        "غ": 102,
+        "ـ": 103,
+        "ف": 104,
+        "ق": 105,
+        "ك": 106,
+        "ل": 107,
+        "م": 108,
+        "ن": 109,
+        "ه": 110,
+        "و": 111,
+        "ي": 112
+    },
+    "itos": {
+        "0": " ",
+        "1": "!",
+        "2": "(",
+        "3": ")",
+        "4": "*",
+        "5": "+",
+        "6": "-",
+        "7": ".",
+        "8": "/",
+        "9": "0",
+        "10": "1",
+        "11": "2",
+        "12": "3",
+        "13": "4",
+        "14": "5",
+        "15": "6",
+        "16": "7",
+        "17": "8",
+        "18": "9",
+        "19": "=",
+        "20": "A",
+        "21": "B",
+        "22": "C",
+        "23": "D",
+        "24": "E",
+        "25": "F",
+        "26": "G",
+        "27": "H",
+        "28": "I",
+        "29": "J",
+        "30": "K",
+        "31": "L",
+        "32": "M",
+        "33": "N",
+        "34": "O",
+        "35": "P",
+        "36": "Q",
+        "37": "R",
+        "38": "S",
+        "39": "T",
+        "40": "U",
+        "41": "V",
+        "42": "W",
+        "43": "X",
+        "44": "Y",
+        "45": "Z",
+        "46": "[",
+        "47": "]",
+        "48": "a",
+        "49": "b",
+        "50": "c",
+        "51": "d",
+        "52": "e",
+        "53": "f",
+        "54": "g",
+        "55": "h",
+        "56": "i",
+        "57": "j",
+        "58": "k",
+        "59": "l",
+        "60": "m",
+        "61": "n",
+        "62": "o",
+        "63": "p",
+        "64": "q",
+        "65": "r",
+        "66": "s",
+        "67": "t",
+        "68": "u",
+        "69": "v",
+        "70": "w",
+        "71": "x",
+        "72": "y",
+        "73": "z",
+        "74": "{",
+        "75": "}",
+        "76": "،",
+        "77": "؟",
+        "78": "ء",
+        "79": "آ",
+        "80": "أ",
+        "81": "ؤ",
+        "82": "إ",
+        "83": "ئ",
+        "84": "ب",
+        "85": "ة",
+        "86": "ت",
+        "87": "ث",
+        "88": "ج",
+        "89": "ح",
+        "90": "خ",
+        "91": "د",
+        "92": "ذ",
+        "93": "ر",
+        "94": "ز",
+        "95": "س",
+        "96": "ش",
+        "97": "ص",
+        "98": "ض",
+        "99": "ط",
+        "100": "ظ",
+        "101": "ع",
+        "102": "غ",
+        "103": "ـ",
+        "104": "ف",
+        "105": "ق",
+        "106": "ك",
+        "107": "ل",
+        "108": "م",
+        "109": "ن",
+        "110": "ه",
+        "111": "و",
+        "112": "ي"
+    },
+    "vocab_size": 115
+}