Added files

README.md

@@ -0,0 +1,32 @@
+---
+language: ar
+tags:
+- roberta
+- language model
+datasets:
+- oscar
+---
+# TavBERT base model
+An Arabic BERT-style masked language model operating over characters, pre-trained by masking spans of characters, similarly to SpanBERT (Joshi et al., 2020).
+
+### How to use
+```python
+import numpy as np
+import torch
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+model = AutoModelForMaskedLM.from_pretrained("tau/tavbert-ar")
+tokenizer = AutoTokenizer.from_pretrained("tau/tavbert-ar")
+
+def mask_sentence(sent, span_len=5):
+    start_pos = np.random.randint(0, len(sent) - span_len)
+    masked_sent = sent[:start_pos] + '[MASK]' * span_len + sent[start_pos + span_len:]
+    print("Masked sentence:", masked_sent)
+    output = model(**tokenizer.encode_plus(masked_sent, 
+                                           return_tensors='pt'))['logits'][0][1:-1]
+    preds = [int(x) for x in torch.argmax(torch.softmax(output, axis=1), axis=1)[start_pos:start_pos + span_len]]
+    pred_sent = sent[:start_pos] + ''.join(tokenizer.convert_ids_to_tokens(preds)) + sent[start_pos + span_len:]
+    print("Model's prediction:", pred_sent)
+```
+## Training data
+OSCAR (Ortiz, 2019) Arabic section (32 GB text, 67 million sentences).

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2050,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.6.0.dev0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 302
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8698054675af696bb0f18c1268ef07d2d2e1497714a96a01551b25347f225197
+size 349911826

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer.json

@@ -0,0 +1 @@
+{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Split","pattern":{"String":""},"behavior":"Isolated","invert":false},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[0],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[2],"tokens":["[SEP]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[CLS]": 0, "[PAD]": 1, "[SEP]": 2, "[UNK]": 3, "unused0": 4, "unused1": 5, "unused2": 6, "unused3": 7, "unused4": 8, "unused5": 9, "unused6": 10, "unused7": 11, "unused8": 12, "unused9": 13, "unused10": 14, "unused11": 15, "unused12": 16, "unused13": 17, "unused14": 18, "unused15": 19, "unused16": 20, "unused17": 21, "unused18": 22, "unused19": 23, "unused20": 24, "unused21": 25, "unused22": 26, "unused23": 27, "unused24": 28, "unused25": 29, "unused26": 30, "unused27": 31, "unused28": 32, "unused29": 33, "unused30": 34, "unused31": 35, "unused32": 36, "unused33": 37, "unused34": 38, "unused35": 39, "unused36": 40, "unused37": 41, "unused38": 42, "unused39": 43, "unused40": 44, "unused41": 45, "unused42": 46, "unused43": 47, "unused44": 48, "unused45": 49, "unused46": 50, "unused47": 51, "unused48": 52, "unused49": 53, "unused50": 54, "unused51": 55, "unused52": 56, "unused53": 57, "unused54": 58, "unused55": 59, "unused56": 60, "unused57": 61, "unused58": 62, "unused59": 63, "unused60": 64, "unused61": 65, "unused62": 66, "unused63": 67, "unused64": 68, "unused65": 69, "unused66": 70, "unused67": 71, "unused68": 72, "unused69": 73, "unused70": 74, "unused71": 75, "unused72": 76, "unused73": 77, "unused74": 78, "unused75": 79, "unused76": 80, "unused77": 81, "unused78": 82, "unused79": 83, "unused80": 84, "unused81": 85, "unused82": 86, "unused83": 87, "unused84": 88, "unused85": 89, "unused86": 90, "unused87": 91, "unused88": 92, "unused89": 93, "unused90": 94, "unused91": 95, "unused92": 96, "unused93": 97, "unused94": 98, "unused95": 99, "unused96": 100, "unused97": 101, "unused98": 102, "unused99": 103, " ": 104, "!": 105, "\"": 106, "#": 107, "$": 108, "%": 109, "&": 110, "'": 111, "(": 112, ")": 113, "*": 114, "+": 115, ",": 116, "-": 117, ".": 118, "/": 119, "0": 120, "1": 121, "2": 122, "3": 123, "4": 124, "5": 125, "6": 126, "7": 127, "8": 128, "9": 129, ":": 130, ";": 131, "<": 132, "=": 133, ">": 134, "?": 135, "@": 136, "A": 137, "B": 138, "C": 139, "D": 140, "E": 141, "F": 142, "G": 143, "H": 144, "I": 145, "J": 146, "K": 147, "L": 148, "M": 149, "N": 150, "O": 151, "P": 152, "Q": 153, "R": 154, "S": 155, "T": 156, "U": 157, "V": 158, "W": 159, "X": 160, "Y": 161, "Z": 162, "[": 163, "\\": 164, "]": 165, "^": 166, "_": 167, "a": 168, "b": 169, "c": 170, "d": 171, "e": 172, "f": 173, "g": 174, "h": 175, "i": 176, "j": 177, "k": 178, "l": 179, "m": 180, "n": 181, "o": 182, "p": 183, "q": 184, "r": 185, "s": 186, "t": 187, "u": 188, "v": 189, "w": 190, "x": 191, "y": 192, "z": 193, "{": 194, "|": 195, "}": 196, "~": 197, "«": 198, "°": 199, "·": 200, "»": 201, "é": 202, "а": 203, "в": 204, "д": 205, "е": 206, "и": 207, "к": 208, "л": 209, "м": 210, "н": 211, "о": 212, "п": 213, "р": 214, "с": 215, "т": 216, "،": 217, "؛": 218, "؟": 219, "ء": 220, "آ": 221, "أ": 222, "ؤ": 223, "إ": 224, "ئ": 225, "ا": 226, "ب": 227, "ة": 228, "ت": 229, "ث": 230, "ج": 231, "ح": 232, "خ": 233, "د": 234, "ذ": 235, "ر": 236, "ز": 237, "س": 238, "ش": 239, "ص": 240, "ض": 241, "ط": 242, "ظ": 243, "ع": 244, "غ": 245, "ـ": 246, "ف": 247, "ق": 248, "ك": 249, "ل": 250, "م": 251, "ن": 252, "ه": 253, "و": 254, "ى": 255, "ي": 256, "ً": 257, "ٌ": 258, "ٍ": 259, "َ": 260, "ُ": 261, "ِ": 262, "ّ": 263, "ْ": 264, "٠": 265, "١": 266, "٢": 267, "٣": 268, "٤": 269, "٥": 270, "٦": 271, "٧": 272, "٨": 273, "٩": 274, "٪": 275, "پ": 276, "ک": 277, "گ": 278, "ھ": 279, "ی": 280, "​": 281, "‌": 282, "‎": 283, "‏": 284, "–": 285, "—": 286, "‘": 287, "’": 288, "“": 289, "”": 290, "•": 291, "…": 292, "‪": 293, "‫": 294, "‬": 295, "‭": 296, "‮": 297, "﴾": 298, "﴿": 299, "�": 300, "[MASK]": 301},"unk_token":"[UNK]"}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"special_tokens_map_file": "./special_tokens_map.json", "name_or_path": ".", "tokenizer_class": "PreTrainedTokenizerFast"}

vocab.json

@@ -0,0 +1 @@
+{"[CLS]": 0, "[PAD]": 1, "[SEP]": 2, "[UNK]": 3, "unused0": 4, "unused1": 5, "unused2": 6, "unused3": 7, "unused4": 8, "unused5": 9, "unused6": 10, "unused7": 11, "unused8": 12, "unused9": 13, "unused10": 14, "unused11": 15, "unused12": 16, "unused13": 17, "unused14": 18, "unused15": 19, "unused16": 20, "unused17": 21, "unused18": 22, "unused19": 23, "unused20": 24, "unused21": 25, "unused22": 26, "unused23": 27, "unused24": 28, "unused25": 29, "unused26": 30, "unused27": 31, "unused28": 32, "unused29": 33, "unused30": 34, "unused31": 35, "unused32": 36, "unused33": 37, "unused34": 38, "unused35": 39, "unused36": 40, "unused37": 41, "unused38": 42, "unused39": 43, "unused40": 44, "unused41": 45, "unused42": 46, "unused43": 47, "unused44": 48, "unused45": 49, "unused46": 50, "unused47": 51, "unused48": 52, "unused49": 53, "unused50": 54, "unused51": 55, "unused52": 56, "unused53": 57, "unused54": 58, "unused55": 59, "unused56": 60, "unused57": 61, "unused58": 62, "unused59": 63, "unused60": 64, "unused61": 65, "unused62": 66, "unused63": 67, "unused64": 68, "unused65": 69, "unused66": 70, "unused67": 71, "unused68": 72, "unused69": 73, "unused70": 74, "unused71": 75, "unused72": 76, "unused73": 77, "unused74": 78, "unused75": 79, "unused76": 80, "unused77": 81, "unused78": 82, "unused79": 83, "unused80": 84, "unused81": 85, "unused82": 86, "unused83": 87, "unused84": 88, "unused85": 89, "unused86": 90, "unused87": 91, "unused88": 92, "unused89": 93, "unused90": 94, "unused91": 95, "unused92": 96, "unused93": 97, "unused94": 98, "unused95": 99, "unused96": 100, "unused97": 101, "unused98": 102, "unused99": 103, " ": 104, "!": 105, "\"": 106, "#": 107, "$": 108, "%": 109, "&": 110, "'": 111, "(": 112, ")": 113, "*": 114, "+": 115, ",": 116, "-": 117, ".": 118, "/": 119, "0": 120, "1": 121, "2": 122, "3": 123, "4": 124, "5": 125, "6": 126, "7": 127, "8": 128, "9": 129, ":": 130, ";": 131, "<": 132, "=": 133, ">": 134, "?": 135, "@": 136, "A": 137, "B": 138, "C": 139, "D": 140, "E": 141, "F": 142, "G": 143, "H": 144, "I": 145, "J": 146, "K": 147, "L": 148, "M": 149, "N": 150, "O": 151, "P": 152, "Q": 153, "R": 154, "S": 155, "T": 156, "U": 157, "V": 158, "W": 159, "X": 160, "Y": 161, "Z": 162, "[": 163, "\\": 164, "]": 165, "^": 166, "_": 167, "a": 168, "b": 169, "c": 170, "d": 171, "e": 172, "f": 173, "g": 174, "h": 175, "i": 176, "j": 177, "k": 178, "l": 179, "m": 180, "n": 181, "o": 182, "p": 183, "q": 184, "r": 185, "s": 186, "t": 187, "u": 188, "v": 189, "w": 190, "x": 191, "y": 192, "z": 193, "{": 194, "|": 195, "}": 196, "~": 197, "«": 198, "°": 199, "·": 200, "»": 201, "é": 202, "а": 203, "в": 204, "д": 205, "е": 206, "и": 207, "к": 208, "л": 209, "м": 210, "н": 211, "о": 212, "п": 213, "р": 214, "с": 215, "т": 216, "،": 217, "؛": 218, "؟": 219, "ء": 220, "آ": 221, "أ": 222, "ؤ": 223, "إ": 224, "ئ": 225, "ا": 226, "ب": 227, "ة": 228, "ت": 229, "ث": 230, "ج": 231, "ح": 232, "خ": 233, "د": 234, "ذ": 235, "ر": 236, "ز": 237, "س": 238, "ش": 239, "ص": 240, "ض": 241, "ط": 242, "ظ": 243, "ع": 244, "غ": 245, "ـ": 246, "ف": 247, "ق": 248, "ك": 249, "ل": 250, "م": 251, "ن": 252, "ه": 253, "و": 254, "ى": 255, "ي": 256, "ً": 257, "ٌ": 258, "ٍ": 259, "َ": 260, "ُ": 261, "ِ": 262, "ّ": 263, "ْ": 264, "٠": 265, "١": 266, "٢": 267, "٣": 268, "٤": 269, "٥": 270, "٦": 271, "٧": 272, "٨": 273, "٩": 274, "٪": 275, "پ": 276, "ک": 277, "گ": 278, "ھ": 279, "ی": 280, "​": 281, "‌": 282, "‎": 283, "‏": 284, "–": 285, "—": 286, "‘": 287, "’": 288, "“": 289, "”": 290, "•": 291, "…": 292, "‪": 293, "‫": 294, "‬": 295, "‭": 296, "‮": 297, "﴾": 298, "﴿": 299, "�": 300, "[MASK]": 301}

vocab.txt

@@ -0,0 +1,302 @@
+[CLS]
+[PAD]
+[SEP]
+[UNK]
+unused0
+unused1
+unused2
+unused3
+unused4
+unused5
+unused6
+unused7
+unused8
+unused9
+unused10
+unused11
+unused12
+unused13
+unused14
+unused15
+unused16
+unused17
+unused18
+unused19
+unused20
+unused21
+unused22
+unused23
+unused24
+unused25
+unused26
+unused27
+unused28
+unused29
+unused30
+unused31
+unused32
+unused33
+unused34
+unused35
+unused36
+unused37
+unused38
+unused39
+unused40
+unused41
+unused42
+unused43
+unused44
+unused45
+unused46
+unused47
+unused48
+unused49
+unused50
+unused51
+unused52
+unused53
+unused54
+unused55
+unused56
+unused57
+unused58
+unused59
+unused60
+unused61
+unused62
+unused63
+unused64
+unused65
+unused66
+unused67
+unused68
+unused69
+unused70
+unused71
+unused72
+unused73
+unused74
+unused75
+unused76
+unused77
+unused78
+unused79
+unused80
+unused81
+unused82
+unused83
+unused84
+unused85
+unused86
+unused87
+unused88
+unused89
+unused90
+unused91
+unused92
+unused93
+unused94
+unused95
+unused96
+unused97
+unused98
+unused99
+ 
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+:
+;
+<
+=
+>
+?
+@
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+[
+\
+]
+^
+_
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+{
+|
+}
+~
+«
+°
+·
+»
+é
+а
+в
+д
+е
+и
+к
+л
+м
+н
+о
+п
+р
+с
+т
+،
+؛
+؟
+ء
+آ
+أ
+ؤ
+إ
+ئ
+ا
+ب
+ة
+ت
+ث
+ج
+ح
+خ
+د
+ذ
+ر
+ز
+س
+ش
+ص
+ض
+ط
+ظ
+ع
+غ
+ـ
+ف
+ق
+ك
+ل
+م
+ن
+ه
+و
+ى
+ي
+ً
+ٌ
+ٍ
+َ
+ُ
+ِ
+ّ
+ْ
+٠
+١
+٢
+٣
+٤
+٥
+٦
+٧
+٨
+٩
+٪
+پ
+ک
+گ
+ھ
+ی
+​
+‌
+‎
+‏
+–
+—
+‘
+’
+“
+”
+•
+…
+‪
+‫
+‬
+‭
+‮
+﴾
+﴿
+�
+[MASK]