Upload config.json with huggingface_hub

config.json

@@ -1,41 +1,34 @@
 {
-  "model_type": "gpt",
+  "model_type": "semitic-gpt",
   "architectures": [
-    "MultilingualGPT"
+    "SemiticGPT"
   ],
   "vocab_size": 32000,
   "hidden_size": 3072,
   "num_hidden_layers": 26,
   "num_attention_heads": 24,
-  "head_dim": 128,
-  "max_position_embeddings": 2048,
   "intermediate_size": 8192,
-  "activation_function": "swiglu",
-  "normalization": "rmsnorm",
-  "position_encoding": "rope",
+  "max_position_embeddings": 2048,
   "rope_theta": 10000.0,
-  "total_params": "3.14B",
-  "tokenizer_type": "sentencepiece",
-  "tokenizer_vocab_size": 32000,
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "pad_token": "<pad>",
-  "special_tokens": [
-    "<|user|>",
-    "<|assistant|>",
-    "<s>",
-    "</s>",
-    "<pad>"
+  "rope_dim": 64,
+  "rms_norm_eps": 1e-06,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "num_parameters": 3140000000,
+  "tokenizer": "sentencepiece",
+  "tokenizer_model": "multilingual_32k.model",
+  "languages": [
+    "Hebrew (he)",
+    "Arabic (ar)",
+    "Farsi/Persian (fa)",
+    "English (en)"
   ],
   "training": {
-    "optimizer": "AdamW",
-    "learning_rate": 0.0003,
-    "schedule": "cosine_decay",
-    "warmup_steps": 2000,
-    "batch_size_tokens": 524288,
-    "weight_decay": 0.1,
-    "gradient_clip": 1.0,
-    "precision": "bf16",
-    "total_tokens": "~20B"
+    "pretraining_tokens": "4.48B",
+    "pretraining_languages": "Hebrew (40%), Arabic (20%), Farsi (20%), English (20%)",
+    "sft_v4_samples": 36980,
+    "sft_optimizer": "AdamW",
+    "sft_lr": "2e-5",
+    "sft_steps": 8000
   }
 }