Initial MeridianAI seed (Base: HuggingFaceTB/SmolLM2-360M, Tokenizer: HuggingFaceTB/SmolLM2-360M)

checkpoint/config.json

@@ -4,41 +4,31 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 1,
-  "capacity_factor_eval": 2.0,
-  "capacity_factor_train": 1.25,
-  "drop_tks": true,
-  "dropout_rate": 0.0,
-  "dtype": "float32",
-  "eos_token_id": 2,
-  "expert_parallel": null,
-  "gated": true,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
   "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 768,
+  "hidden_size": 960,
   "initializer_range": 0.02,
-  "intermediate_size": 2048,
-  "layer_norm_epsilon": 1e-06,
-  "max_position_embeddings": 2048,
-  "min_capacity": 4,
+  "intermediate_size": 2560,
+  "is_llama_config": true,
+  "max_position_embeddings": 8192,
   "mlp_bias": false,
   "model_type": "llama",
-  "moe_layer_interval": 4,
-  "noisy_policy": null,
-  "num_attention_heads": 12,
-  "num_experts": 16,
-  "num_hidden_layers": 12,
-  "num_key_value_heads": 12,
-  "pad_token_id": 0,
+  "num_attention_heads": 15,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 5,
+  "pad_token_id": null,
   "pretraining_tp": 1,
-  "rms_norm_eps": 1e-06,
+  "rms_norm_eps": 1e-05,
+  "rope_interleaved": false,
   "rope_parameters": {
-    "rope_theta": 10000.0,
+    "rope_theta": 100000,
     "rope_type": "default"
   },
-  "tie_word_embeddings": false,
-  "topk": 2,
+  "tie_word_embeddings": true,
   "transformers_version": "5.3.0",
-  "use_cache": false,
-  "vocab_size": 256384
+  "use_cache": true,
+  "vocab_size": 49152
 }

checkpoint/generation_config.json

@@ -1,7 +1,6 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
-  "pad_token_id": 0,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
   "transformers_version": "5.3.0"
 }

checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c7906faca1e4ee4fa759182b65591628aa965ea23df3c80108982ed7f81c88c
-size 1915051264
+oid sha256:7aaff6661428bed033abba9522bec81938678642cca3181fe752b6ca9e1e540f
+size 723674912

checkpoint/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:39daf1fcb0db6fdaac7e1a1902b598c2f147e2ca6720614761d20924b631bc60
-size 16853111
+oid sha256:bf346d64f6f0fbcefb4c1b6928a98241467dff36c6fbae5fe1785c4ff90667f4
+size 3522871

checkpoint/tokenizer_config.json

@@ -1,316 +1,33 @@
 {
+  "add_prefix_space": false,
   "backend": "tokenizers",
-  "bos_token": "<s>",
-  "clean_up_tokenization_spaces": true,
-  "eos_token": "</s>",
-  "extra_ids": 300,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
   "extra_special_tokens": [
-    "<extra_id_0>",
-    "<extra_id_1>",
-    "<extra_id_2>",
-    "<extra_id_3>",
-    "<extra_id_4>",
-    "<extra_id_5>",
-    "<extra_id_6>",
-    "<extra_id_7>",
-    "<extra_id_8>",
-    "<extra_id_9>",
-    "<extra_id_10>",
-    "<extra_id_11>",
-    "<extra_id_12>",
-    "<extra_id_13>",
-    "<extra_id_14>",
-    "<extra_id_15>",
-    "<extra_id_16>",
-    "<extra_id_17>",
-    "<extra_id_18>",
-    "<extra_id_19>",
-    "<extra_id_20>",
-    "<extra_id_21>",
-    "<extra_id_22>",
-    "<extra_id_23>",
-    "<extra_id_24>",
-    "<extra_id_25>",
-    "<extra_id_26>",
-    "<extra_id_27>",
-    "<extra_id_28>",
-    "<extra_id_29>",
-    "<extra_id_30>",
-    "<extra_id_31>",
-    "<extra_id_32>",
-    "<extra_id_33>",
-    "<extra_id_34>",
-    "<extra_id_35>",
-    "<extra_id_36>",
-    "<extra_id_37>",
-    "<extra_id_38>",
-    "<extra_id_39>",
-    "<extra_id_40>",
-    "<extra_id_41>",
-    "<extra_id_42>",
-    "<extra_id_43>",
-    "<extra_id_44>",
-    "<extra_id_45>",
-    "<extra_id_46>",
-    "<extra_id_47>",
-    "<extra_id_48>",
-    "<extra_id_49>",
-    "<extra_id_50>",
-    "<extra_id_51>",
-    "<extra_id_52>",
-    "<extra_id_53>",
-    "<extra_id_54>",
-    "<extra_id_55>",
-    "<extra_id_56>",
-    "<extra_id_57>",
-    "<extra_id_58>",
-    "<extra_id_59>",
-    "<extra_id_60>",
-    "<extra_id_61>",
-    "<extra_id_62>",
-    "<extra_id_63>",
-    "<extra_id_64>",
-    "<extra_id_65>",
-    "<extra_id_66>",
-    "<extra_id_67>",
-    "<extra_id_68>",
-    "<extra_id_69>",
-    "<extra_id_70>",
-    "<extra_id_71>",
-    "<extra_id_72>",
-    "<extra_id_73>",
-    "<extra_id_74>",
-    "<extra_id_75>",
-    "<extra_id_76>",
-    "<extra_id_77>",
-    "<extra_id_78>",
-    "<extra_id_79>",
-    "<extra_id_80>",
-    "<extra_id_81>",
-    "<extra_id_82>",
-    "<extra_id_83>",
-    "<extra_id_84>",
-    "<extra_id_85>",
-    "<extra_id_86>",
-    "<extra_id_87>",
-    "<extra_id_88>",
-    "<extra_id_89>",
-    "<extra_id_90>",
-    "<extra_id_91>",
-    "<extra_id_92>",
-    "<extra_id_93>",
-    "<extra_id_94>",
-    "<extra_id_95>",
-    "<extra_id_96>",
-    "<extra_id_97>",
-    "<extra_id_98>",
-    "<extra_id_99>",
-    "<extra_id_100>",
-    "<extra_id_101>",
-    "<extra_id_102>",
-    "<extra_id_103>",
-    "<extra_id_104>",
-    "<extra_id_105>",
-    "<extra_id_106>",
-    "<extra_id_107>",
-    "<extra_id_108>",
-    "<extra_id_109>",
-    "<extra_id_110>",
-    "<extra_id_111>",
-    "<extra_id_112>",
-    "<extra_id_113>",
-    "<extra_id_114>",
-    "<extra_id_115>",
-    "<extra_id_116>",
-    "<extra_id_117>",
-    "<extra_id_118>",
-    "<extra_id_119>",
-    "<extra_id_120>",
-    "<extra_id_121>",
-    "<extra_id_122>",
-    "<extra_id_123>",
-    "<extra_id_124>",
-    "<extra_id_125>",
-    "<extra_id_126>",
-    "<extra_id_127>",
-    "<extra_id_128>",
-    "<extra_id_129>",
-    "<extra_id_130>",
-    "<extra_id_131>",
-    "<extra_id_132>",
-    "<extra_id_133>",
-    "<extra_id_134>",
-    "<extra_id_135>",
-    "<extra_id_136>",
-    "<extra_id_137>",
-    "<extra_id_138>",
-    "<extra_id_139>",
-    "<extra_id_140>",
-    "<extra_id_141>",
-    "<extra_id_142>",
-    "<extra_id_143>",
-    "<extra_id_144>",
-    "<extra_id_145>",
-    "<extra_id_146>",
-    "<extra_id_147>",
-    "<extra_id_148>",
-    "<extra_id_149>",
-    "<extra_id_150>",
-    "<extra_id_151>",
-    "<extra_id_152>",
-    "<extra_id_153>",
-    "<extra_id_154>",
-    "<extra_id_155>",
-    "<extra_id_156>",
-    "<extra_id_157>",
-    "<extra_id_158>",
-    "<extra_id_159>",
-    "<extra_id_160>",
-    "<extra_id_161>",
-    "<extra_id_162>",
-    "<extra_id_163>",
-    "<extra_id_164>",
-    "<extra_id_165>",
-    "<extra_id_166>",
-    "<extra_id_167>",
-    "<extra_id_168>",
-    "<extra_id_169>",
-    "<extra_id_170>",
-    "<extra_id_171>",
-    "<extra_id_172>",
-    "<extra_id_173>",
-    "<extra_id_174>",
-    "<extra_id_175>",
-    "<extra_id_176>",
-    "<extra_id_177>",
-    "<extra_id_178>",
-    "<extra_id_179>",
-    "<extra_id_180>",
-    "<extra_id_181>",
-    "<extra_id_182>",
-    "<extra_id_183>",
-    "<extra_id_184>",
-    "<extra_id_185>",
-    "<extra_id_186>",
-    "<extra_id_187>",
-    "<extra_id_188>",
-    "<extra_id_189>",
-    "<extra_id_190>",
-    "<extra_id_191>",
-    "<extra_id_192>",
-    "<extra_id_193>",
-    "<extra_id_194>",
-    "<extra_id_195>",
-    "<extra_id_196>",
-    "<extra_id_197>",
-    "<extra_id_198>",
-    "<extra_id_199>",
-    "<extra_id_200>",
-    "<extra_id_201>",
-    "<extra_id_202>",
-    "<extra_id_203>",
-    "<extra_id_204>",
-    "<extra_id_205>",
-    "<extra_id_206>",
-    "<extra_id_207>",
-    "<extra_id_208>",
-    "<extra_id_209>",
-    "<extra_id_210>",
-    "<extra_id_211>",
-    "<extra_id_212>",
-    "<extra_id_213>",
-    "<extra_id_214>",
-    "<extra_id_215>",
-    "<extra_id_216>",
-    "<extra_id_217>",
-    "<extra_id_218>",
-    "<extra_id_219>",
-    "<extra_id_220>",
-    "<extra_id_221>",
-    "<extra_id_222>",
-    "<extra_id_223>",
-    "<extra_id_224>",
-    "<extra_id_225>",
-    "<extra_id_226>",
-    "<extra_id_227>",
-    "<extra_id_228>",
-    "<extra_id_229>",
-    "<extra_id_230>",
-    "<extra_id_231>",
-    "<extra_id_232>",
-    "<extra_id_233>",
-    "<extra_id_234>",
-    "<extra_id_235>",
-    "<extra_id_236>",
-    "<extra_id_237>",
-    "<extra_id_238>",
-    "<extra_id_239>",
-    "<extra_id_240>",
-    "<extra_id_241>",
-    "<extra_id_242>",
-    "<extra_id_243>",
-    "<extra_id_244>",
-    "<extra_id_245>",
-    "<extra_id_246>",
-    "<extra_id_247>",
-    "<extra_id_248>",
-    "<extra_id_249>",
-    "<extra_id_250>",
-    "<extra_id_251>",
-    "<extra_id_252>",
-    "<extra_id_253>",
-    "<extra_id_254>",
-    "<extra_id_255>",
-    "<extra_id_256>",
-    "<extra_id_257>",
-    "<extra_id_258>",
-    "<extra_id_259>",
-    "<extra_id_260>",
-    "<extra_id_261>",
-    "<extra_id_262>",
-    "<extra_id_263>",
-    "<extra_id_264>",
-    "<extra_id_265>",
-    "<extra_id_266>",
-    "<extra_id_267>",
-    "<extra_id_268>",
-    "<extra_id_269>",
-    "<extra_id_270>",
-    "<extra_id_271>",
-    "<extra_id_272>",
-    "<extra_id_273>",
-    "<extra_id_274>",
-    "<extra_id_275>",
-    "<extra_id_276>",
-    "<extra_id_277>",
-    "<extra_id_278>",
-    "<extra_id_279>",
-    "<extra_id_280>",
-    "<extra_id_281>",
-    "<extra_id_282>",
-    "<extra_id_283>",
-    "<extra_id_284>",
-    "<extra_id_285>",
-    "<extra_id_286>",
-    "<extra_id_287>",
-    "<extra_id_288>",
-    "<extra_id_289>",
-    "<extra_id_290>",
-    "<extra_id_291>",
-    "<extra_id_292>",
-    "<extra_id_293>",
-    "<extra_id_294>",
-    "<extra_id_295>",
-    "<extra_id_296>",
-    "<extra_id_297>",
-    "<extra_id_298>",
-    "<extra_id_299>"
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<repo_name>",
+    "<reponame>",
+    "<file_sep>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>"
   ],
   "is_local": false,
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<pad>",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "T5Tokenizer",
-  "unk_token": "<unk>"
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
 }