epoch 22534 | block 8112294 | steps 175 | loss 4.0805 | sim_abs 0.103 sq_acc 0.00

config.json

@@ -3,7 +3,7 @@
     "Mamba2ForCausalLM"
   ],
   "bos_token_id": null,
-  "chunk_size": 512,
+  "chunk_size": 256,
   "conv_kernel": 4,
   "dtype": "bfloat16",
   "eos_token_id": 151645,
@@ -11,14 +11,14 @@
   "head_dim": 64,
   "hidden_act": "silu",
   "hidden_size": 1024,
-  "initializer_range": 0.02,
+  "initializer_range": 0.1,
   "layer_norm_epsilon": 1e-05,
   "model_type": "mamba2",
   "n_groups": 1,
   "num_heads": 32,
   "num_hidden_layers": 48,
   "pad_token_id": 151643,
-  "rescale_prenorm_residual": true,
+  "rescale_prenorm_residual": false,
   "residual_in_fp32": true,
   "rms_norm": true,
   "state_size": 128,

generation_config.json

@@ -1,11 +1,8 @@
 {
   "_from_model_config": true,
-  "eos_token_id": [
-    151645
-  ],
-  "output_attentions": false,
-  "output_hidden_states": false,
+  "bos_token_id": 0,
+  "eos_token_id": 151645,
   "pad_token_id": 151643,
   "transformers_version": "5.7.0",
-  "use_cache": false
+  "use_cache": true
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1714930c07abd3051b1ede301f09cdf97ae7efb25225e9597c08336fe7d9dc6a
+oid sha256:2e4c35ee9ef90c9a4785261dc3e934e01f2fb493b951452f3c6459c0c8abb233
 size 944367904