| { |
| "version": 2, |
| "model_id": "moonshotai/Kimi-K2-Instruct", |
| "model_config": { |
| "moe": { |
| "model_type": "kimi_k2", |
| "num_layers": 61, |
| "hidden_size": 7168, |
| "intermediate_size": 18432, |
| "moe_intermediate_size": 2048, |
| "n_routed_experts": 384, |
| "n_shared_experts": 1, |
| "num_experts_per_tok": 8, |
| "first_k_dense_replace": 1, |
| "torch_dtype": "bfloat16", |
| "quant_method": "fp8" |
| } |
| }, |
| "num_feats": 64, |
| "dtype": "float32", |
| "gate_vector_semantics": "right_singular_vectors_of_gate_proj", |
| "layers": { |
| "1": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 0 |
| }, |
| "2": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 704643072 |
| }, |
| "3": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 1409286144 |
| }, |
| "4": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 2113929216 |
| }, |
| "5": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 2818572288 |
| }, |
| "6": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 3523215360 |
| }, |
| "7": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 4227858432 |
| }, |
| "8": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 4932501504 |
| }, |
| "9": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 5637144576 |
| }, |
| "10": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 6341787648 |
| }, |
| "11": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 7046430720 |
| }, |
| "12": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 7751073792 |
| }, |
| "13": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 8455716864 |
| }, |
| "14": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 9160359936 |
| }, |
| "15": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 9865003008 |
| }, |
| "16": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 10569646080 |
| }, |
| "17": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 11274289152 |
| }, |
| "18": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 11978932224 |
| }, |
| "19": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 12683575296 |
| }, |
| "20": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 13388218368 |
| }, |
| "21": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 14092861440 |
| }, |
| "22": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 14797504512 |
| }, |
| "23": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 15502147584 |
| }, |
| "24": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 16206790656 |
| }, |
| "25": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 16911433728 |
| }, |
| "26": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 17616076800 |
| }, |
| "27": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 18320719872 |
| }, |
| "28": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 19025362944 |
| }, |
| "29": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 19730006016 |
| }, |
| "30": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 20434649088 |
| }, |
| "31": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 21139292160 |
| }, |
| "32": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 21843935232 |
| }, |
| "33": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 22548578304 |
| }, |
| "34": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 23253221376 |
| }, |
| "35": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 23957864448 |
| }, |
| "36": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 24662507520 |
| }, |
| "37": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 25367150592 |
| }, |
| "38": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 26071793664 |
| }, |
| "39": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 26776436736 |
| }, |
| "40": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 27481079808 |
| }, |
| "41": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 28185722880 |
| }, |
| "42": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 28890365952 |
| }, |
| "43": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 29595009024 |
| }, |
| "44": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 30299652096 |
| }, |
| "45": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 31004295168 |
| }, |
| "46": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 31708938240 |
| }, |
| "47": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 32413581312 |
| }, |
| "48": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 33118224384 |
| }, |
| "49": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 33822867456 |
| }, |
| "50": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 34527510528 |
| }, |
| "51": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 35232153600 |
| }, |
| "52": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 35936796672 |
| }, |
| "53": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 36641439744 |
| }, |
| "54": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 37346082816 |
| }, |
| "55": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 38050725888 |
| }, |
| "56": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 38755368960 |
| }, |
| "57": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 39460012032 |
| }, |
| "58": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 40164655104 |
| }, |
| "59": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 40869298176 |
| }, |
| "60": { |
| "shape": [ |
| 384, |
| 64, |
| 7168 |
| ], |
| "file_offset": 41573941248 |
| } |
| }, |
| "layer_stats": { |
| "1": { |
| "median_var64": 0.0946, |
| "q25_var64": 0.0858, |
| "q75_var64": 0.1024, |
| "n_experts": 384 |
| }, |
| "2": { |
| "median_var64": 0.0968, |
| "q25_var64": 0.0927, |
| "q75_var64": 0.1005, |
| "n_experts": 384 |
| }, |
| "3": { |
| "median_var64": 0.0954, |
| "q25_var64": 0.0927, |
| "q75_var64": 0.0991, |
| "n_experts": 384 |
| }, |
| "4": { |
| "median_var64": 0.0963, |
| "q25_var64": 0.0928, |
| "q75_var64": 0.1008, |
| "n_experts": 384 |
| }, |
| "5": { |
| "median_var64": 0.098, |
| "q25_var64": 0.0944, |
| "q75_var64": 0.1022, |
| "n_experts": 384 |
| }, |
| "6": { |
| "median_var64": 0.0944, |
| "q25_var64": 0.0913, |
| "q75_var64": 0.0975, |
| "n_experts": 384 |
| }, |
| "7": { |
| "median_var64": 0.0925, |
| "q25_var64": 0.0895, |
| "q75_var64": 0.0965, |
| "n_experts": 384 |
| }, |
| "8": { |
| "median_var64": 0.0925, |
| "q25_var64": 0.0889, |
| "q75_var64": 0.0971, |
| "n_experts": 384 |
| }, |
| "9": { |
| "median_var64": 0.091, |
| "q25_var64": 0.0869, |
| "q75_var64": 0.096, |
| "n_experts": 384 |
| }, |
| "10": { |
| "median_var64": 0.0926, |
| "q25_var64": 0.0876, |
| "q75_var64": 0.0984, |
| "n_experts": 384 |
| }, |
| "11": { |
| "median_var64": 0.0927, |
| "q25_var64": 0.0874, |
| "q75_var64": 0.0996, |
| "n_experts": 384 |
| }, |
| "12": { |
| "median_var64": 0.0961, |
| "q25_var64": 0.089, |
| "q75_var64": 0.1033, |
| "n_experts": 384 |
| }, |
| "13": { |
| "median_var64": 0.1009, |
| "q25_var64": 0.0925, |
| "q75_var64": 0.1102, |
| "n_experts": 384 |
| }, |
| "14": { |
| "median_var64": 0.1015, |
| "q25_var64": 0.0919, |
| "q75_var64": 0.1102, |
| "n_experts": 384 |
| }, |
| "15": { |
| "median_var64": 0.1003, |
| "q25_var64": 0.0913, |
| "q75_var64": 0.1137, |
| "n_experts": 384 |
| }, |
| "16": { |
| "median_var64": 0.1007, |
| "q25_var64": 0.0912, |
| "q75_var64": 0.1146, |
| "n_experts": 384 |
| }, |
| "17": { |
| "median_var64": 0.1047, |
| "q25_var64": 0.0924, |
| "q75_var64": 0.1171, |
| "n_experts": 384 |
| }, |
| "18": { |
| "median_var64": 0.1034, |
| "q25_var64": 0.0902, |
| "q75_var64": 0.1158, |
| "n_experts": 384 |
| }, |
| "19": { |
| "median_var64": 0.1018, |
| "q25_var64": 0.0888, |
| "q75_var64": 0.1168, |
| "n_experts": 384 |
| }, |
| "20": { |
| "median_var64": 0.1023, |
| "q25_var64": 0.0882, |
| "q75_var64": 0.1181, |
| "n_experts": 384 |
| }, |
| "21": { |
| "median_var64": 0.1061, |
| "q25_var64": 0.09, |
| "q75_var64": 0.1209, |
| "n_experts": 384 |
| }, |
| "22": { |
| "median_var64": 0.106, |
| "q25_var64": 0.09, |
| "q75_var64": 0.1248, |
| "n_experts": 384 |
| }, |
| "23": { |
| "median_var64": 0.1065, |
| "q25_var64": 0.0882, |
| "q75_var64": 0.1258, |
| "n_experts": 384 |
| }, |
| "24": { |
| "median_var64": 0.1079, |
| "q25_var64": 0.0916, |
| "q75_var64": 0.1298, |
| "n_experts": 384 |
| }, |
| "25": { |
| "median_var64": 0.106, |
| "q25_var64": 0.0895, |
| "q75_var64": 0.1251, |
| "n_experts": 384 |
| }, |
| "26": { |
| "median_var64": 0.1042, |
| "q25_var64": 0.09, |
| "q75_var64": 0.1263, |
| "n_experts": 384 |
| }, |
| "27": { |
| "median_var64": 0.1013, |
| "q25_var64": 0.089, |
| "q75_var64": 0.1229, |
| "n_experts": 384 |
| }, |
| "28": { |
| "median_var64": 0.0995, |
| "q25_var64": 0.0875, |
| "q75_var64": 0.1186, |
| "n_experts": 384 |
| }, |
| "29": { |
| "median_var64": 0.1005, |
| "q25_var64": 0.0873, |
| "q75_var64": 0.1157, |
| "n_experts": 384 |
| }, |
| "30": { |
| "median_var64": 0.0955, |
| "q25_var64": 0.0864, |
| "q75_var64": 0.1129, |
| "n_experts": 384 |
| }, |
| "31": { |
| "median_var64": 0.0927, |
| "q25_var64": 0.0856, |
| "q75_var64": 0.1083, |
| "n_experts": 384 |
| }, |
| "32": { |
| "median_var64": 0.09, |
| "q25_var64": 0.0843, |
| "q75_var64": 0.1049, |
| "n_experts": 384 |
| }, |
| "33": { |
| "median_var64": 0.0903, |
| "q25_var64": 0.0833, |
| "q75_var64": 0.1028, |
| "n_experts": 384 |
| }, |
| "34": { |
| "median_var64": 0.0893, |
| "q25_var64": 0.0844, |
| "q75_var64": 0.0996, |
| "n_experts": 384 |
| }, |
| "35": { |
| "median_var64": 0.0874, |
| "q25_var64": 0.0829, |
| "q75_var64": 0.0984, |
| "n_experts": 384 |
| }, |
| "36": { |
| "median_var64": 0.0879, |
| "q25_var64": 0.083, |
| "q75_var64": 0.0988, |
| "n_experts": 384 |
| }, |
| "37": { |
| "median_var64": 0.0866, |
| "q25_var64": 0.0821, |
| "q75_var64": 0.0966, |
| "n_experts": 384 |
| }, |
| "38": { |
| "median_var64": 0.0867, |
| "q25_var64": 0.0824, |
| "q75_var64": 0.0939, |
| "n_experts": 384 |
| }, |
| "39": { |
| "median_var64": 0.0848, |
| "q25_var64": 0.0812, |
| "q75_var64": 0.0916, |
| "n_experts": 384 |
| }, |
| "40": { |
| "median_var64": 0.0835, |
| "q25_var64": 0.0806, |
| "q75_var64": 0.0903, |
| "n_experts": 384 |
| }, |
| "41": { |
| "median_var64": 0.0832, |
| "q25_var64": 0.0802, |
| "q75_var64": 0.0892, |
| "n_experts": 384 |
| }, |
| "42": { |
| "median_var64": 0.0832, |
| "q25_var64": 0.0804, |
| "q75_var64": 0.0898, |
| "n_experts": 384 |
| }, |
| "43": { |
| "median_var64": 0.083, |
| "q25_var64": 0.0804, |
| "q75_var64": 0.0892, |
| "n_experts": 384 |
| }, |
| "44": { |
| "median_var64": 0.0832, |
| "q25_var64": 0.0805, |
| "q75_var64": 0.0905, |
| "n_experts": 384 |
| }, |
| "45": { |
| "median_var64": 0.0831, |
| "q25_var64": 0.0802, |
| "q75_var64": 0.0889, |
| "n_experts": 384 |
| }, |
| "46": { |
| "median_var64": 0.0834, |
| "q25_var64": 0.0802, |
| "q75_var64": 0.0907, |
| "n_experts": 384 |
| }, |
| "47": { |
| "median_var64": 0.0833, |
| "q25_var64": 0.0799, |
| "q75_var64": 0.0901, |
| "n_experts": 384 |
| }, |
| "48": { |
| "median_var64": 0.0834, |
| "q25_var64": 0.0798, |
| "q75_var64": 0.0901, |
| "n_experts": 384 |
| }, |
| "49": { |
| "median_var64": 0.0835, |
| "q25_var64": 0.0799, |
| "q75_var64": 0.0921, |
| "n_experts": 384 |
| }, |
| "50": { |
| "median_var64": 0.0848, |
| "q25_var64": 0.0806, |
| "q75_var64": 0.0921, |
| "n_experts": 384 |
| }, |
| "51": { |
| "median_var64": 0.0856, |
| "q25_var64": 0.0808, |
| "q75_var64": 0.0951, |
| "n_experts": 384 |
| }, |
| "52": { |
| "median_var64": 0.0872, |
| "q25_var64": 0.081, |
| "q75_var64": 0.097, |
| "n_experts": 384 |
| }, |
| "53": { |
| "median_var64": 0.0872, |
| "q25_var64": 0.0817, |
| "q75_var64": 0.1009, |
| "n_experts": 384 |
| }, |
| "54": { |
| "median_var64": 0.0911, |
| "q25_var64": 0.0827, |
| "q75_var64": 0.1038, |
| "n_experts": 384 |
| }, |
| "55": { |
| "median_var64": 0.0932, |
| "q25_var64": 0.0845, |
| "q75_var64": 0.1058, |
| "n_experts": 384 |
| }, |
| "56": { |
| "median_var64": 0.0944, |
| "q25_var64": 0.0844, |
| "q75_var64": 0.109, |
| "n_experts": 384 |
| }, |
| "57": { |
| "median_var64": 0.096, |
| "q25_var64": 0.0873, |
| "q75_var64": 0.1137, |
| "n_experts": 384 |
| }, |
| "58": { |
| "median_var64": 0.0959, |
| "q25_var64": 0.0863, |
| "q75_var64": 0.1112, |
| "n_experts": 384 |
| }, |
| "59": { |
| "median_var64": 0.0955, |
| "q25_var64": 0.0866, |
| "q75_var64": 0.1091, |
| "n_experts": 384 |
| }, |
| "60": { |
| "median_var64": 0.0975, |
| "q25_var64": 0.0884, |
| "q75_var64": 0.1075, |
| "n_experts": 384 |
| } |
| } |
| } |
