| { | |
| "model_id": "Qwen/Qwen3-8B", | |
| "model_family": "qwen3_8b", | |
| "num_layers": 36, | |
| "hidden_size": 4096, | |
| "intermediate": 12288, | |
| "vocab_size": 151936, | |
| "num_feats": 128, | |
| "dtype": "float16", | |
| "layers": [ | |
| { | |
| "layer": 0, | |
| "num_features": 128, | |
| "gate_var": 0.2108, | |
| "down_var": 0.1144 | |
| }, | |
| { | |
| "layer": 1, | |
| "num_features": 128, | |
| "gate_var": 0.3322, | |
| "down_var": 0.1848 | |
| }, | |
| { | |
| "layer": 2, | |
| "num_features": 128, | |
| "gate_var": 0.2593, | |
| "down_var": 0.2129 | |
| }, | |
| { | |
| "layer": 3, | |
| "num_features": 128, | |
| "gate_var": 0.2302, | |
| "down_var": 0.132 | |
| }, | |
| { | |
| "layer": 4, | |
| "num_features": 128, | |
| "gate_var": 0.2146, | |
| "down_var": 0.1095 | |
| }, | |
| { | |
| "layer": 5, | |
| "num_features": 128, | |
| "gate_var": 0.211, | |
| "down_var": 0.1062 | |
| }, | |
| { | |
| "layer": 6, | |
| "num_features": 128, | |
| "gate_var": 0.2153, | |
| "down_var": 0.1017 | |
| }, | |
| { | |
| "layer": 7, | |
| "num_features": 128, | |
| "gate_var": 0.1735, | |
| "down_var": 0.1153 | |
| }, | |
| { | |
| "layer": 8, | |
| "num_features": 128, | |
| "gate_var": 0.1409, | |
| "down_var": 0.1171 | |
| }, | |
| { | |
| "layer": 9, | |
| "num_features": 128, | |
| "gate_var": 0.1871, | |
| "down_var": 0.1173 | |
| }, | |
| { | |
| "layer": 10, | |
| "num_features": 128, | |
| "gate_var": 0.1549, | |
| "down_var": 0.0986 | |
| }, | |
| { | |
| "layer": 11, | |
| "num_features": 128, | |
| "gate_var": 0.1394, | |
| "down_var": 0.1049 | |
| }, | |
| { | |
| "layer": 12, | |
| "num_features": 128, | |
| "gate_var": 0.136, | |
| "down_var": 0.1099 | |
| }, | |
| { | |
| "layer": 13, | |
| "num_features": 128, | |
| "gate_var": 0.1335, | |
| "down_var": 0.1168 | |
| }, | |
| { | |
| "layer": 14, | |
| "num_features": 128, | |
| "gate_var": 0.1396, | |
| "down_var": 0.1095 | |
| }, | |
| { | |
| "layer": 15, | |
| "num_features": 128, | |
| "gate_var": 0.139, | |
| "down_var": 0.119 | |
| }, | |
| { | |
| "layer": 16, | |
| "num_features": 128, | |
| "gate_var": 0.1477, | |
| "down_var": 0.108 | |
| }, | |
| { | |
| "layer": 17, | |
| "num_features": 128, | |
| "gate_var": 0.1427, | |
| "down_var": 0.1133 | |
| }, | |
| { | |
| "layer": 18, | |
| "num_features": 128, | |
| "gate_var": 0.1383, | |
| "down_var": 0.1209 | |
| }, | |
| { | |
| "layer": 19, | |
| "num_features": 128, | |
| "gate_var": 0.1346, | |
| "down_var": 0.1264 | |
| }, | |
| { | |
| "layer": 20, | |
| "num_features": 128, | |
| "gate_var": 0.1365, | |
| "down_var": 0.1227 | |
| }, | |
| { | |
| "layer": 21, | |
| "num_features": 128, | |
| "gate_var": 0.1238, | |
| "down_var": 0.1255 | |
| }, | |
| { | |
| "layer": 22, | |
| "num_features": 128, | |
| "gate_var": 0.1219, | |
| "down_var": 0.1029 | |
| }, | |
| { | |
| "layer": 23, | |
| "num_features": 128, | |
| "gate_var": 0.1189, | |
| "down_var": 0.0943 | |
| }, | |
| { | |
| "layer": 24, | |
| "num_features": 128, | |
| "gate_var": 0.1153, | |
| "down_var": 0.0879 | |
| }, | |
| { | |
| "layer": 25, | |
| "num_features": 128, | |
| "gate_var": 0.112, | |
| "down_var": 0.085 | |
| }, | |
| { | |
| "layer": 26, | |
| "num_features": 128, | |
| "gate_var": 0.1121, | |
| "down_var": 0.084 | |
| }, | |
| { | |
| "layer": 27, | |
| "num_features": 128, | |
| "gate_var": 0.1101, | |
| "down_var": 0.0867 | |
| }, | |
| { | |
| "layer": 28, | |
| "num_features": 128, | |
| "gate_var": 0.107, | |
| "down_var": 0.0878 | |
| }, | |
| { | |
| "layer": 29, | |
| "num_features": 128, | |
| "gate_var": 0.1114, | |
| "down_var": 0.0895 | |
| }, | |
| { | |
| "layer": 30, | |
| "num_features": 128, | |
| "gate_var": 0.1075, | |
| "down_var": 0.0972 | |
| }, | |
| { | |
| "layer": 31, | |
| "num_features": 128, | |
| "gate_var": 0.107, | |
| "down_var": 0.1057 | |
| }, | |
| { | |
| "layer": 32, | |
| "num_features": 128, | |
| "gate_var": 0.1089, | |
| "down_var": 0.1063 | |
| }, | |
| { | |
| "layer": 33, | |
| "num_features": 128, | |
| "gate_var": 0.1152, | |
| "down_var": 0.0987 | |
| }, | |
| { | |
| "layer": 34, | |
| "num_features": 128, | |
| "gate_var": 0.1201, | |
| "down_var": 0.0978 | |
| }, | |
| { | |
| "layer": 35, | |
| "num_features": 128, | |
| "gate_var": 0.129, | |
| "down_var": 0.093 | |
| } | |
| ] | |
| } |