Hugging Face's logo

Hiibb
/
factor2

Model card Files
xet
 Community
Hiibb commited on
Commit
03ba413
·
verified ·
1 Parent(s): 511c88a

Upload folder using huggingface_hub

Browse files
Files changed (18) hide show
  1. resid_post_layer_12/trainer_0/ae.pt +3 -0
  2. resid_post_layer_12/trainer_0/config.json +53 -0
  3. resid_post_layer_12/trainer_0/eval_results.json +1 -0
  4. resid_post_layer_12/trainer_1/ae.pt +3 -0
  5. resid_post_layer_12/trainer_1/config.json +53 -0
  6. resid_post_layer_12/trainer_1/eval_results.json +1 -0
  7. resid_post_layer_12/trainer_2/ae.pt +3 -0
  8. resid_post_layer_12/trainer_2/config.json +53 -0
  9. resid_post_layer_12/trainer_2/eval_results.json +1 -0
  10. resid_post_layer_12/trainer_3/ae.pt +3 -0
  11. resid_post_layer_12/trainer_3/config.json +53 -0
  12. resid_post_layer_12/trainer_3/eval_results.json +1 -0
  13. resid_post_layer_12/trainer_4/ae.pt +3 -0
  14. resid_post_layer_12/trainer_4/config.json +53 -0
  15. resid_post_layer_12/trainer_4/eval_results.json +1 -0
  16. resid_post_layer_12/trainer_5/ae.pt +3 -0
  17. resid_post_layer_12/trainer_5/config.json +53 -0
  18. resid_post_layer_12/trainer_5/eval_results.json +1 -0
resid_post_layer_12/trainer_0/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d1c4c9b845c9583bafffb93573967146726bffcf3ba5fc34bada68c41e94f47b
3
+ size 302066957
resid_post_layer_12/trainer_0/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.03125,
18
+ 0.0625,
19
+ 0.125,
20
+ 0.25,
21
+ 0.53125
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 512,
32
+ 1024,
33
+ 2048,
34
+ 4096,
35
+ 8704
36
+ ],
37
+ "k": 20,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_0",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_0/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 202.95, "l1_loss": 5027.68, "l0": 3290.2006640625, "frac_variance_explained": -0.5823046875, "cossim": 0.8101953125, "l2_ratio": 2.088359375, "relative_reconstruction_bias": 2.3084765625, "loss_original": 2.152998046875, "loss_reconstructed": 2.6158984375, "loss_zero": 12.4375, "frac_recovered": 0.95478515625, "frac_alive": 1.0, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_1/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd52e4574fac92eabb1aeb0152da7da0adb2e2faf4ae849976a092f34642b0b5
3
+ size 302066957
resid_post_layer_12/trainer_1/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.03125,
18
+ 0.0625,
19
+ 0.125,
20
+ 0.25,
21
+ 0.53125
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 512,
32
+ 1024,
33
+ 2048,
34
+ 4096,
35
+ 8704
36
+ ],
37
+ "k": 40,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_1",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_1/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 118.18, "l1_loss": 2646.24, "l0": 1821.313896484375, "frac_variance_explained": 0.42705078125, "cossim": 0.853984375, "l2_ratio": 1.4846875, "relative_reconstruction_bias": 1.6, "loss_original": 2.152998046875, "loss_reconstructed": 2.41458984375, "loss_zero": 12.4375, "frac_recovered": 0.9745703125, "frac_alive": 1.0, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_2/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40be4936b7357ca54ae634ef90616e1107b7a59637afcf8fc0f2ac56dc0b648d
3
+ size 302066957
resid_post_layer_12/trainer_2/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.03125,
18
+ 0.0625,
19
+ 0.125,
20
+ 0.25,
21
+ 0.53125
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 512,
32
+ 1024,
33
+ 2048,
34
+ 4096,
35
+ 8704
36
+ ],
37
+ "k": 80,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_2",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_2/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 89.615, "l1_loss": 2230.72, "l0": 1531.2031103515626, "frac_variance_explained": 0.64884765625, "cossim": 0.88275390625, "l2_ratio": 1.2948046875, "relative_reconstruction_bias": 1.3678125, "loss_original": 2.152998046875, "loss_reconstructed": 2.31990234375, "loss_zero": 12.4375, "frac_recovered": 0.983671875, "frac_alive": 1.0, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_3/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03bc31a8b0ec0b41af83395b98f746a755e42161e734619163733ba5cad8bd3c
3
+ size 302066957
resid_post_layer_12/trainer_3/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.03125,
18
+ 0.0625,
19
+ 0.125,
20
+ 0.25,
21
+ 0.53125
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 512,
32
+ 1024,
33
+ 2048,
34
+ 4096,
35
+ 8704
36
+ ],
37
+ "k": 160,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_3",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_3/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 71.5675, "l1_loss": 2229.76, "l0": 1558.9755126953125, "frac_variance_explained": 0.7692578125, "cossim": 0.90623046875, "l2_ratio": 1.179296875, "relative_reconstruction_bias": 1.2312890625, "loss_original": 2.152998046875, "loss_reconstructed": 2.2579296875, "loss_zero": 12.4375, "frac_recovered": 0.98990234375, "frac_alive": 1.0, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_4/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c01d73dd3d1ffa706f0557ba2482f1a67147461290a76a541eecc73bf9274e6
3
+ size 302066957
resid_post_layer_12/trainer_4/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.03125,
18
+ 0.0625,
19
+ 0.125,
20
+ 0.25,
21
+ 0.53125
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 512,
32
+ 1024,
33
+ 2048,
34
+ 4096,
35
+ 8704
36
+ ],
37
+ "k": 320,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_4",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_4/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 55.41125, "l1_loss": 2431.2, "l0": 1631.423515625, "frac_variance_explained": 0.8597265625, "cossim": 0.9296875, "l2_ratio": 1.056484375, "relative_reconstruction_bias": 1.1066015625, "loss_original": 2.152998046875, "loss_reconstructed": 2.217705078125, "loss_zero": 12.4375, "frac_recovered": 0.993046875, "frac_alive": 0.997314453125, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}
resid_post_layer_12/trainer_5/ae.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6db8036a2673a1601ab6ea157a2b5eef48e0dea3c3d40ab40b4361825c93ec8
3
+ size 302066957
resid_post_layer_12/trainer_5/config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "MatroyshkaBatchTopKTrainer",
4
+ "dict_class": "MatroyshkaBatchTopKSAE",
5
+ "lr": 0.0003,
6
+ "steps": 97656,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": 78124,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 1152,
13
+ "seed": 0,
14
+ "activation_dim": 2304,
15
+ "dict_size": 16384,
16
+ "group_fractions": [
17
+ 0.03125,
18
+ 0.0625,
19
+ 0.125,
20
+ 0.25,
21
+ 0.53125
22
+ ],
23
+ "group_weights": [
24
+ 0.2,
25
+ 0.2,
26
+ 0.2,
27
+ 0.2,
28
+ 0.2
29
+ ],
30
+ "group_sizes": [
31
+ 512,
32
+ 1024,
33
+ 2048,
34
+ 4096,
35
+ 8704
36
+ ],
37
+ "k": 640,
38
+ "device": "cuda:0",
39
+ "layer": 12,
40
+ "lm_name": "google/gemma-2-2b",
41
+ "wandb_name": "MatroyshkaBatchTopKTrainer-google/gemma-2-2b-resid_post_layer_12_trainer_5",
42
+ "submodule_name": "resid_post_layer_12"
43
+ },
44
+ "buffer": {
45
+ "d_submodule": 2304,
46
+ "io": "out",
47
+ "n_ctxs": 244,
48
+ "ctx_len": 1024,
49
+ "refresh_batch_size": 4,
50
+ "out_batch_size": 2048,
51
+ "device": "cuda:0"
52
+ }
53
+ }
resid_post_layer_12/trainer_5/eval_results.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"l2_loss": 43.2775, "l1_loss": 3167.84, "l0": 1587.0543212890625, "frac_variance_explained": 0.910390625, "cossim": 0.953125, "l2_ratio": 0.937734375, "relative_reconstruction_bias": 0.9917578125, "loss_original": 2.152998046875, "loss_reconstructed": 2.19259765625, "loss_zero": 12.4375, "frac_recovered": 0.99556640625, "frac_alive": 0.97613525390625, "hyperparameters": {"n_inputs": 200, "context_length": 1024}}