Upload absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.22.hook_resid_post_eval_results.json with huggingface_hub

absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.22.hook_resid_post_eval_results.json

@@ -0,0 +1,275 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50,
+    "min_GT_probe_f1": 0.6,
+    "min_feats_for_eval": 20
+  },
+  "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6",
+  "datetime_epoch_millis": 1745739290596,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.16626212721800512,
+      "mean_full_absorption_score": 0.1501538194341297,
+      "mean_num_split_features": 1.3846153846153846,
+      "std_dev_absorption_fraction_score": 0.15718811420332557,
+      "std_dev_full_absorption_score": 0.16053207266898356,
+      "std_dev_num_split_features": 0.7524319545007683
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.45808036761866044,
+      "full_absorption_rate": 0.41618969746524936,
+      "num_full_absorption": 1018,
+      "num_probe_true_positives": 2446,
+      "num_split_features": 4
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.06255062010649774,
+      "full_absorption_rate": 0.07979407979407979,
+      "num_full_absorption": 124,
+      "num_probe_true_positives": 1554,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.5039659769651796,
+      "full_absorption_rate": 0.509941089837997,
+      "num_full_absorption": 1385,
+      "num_probe_true_positives": 2716,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.17206178236693098,
+      "full_absorption_rate": 0.16308470290771176,
+      "num_full_absorption": 258,
+      "num_probe_true_positives": 1582,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.17212770018911805,
+      "full_absorption_rate": 0.18348045397225726,
+      "num_full_absorption": 291,
+      "num_probe_true_positives": 1586,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.10375983969655311,
+      "full_absorption_rate": 0.11541701769165964,
+      "num_full_absorption": 137,
+      "num_probe_true_positives": 1187,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.03092594259882946,
+      "full_absorption_rate": 0.01526032315978456,
+      "num_full_absorption": 17,
+      "num_probe_true_positives": 1114,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.045683150633427184,
+      "full_absorption_rate": 0.07807486631016043,
+      "num_full_absorption": 73,
+      "num_probe_true_positives": 935,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.0737213840331355,
+      "full_absorption_rate": 0.0947176684881603,
+      "num_full_absorption": 156,
+      "num_probe_true_positives": 1647,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.007506573728545132,
+      "full_absorption_rate": 0.004640371229698376,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 431,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.024664873322650122,
+      "full_absorption_rate": 0.024566473988439308,
+      "num_full_absorption": 17,
+      "num_probe_true_positives": 692,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.06627564175651028,
+      "full_absorption_rate": 0.054673721340388004,
+      "num_full_absorption": 62,
+      "num_probe_true_positives": 1134,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.14713849256993894,
+      "full_absorption_rate": 0.12684869169510807,
+      "num_full_absorption": 223,
+      "num_probe_true_positives": 1758,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.1587078650111927,
+      "full_absorption_rate": 0.06892382103990327,
+      "num_full_absorption": 57,
+      "num_probe_true_positives": 827,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.09099456768562256,
+      "full_absorption_rate": 0.1028225806451613,
+      "num_full_absorption": 102,
+      "num_probe_true_positives": 992,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.3781165983871244,
+      "full_absorption_rate": 0.37505451373746185,
+      "num_full_absorption": 860,
+      "num_probe_true_positives": 2293,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.1984926113784496,
+      "full_absorption_rate": 0.11494252873563218,
+      "num_full_absorption": 20,
+      "num_probe_true_positives": 174,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.11193284845594453,
+      "full_absorption_rate": 0.11416361416361416,
+      "num_full_absorption": 187,
+      "num_probe_true_positives": 1638,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.5783564144892005,
+      "full_absorption_rate": 0.611731843575419,
+      "num_full_absorption": 1752,
+      "num_probe_true_positives": 2864,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.3154526614323926,
+      "full_absorption_rate": 0.3022700119474313,
+      "num_full_absorption": 506,
+      "num_probe_true_positives": 1674,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.09121967498793804,
+      "full_absorption_rate": 0.06754966887417219,
+      "num_full_absorption": 51,
+      "num_probe_true_positives": 755,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.21617630098275054,
+      "full_absorption_rate": 0.1505235602094241,
+      "num_full_absorption": 115,
+      "num_probe_true_positives": 764,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.016801821594329087,
+      "full_absorption_rate": 0.022865853658536585,
+      "num_full_absorption": 15,
+      "num_probe_true_positives": 656,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.19781591501965576,
+      "full_absorption_rate": 0.07920792079207921,
+      "num_full_absorption": 8,
+      "num_probe_true_positives": 101,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.078746115457656,
+      "full_absorption_rate": 0.018633540372670808,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 161,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.021539567199900623,
+      "full_absorption_rate": 0.008620689655172414,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 232,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "Unknown",
+  "sae_lens_id": "blocks.22.hook_resid_post",
+  "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc",
+  "sae_lens_version": "5.9.1",
+  "sae_cfg_dict": {
+    "architecture": "jumprelu",
+    "d_in": 2304,
+    "d_sae": 32768,
+    "activation_fn_str": "relu",
+    "apply_b_dec_to_input": true,
+    "finetuning_scaling_factor": false,
+    "context_size": 1024,
+    "model_name": "gemma-2-2b",
+    "hook_name": "blocks.22.hook_resid_post",
+    "hook_layer": 22,
+    "hook_head_index": null,
+    "prepend_bos": true,
+    "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B",
+    "dataset_trust_remote_code": true,
+    "normalize_activations": "none",
+    "dtype": "torch.bfloat16",
+    "device": "cuda",
+    "sae_lens_training_version": "5.5.1",
+    "activation_fn_kwargs": {
+      "k": 40
+    },
+    "neuronpedia_id": null,
+    "model_from_pretrained_kwargs": {
+      "center_writing_weights": false
+    },
+    "seqpos_slice": [
+      null
+    ]
+  },
+  "eval_result_unstructured": null
+}