| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6", |
| "datetime_epoch_millis": 1745720443293, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.016763214362445024, |
| "mean_full_absorption_score": 0.011630179155322022, |
| "mean_num_split_features": 1.0769230769230769, |
| "std_dev_absorption_fraction_score": 0.018035996941677988, |
| "std_dev_full_absorption_score": 0.012513153528487522, |
| "std_dev_num_split_features": 0.271746488194703 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.005612017404935165, |
| "full_absorption_rate": 0.011276681433749497, |
| "num_full_absorption": 28, |
| "num_probe_true_positives": 2483, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.0019013184680206493, |
| "full_absorption_rate": 0.006426735218508998, |
| "num_full_absorption": 10, |
| "num_probe_true_positives": 1556, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.008538887855545522, |
| "full_absorption_rate": 0.0014280614066404856, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 2801, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.006077426553736407, |
| "full_absorption_rate": 0.007629107981220657, |
| "num_full_absorption": 13, |
| "num_probe_true_positives": 1704, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.005155292010909538, |
| "full_absorption_rate": 0.008064516129032258, |
| "num_full_absorption": 13, |
| "num_probe_true_positives": 1612, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.02882973862104545, |
| "full_absorption_rate": 0.014802631578947368, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 1216, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.010387540656820503, |
| "full_absorption_rate": 0.00684931506849315, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 1168, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.0033697331640740818, |
| "full_absorption_rate": 0.0029469548133595285, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 1018, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.0058956206780094765, |
| "full_absorption_rate": 0.008373205741626795, |
| "num_full_absorption": 14, |
| "num_probe_true_positives": 1672, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0009022482263360316, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 419, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0015337423312883436, |
| "full_absorption_rate": 0.0015337423312883436, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 652, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.0010694553379880585, |
| "full_absorption_rate": 0.0008116883116883117, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 1232, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.004801515378917633, |
| "full_absorption_rate": 0.022900763358778626, |
| "num_full_absorption": 42, |
| "num_probe_true_positives": 1834, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.03300999167683128, |
| "full_absorption_rate": 0.01773049645390071, |
| "num_full_absorption": 15, |
| "num_probe_true_positives": 846, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.06422354686962979, |
| "full_absorption_rate": 0.05595667870036101, |
| "num_full_absorption": 62, |
| "num_probe_true_positives": 1108, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.038185373024373234, |
| "full_absorption_rate": 0.015859408486926702, |
| "num_full_absorption": 37, |
| "num_probe_true_positives": 2333, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.016001694992939925, |
| "full_absorption_rate": 0.004784688995215311, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 209, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.028011758692352895, |
| "full_absorption_rate": 0.01651376146788991, |
| "num_full_absorption": 27, |
| "num_probe_true_positives": 1635, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.04626430298599293, |
| "full_absorption_rate": 0.03318197694725812, |
| "num_full_absorption": 95, |
| "num_probe_true_positives": 2863, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.011768183755531406, |
| "full_absorption_rate": 0.00794135613927917, |
| "num_full_absorption": 13, |
| "num_probe_true_positives": 1637, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.05673816030794099, |
| "full_absorption_rate": 0.0267022696929239, |
| "num_full_absorption": 20, |
| "num_probe_true_positives": 749, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.0012706480304955528, |
| "full_absorption_rate": 0.005082592121982211, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 787, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.026834373779582775, |
| "full_absorption_rate": 0.017391304347826087, |
| "num_full_absorption": 12, |
| "num_probe_true_positives": 690, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.009746281257930854, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 101, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.01387135558365356, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 165, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.005843365778688525, |
| "full_absorption_rate": 0.00819672131147541, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 244, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.9.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.9.hook_resid_post", |
| "hook_layer": 9, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
