| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745794402915, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.17200542151017267, |
| "mean_full_absorption_score": 0.1469911774578058, |
| "mean_num_split_features": 1.1923076923076923, |
| "std_dev_absorption_fraction_score": 0.18922825103251242, |
| "std_dev_full_absorption_score": 0.17188533164556716, |
| "std_dev_num_split_features": 0.4019184762342502 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.3856598207008102, |
| "full_absorption_rate": 0.360546875, |
| "num_full_absorption": 923, |
| "num_probe_true_positives": 2560, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.01388767172427744, |
| "full_absorption_rate": 0.010081915563957152, |
| "num_full_absorption": 16, |
| "num_probe_true_positives": 1587, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.4136481014844617, |
| "full_absorption_rate": 0.33878589603780446, |
| "num_full_absorption": 932, |
| "num_probe_true_positives": 2751, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.247468155348793, |
| "full_absorption_rate": 0.1855541718555417, |
| "num_full_absorption": 298, |
| "num_probe_true_positives": 1606, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.40524397310832216, |
| "full_absorption_rate": 0.3870967741935484, |
| "num_full_absorption": 624, |
| "num_probe_true_positives": 1612, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.026920160123395348, |
| "full_absorption_rate": 0.02003338898163606, |
| "num_full_absorption": 24, |
| "num_probe_true_positives": 1198, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.05022462281430055, |
| "full_absorption_rate": 0.021607605877268798, |
| "num_full_absorption": 25, |
| "num_probe_true_positives": 1157, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.08872445113202193, |
| "full_absorption_rate": 0.07984031936127745, |
| "num_full_absorption": 80, |
| "num_probe_true_positives": 1002, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.3522506625670308, |
| "full_absorption_rate": 0.30350665054413545, |
| "num_full_absorption": 502, |
| "num_probe_true_positives": 1654, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.01771520517385742, |
| "full_absorption_rate": 0.017977528089887642, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 445, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.003923444694579765, |
| "full_absorption_rate": 0.008849557522123894, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 678, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.19849784084035754, |
| "full_absorption_rate": 0.18945634266886327, |
| "num_full_absorption": 230, |
| "num_probe_true_positives": 1214, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.020848132854383195, |
| "full_absorption_rate": 0.012749445676274944, |
| "num_full_absorption": 23, |
| "num_probe_true_positives": 1804, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.04700937891449542, |
| "full_absorption_rate": 0.032663316582914576, |
| "num_full_absorption": 26, |
| "num_probe_true_positives": 796, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.020196344741402875, |
| "full_absorption_rate": 0.029657089898053754, |
| "num_full_absorption": 32, |
| "num_probe_true_positives": 1079, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.3033090603838104, |
| "full_absorption_rate": 0.2417247386759582, |
| "num_full_absorption": 555, |
| "num_probe_true_positives": 2296, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.005596453131192108, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 188, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.32794636459594023, |
| "full_absorption_rate": 0.2563176895306859, |
| "num_full_absorption": 426, |
| "num_probe_true_positives": 1662, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.155864409128586, |
| "full_absorption_rate": 0.13556953179594688, |
| "num_full_absorption": 388, |
| "num_probe_true_positives": 2862, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.31537006971741754, |
| "full_absorption_rate": 0.27751196172248804, |
| "num_full_absorption": 464, |
| "num_probe_true_positives": 1672, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.7683763370357332, |
| "full_absorption_rate": 0.7077509529860229, |
| "num_full_absorption": 557, |
| "num_probe_true_positives": 787, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.06718868080607363, |
| "full_absorption_rate": 0.055288461538461536, |
| "num_full_absorption": 46, |
| "num_probe_true_positives": 832, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.11206665906780537, |
| "full_absorption_rate": 0.07391304347826087, |
| "num_full_absorption": 51, |
| "num_probe_true_positives": 690, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.01907612770546518, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 101, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.09863382377523693, |
| "full_absorption_rate": 0.06666666666666667, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 165, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.0064950076947398305, |
| "full_absorption_rate": 0.008620689655172414, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 232, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.13.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.13.hook_resid_post", |
| "hook_layer": 13, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
