| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745798937404, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.1576248552155899, |
| "mean_full_absorption_score": 0.13315013186374974, |
| "mean_num_split_features": 1.3846153846153846, |
| "std_dev_absorption_fraction_score": 0.1320272826049921, |
| "std_dev_full_absorption_score": 0.13063227231734018, |
| "std_dev_num_split_features": 0.9829312519977408 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.3969816540620667, |
| "full_absorption_rate": 0.3641434262948207, |
| "num_full_absorption": 914, |
| "num_probe_true_positives": 2510, |
| "num_split_features": 4 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.04433437167046504, |
| "full_absorption_rate": 0.03599476439790576, |
| "num_full_absorption": 55, |
| "num_probe_true_positives": 1528, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.3626846041925525, |
| "full_absorption_rate": 0.3620754012691303, |
| "num_full_absorption": 970, |
| "num_probe_true_positives": 2679, |
| "num_split_features": 4 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.15235350660743732, |
| "full_absorption_rate": 0.13817480719794345, |
| "num_full_absorption": 215, |
| "num_probe_true_positives": 1556, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.2020876376508008, |
| "full_absorption_rate": 0.18286445012787725, |
| "num_full_absorption": 286, |
| "num_probe_true_positives": 1564, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.0751804963263191, |
| "full_absorption_rate": 0.06734006734006734, |
| "num_full_absorption": 80, |
| "num_probe_true_positives": 1188, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.05477271220896778, |
| "full_absorption_rate": 0.03588290840415486, |
| "num_full_absorption": 38, |
| "num_probe_true_positives": 1059, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.037082632612917936, |
| "full_absorption_rate": 0.049682875264270614, |
| "num_full_absorption": 47, |
| "num_probe_true_positives": 946, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.04872640057340436, |
| "full_absorption_rate": 0.03676913803496082, |
| "num_full_absorption": 61, |
| "num_probe_true_positives": 1659, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.003247629017503534, |
| "full_absorption_rate": 0.004555808656036446, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 439, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0924326131800674, |
| "full_absorption_rate": 0.028694404591104734, |
| "num_full_absorption": 20, |
| "num_probe_true_positives": 697, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.16202589064785036, |
| "full_absorption_rate": 0.08413672217353199, |
| "num_full_absorption": 96, |
| "num_probe_true_positives": 1141, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.18467445682686331, |
| "full_absorption_rate": 0.17030812324929973, |
| "num_full_absorption": 304, |
| "num_probe_true_positives": 1785, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.07494133728444578, |
| "full_absorption_rate": 0.06641604010025062, |
| "num_full_absorption": 53, |
| "num_probe_true_positives": 798, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.030354407666580603, |
| "full_absorption_rate": 0.045090180360721446, |
| "num_full_absorption": 45, |
| "num_probe_true_positives": 998, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.35589146119442794, |
| "full_absorption_rate": 0.34765625, |
| "num_full_absorption": 801, |
| "num_probe_true_positives": 2304, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.2524863041942863, |
| "full_absorption_rate": 0.15853658536585366, |
| "num_full_absorption": 26, |
| "num_probe_true_positives": 164, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.16625837002608707, |
| "full_absorption_rate": 0.15164698570540708, |
| "num_full_absorption": 244, |
| "num_probe_true_positives": 1609, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.4938326664152507, |
| "full_absorption_rate": 0.5122037495578352, |
| "num_full_absorption": 1448, |
| "num_probe_true_positives": 2827, |
| "num_split_features": 4 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.10617749473804891, |
| "full_absorption_rate": 0.11398644485520641, |
| "num_full_absorption": 185, |
| "num_probe_true_positives": 1623, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.15981488791698337, |
| "full_absorption_rate": 0.1325136612021858, |
| "num_full_absorption": 97, |
| "num_probe_true_positives": 732, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.32143711063303204, |
| "full_absorption_rate": 0.20491803278688525, |
| "num_full_absorption": 175, |
| "num_probe_true_positives": 854, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.025026744792846563, |
| "full_absorption_rate": 0.016129032258064516, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 682, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.1406226019169875, |
| "full_absorption_rate": 0.027522935779816515, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 109, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.12389382534510339, |
| "full_absorption_rate": 0.10897435897435898, |
| "num_full_absorption": 17, |
| "num_probe_true_positives": 156, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.030924417904040963, |
| "full_absorption_rate": 0.01568627450980392, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 255, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.23.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.23.hook_resid_post", |
| "hook_layer": 23, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
