| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745789855637, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.059579364180252614, |
| "mean_full_absorption_score": 0.03550500003745534, |
| "mean_num_split_features": 1.0384615384615385, |
| "std_dev_absorption_fraction_score": 0.16772193244683573, |
| "std_dev_full_absorption_score": 0.11778278098675365, |
| "std_dev_num_split_features": 0.19611613513818404 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.0005864436090519191, |
| "full_absorption_rate": 0.004334121355397951, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 2538, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.001256281407035176, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 1592, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.0005793239737840822, |
| "full_absorption_rate": 0.0007230657989877079, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 2766, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.0005384237616636382, |
| "full_absorption_rate": 0.0036101083032490976, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 1662, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.001169802679395366, |
| "full_absorption_rate": 0.009950248756218905, |
| "num_full_absorption": 16, |
| "num_probe_true_positives": 1608, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.0040582413486957855, |
| "full_absorption_rate": 0.0008130081300813008, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 1230, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.05434896300678841, |
| "full_absorption_rate": 0.019861830742659757, |
| "num_full_absorption": 23, |
| "num_probe_true_positives": 1158, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.0008720024725657045, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1030, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.000874516268366849, |
| "full_absorption_rate": 0.004110393423370522, |
| "num_full_absorption": 7, |
| "num_probe_true_positives": 1703, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.002079002079002079, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 481, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.001177144563804059, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 698, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 1.56385008640553e-05, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1240, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.00015587956005406727, |
| "full_absorption_rate": 0.00446677833612507, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 1791, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 850, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.4301750807386787, |
| "full_absorption_rate": 0.18529411764705883, |
| "num_full_absorption": 189, |
| "num_probe_true_positives": 1020, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0008417508417508417, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 2376, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.008217496254637647, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 183, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.0008780347234830504, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1681, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.01654520699976161, |
| "full_absorption_rate": 0.0063025210084033615, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 2856, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.004376662983759029, |
| "full_absorption_rate": 0.0023584905660377358, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 1696, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.7519211616933751, |
| "full_absorption_rate": 0.5830065359477125, |
| "num_full_absorption": 446, |
| "num_probe_true_positives": 765, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.0009681113794864628, |
| "full_absorption_rate": 0.0012391573729863693, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 807, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.028443543967051545, |
| "full_absorption_rate": 0.01447178002894356, |
| "num_full_absorption": 10, |
| "num_probe_true_positives": 691, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.015518605319734256, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 117, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.1999148258831297, |
| "full_absorption_rate": 0.06626506024096386, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 166, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.027728358998436935, |
| "full_absorption_rate": 0.012145748987854251, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 247, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.3.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.3.hook_resid_post", |
| "hook_layer": 3, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
