| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745797123581, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.17631391845600158, |
| "mean_full_absorption_score": 0.15224585436894475, |
| "mean_num_split_features": 1.6153846153846154, |
| "std_dev_absorption_fraction_score": 0.20107255988480455, |
| "std_dev_full_absorption_score": 0.18954475486419284, |
| "std_dev_num_split_features": 1.5511782122483047 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.4610722430895099, |
| "full_absorption_rate": 0.3884780070066174, |
| "num_full_absorption": 998, |
| "num_probe_true_positives": 2569, |
| "num_split_features": 5 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.16059750387966845, |
| "full_absorption_rate": 0.12204989885367498, |
| "num_full_absorption": 181, |
| "num_probe_true_positives": 1483, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.7735502426799653, |
| "full_absorption_rate": 0.7546164772727273, |
| "num_full_absorption": 2125, |
| "num_probe_true_positives": 2816, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.2161027346097252, |
| "full_absorption_rate": 0.1939655172413793, |
| "num_full_absorption": 315, |
| "num_probe_true_positives": 1624, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.18299261514245085, |
| "full_absorption_rate": 0.15675675675675677, |
| "num_full_absorption": 261, |
| "num_probe_true_positives": 1665, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.19440988947931143, |
| "full_absorption_rate": 0.17301324503311258, |
| "num_full_absorption": 209, |
| "num_probe_true_positives": 1208, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.1846170861927906, |
| "full_absorption_rate": 0.1431095406360424, |
| "num_full_absorption": 162, |
| "num_probe_true_positives": 1132, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.0627535059445316, |
| "full_absorption_rate": 0.04639684106614018, |
| "num_full_absorption": 47, |
| "num_probe_true_positives": 1013, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.36041211792760597, |
| "full_absorption_rate": 0.27808471454880296, |
| "num_full_absorption": 453, |
| "num_probe_true_positives": 1629, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 9.142844462341788e-05, |
| "full_absorption_rate": 0.002421307506053269, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 413, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0010477684752322513, |
| "full_absorption_rate": 0.002881844380403458, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 694, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.04633723524255254, |
| "full_absorption_rate": 0.046379170056956874, |
| "num_full_absorption": 57, |
| "num_probe_true_positives": 1229, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.47602906717289256, |
| "full_absorption_rate": 0.41437394722066256, |
| "num_full_absorption": 738, |
| "num_probe_true_positives": 1781, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.04614816299936556, |
| "full_absorption_rate": 0.03271028037383177, |
| "num_full_absorption": 28, |
| "num_probe_true_positives": 856, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.04902687843016666, |
| "full_absorption_rate": 0.0643611911623439, |
| "num_full_absorption": 67, |
| "num_probe_true_positives": 1041, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.5880462364453096, |
| "full_absorption_rate": 0.571173583221776, |
| "num_full_absorption": 1280, |
| "num_probe_true_positives": 2241, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.0059541735093205685, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 182, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.07804074186011996, |
| "full_absorption_rate": 0.08069522036002483, |
| "num_full_absorption": 130, |
| "num_probe_true_positives": 1611, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.1995955047437944, |
| "full_absorption_rate": 0.19, |
| "num_full_absorption": 532, |
| "num_probe_true_positives": 2800, |
| "num_split_features": 8 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.06809607028974686, |
| "full_absorption_rate": 0.0762081784386617, |
| "num_full_absorption": 123, |
| "num_probe_true_positives": 1614, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.23970341729142033, |
| "full_absorption_rate": 0.12907608695652173, |
| "num_full_absorption": 95, |
| "num_probe_true_positives": 736, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.04229398741711368, |
| "full_absorption_rate": 0.03109452736318408, |
| "num_full_absorption": 25, |
| "num_probe_true_positives": 804, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.04619636648664271, |
| "full_absorption_rate": 0.0440251572327044, |
| "num_full_absorption": 28, |
| "num_probe_true_positives": 636, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.06513638278670303, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 90, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.03198229866394904, |
| "full_absorption_rate": 0.012422360248447204, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 161, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.003928220651528832, |
| "full_absorption_rate": 0.004098360655737705, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 244, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.19.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.19.hook_resid_post", |
| "hook_layer": 19, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
