| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745793503274, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.057870298906844156, |
| "mean_full_absorption_score": 0.04359829642754091, |
| "mean_num_split_features": 1.2307692307692308, |
| "std_dev_absorption_fraction_score": 0.08397501796906298, |
| "std_dev_full_absorption_score": 0.06925035498718567, |
| "std_dev_num_split_features": 0.5144077999169381 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.00864086662479927, |
| "full_absorption_rate": 0.01276595744680851, |
| "num_full_absorption": 33, |
| "num_probe_true_positives": 2585, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.015176548679732173, |
| "full_absorption_rate": 0.012468827930174564, |
| "num_full_absorption": 20, |
| "num_probe_true_positives": 1604, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.2001803163019912, |
| "full_absorption_rate": 0.15644741731698253, |
| "num_full_absorption": 421, |
| "num_probe_true_positives": 2691, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.2661801560789568, |
| "full_absorption_rate": 0.22095959595959597, |
| "num_full_absorption": 350, |
| "num_probe_true_positives": 1584, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.032885305259062, |
| "full_absorption_rate": 0.030207677784770296, |
| "num_full_absorption": 48, |
| "num_probe_true_positives": 1589, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.01437438601752191, |
| "full_absorption_rate": 0.010833333333333334, |
| "num_full_absorption": 13, |
| "num_probe_true_positives": 1200, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.026321211515251252, |
| "full_absorption_rate": 0.0156794425087108, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 1148, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.09932245852649435, |
| "full_absorption_rate": 0.05916585838991271, |
| "num_full_absorption": 61, |
| "num_probe_true_positives": 1031, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.32316368911542326, |
| "full_absorption_rate": 0.27314814814814814, |
| "num_full_absorption": 472, |
| "num_probe_true_positives": 1728, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0021691973969631237, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 461, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0022512778174862973, |
| "full_absorption_rate": 0.004424778761061947, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 678, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.00840439970644219, |
| "full_absorption_rate": 0.004240882103477523, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 1179, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.003426768156911309, |
| "full_absorption_rate": 0.002224694104560623, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 1798, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.02670078866436682, |
| "full_absorption_rate": 0.01488833746898263, |
| "num_full_absorption": 12, |
| "num_probe_true_positives": 806, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.028513237253876663, |
| "full_absorption_rate": 0.021677662582469368, |
| "num_full_absorption": 23, |
| "num_probe_true_positives": 1061, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.11418235216000715, |
| "full_absorption_rate": 0.0782312925170068, |
| "num_full_absorption": 184, |
| "num_probe_true_positives": 2352, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.005681818181818182, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 176, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.01760684877226353, |
| "full_absorption_rate": 0.010532475131655939, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 1709, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.08354297284885323, |
| "full_absorption_rate": 0.05917159763313609, |
| "num_full_absorption": 170, |
| "num_probe_true_positives": 2873, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.05175124192946949, |
| "full_absorption_rate": 0.04642857142857143, |
| "num_full_absorption": 78, |
| "num_probe_true_positives": 1680, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.07896046916914314, |
| "full_absorption_rate": 0.038461538461538464, |
| "num_full_absorption": 29, |
| "num_probe_true_positives": 754, |
| "num_split_features": 3 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.006206771743384643, |
| "full_absorption_rate": 0.007585335018963337, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 791, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.06788182174468654, |
| "full_absorption_rate": 0.045925925925925926, |
| "num_full_absorption": 31, |
| "num_probe_true_positives": 675, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.016918990288025192, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 86, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.006353075021981416, |
| "full_absorption_rate": 0.005917159763313609, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 169, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 228, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.11.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.11.hook_resid_post", |
| "hook_layer": 11, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
