| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745793054342, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.043332548252255534, |
| "mean_full_absorption_score": 0.031043553844892487, |
| "mean_num_split_features": 1.1538461538461537, |
| "std_dev_absorption_fraction_score": 0.09064406063392703, |
| "std_dev_full_absorption_score": 0.06805812714584618, |
| "std_dev_num_split_features": 0.36794648440311994 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 8.297898795377763e-05, |
| "full_absorption_rate": 0.0026985350809560524, |
| "num_full_absorption": 7, |
| "num_probe_true_positives": 2594, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.003024408788379693, |
| "full_absorption_rate": 0.005663939584644431, |
| "num_full_absorption": 9, |
| "num_probe_true_positives": 1589, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.023211488312181304, |
| "full_absorption_rate": 0.008664259927797834, |
| "num_full_absorption": 24, |
| "num_probe_true_positives": 2770, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.01810960173618097, |
| "full_absorption_rate": 0.007476635514018692, |
| "num_full_absorption": 12, |
| "num_probe_true_positives": 1605, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.004381251949082855, |
| "full_absorption_rate": 0.010685103708359522, |
| "num_full_absorption": 17, |
| "num_probe_true_positives": 1591, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.017010480974966494, |
| "full_absorption_rate": 0.004842615012106538, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 1239, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.006095879726494826, |
| "full_absorption_rate": 0.0026642984014209592, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 1126, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.004591353022879804, |
| "full_absorption_rate": 0.006505576208178439, |
| "num_full_absorption": 7, |
| "num_probe_true_positives": 1076, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.027404411903801502, |
| "full_absorption_rate": 0.02916160388821385, |
| "num_full_absorption": 48, |
| "num_probe_true_positives": 1646, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.004576659038901602, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 437, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.002094954013209374, |
| "full_absorption_rate": 0.0014749262536873156, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 678, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.3939788038833844, |
| "full_absorption_rate": 0.29900332225913623, |
| "num_full_absorption": 360, |
| "num_probe_true_positives": 1204, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.00594680868599322, |
| "full_absorption_rate": 0.0016602102933038186, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 1807, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.021926420524952285, |
| "full_absorption_rate": 0.01373283395755306, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 801, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.02481109619908078, |
| "full_absorption_rate": 0.018518518518518517, |
| "num_full_absorption": 20, |
| "num_probe_true_positives": 1080, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.056073440508186044, |
| "full_absorption_rate": 0.03365384615384615, |
| "num_full_absorption": 77, |
| "num_probe_true_positives": 2288, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.006982175004683158, |
| "full_absorption_rate": 0.005208333333333333, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 192, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.020354541685387403, |
| "full_absorption_rate": 0.013914095583787053, |
| "num_full_absorption": 23, |
| "num_probe_true_positives": 1653, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.28199045451150656, |
| "full_absorption_rate": 0.21019771071800208, |
| "num_full_absorption": 606, |
| "num_probe_true_positives": 2883, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.060092696508166114, |
| "full_absorption_rate": 0.039015606242496996, |
| "num_full_absorption": 65, |
| "num_probe_true_positives": 1666, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.03844998751590955, |
| "full_absorption_rate": 0.019329896907216496, |
| "num_full_absorption": 15, |
| "num_probe_true_positives": 776, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.004294114447358905, |
| "full_absorption_rate": 0.004819277108433735, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 830, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.0863585648746041, |
| "full_absorption_rate": 0.04192546583850932, |
| "num_full_absorption": 27, |
| "num_probe_true_positives": 644, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.00917585172389931, |
| "full_absorption_rate": 0.021739130434782608, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 92, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.010204489070401427, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 149, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 287, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.10.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.10.hook_resid_post", |
| "hook_layer": 10, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
