| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745798486637, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.18548205958590785, |
| "mean_full_absorption_score": 0.16676032384209613, |
| "mean_num_split_features": 1.3461538461538463, |
| "std_dev_absorption_fraction_score": 0.15259194584122465, |
| "std_dev_full_absorption_score": 0.15251463104026725, |
| "std_dev_num_split_features": 0.891843380524078 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.4599548374687448, |
| "full_absorption_rate": 0.42395748160261654, |
| "num_full_absorption": 1037, |
| "num_probe_true_positives": 2446, |
| "num_split_features": 5 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.03211089695747652, |
| "full_absorption_rate": 0.035392535392535396, |
| "num_full_absorption": 55, |
| "num_probe_true_positives": 1554, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.5896895327032187, |
| "full_absorption_rate": 0.5699558173784978, |
| "num_full_absorption": 1548, |
| "num_probe_true_positives": 2716, |
| "num_split_features": 3 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.17990600227470654, |
| "full_absorption_rate": 0.1529709228824273, |
| "num_full_absorption": 242, |
| "num_probe_true_positives": 1582, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.22564576065331274, |
| "full_absorption_rate": 0.22068095838587642, |
| "num_full_absorption": 350, |
| "num_probe_true_positives": 1586, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.0902947875088285, |
| "full_absorption_rate": 0.09856781802864364, |
| "num_full_absorption": 117, |
| "num_probe_true_positives": 1187, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.04109348015114809, |
| "full_absorption_rate": 0.02423698384201077, |
| "num_full_absorption": 27, |
| "num_probe_true_positives": 1114, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.31612499818453105, |
| "full_absorption_rate": 0.28556149732620323, |
| "num_full_absorption": 267, |
| "num_probe_true_positives": 935, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.12944712464613606, |
| "full_absorption_rate": 0.13296903460837886, |
| "num_full_absorption": 219, |
| "num_probe_true_positives": 1647, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.023155699483006973, |
| "full_absorption_rate": 0.0069605568445475635, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 431, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.06247064911320011, |
| "full_absorption_rate": 0.031791907514450865, |
| "num_full_absorption": 22, |
| "num_probe_true_positives": 692, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.3868048067607811, |
| "full_absorption_rate": 0.30952380952380953, |
| "num_full_absorption": 351, |
| "num_probe_true_positives": 1134, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.17428893609953175, |
| "full_absorption_rate": 0.14846416382252559, |
| "num_full_absorption": 261, |
| "num_probe_true_positives": 1758, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.08186370056843241, |
| "full_absorption_rate": 0.07496977025392987, |
| "num_full_absorption": 62, |
| "num_probe_true_positives": 827, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.11461795009436364, |
| "full_absorption_rate": 0.13306451612903225, |
| "num_full_absorption": 132, |
| "num_probe_true_positives": 992, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.4062782929505677, |
| "full_absorption_rate": 0.39206279982555603, |
| "num_full_absorption": 899, |
| "num_probe_true_positives": 2293, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.20745238737977303, |
| "full_absorption_rate": 0.1206896551724138, |
| "num_full_absorption": 21, |
| "num_probe_true_positives": 174, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.19105293797542694, |
| "full_absorption_rate": 0.19291819291819293, |
| "num_full_absorption": 316, |
| "num_probe_true_positives": 1638, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.3846787158500273, |
| "full_absorption_rate": 0.45879888268156427, |
| "num_full_absorption": 1314, |
| "num_probe_true_positives": 2864, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.15297970624925938, |
| "full_absorption_rate": 0.17323775388291518, |
| "num_full_absorption": 290, |
| "num_probe_true_positives": 1674, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.1205468045976172, |
| "full_absorption_rate": 0.10331125827814569, |
| "num_full_absorption": 78, |
| "num_probe_true_positives": 755, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.028922173795110126, |
| "full_absorption_rate": 0.031413612565445025, |
| "num_full_absorption": 24, |
| "num_probe_true_positives": 764, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.05017034975840341, |
| "full_absorption_rate": 0.06707317073170732, |
| "num_full_absorption": 44, |
| "num_probe_true_positives": 656, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.24021020046299707, |
| "full_absorption_rate": 0.10891089108910891, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 101, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.09646304681998517, |
| "full_absorption_rate": 0.012422360248447204, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 161, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.03630977072701775, |
| "full_absorption_rate": 0.02586206896551724, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 232, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.22.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.22.hook_resid_post", |
| "hook_layer": 22, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
