| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745799386022, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.21526497177610726, |
| "mean_full_absorption_score": 0.2805231207323319, |
| "mean_num_split_features": 1.9230769230769231, |
| "std_dev_absorption_fraction_score": 0.12272353957570045, |
| "std_dev_full_absorption_score": 0.15669302565674426, |
| "std_dev_num_split_features": 1.3834182859302366 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.3859504439163319, |
| "full_absorption_rate": 0.49003189792663476, |
| "num_full_absorption": 1229, |
| "num_probe_true_positives": 2508, |
| "num_split_features": 4 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.2700639248619282, |
| "full_absorption_rate": 0.38312123241795043, |
| "num_full_absorption": 572, |
| "num_probe_true_positives": 1493, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.18158540458290628, |
| "full_absorption_rate": 0.4320029563932003, |
| "num_full_absorption": 1169, |
| "num_probe_true_positives": 2706, |
| "num_split_features": 6 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.2770401962256209, |
| "full_absorption_rate": 0.2923898531375167, |
| "num_full_absorption": 438, |
| "num_probe_true_positives": 1498, |
| "num_split_features": 4 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.2711825402668218, |
| "full_absorption_rate": 0.34319162045594576, |
| "num_full_absorption": 557, |
| "num_probe_true_positives": 1623, |
| "num_split_features": 3 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.31167663481691327, |
| "full_absorption_rate": 0.4152910512597741, |
| "num_full_absorption": 478, |
| "num_probe_true_positives": 1151, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.26815582709401564, |
| "full_absorption_rate": 0.26174496644295303, |
| "num_full_absorption": 312, |
| "num_probe_true_positives": 1192, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.31583451913869115, |
| "full_absorption_rate": 0.3437806072477963, |
| "num_full_absorption": 351, |
| "num_probe_true_positives": 1021, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.2519339017839644, |
| "full_absorption_rate": 0.34375, |
| "num_full_absorption": 528, |
| "num_probe_true_positives": 1536, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0071534068676855946, |
| "full_absorption_rate": 0.03325942350332594, |
| "num_full_absorption": 15, |
| "num_probe_true_positives": 451, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.04946763727018178, |
| "full_absorption_rate": 0.07776049766718507, |
| "num_full_absorption": 50, |
| "num_probe_true_positives": 643, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.42309128607391, |
| "full_absorption_rate": 0.4572425828970332, |
| "num_full_absorption": 524, |
| "num_probe_true_positives": 1146, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.4182416216138434, |
| "full_absorption_rate": 0.47442680776014107, |
| "num_full_absorption": 807, |
| "num_probe_true_positives": 1701, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.29910164257445954, |
| "full_absorption_rate": 0.3231552162849873, |
| "num_full_absorption": 254, |
| "num_probe_true_positives": 786, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.20617769280553322, |
| "full_absorption_rate": 0.28907048008171604, |
| "num_full_absorption": 283, |
| "num_probe_true_positives": 979, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.2635898232329038, |
| "full_absorption_rate": 0.42787610619469024, |
| "num_full_absorption": 967, |
| "num_probe_true_positives": 2260, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.17135160607736144, |
| "full_absorption_rate": 0.13580246913580246, |
| "num_full_absorption": 22, |
| "num_probe_true_positives": 162, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.26521624858203446, |
| "full_absorption_rate": 0.32448733413751507, |
| "num_full_absorption": 538, |
| "num_probe_true_positives": 1658, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.23816714465033076, |
| "full_absorption_rate": 0.45354998233839633, |
| "num_full_absorption": 1284, |
| "num_probe_true_positives": 2831, |
| "num_split_features": 5 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.24674520304302952, |
| "full_absorption_rate": 0.37475976937860345, |
| "num_full_absorption": 585, |
| "num_probe_true_positives": 1561, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.214926624577281, |
| "full_absorption_rate": 0.2740740740740741, |
| "num_full_absorption": 185, |
| "num_probe_true_positives": 675, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.05107489726501669, |
| "full_absorption_rate": 0.07477820025348543, |
| "num_full_absorption": 59, |
| "num_probe_true_positives": 789, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.1330119947575182, |
| "full_absorption_rate": 0.20239880059970014, |
| "num_full_absorption": 135, |
| "num_probe_true_positives": 667, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.036042530876978264, |
| "full_absorption_rate": 0.01020408163265306, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 98, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.02797605110842932, |
| "full_absorption_rate": 0.019736842105263157, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 152, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.012130462115098535, |
| "full_absorption_rate": 0.03571428571428571, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 224, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.24.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.24.hook_resid_post", |
| "hook_layer": 24, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
