| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745797579234, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.21342511542943934, |
| "mean_full_absorption_score": 0.18598426121803746, |
| "mean_num_split_features": 1.2692307692307692, |
| "std_dev_absorption_fraction_score": 0.2553427420423028, |
| "std_dev_full_absorption_score": 0.2420585309725433, |
| "std_dev_num_split_features": 0.533493565673837 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.33082372413170374, |
| "full_absorption_rate": 0.25265225933202357, |
| "num_full_absorption": 643, |
| "num_probe_true_positives": 2545, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.1524404219476543, |
| "full_absorption_rate": 0.11090225563909774, |
| "num_full_absorption": 177, |
| "num_probe_true_positives": 1596, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.5049150717161012, |
| "full_absorption_rate": 0.46023564064801176, |
| "num_full_absorption": 1250, |
| "num_probe_true_positives": 2716, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.17806306279643594, |
| "full_absorption_rate": 0.15127092374457532, |
| "num_full_absorption": 244, |
| "num_probe_true_positives": 1613, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.15561198492359832, |
| "full_absorption_rate": 0.15040397762585456, |
| "num_full_absorption": 242, |
| "num_probe_true_positives": 1609, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.1774656475503202, |
| "full_absorption_rate": 0.15103734439834024, |
| "num_full_absorption": 182, |
| "num_probe_true_positives": 1205, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.04563388313001719, |
| "full_absorption_rate": 0.028745644599303136, |
| "num_full_absorption": 33, |
| "num_probe_true_positives": 1148, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.04946583375206639, |
| "full_absorption_rate": 0.048503611971104234, |
| "num_full_absorption": 47, |
| "num_probe_true_positives": 969, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.17653161862457492, |
| "full_absorption_rate": 0.18044659022329512, |
| "num_full_absorption": 299, |
| "num_probe_true_positives": 1657, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 3.072431234371448e-05, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 415, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0036572058006725935, |
| "full_absorption_rate": 0.0030627871362940277, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 653, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.1496291794186504, |
| "full_absorption_rate": 0.14156378600823044, |
| "num_full_absorption": 172, |
| "num_probe_true_positives": 1215, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.3325867333915395, |
| "full_absorption_rate": 0.2749445676274945, |
| "num_full_absorption": 496, |
| "num_probe_true_positives": 1804, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.04191789764464963, |
| "full_absorption_rate": 0.04187192118226601, |
| "num_full_absorption": 34, |
| "num_probe_true_positives": 812, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.06027940422095779, |
| "full_absorption_rate": 0.08527131782945736, |
| "num_full_absorption": 88, |
| "num_probe_true_positives": 1032, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.17716080390336, |
| "full_absorption_rate": 0.16330904414916417, |
| "num_full_absorption": 381, |
| "num_probe_true_positives": 2333, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 199, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.027809109885836673, |
| "full_absorption_rate": 0.03371501272264631, |
| "num_full_absorption": 53, |
| "num_probe_true_positives": 1572, |
| "num_split_features": 3 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.8186751008065684, |
| "full_absorption_rate": 0.8231292517006803, |
| "num_full_absorption": 2299, |
| "num_probe_true_positives": 2793, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.2060006947973954, |
| "full_absorption_rate": 0.18070067609096496, |
| "num_full_absorption": 294, |
| "num_probe_true_positives": 1627, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.20808960266963086, |
| "full_absorption_rate": 0.09986859395532194, |
| "num_full_absorption": 76, |
| "num_probe_true_positives": 761, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0024125452352231603, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 829, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.08197207883170882, |
| "full_absorption_rate": 0.054380664652567974, |
| "num_full_absorption": 36, |
| "num_probe_true_positives": 662, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.9454545454545454, |
| "full_absorption_rate": 0.9636363636363636, |
| "num_full_absorption": 106, |
| "num_probe_true_positives": 110, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.6893315709316808, |
| "full_absorption_rate": 0.43352601156069365, |
| "num_full_absorption": 75, |
| "num_probe_true_positives": 173, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.03550710052341046, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 219, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.20.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.20.hook_resid_post", |
| "hook_layer": 20, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
