| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745792150612, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.03162603772930549, |
| "mean_full_absorption_score": 0.02188986687902274, |
| "mean_num_split_features": 1.0769230769230769, |
| "std_dev_absorption_fraction_score": 0.11340603630221202, |
| "std_dev_full_absorption_score": 0.07666421505893235, |
| "std_dev_num_split_features": 0.271746488194703 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.0031484030668817026, |
| "full_absorption_rate": 0.01266308518802763, |
| "num_full_absorption": 33, |
| "num_probe_true_positives": 2606, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.001276098214550636, |
| "full_absorption_rate": 0.006740196078431373, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 1632, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 2.575238731590523e-05, |
| "full_absorption_rate": 0.0003619254433586681, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 2763, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.004667444574095682, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 1714, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.010342097912838065, |
| "full_absorption_rate": 0.013630731102850062, |
| "num_full_absorption": 22, |
| "num_probe_true_positives": 1614, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.009209890954117131, |
| "full_absorption_rate": 0.003341687552213868, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 1197, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.02234008731072798, |
| "full_absorption_rate": 0.0034904013961605585, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 1146, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.0055895790464381735, |
| "full_absorption_rate": 0.004830917874396135, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 1035, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.0033317778428189958, |
| "full_absorption_rate": 0.004648460197559558, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 1721, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0024672854463755204, |
| "full_absorption_rate": 0.006818181818181818, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 440, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.001479034656094021, |
| "full_absorption_rate": 0.001451378809869376, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 689, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.003063996578860352, |
| "full_absorption_rate": 0.000819672131147541, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 1220, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.0021639297461292935, |
| "full_absorption_rate": 0.0011435105774728416, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 1749, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.0012860179183422592, |
| "full_absorption_rate": 0.0011148272017837235, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 897, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.07532045285262642, |
| "full_absorption_rate": 0.057197330791229746, |
| "num_full_absorption": 60, |
| "num_probe_true_positives": 1049, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.001282051282051282, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 2340, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.006134969325153374, |
| "full_absorption_rate": 0.006134969325153374, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 163, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.03340625463542878, |
| "full_absorption_rate": 0.02074688796680498, |
| "num_full_absorption": 35, |
| "num_probe_true_positives": 1687, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.0016649926902313717, |
| "full_absorption_rate": 0.0007127583749109052, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 2806, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.01753815261068991, |
| "full_absorption_rate": 0.009057971014492754, |
| "num_full_absorption": 15, |
| "num_probe_true_positives": 1656, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.5822096640151566, |
| "full_absorption_rate": 0.39363057324840767, |
| "num_full_absorption": 309, |
| "num_probe_true_positives": 785, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0013280212483399733, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 753, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.016498634425541055, |
| "full_absorption_rate": 0.007112375533428165, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 703, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.01925389236283221, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 109, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.003113260780969229, |
| "full_absorption_rate": 0.006211180124223602, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 161, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.001412756181823613, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 227, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.8.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.8.hook_resid_post", |
| "hook_layer": 8, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
