| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745796216420, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.08071898043571245, |
| "mean_full_absorption_score": 0.06482811423030974, |
| "mean_num_split_features": 1.5, |
| "std_dev_absorption_fraction_score": 0.13331686187174158, |
| "std_dev_full_absorption_score": 0.11795834263280236, |
| "std_dev_num_split_features": 0.9486832980505138 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.47115536743999764, |
| "full_absorption_rate": 0.36902927580893685, |
| "num_full_absorption": 958, |
| "num_probe_true_positives": 2596, |
| "num_split_features": 5 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.021268804478020376, |
| "full_absorption_rate": 0.005600497822028625, |
| "num_full_absorption": 9, |
| "num_probe_true_positives": 1607, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.44647716349024624, |
| "full_absorption_rate": 0.4262413918086263, |
| "num_full_absorption": 1176, |
| "num_probe_true_positives": 2759, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.012766019627623738, |
| "full_absorption_rate": 0.014943215780035863, |
| "num_full_absorption": 25, |
| "num_probe_true_positives": 1673, |
| "num_split_features": 3 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.04693591178248158, |
| "full_absorption_rate": 0.037476577139287946, |
| "num_full_absorption": 60, |
| "num_probe_true_positives": 1601, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.013185431136918852, |
| "full_absorption_rate": 0.009401709401709401, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 1170, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.005649644248803581, |
| "full_absorption_rate": 0.005208333333333333, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 1152, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.009265835590070019, |
| "full_absorption_rate": 0.0019193857965451055, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 1042, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.0016563884916560485, |
| "full_absorption_rate": 0.011097410604192354, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 1622, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.023231398504175268, |
| "full_absorption_rate": 0.0022675736961451248, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 441, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.018421467245446604, |
| "full_absorption_rate": 0.007776049766718507, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 643, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.001706620220642646, |
| "full_absorption_rate": 0.0032520325203252032, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 1230, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.0891520023620283, |
| "full_absorption_rate": 0.06703601108033241, |
| "num_full_absorption": 121, |
| "num_probe_true_positives": 1805, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.05740152230151289, |
| "full_absorption_rate": 0.026570048309178744, |
| "num_full_absorption": 22, |
| "num_probe_true_positives": 828, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.23268318618546108, |
| "full_absorption_rate": 0.1798631476050831, |
| "num_full_absorption": 184, |
| "num_probe_true_positives": 1023, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.013209853684975117, |
| "full_absorption_rate": 0.003036876355748373, |
| "num_full_absorption": 7, |
| "num_probe_true_positives": 2305, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0055248618784530384, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 181, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.03563459372103949, |
| "full_absorption_rate": 0.015432098765432098, |
| "num_full_absorption": 25, |
| "num_probe_true_positives": 1620, |
| "num_split_features": 3 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.27524885254141523, |
| "full_absorption_rate": 0.2732142857142857, |
| "num_full_absorption": 765, |
| "num_probe_true_positives": 2800, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.0076875923133337696, |
| "full_absorption_rate": 0.014010507880910683, |
| "num_full_absorption": 24, |
| "num_probe_true_positives": 1713, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.02730413567253992, |
| "full_absorption_rate": 0.017310252996005325, |
| "num_full_absorption": 13, |
| "num_probe_true_positives": 751, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.20458025698905868, |
| "full_absorption_rate": 0.15375153751537515, |
| "num_full_absorption": 125, |
| "num_probe_true_positives": 813, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.034713309718603744, |
| "full_absorption_rate": 0.02465331278890601, |
| "num_full_absorption": 16, |
| "num_probe_true_positives": 649, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.00539698430448098, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 94, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.043961149277991704, |
| "full_absorption_rate": 0.006993006993006993, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 143, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.00392156862745098, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 255, |
| "num_split_features": 2 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.17.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.17.hook_resid_post", |
| "hook_layer": 17, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
