| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745798033939, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.16674315723085067, |
| "mean_full_absorption_score": 0.14565752359280973, |
| "mean_num_split_features": 1.2692307692307692, |
| "std_dev_absorption_fraction_score": 0.1570970287807047, |
| "std_dev_full_absorption_score": 0.1636264800961526, |
| "std_dev_num_split_features": 0.4523443208612048 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.46454876272810397, |
| "full_absorption_rate": 0.4304791830322074, |
| "num_full_absorption": 1096, |
| "num_probe_true_positives": 2546, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.14653548806375244, |
| "full_absorption_rate": 0.07633098139833226, |
| "num_full_absorption": 119, |
| "num_probe_true_positives": 1559, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.59272495924861, |
| "full_absorption_rate": 0.6418992388546575, |
| "num_full_absorption": 1771, |
| "num_probe_true_positives": 2759, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.3112109082909835, |
| "full_absorption_rate": 0.2901002506265664, |
| "num_full_absorption": 463, |
| "num_probe_true_positives": 1596, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.15575208722363373, |
| "full_absorption_rate": 0.1341614906832298, |
| "num_full_absorption": 216, |
| "num_probe_true_positives": 1610, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.021417952489556878, |
| "full_absorption_rate": 0.01452991452991453, |
| "num_full_absorption": 17, |
| "num_probe_true_positives": 1170, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.05752603042113519, |
| "full_absorption_rate": 0.05789473684210526, |
| "num_full_absorption": 66, |
| "num_probe_true_positives": 1140, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.006991899088980313, |
| "full_absorption_rate": 0.009424083769633508, |
| "num_full_absorption": 9, |
| "num_probe_true_positives": 955, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.16502262573206747, |
| "full_absorption_rate": 0.14950980392156862, |
| "num_full_absorption": 244, |
| "num_probe_true_positives": 1632, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.016800894753446226, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 425, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.11674769221448418, |
| "full_absorption_rate": 0.03265940902021773, |
| "num_full_absorption": 21, |
| "num_probe_true_positives": 643, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.1186901343539054, |
| "full_absorption_rate": 0.11101766190075694, |
| "num_full_absorption": 132, |
| "num_probe_true_positives": 1189, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.23532540367946916, |
| "full_absorption_rate": 0.20271800679501698, |
| "num_full_absorption": 358, |
| "num_probe_true_positives": 1766, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.06301223117908614, |
| "full_absorption_rate": 0.05372405372405373, |
| "num_full_absorption": 44, |
| "num_probe_true_positives": 819, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.10397296534972689, |
| "full_absorption_rate": 0.1111111111111111, |
| "num_full_absorption": 117, |
| "num_probe_true_positives": 1053, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.3760208886268326, |
| "full_absorption_rate": 0.3612450679526523, |
| "num_full_absorption": 824, |
| "num_probe_true_positives": 2281, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.05206112220482297, |
| "full_absorption_rate": 0.0223463687150838, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 179, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.13986132789950478, |
| "full_absorption_rate": 0.14745972738537794, |
| "num_full_absorption": 238, |
| "num_probe_true_positives": 1614, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.40210225194793314, |
| "full_absorption_rate": 0.4288224956063269, |
| "num_full_absorption": 1220, |
| "num_probe_true_positives": 2845, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.13926184954704016, |
| "full_absorption_rate": 0.1387678904791537, |
| "num_full_absorption": 223, |
| "num_probe_true_positives": 1607, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.05911038302906191, |
| "full_absorption_rate": 0.05485232067510549, |
| "num_full_absorption": 39, |
| "num_probe_true_positives": 711, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.0358826723480427, |
| "full_absorption_rate": 0.014616321559074299, |
| "num_full_absorption": 12, |
| "num_probe_true_positives": 821, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.0901428035348417, |
| "full_absorption_rate": 0.03829160530191458, |
| "num_full_absorption": 26, |
| "num_probe_true_positives": 679, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.3724676311793158, |
| "full_absorption_rate": 0.24285714285714285, |
| "num_full_absorption": 17, |
| "num_probe_true_positives": 70, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.05882034261391029, |
| "full_absorption_rate": 0.018292682926829267, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 164, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.03331078025387009, |
| "full_absorption_rate": 0.00398406374501992, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 251, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.21.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.21.hook_resid_post", |
| "hook_layer": 21, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
