| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745795306774, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.05733850527245465, |
| "mean_full_absorption_score": 0.047500238355696084, |
| "mean_num_split_features": 1.1153846153846154, |
| "std_dev_absorption_fraction_score": 0.14224161999239068, |
| "std_dev_full_absorption_score": 0.12597164114707574, |
| "std_dev_num_split_features": 0.3258125936084211 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.006097175357630516, |
| "full_absorption_rate": 0.005496662740478995, |
| "num_full_absorption": 14, |
| "num_probe_true_positives": 2547, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.00667292182333627, |
| "full_absorption_rate": 0.01454775458570525, |
| "num_full_absorption": 23, |
| "num_probe_true_positives": 1581, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.00482835467567293, |
| "full_absorption_rate": 0.007658643326039387, |
| "num_full_absorption": 21, |
| "num_probe_true_positives": 2742, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.010852619199211593, |
| "full_absorption_rate": 0.014032946918852958, |
| "num_full_absorption": 23, |
| "num_probe_true_positives": 1639, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.001500648024292079, |
| "full_absorption_rate": 0.010935601458080195, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 1646, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.01357219938687626, |
| "full_absorption_rate": 0.008210180623973728, |
| "num_full_absorption": 10, |
| "num_probe_true_positives": 1218, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.027264368149825056, |
| "full_absorption_rate": 0.015356820234869015, |
| "num_full_absorption": 17, |
| "num_probe_true_positives": 1107, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.005981503427704533, |
| "full_absorption_rate": 0.005853658536585366, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 1025, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.011314657336541105, |
| "full_absorption_rate": 0.01092896174863388, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 1647, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.011725584852973479, |
| "full_absorption_rate": 0.018957345971563982, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 422, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0005410964746647586, |
| "full_absorption_rate": 0.0015105740181268882, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 662, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.06634547545711322, |
| "full_absorption_rate": 0.038818565400843885, |
| "num_full_absorption": 46, |
| "num_probe_true_positives": 1185, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.009983123206527864, |
| "full_absorption_rate": 0.01615598885793872, |
| "num_full_absorption": 29, |
| "num_probe_true_positives": 1795, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.04845569983420948, |
| "full_absorption_rate": 0.029036004645760744, |
| "num_full_absorption": 25, |
| "num_probe_true_positives": 861, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.170130212888805, |
| "full_absorption_rate": 0.12987012987012986, |
| "num_full_absorption": 140, |
| "num_probe_true_positives": 1078, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.01574963892725152, |
| "full_absorption_rate": 0.011558219178082191, |
| "num_full_absorption": 27, |
| "num_probe_true_positives": 2336, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.012608662016035459, |
| "full_absorption_rate": 0.005555555555555556, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 180, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.03865367197342095, |
| "full_absorption_rate": 0.03415783274440518, |
| "num_full_absorption": 58, |
| "num_probe_true_positives": 1698, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.05442648472294919, |
| "full_absorption_rate": 0.03793843951324266, |
| "num_full_absorption": 106, |
| "num_probe_true_positives": 2794, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.06361612951094564, |
| "full_absorption_rate": 0.04747774480712166, |
| "num_full_absorption": 80, |
| "num_probe_true_positives": 1685, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.7313507594669242, |
| "full_absorption_rate": 0.6522875816993464, |
| "num_full_absorption": 499, |
| "num_probe_true_positives": 765, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.004085341141374133, |
| "full_absorption_rate": 0.007352941176470588, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 816, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.06118063907549146, |
| "full_absorption_rate": 0.04035874439461883, |
| "num_full_absorption": 27, |
| "num_probe_true_positives": 669, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.053551806390596146, |
| "full_absorption_rate": 0.033707865168539325, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 89, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.060312363763448056, |
| "full_absorption_rate": 0.033707865168539325, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 178, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0035335689045936395, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 283, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.15.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.15.hook_resid_post", |
| "hook_layer": 15, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
