| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6", |
| "datetime_epoch_millis": 1745715300314, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.028491299664495845, |
| "mean_full_absorption_score": 0.010148402753251037, |
| "mean_num_split_features": 1.1153846153846154, |
| "std_dev_absorption_fraction_score": 0.11438444432669825, |
| "std_dev_full_absorption_score": 0.0371814953839519, |
| "std_dev_num_split_features": 0.3258125936084211 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.003611971141358597, |
| "full_absorption_rate": 0.008155339805825243, |
| "num_full_absorption": 21, |
| "num_probe_true_positives": 2575, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0018281535648994515, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 1641, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.00030878827714068144, |
| "full_absorption_rate": 0.00035587188612099647, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 2810, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.002986857825567503, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 1674, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0043397396156230625, |
| "num_full_absorption": 7, |
| "num_probe_true_positives": 1613, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.0007744174949998641, |
| "full_absorption_rate": 0.0016220600162206002, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 1233, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.043234746814266485, |
| "full_absorption_rate": 0.009540329575021683, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 1153, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.0004039064581796007, |
| "full_absorption_rate": 0.0009633911368015414, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 1038, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.0003431586586339689, |
| "full_absorption_rate": 0.0005704506560182544, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 1753, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0037303050471264526, |
| "full_absorption_rate": 0.004454342984409799, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 449, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0065926497632613525, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 692, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.00015344639723778797, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1208, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.0016512550783602156, |
| "full_absorption_rate": 0.0017201834862385322, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 1744, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.010410229726142903, |
| "full_absorption_rate": 0.0023446658851113715, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 853, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.5871499978419803, |
| "full_absorption_rate": 0.1919191919191919, |
| "num_full_absorption": 209, |
| "num_probe_true_positives": 1089, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 2351, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.00712739162217137, |
| "full_absorption_rate": 0.005263157894736842, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 190, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.0007979163237209857, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1633, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.002060894307223809, |
| "full_absorption_rate": 0.0007104795737122557, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 2815, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.0034319078400795013, |
| "full_absorption_rate": 0.0005991611743559018, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 1669, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.026832953876707505, |
| "full_absorption_rate": 0.00651890482398957, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 767, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.002506265664160401, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 798, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.011486860759853433, |
| "full_absorption_rate": 0.007363770250368188, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 679, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.000319212461038081, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 94, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.01861206081539049, |
| "full_absorption_rate": 0.00625, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 160, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.011739720572018586, |
| "full_absorption_rate": 0.0038461538461538464, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 260, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.4.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.4.hook_resid_post", |
| "hook_layer": 4, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
