| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6", |
| "datetime_epoch_millis": 1745712369823, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.16721133547328304, |
| "mean_full_absorption_score": 0.0919159506843662, |
| "mean_num_split_features": 1.2692307692307692, |
| "std_dev_absorption_fraction_score": 0.17135516176932297, |
| "std_dev_full_absorption_score": 0.09573599187633346, |
| "std_dev_num_split_features": 0.6667948594698258 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.2032784308144539, |
| "full_absorption_rate": 0.13490853658536586, |
| "num_full_absorption": 354, |
| "num_probe_true_positives": 2624, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.05486946295185843, |
| "full_absorption_rate": 0.038853503184713374, |
| "num_full_absorption": 61, |
| "num_probe_true_positives": 1570, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.006948416251906628, |
| "full_absorption_rate": 0.008014571948998178, |
| "num_full_absorption": 22, |
| "num_probe_true_positives": 2745, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.24422305540024689, |
| "full_absorption_rate": 0.16046369737644905, |
| "num_full_absorption": 263, |
| "num_probe_true_positives": 1639, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.04049069885651504, |
| "full_absorption_rate": 0.03094059405940594, |
| "num_full_absorption": 50, |
| "num_probe_true_positives": 1616, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.0954804626559779, |
| "full_absorption_rate": 0.06233538191395961, |
| "num_full_absorption": 71, |
| "num_probe_true_positives": 1139, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.11895808655464311, |
| "full_absorption_rate": 0.05847457627118644, |
| "num_full_absorption": 69, |
| "num_probe_true_positives": 1180, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.24551650436369388, |
| "full_absorption_rate": 0.13314447592067988, |
| "num_full_absorption": 141, |
| "num_probe_true_positives": 1059, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.34446842501174796, |
| "full_absorption_rate": 0.20071471113758188, |
| "num_full_absorption": 337, |
| "num_probe_true_positives": 1679, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.33057020079953253, |
| "full_absorption_rate": 0.1445221445221445, |
| "num_full_absorption": 62, |
| "num_probe_true_positives": 429, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.04683802261524384, |
| "full_absorption_rate": 0.017569546120058566, |
| "num_full_absorption": 12, |
| "num_probe_true_positives": 683, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.5275514952812571, |
| "full_absorption_rate": 0.3099921321793863, |
| "num_full_absorption": 394, |
| "num_probe_true_positives": 1271, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.013335317984867959, |
| "full_absorption_rate": 0.01082004555808656, |
| "num_full_absorption": 19, |
| "num_probe_true_positives": 1756, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.17666150083727022, |
| "full_absorption_rate": 0.08472553699284009, |
| "num_full_absorption": 71, |
| "num_probe_true_positives": 838, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.24784997161038702, |
| "full_absorption_rate": 0.13189897100093545, |
| "num_full_absorption": 141, |
| "num_probe_true_positives": 1069, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.001345719856698647, |
| "full_absorption_rate": 0.007352941176470588, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 2448, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.016708742873230268, |
| "full_absorption_rate": 0.016042780748663103, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 187, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.14373508309131128, |
| "full_absorption_rate": 0.09791921664626684, |
| "num_full_absorption": 160, |
| "num_probe_true_positives": 1634, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.007388886122852638, |
| "full_absorption_rate": 0.009494743981010512, |
| "num_full_absorption": 28, |
| "num_probe_true_positives": 2949, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.029410282984506448, |
| "full_absorption_rate": 0.028874484384207425, |
| "num_full_absorption": 49, |
| "num_probe_true_positives": 1697, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.6855802161297957, |
| "full_absorption_rate": 0.3800786369593709, |
| "num_full_absorption": 290, |
| "num_probe_true_positives": 763, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.3342004323825742, |
| "full_absorption_rate": 0.18058968058968058, |
| "num_full_absorption": 147, |
| "num_probe_true_positives": 814, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.12197212395602165, |
| "full_absorption_rate": 0.07244318181818182, |
| "num_full_absorption": 51, |
| "num_probe_true_positives": 704, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.016580599639673347, |
| "full_absorption_rate": 0.01020408163265306, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 98, |
| "num_split_features": 4 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.21288844129978676, |
| "full_absorption_rate": 0.029585798816568046, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 169, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.08064414197930604, |
| "full_absorption_rate": 0.029850746268656716, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 268, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.1.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.1.hook_resid_post", |
| "hook_layer": 1, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
