| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745793953759, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.06999524871358496, |
| "mean_full_absorption_score": 0.0581646353897541, |
| "mean_num_split_features": 1.0384615384615385, |
| "std_dev_absorption_fraction_score": 0.11231675527401844, |
| "std_dev_full_absorption_score": 0.10174778232192411, |
| "std_dev_num_split_features": 0.19611613513818404 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.548867516527537, |
| "full_absorption_rate": 0.5121028744326778, |
| "num_full_absorption": 1354, |
| "num_probe_true_positives": 2644, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.0029041122507529535, |
| "full_absorption_rate": 0.0050858232676414495, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 1573, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.07113309215962084, |
| "full_absorption_rate": 0.04013015184381779, |
| "num_full_absorption": 111, |
| "num_probe_true_positives": 2766, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.0427561960561401, |
| "full_absorption_rate": 0.041371158392434985, |
| "num_full_absorption": 70, |
| "num_probe_true_positives": 1692, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.03020719492952968, |
| "full_absorption_rate": 0.03708523096942095, |
| "num_full_absorption": 57, |
| "num_probe_true_positives": 1537, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.006551020390014473, |
| "full_absorption_rate": 0.005050505050505051, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 1188, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.011590907048010834, |
| "full_absorption_rate": 0.011314186248912098, |
| "num_full_absorption": 13, |
| "num_probe_true_positives": 1149, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.014590863787782984, |
| "full_absorption_rate": 0.011616650532429816, |
| "num_full_absorption": 12, |
| "num_probe_true_positives": 1033, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.028338278330986007, |
| "full_absorption_rate": 0.03302433371958285, |
| "num_full_absorption": 57, |
| "num_probe_true_positives": 1726, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.012701728100696207, |
| "full_absorption_rate": 0.009389671361502348, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 426, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.007870905431712736, |
| "full_absorption_rate": 0.008695652173913044, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 690, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.012034846964941079, |
| "full_absorption_rate": 0.00846740050804403, |
| "num_full_absorption": 10, |
| "num_probe_true_positives": 1181, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.014444501122528012, |
| "full_absorption_rate": 0.005534034311012728, |
| "num_full_absorption": 10, |
| "num_probe_true_positives": 1807, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.04550967614378086, |
| "full_absorption_rate": 0.028950542822677925, |
| "num_full_absorption": 24, |
| "num_probe_true_positives": 829, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.06436106801984783, |
| "full_absorption_rate": 0.05572139303482587, |
| "num_full_absorption": 56, |
| "num_probe_true_positives": 1005, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.12518823782451677, |
| "full_absorption_rate": 0.08354866494401378, |
| "num_full_absorption": 194, |
| "num_probe_true_positives": 2322, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.011299435028248588, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 177, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.05290789442306641, |
| "full_absorption_rate": 0.03486997635933806, |
| "num_full_absorption": 59, |
| "num_probe_true_positives": 1692, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.18613462439459597, |
| "full_absorption_rate": 0.16398601398601398, |
| "num_full_absorption": 469, |
| "num_probe_true_positives": 2860, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.05762842535543915, |
| "full_absorption_rate": 0.042874396135265704, |
| "num_full_absorption": 71, |
| "num_probe_true_positives": 1656, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.12329078231106516, |
| "full_absorption_rate": 0.08758169934640522, |
| "num_full_absorption": 67, |
| "num_probe_true_positives": 765, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.08737485573904961, |
| "full_absorption_rate": 0.07924528301886792, |
| "num_full_absorption": 63, |
| "num_probe_true_positives": 795, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.02517904212863916, |
| "full_absorption_rate": 0.023529411764705882, |
| "num_full_absorption": 16, |
| "num_probe_true_positives": 680, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.03267084150756558, |
| "full_absorption_rate": 0.009708737864077669, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 103, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.21146841344776193, |
| "full_absorption_rate": 0.1497005988023952, |
| "num_full_absorption": 25, |
| "num_probe_true_positives": 167, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.004171442157627781, |
| "full_absorption_rate": 0.012396694214876033, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 242, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.12.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.12.hook_resid_post", |
| "hook_layer": 12, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": "gemma-2-2b/12-res-matryoshka-dc", |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
