| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745794853523, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.0541222248287161, |
| "mean_full_absorption_score": 0.045247108070390785, |
| "mean_num_split_features": 1.0769230769230769, |
| "std_dev_absorption_fraction_score": 0.09085759453660358, |
| "std_dev_full_absorption_score": 0.07518258378534703, |
| "std_dev_num_split_features": 0.271746488194703 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.0051880894572183455, |
| "full_absorption_rate": 0.0050820953870211105, |
| "num_full_absorption": 13, |
| "num_probe_true_positives": 2558, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.0026063765220943618, |
| "full_absorption_rate": 0.002997601918465228, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 1668, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.00845109958142273, |
| "full_absorption_rate": 0.00646551724137931, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 2784, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.2627493915237698, |
| "full_absorption_rate": 0.19619865113427346, |
| "num_full_absorption": 320, |
| "num_probe_true_positives": 1631, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.024851221416786046, |
| "full_absorption_rate": 0.021831412977562158, |
| "num_full_absorption": 36, |
| "num_probe_true_positives": 1649, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.2258265779360939, |
| "full_absorption_rate": 0.1888412017167382, |
| "num_full_absorption": 220, |
| "num_probe_true_positives": 1165, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.006460477035829956, |
| "full_absorption_rate": 0.009208103130755065, |
| "num_full_absorption": 10, |
| "num_probe_true_positives": 1086, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.027089544504868248, |
| "full_absorption_rate": 0.018357487922705314, |
| "num_full_absorption": 19, |
| "num_probe_true_positives": 1035, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.0007949756034277328, |
| "full_absorption_rate": 0.010688836104513063, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 1684, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0024937655860349127, |
| "full_absorption_rate": 0.007481296758104738, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 401, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.002448599099348701, |
| "full_absorption_rate": 0.004398826979472141, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 682, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.007139257220901455, |
| "full_absorption_rate": 0.003276003276003276, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 1221, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.003882223170096328, |
| "full_absorption_rate": 0.013785180930499713, |
| "num_full_absorption": 24, |
| "num_probe_true_positives": 1741, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.08484204730665265, |
| "full_absorption_rate": 0.046750285062713795, |
| "num_full_absorption": 41, |
| "num_probe_true_positives": 877, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.018798725206087268, |
| "full_absorption_rate": 0.017839444995044598, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 1009, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.14300970625601755, |
| "full_absorption_rate": 0.10932754880694143, |
| "num_full_absorption": 252, |
| "num_probe_true_positives": 2305, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 184, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.006822015823818754, |
| "full_absorption_rate": 0.004744958481613286, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 1686, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.02634560595765921, |
| "full_absorption_rate": 0.01806182702327197, |
| "num_full_absorption": 52, |
| "num_probe_true_positives": 2879, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.025318628633904224, |
| "full_absorption_rate": 0.020833333333333332, |
| "num_full_absorption": 35, |
| "num_probe_true_positives": 1680, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.34585909653880353, |
| "full_absorption_rate": 0.3087071240105541, |
| "num_full_absorption": 234, |
| "num_probe_true_positives": 758, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 1.4790044307196562e-05, |
| "full_absorption_rate": 0.005263157894736842, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 760, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.0893898108030974, |
| "full_absorption_rate": 0.07786259541984733, |
| "num_full_absorption": 51, |
| "num_probe_true_positives": 655, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.029662234912157335, |
| "full_absorption_rate": 0.02830188679245283, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 106, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.05520379838436618, |
| "full_absorption_rate": 0.0379746835443038, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 158, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.001929787021854715, |
| "full_absorption_rate": 0.012145748987854251, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 247, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.14.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.14.hook_resid_post", |
| "hook_layer": 14, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
