| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "a15aaf76-1ebc-4334-8744-86b1c8367c42", |
| "datetime_epoch_millis": 1745796671224, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.1424107673166, |
| "mean_full_absorption_score": 0.12510257067657027, |
| "mean_num_split_features": 1.4230769230769231, |
| "std_dev_absorption_fraction_score": 0.1886807126127507, |
| "std_dev_full_absorption_score": 0.1826445961743382, |
| "std_dev_num_split_features": 0.986836437230686 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.5328801123530625, |
| "full_absorption_rate": 0.4609069585613761, |
| "num_full_absorption": 1179, |
| "num_probe_true_positives": 2558, |
| "num_split_features": 4 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.08956793828369938, |
| "full_absorption_rate": 0.044859813084112146, |
| "num_full_absorption": 72, |
| "num_probe_true_positives": 1605, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.7127686456185302, |
| "full_absorption_rate": 0.7023853211009174, |
| "num_full_absorption": 1914, |
| "num_probe_true_positives": 2725, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.13557929372505884, |
| "full_absorption_rate": 0.08487084870848709, |
| "num_full_absorption": 138, |
| "num_probe_true_positives": 1626, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.20619154600483394, |
| "full_absorption_rate": 0.19260204081632654, |
| "num_full_absorption": 302, |
| "num_probe_true_positives": 1568, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.09095697692142744, |
| "full_absorption_rate": 0.06448911222780569, |
| "num_full_absorption": 77, |
| "num_probe_true_positives": 1194, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.005152931679235823, |
| "full_absorption_rate": 0.009769094138543518, |
| "num_full_absorption": 11, |
| "num_probe_true_positives": 1126, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.007743445860113119, |
| "full_absorption_rate": 0.006060606060606061, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 990, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.06884379827803416, |
| "full_absorption_rate": 0.057356608478802994, |
| "num_full_absorption": 92, |
| "num_probe_true_positives": 1604, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 455, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.005084256830246763, |
| "full_absorption_rate": 0.008849557522123894, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 678, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.06506943714982681, |
| "full_absorption_rate": 0.04991394148020654, |
| "num_full_absorption": 58, |
| "num_probe_true_positives": 1162, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.22891553059761788, |
| "full_absorption_rate": 0.18057921635434412, |
| "num_full_absorption": 318, |
| "num_probe_true_positives": 1761, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.07043036537933946, |
| "full_absorption_rate": 0.0364963503649635, |
| "num_full_absorption": 30, |
| "num_probe_true_positives": 822, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.09204694481402126, |
| "full_absorption_rate": 0.10933081998114987, |
| "num_full_absorption": 116, |
| "num_probe_true_positives": 1061, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.397383525427396, |
| "full_absorption_rate": 0.3700887198986058, |
| "num_full_absorption": 876, |
| "num_probe_true_positives": 2367, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.0009224929378531073, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 177, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.12721780451576514, |
| "full_absorption_rate": 0.13096695226438188, |
| "num_full_absorption": 214, |
| "num_probe_true_positives": 1634, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.5164213398692016, |
| "full_absorption_rate": 0.5086206896551724, |
| "num_full_absorption": 1416, |
| "num_probe_true_positives": 2784, |
| "num_split_features": 5 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.12789020911615354, |
| "full_absorption_rate": 0.11675741076830005, |
| "num_full_absorption": 193, |
| "num_probe_true_positives": 1653, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.10128583813911718, |
| "full_absorption_rate": 0.07472826086956522, |
| "num_full_absorption": 55, |
| "num_probe_true_positives": 736, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.02339272024828267, |
| "full_absorption_rate": 0.022857142857142857, |
| "num_full_absorption": 20, |
| "num_probe_true_positives": 875, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.01206131369374057, |
| "full_absorption_rate": 0.009188361408882083, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 653, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.07067388653125427, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 96, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.005526205377930516, |
| "full_absorption_rate": 0.01098901098901099, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 182, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.008673390879857475, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 230, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.18.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.18.hook_resid_post", |
| "hook_layer": 18, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
