| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6", |
| "datetime_epoch_millis": 1745714326463, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.06252438067410315, |
| "mean_full_absorption_score": 0.03032837603770599, |
| "mean_num_split_features": 1.1153846153846154, |
| "std_dev_absorption_fraction_score": 0.17523681810629935, |
| "std_dev_full_absorption_score": 0.08994997946680468, |
| "std_dev_num_split_features": 0.3258125936084211 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 6.745479887436016e-05, |
| "full_absorption_rate": 0.00591016548463357, |
| "num_full_absorption": 15, |
| "num_probe_true_positives": 2538, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.001256281407035176, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 1592, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.0005428242028196644, |
| "full_absorption_rate": 0.0007230657989877079, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 2766, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.0007365244729527473, |
| "full_absorption_rate": 0.0018050541516245488, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 1662, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.0023231423532737313, |
| "full_absorption_rate": 0.0037313432835820895, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 1608, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.002298539790103218, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1230, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.0467382967641465, |
| "full_absorption_rate": 0.010362694300518135, |
| "num_full_absorption": 12, |
| "num_probe_true_positives": 1158, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.003653998530967071, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1030, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.02189164337173642, |
| "full_absorption_rate": 0.011156782149148562, |
| "num_full_absorption": 19, |
| "num_probe_true_positives": 1703, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.00029991001094413993, |
| "full_absorption_rate": 0.002079002079002079, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 481, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0025307596845225355, |
| "full_absorption_rate": 0.0014326647564469914, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 698, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.001670328153269901, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1240, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.0026368777361244107, |
| "full_absorption_rate": 0.007816862088218872, |
| "num_full_absorption": 14, |
| "num_probe_true_positives": 1791, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.014327672359284787, |
| "full_absorption_rate": 0.002352941176470588, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 850, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.6964378812523078, |
| "full_absorption_rate": 0.35980392156862745, |
| "num_full_absorption": 367, |
| "num_probe_true_positives": 1020, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0008417508417508417, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 2376, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.002043754694139116, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 183, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.004093797083668226, |
| "full_absorption_rate": 0.001189767995240928, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 1681, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.005696734018731291, |
| "full_absorption_rate": 0.0021008403361344537, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 2856, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.0049314909971503885, |
| "full_absorption_rate": 0.00294811320754717, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 1696, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.04942735184557663, |
| "full_absorption_rate": 0.00784313725490196, |
| "num_full_absorption": 6, |
| "num_probe_true_positives": 765, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.001089946284004534, |
| "full_absorption_rate": 0.0012391573729863693, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 807, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.09739403070779135, |
| "full_absorption_rate": 0.04052098408104197, |
| "num_full_absorption": 28, |
| "num_probe_true_positives": 691, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.03159556645362061, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 117, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.606364766354413, |
| "full_absorption_rate": 0.3072289156626506, |
| "num_full_absorption": 51, |
| "num_probe_true_positives": 166, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.026840605606259556, |
| "full_absorption_rate": 0.016194331983805668, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 247, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.3.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.3.hook_resid_post", |
| "hook_layer": 3, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
