| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6", |
| "datetime_epoch_millis": 1745713344966, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.030641738503430955, |
| "mean_full_absorption_score": 0.010620377435913441, |
| "mean_num_split_features": 1.2692307692307692, |
| "std_dev_absorption_fraction_score": 0.08017005200130402, |
| "std_dev_full_absorption_score": 0.026213172490360672, |
| "std_dev_num_split_features": 0.533493565673837 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.005798810692021083, |
| "full_absorption_rate": 0.013471901462663588, |
| "num_full_absorption": 35, |
| "num_probe_true_positives": 2598, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.00367064709567123, |
| "full_absorption_rate": 0.001890359168241966, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 1587, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.00013667976900869454, |
| "full_absorption_rate": 0.0014998125234345708, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 2667, |
| "num_split_features": 3 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.0036121105863952383, |
| "full_absorption_rate": 0.004347826086956522, |
| "num_full_absorption": 7, |
| "num_probe_true_positives": 1610, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.007544122017269414, |
| "full_absorption_rate": 0.006199628022318661, |
| "num_full_absorption": 10, |
| "num_probe_true_positives": 1613, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.027624098709002357, |
| "full_absorption_rate": 0.004262574595055414, |
| "num_full_absorption": 5, |
| "num_probe_true_positives": 1173, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.012815803074061492, |
| "full_absorption_rate": 0.0033500837520938024, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 1194, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.0015463271322329441, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1021, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.033200589252298865, |
| "full_absorption_rate": 0.008928571428571428, |
| "num_full_absorption": 15, |
| "num_probe_true_positives": 1680, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.0002974403071995341, |
| "full_absorption_rate": 0.0021929824561403508, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 456, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.0001042819146698526, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 716, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 7.089382940108893e-05, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 1178, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.005401082493288296, |
| "full_absorption_rate": 0.004909983633387889, |
| "num_full_absorption": 9, |
| "num_probe_true_positives": 1833, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.10979792798809655, |
| "full_absorption_rate": 0.03837719298245614, |
| "num_full_absorption": 35, |
| "num_probe_true_positives": 912, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.017309150233650282, |
| "full_absorption_rate": 0.0018674136321195146, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 1071, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.014813211210475526, |
| "full_absorption_rate": 0.003426124197002141, |
| "num_full_absorption": 8, |
| "num_probe_true_positives": 2335, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.0, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 196, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.00724679768500131, |
| "full_absorption_rate": 0.001791044776119403, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 1675, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.015288309618132295, |
| "full_absorption_rate": 0.006456241032998565, |
| "num_full_absorption": 18, |
| "num_probe_true_positives": 2788, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.01178612202204936, |
| "full_absorption_rate": 0.008097165991902834, |
| "num_full_absorption": 14, |
| "num_probe_true_positives": 1729, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.026094149231887118, |
| "full_absorption_rate": 0.0024906600249066002, |
| "num_full_absorption": 2, |
| "num_probe_true_positives": 803, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.002677742701824105, |
| "full_absorption_rate": 0.0034762456546929316, |
| "num_full_absorption": 3, |
| "num_probe_true_positives": 863, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.042907618908085636, |
| "full_absorption_rate": 0.02269503546099291, |
| "num_full_absorption": 16, |
| "num_probe_true_positives": 705, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.0049581790704484605, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 88, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.40765481891445277, |
| "full_absorption_rate": 0.13259668508287292, |
| "num_full_absorption": 24, |
| "num_probe_true_positives": 181, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.03432828663258132, |
| "full_absorption_rate": 0.0038022813688212928, |
| "num_full_absorption": 1, |
| "num_probe_true_positives": 263, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.2.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.2.hook_resid_post", |
| "hook_layer": 2, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
