| { |
| "eval_type_id": "tpp", |
| "eval_config": { |
| "random_seed": 42, |
| "dataset_names": [ |
| "LabHC/bias_in_bios_class_set1", |
| "canrager/amazon_reviews_mcauley_1and5" |
| ], |
| "perform_scr": false, |
| "early_stopping_patience": 20, |
| "train_set_size": 4000, |
| "test_set_size": 1000, |
| "context_length": 128, |
| "probe_train_batch_size": 16, |
| "probe_test_batch_size": 500, |
| "probe_epochs": 20, |
| "probe_lr": 0.001, |
| "probe_l1_penalty": 0.001, |
| "sae_batch_size": 125, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "lower_vram_usage": false, |
| "model_name": "gemma-2-2b", |
| "n_values": [ |
| 2, |
| 5, |
| 10, |
| 20, |
| 50, |
| 100, |
| 500 |
| ], |
| "column1_vals_lookup": { |
| "LabHC/bias_in_bios_class_set1": [ |
| [ |
| "professor", |
| "nurse" |
| ], |
| [ |
| "architect", |
| "journalist" |
| ], |
| [ |
| "surgeon", |
| "psychologist" |
| ], |
| [ |
| "attorney", |
| "teacher" |
| ] |
| ], |
| "canrager/amazon_reviews_mcauley_1and5": [ |
| [ |
| "Books", |
| "CDs_and_Vinyl" |
| ], |
| [ |
| "Software", |
| "Electronics" |
| ], |
| [ |
| "Pet_Supplies", |
| "Office_Products" |
| ], |
| [ |
| "Industrial_and_Scientific", |
| "Toys_and_Games" |
| ] |
| ] |
| } |
| }, |
| "eval_id": "65b37170-42a3-4f63-bfdc-b4728a912b3a", |
| "datetime_epoch_millis": 1745617889770, |
| "eval_result_metrics": { |
| "tpp_metrics": { |
| "tpp_threshold_2_total_metric": 0.034275001287460326, |
| "tpp_threshold_2_intended_diff_only": 0.04880000948905945, |
| "tpp_threshold_2_unintended_diff_only": 0.01452500820159912, |
| "tpp_threshold_5_total_metric": 0.09335000514984132, |
| "tpp_threshold_5_intended_diff_only": 0.12610000967979432, |
| "tpp_threshold_5_unintended_diff_only": 0.032750004529953004, |
| "tpp_threshold_10_total_metric": 0.15155001133680346, |
| "tpp_threshold_10_intended_diff_only": 0.22580001950263978, |
| "tpp_threshold_10_unintended_diff_only": 0.07425000816583634, |
| "tpp_threshold_20_total_metric": 0.17910000532865522, |
| "tpp_threshold_20_intended_diff_only": 0.2895000159740448, |
| "tpp_threshold_20_unintended_diff_only": 0.11040001064538957, |
| "tpp_threshold_50_total_metric": 0.23420000821352005, |
| "tpp_threshold_50_intended_diff_only": 0.3863000214099884, |
| "tpp_threshold_50_unintended_diff_only": 0.15210001319646835, |
| "tpp_threshold_100_total_metric": 0.2484750121831894, |
| "tpp_threshold_100_intended_diff_only": 0.42960002422332766, |
| "tpp_threshold_100_unintended_diff_only": 0.18112501204013823, |
| "tpp_threshold_500_total_metric": 0.21202503293752673, |
| "tpp_threshold_500_intended_diff_only": 0.4476000487804413, |
| "tpp_threshold_500_unintended_diff_only": 0.23557501584291457 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", |
| "tpp_threshold_2_total_metric": 0.04569999873638153, |
| "tpp_threshold_2_intended_diff_only": 0.07200000286102295, |
| "tpp_threshold_2_unintended_diff_only": 0.026300004124641417, |
| "tpp_threshold_5_total_metric": 0.1410499930381775, |
| "tpp_threshold_5_intended_diff_only": 0.20260000228881836, |
| "tpp_threshold_5_unintended_diff_only": 0.06155000925064087, |
| "tpp_threshold_10_total_metric": 0.22605000138282777, |
| "tpp_threshold_10_intended_diff_only": 0.3658000111579895, |
| "tpp_threshold_10_unintended_diff_only": 0.13975000977516175, |
| "tpp_threshold_20_total_metric": 0.2153500109910965, |
| "tpp_threshold_20_intended_diff_only": 0.4216000199317932, |
| "tpp_threshold_20_unintended_diff_only": 0.20625000894069673, |
| "tpp_threshold_50_total_metric": 0.1738000065088272, |
| "tpp_threshold_50_intended_diff_only": 0.456600022315979, |
| "tpp_threshold_50_unintended_diff_only": 0.2828000158071518, |
| "tpp_threshold_100_total_metric": 0.13885000348091125, |
| "tpp_threshold_100_intended_diff_only": 0.46760002374649046, |
| "tpp_threshold_100_unintended_diff_only": 0.3287500202655792, |
| "tpp_threshold_500_total_metric": 0.08020002245903016, |
| "tpp_threshold_500_intended_diff_only": 0.4704000473022461, |
| "tpp_threshold_500_unintended_diff_only": 0.3902000248432159 |
| }, |
| { |
| "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", |
| "tpp_threshold_2_total_metric": 0.022850003838539124, |
| "tpp_threshold_2_intended_diff_only": 0.025600016117095947, |
| "tpp_threshold_2_unintended_diff_only": 0.002750012278556824, |
| "tpp_threshold_5_total_metric": 0.045650017261505124, |
| "tpp_threshold_5_intended_diff_only": 0.04960001707077026, |
| "tpp_threshold_5_unintended_diff_only": 0.003949999809265137, |
| "tpp_threshold_10_total_metric": 0.07705002129077912, |
| "tpp_threshold_10_intended_diff_only": 0.08580002784729004, |
| "tpp_threshold_10_unintended_diff_only": 0.008750006556510925, |
| "tpp_threshold_20_total_metric": 0.14284999966621398, |
| "tpp_threshold_20_intended_diff_only": 0.1574000120162964, |
| "tpp_threshold_20_unintended_diff_only": 0.014550012350082398, |
| "tpp_threshold_50_total_metric": 0.2946000099182129, |
| "tpp_threshold_50_intended_diff_only": 0.3160000205039978, |
| "tpp_threshold_50_unintended_diff_only": 0.02140001058578491, |
| "tpp_threshold_100_total_metric": 0.35810002088546755, |
| "tpp_threshold_100_intended_diff_only": 0.3916000247001648, |
| "tpp_threshold_100_unintended_diff_only": 0.03350000381469727, |
| "tpp_threshold_500_total_metric": 0.34385004341602327, |
| "tpp_threshold_500_intended_diff_only": 0.4248000502586365, |
| "tpp_threshold_500_unintended_diff_only": 0.08095000684261322 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.8.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.8.hook_resid_post", |
| "hook_layer": 8, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": { |
| "LabHC/bias_in_bios_class_set1": { |
| "0": { |
| "tpp_threshold_2_total_metric": 0.12300002574920654, |
| "tpp_threshold_2_intended_diff_only": 0.19600003957748413, |
| "tpp_threshold_2_unintended_diff_only": 0.07300001382827759, |
| "tpp_threshold_5_total_metric": 0.16875003278255463, |
| "tpp_threshold_5_intended_diff_only": 0.26500004529953003, |
| "tpp_threshold_5_unintended_diff_only": 0.0962500125169754, |
| "tpp_threshold_10_total_metric": 0.19075001776218414, |
| "tpp_threshold_10_intended_diff_only": 0.3370000123977661, |
| "tpp_threshold_10_unintended_diff_only": 0.14624999463558197, |
| "tpp_threshold_20_total_metric": 0.17150002717971802, |
| "tpp_threshold_20_intended_diff_only": 0.38700002431869507, |
| "tpp_threshold_20_unintended_diff_only": 0.21549999713897705, |
| "tpp_threshold_50_total_metric": 0.14350001513957977, |
| "tpp_threshold_50_intended_diff_only": 0.43300002813339233, |
| "tpp_threshold_50_unintended_diff_only": 0.28950001299381256, |
| "tpp_threshold_100_total_metric": 0.12699998915195465, |
| "tpp_threshold_100_intended_diff_only": 0.44700002670288086, |
| "tpp_threshold_100_unintended_diff_only": 0.3200000375509262, |
| "tpp_threshold_500_total_metric": 0.05675004422664642, |
| "tpp_threshold_500_intended_diff_only": 0.4530000686645508, |
| "tpp_threshold_500_unintended_diff_only": 0.39625002443790436 |
| }, |
| "1": { |
| "tpp_threshold_2_total_metric": 0.07025003433227539, |
| "tpp_threshold_2_intended_diff_only": 0.10000002384185791, |
| "tpp_threshold_2_unintended_diff_only": 0.02974998950958252, |
| "tpp_threshold_5_total_metric": 0.11925002932548523, |
| "tpp_threshold_5_intended_diff_only": 0.1510000228881836, |
| "tpp_threshold_5_unintended_diff_only": 0.031749993562698364, |
| "tpp_threshold_10_total_metric": 0.22475002706050873, |
| "tpp_threshold_10_intended_diff_only": 0.2990000247955322, |
| "tpp_threshold_10_unintended_diff_only": 0.0742499977350235, |
| "tpp_threshold_20_total_metric": 0.26175007224082947, |
| "tpp_threshold_20_intended_diff_only": 0.3890000581741333, |
| "tpp_threshold_20_unintended_diff_only": 0.12724998593330383, |
| "tpp_threshold_50_total_metric": 0.20500002801418304, |
| "tpp_threshold_50_intended_diff_only": 0.44200003147125244, |
| "tpp_threshold_50_unintended_diff_only": 0.2370000034570694, |
| "tpp_threshold_100_total_metric": 0.1220000684261322, |
| "tpp_threshold_100_intended_diff_only": 0.4580000638961792, |
| "tpp_threshold_100_unintended_diff_only": 0.335999995470047, |
| "tpp_threshold_500_total_metric": 0.06700006127357483, |
| "tpp_threshold_500_intended_diff_only": 0.4620000720024109, |
| "tpp_threshold_500_unintended_diff_only": 0.39500001072883606 |
| }, |
| "2": { |
| "tpp_threshold_2_total_metric": 0.00875002145767212, |
| "tpp_threshold_2_intended_diff_only": 0.017000019550323486, |
| "tpp_threshold_2_unintended_diff_only": 0.008249998092651367, |
| "tpp_threshold_5_total_metric": 0.0742499828338623, |
| "tpp_threshold_5_intended_diff_only": 0.12099999189376831, |
| "tpp_threshold_5_unintended_diff_only": 0.046750009059906006, |
| "tpp_threshold_10_total_metric": 0.14274998009204865, |
| "tpp_threshold_10_intended_diff_only": 0.3100000023841858, |
| "tpp_threshold_10_unintended_diff_only": 0.16725002229213715, |
| "tpp_threshold_20_total_metric": 0.13974995911121368, |
| "tpp_threshold_20_intended_diff_only": 0.38099998235702515, |
| "tpp_threshold_20_unintended_diff_only": 0.24125002324581146, |
| "tpp_threshold_50_total_metric": 0.13649998605251312, |
| "tpp_threshold_50_intended_diff_only": 0.4390000104904175, |
| "tpp_threshold_50_unintended_diff_only": 0.30250002443790436, |
| "tpp_threshold_100_total_metric": 0.10924999415874481, |
| "tpp_threshold_100_intended_diff_only": 0.4570000171661377, |
| "tpp_threshold_100_unintended_diff_only": 0.3477500230073929, |
| "tpp_threshold_500_total_metric": 0.05300001800060272, |
| "tpp_threshold_500_intended_diff_only": 0.46000003814697266, |
| "tpp_threshold_500_unintended_diff_only": 0.40700002014636993 |
| }, |
| "6": { |
| "tpp_threshold_2_total_metric": 0.013999968767166138, |
| "tpp_threshold_2_intended_diff_only": 0.0339999794960022, |
| "tpp_threshold_2_unintended_diff_only": 0.02000001072883606, |
| "tpp_threshold_5_total_metric": 0.1484999805688858, |
| "tpp_threshold_5_intended_diff_only": 0.19099998474121094, |
| "tpp_threshold_5_unintended_diff_only": 0.042500004172325134, |
| "tpp_threshold_10_total_metric": 0.3632500320672989, |
| "tpp_threshold_10_intended_diff_only": 0.46900004148483276, |
| "tpp_threshold_10_unintended_diff_only": 0.10575000941753387, |
| "tpp_threshold_20_total_metric": 0.29200002551078796, |
| "tpp_threshold_20_intended_diff_only": 0.4830000400543213, |
| "tpp_threshold_20_unintended_diff_only": 0.19100001454353333, |
| "tpp_threshold_50_total_metric": 0.2172500044107437, |
| "tpp_threshold_50_intended_diff_only": 0.48900002241134644, |
| "tpp_threshold_50_unintended_diff_only": 0.2717500180006027, |
| "tpp_threshold_100_total_metric": 0.19999997317790985, |
| "tpp_threshold_100_intended_diff_only": 0.4959999918937683, |
| "tpp_threshold_100_unintended_diff_only": 0.29600001871585846, |
| "tpp_threshold_500_total_metric": 0.13450001180171967, |
| "tpp_threshold_500_intended_diff_only": 0.4970000386238098, |
| "tpp_threshold_500_unintended_diff_only": 0.36250002682209015 |
| }, |
| "9": { |
| "tpp_threshold_2_total_metric": 0.012499943375587463, |
| "tpp_threshold_2_intended_diff_only": 0.012999951839447021, |
| "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, |
| "tpp_threshold_5_total_metric": 0.1944999396800995, |
| "tpp_threshold_5_intended_diff_only": 0.2849999666213989, |
| "tpp_threshold_5_unintended_diff_only": 0.09050002694129944, |
| "tpp_threshold_10_total_metric": 0.2087499499320984, |
| "tpp_threshold_10_intended_diff_only": 0.4139999747276306, |
| "tpp_threshold_10_unintended_diff_only": 0.20525002479553223, |
| "tpp_threshold_20_total_metric": 0.21174997091293335, |
| "tpp_threshold_20_intended_diff_only": 0.46799999475479126, |
| "tpp_threshold_20_unintended_diff_only": 0.2562500238418579, |
| "tpp_threshold_50_total_metric": 0.1667499989271164, |
| "tpp_threshold_50_intended_diff_only": 0.48000001907348633, |
| "tpp_threshold_50_unintended_diff_only": 0.31325002014636993, |
| "tpp_threshold_100_total_metric": 0.13599999248981476, |
| "tpp_threshold_100_intended_diff_only": 0.48000001907348633, |
| "tpp_threshold_100_unintended_diff_only": 0.34400002658367157, |
| "tpp_threshold_500_total_metric": 0.08974997699260712, |
| "tpp_threshold_500_intended_diff_only": 0.48000001907348633, |
| "tpp_threshold_500_unintended_diff_only": 0.3902500420808792 |
| } |
| }, |
| "canrager/amazon_reviews_mcauley_1and5": { |
| "1": { |
| "tpp_threshold_2_total_metric": 0.0022500604391098022, |
| "tpp_threshold_2_intended_diff_only": 0.001000046730041504, |
| "tpp_threshold_2_unintended_diff_only": -0.0012500137090682983, |
| "tpp_threshold_5_total_metric": 0.003750056028366089, |
| "tpp_threshold_5_intended_diff_only": 0.005000054836273193, |
| "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, |
| "tpp_threshold_10_total_metric": 0.021250054240226746, |
| "tpp_threshold_10_intended_diff_only": 0.028000056743621826, |
| "tpp_threshold_10_unintended_diff_only": 0.006750002503395081, |
| "tpp_threshold_20_total_metric": 0.07750004529953003, |
| "tpp_threshold_20_intended_diff_only": 0.08800005912780762, |
| "tpp_threshold_20_unintended_diff_only": 0.010500013828277588, |
| "tpp_threshold_50_total_metric": 0.23975002765655518, |
| "tpp_threshold_50_intended_diff_only": 0.26100003719329834, |
| "tpp_threshold_50_unintended_diff_only": 0.021250009536743164, |
| "tpp_threshold_100_total_metric": 0.35475003719329834, |
| "tpp_threshold_100_intended_diff_only": 0.39500004053115845, |
| "tpp_threshold_100_unintended_diff_only": 0.04025000333786011, |
| "tpp_threshold_500_total_metric": 0.39375007152557373, |
| "tpp_threshold_500_intended_diff_only": 0.44800007343292236, |
| "tpp_threshold_500_unintended_diff_only": 0.05425000190734863 |
| }, |
| "2": { |
| "tpp_threshold_2_total_metric": 0.003000020980834961, |
| "tpp_threshold_2_intended_diff_only": 0.012000024318695068, |
| "tpp_threshold_2_unintended_diff_only": 0.009000003337860107, |
| "tpp_threshold_5_total_metric": 0.02799999713897705, |
| "tpp_threshold_5_intended_diff_only": 0.03799998760223389, |
| "tpp_threshold_5_unintended_diff_only": 0.009999990463256836, |
| "tpp_threshold_10_total_metric": 0.07700000703334808, |
| "tpp_threshold_10_intended_diff_only": 0.09000003337860107, |
| "tpp_threshold_10_unintended_diff_only": 0.01300002634525299, |
| "tpp_threshold_20_total_metric": 0.14699998497962952, |
| "tpp_threshold_20_intended_diff_only": 0.17100000381469727, |
| "tpp_threshold_20_unintended_diff_only": 0.02400001883506775, |
| "tpp_threshold_50_total_metric": 0.37975001335144043, |
| "tpp_threshold_50_intended_diff_only": 0.4180000424385071, |
| "tpp_threshold_50_unintended_diff_only": 0.03825002908706665, |
| "tpp_threshold_100_total_metric": 0.39100004732608795, |
| "tpp_threshold_100_intended_diff_only": 0.4410000443458557, |
| "tpp_threshold_100_unintended_diff_only": 0.04999999701976776, |
| "tpp_threshold_500_total_metric": 0.3580000251531601, |
| "tpp_threshold_500_intended_diff_only": 0.4410000443458557, |
| "tpp_threshold_500_unintended_diff_only": 0.08300001919269562 |
| }, |
| "3": { |
| "tpp_threshold_2_total_metric": 0.001999974250793457, |
| "tpp_threshold_2_intended_diff_only": 0.0, |
| "tpp_threshold_2_unintended_diff_only": -0.001999974250793457, |
| "tpp_threshold_5_total_metric": 0.012000054121017456, |
| "tpp_threshold_5_intended_diff_only": 0.01500004529953003, |
| "tpp_threshold_5_unintended_diff_only": 0.0029999911785125732, |
| "tpp_threshold_10_total_metric": 0.02100004255771637, |
| "tpp_threshold_10_intended_diff_only": 0.029000043869018555, |
| "tpp_threshold_10_unintended_diff_only": 0.008000001311302185, |
| "tpp_threshold_20_total_metric": 0.07100000977516174, |
| "tpp_threshold_20_intended_diff_only": 0.08300000429153442, |
| "tpp_threshold_20_unintended_diff_only": 0.01199999451637268, |
| "tpp_threshold_50_total_metric": 0.1847500205039978, |
| "tpp_threshold_50_intended_diff_only": 0.19700002670288086, |
| "tpp_threshold_50_unintended_diff_only": 0.012250006198883057, |
| "tpp_threshold_100_total_metric": 0.294500008225441, |
| "tpp_threshold_100_intended_diff_only": 0.3190000057220459, |
| "tpp_threshold_100_unintended_diff_only": 0.02449999749660492, |
| "tpp_threshold_500_total_metric": 0.26575006544589996, |
| "tpp_threshold_500_intended_diff_only": 0.42100006341934204, |
| "tpp_threshold_500_unintended_diff_only": 0.15524999797344208 |
| }, |
| "5": { |
| "tpp_threshold_2_total_metric": 0.0755000114440918, |
| "tpp_threshold_2_intended_diff_only": 0.08100003004074097, |
| "tpp_threshold_2_unintended_diff_only": 0.00550001859664917, |
| "tpp_threshold_5_total_metric": 0.11924998462200165, |
| "tpp_threshold_5_intended_diff_only": 0.12099999189376831, |
| "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626, |
| "tpp_threshold_10_total_metric": 0.18275001645088196, |
| "tpp_threshold_10_intended_diff_only": 0.18800002336502075, |
| "tpp_threshold_10_unintended_diff_only": 0.005250006914138794, |
| "tpp_threshold_20_total_metric": 0.24925000965595245, |
| "tpp_threshold_20_intended_diff_only": 0.26200002431869507, |
| "tpp_threshold_20_unintended_diff_only": 0.012750014662742615, |
| "tpp_threshold_50_total_metric": 0.37824998795986176, |
| "tpp_threshold_50_intended_diff_only": 0.39399999380111694, |
| "tpp_threshold_50_unintended_diff_only": 0.015750005841255188, |
| "tpp_threshold_100_total_metric": 0.3997500091791153, |
| "tpp_threshold_100_intended_diff_only": 0.42900002002716064, |
| "tpp_threshold_100_unintended_diff_only": 0.02925001084804535, |
| "tpp_threshold_500_total_metric": 0.36400003731250763, |
| "tpp_threshold_500_intended_diff_only": 0.4320000410079956, |
| "tpp_threshold_500_unintended_diff_only": 0.06800000369548798 |
| }, |
| "6": { |
| "tpp_threshold_2_total_metric": 0.0314999520778656, |
| "tpp_threshold_2_intended_diff_only": 0.0339999794960022, |
| "tpp_threshold_2_unintended_diff_only": 0.0025000274181365967, |
| "tpp_threshold_5_total_metric": 0.06524999439716339, |
| "tpp_threshold_5_intended_diff_only": 0.0690000057220459, |
| "tpp_threshold_5_unintended_diff_only": 0.0037500113248825073, |
| "tpp_threshold_10_total_metric": 0.08324998617172241, |
| "tpp_threshold_10_intended_diff_only": 0.09399998188018799, |
| "tpp_threshold_10_unintended_diff_only": 0.010749995708465576, |
| "tpp_threshold_20_total_metric": 0.1694999486207962, |
| "tpp_threshold_20_intended_diff_only": 0.18299996852874756, |
| "tpp_threshold_20_unintended_diff_only": 0.013500019907951355, |
| "tpp_threshold_50_total_metric": 0.2905000001192093, |
| "tpp_threshold_50_intended_diff_only": 0.3100000023841858, |
| "tpp_threshold_50_unintended_diff_only": 0.0195000022649765, |
| "tpp_threshold_100_total_metric": 0.3505000025033951, |
| "tpp_threshold_100_intended_diff_only": 0.37400001287460327, |
| "tpp_threshold_100_unintended_diff_only": 0.02350001037120819, |
| "tpp_threshold_500_total_metric": 0.33775001764297485, |
| "tpp_threshold_500_intended_diff_only": 0.38200002908706665, |
| "tpp_threshold_500_unintended_diff_only": 0.0442500114440918 |
| } |
| } |
| } |
| } |
