{
  "results": {
    "crows_pairs_english_disability": {
      "likelihood_difference": 7.950961538461539,
      "likelihood_difference_stderr": 1.2464549281452417,
      "pct_stereotype": 0.5384615384615384,
      "pct_stereotype_stderr": 0.06231481440776789
    },
    "crows_pairs_french_sexual_orientation": {
      "likelihood_difference": 14.897664835164836,
      "likelihood_difference_stderr": 1.023996890557528,
      "pct_stereotype": 0.8021978021978022,
      "pct_stereotype_stderr": 0.04198895203196222
    },
    "logiqa": {
      "acc": 0.22887864823348694,
      "acc_stderr": 0.01647810727631327,
      "acc_norm": 0.24731182795698925,
      "acc_norm_stderr": 0.016922842446712393
    },
    "sciq": {
      "acc": 0.194,
      "acc_stderr": 0.012510816141264368,
      "acc_norm": 0.216,
      "acc_norm_stderr": 0.013019735539307789
    },
    "arc_easy": {
      "acc": 0.27104377104377103,
      "acc_stderr": 0.009120919741760602,
      "acc_norm": 0.2516835016835017,
      "acc_norm_stderr": 0.008905088235948768
    },
    "hendrycksTest-business_ethics": {
      "acc": 0.23,
      "acc_stderr": 0.04229525846816505,
      "acc_norm": 0.25,
      "acc_norm_stderr": 0.04351941398892446
    },
    "hendrycksTest-world_religions": {
      "acc": 0.19298245614035087,
      "acc_stderr": 0.03026745755489847,
      "acc_norm": 0.23391812865497075,
      "acc_norm_stderr": 0.03246721765117825
    },
    "hendrycksTest-philosophy": {
      "acc": 0.22508038585209003,
      "acc_stderr": 0.023720088516179027,
      "acc_norm": 0.2958199356913183,
      "acc_norm_stderr": 0.025922371788818784
    },
    "crows_pairs_english_sexual_orientation": {
      "likelihood_difference": 5.842741935483871,
      "likelihood_difference_stderr": 0.7853109235496362,
      "pct_stereotype": 0.44086021505376344,
      "pct_stereotype_stderr": 0.051762678118979284
    },
    "crows_pairs_english": {
      "likelihood_difference": 5.816692754919499,
      "likelihood_difference_stderr": 0.2005617516277463,
      "pct_stereotype": 0.45915324985092426,
      "pct_stereotype_stderr": 0.012172476264191387
    },
    "hendrycksTest-high_school_world_history": {
      "acc": 0.17721518987341772,
      "acc_stderr": 0.024856364184503217,
      "acc_norm": 0.21518987341772153,
      "acc_norm_stderr": 0.02675082699467616
    },
    "hendrycksTest-international_law": {
      "acc": 0.08264462809917356,
      "acc_stderr": 0.02513538235660422,
      "acc_norm": 0.2066115702479339,
      "acc_norm_stderr": 0.03695980128098824
    },
    "hendrycksTest-machine_learning": {
      "acc": 0.3125,
      "acc_stderr": 0.043994650575715215,
      "acc_norm": 0.30357142857142855,
      "acc_norm_stderr": 0.04364226155841044
    },
    "hendrycksTest-conceptual_physics": {
      "acc": 0.2553191489361702,
      "acc_stderr": 0.0285048564705142,
      "acc_norm": 0.25957446808510637,
      "acc_norm_stderr": 0.02865917937429232
    },
    "arc_challenge": {
      "acc": 0.21416382252559726,
      "acc_stderr": 0.011988383205966508,
      "acc_norm": 0.24658703071672355,
      "acc_norm_stderr": 0.01259572626879014
    },
    "crows_pairs_french_religion": {
      "likelihood_difference": 12.441304347826087,
      "likelihood_difference_stderr": 0.9700478612500396,
      "pct_stereotype": 0.6086956521739131,
      "pct_stereotype_stderr": 0.045709346351117126
    },
    "crows_pairs_english_race_color": {
      "likelihood_difference": 5.947957677165355,
      "likelihood_difference_stderr": 0.3647056359584575,
      "pct_stereotype": 0.3641732283464567,
      "pct_stereotype_stderr": 0.021370733739311764
    },
    "hendrycksTest-abstract_algebra": {
      "acc": 0.25,
      "acc_stderr": 0.04351941398892446,
      "acc_norm": 0.23,
      "acc_norm_stderr": 0.04229525846816506
    },
    "hendrycksTest-miscellaneous": {
      "acc": 0.2503192848020434,
      "acc_stderr": 0.01549108895149458,
      "acc_norm": 0.26436781609195403,
      "acc_norm_stderr": 0.015769984840690518
    },
    "hendrycksTest-college_medicine": {
      "acc": 0.20809248554913296,
      "acc_stderr": 0.03095289021774988,
      "acc_norm": 0.2658959537572254,
      "acc_norm_stderr": 0.03368762932259431
    },
    "hendrycksTest-global_facts": {
      "acc": 0.26,
      "acc_stderr": 0.04408440022768078,
      "acc_norm": 0.27,
      "acc_norm_stderr": 0.044619604333847394
    },
    "lambada_openai": {
      "ppl": 3683856.625077963,
      "ppl_stderr": 360078.72745232895,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "winogrande": {
      "acc": 0.4980268350434096,
      "acc_stderr": 0.014052376259225632
    },
    "hendrycksTest-marketing": {
      "acc": 0.23504273504273504,
      "acc_stderr": 0.027778835904935437,
      "acc_norm": 0.2564102564102564,
      "acc_norm_stderr": 0.028605953702004243
    },
    "hendrycksTest-high_school_physics": {
      "acc": 0.18543046357615894,
      "acc_stderr": 0.031732843842942865,
      "acc_norm": 0.31125827814569534,
      "acc_norm_stderr": 0.03780445850526733
    },
    "hendrycksTest-nutrition": {
      "acc": 0.17973856209150327,
      "acc_stderr": 0.021986032182064148,
      "acc_norm": 0.238562091503268,
      "acc_norm_stderr": 0.024404394928087877
    },
    "hendrycksTest-jurisprudence": {
      "acc": 0.1574074074074074,
      "acc_stderr": 0.03520703990517965,
      "acc_norm": 0.25925925925925924,
      "acc_norm_stderr": 0.042365112580946315
    },
    "crows_pairs_french_socioeconomic": {
      "likelihood_difference": 13.086854272959183,
      "likelihood_difference_stderr": 0.8661141987300517,
      "pct_stereotype": 0.413265306122449,
      "pct_stereotype_stderr": 0.035262902194360866
    },
    "hendrycksTest-college_computer_science": {
      "acc": 0.26,
      "acc_stderr": 0.04408440022768079,
      "acc_norm": 0.19,
      "acc_norm_stderr": 0.03942772444036624
    },
    "hendrycksTest-electrical_engineering": {
      "acc": 0.19310344827586207,
      "acc_stderr": 0.03289445522127401,
      "acc_norm": 0.19310344827586207,
      "acc_norm_stderr": 0.03289445522127401
    },
    "hendrycksTest-college_mathematics": {
      "acc": 0.18,
      "acc_stderr": 0.03861229196653695,
      "acc_norm": 0.17,
      "acc_norm_stderr": 0.03775251680686371
    },
    "crows_pairs_english_autre": {
      "likelihood_difference": 6.284090909090909,
      "likelihood_difference_stderr": 3.185464631314461,
      "pct_stereotype": 0.45454545454545453,
      "pct_stereotype_stderr": 0.15745916432444335
    },
    "hendrycksTest-management": {
      "acc": 0.2524271844660194,
      "acc_stderr": 0.04301250399690879,
      "acc_norm": 0.2912621359223301,
      "acc_norm_stderr": 0.044986763205729245
    },
    "hendrycksTest-security_studies": {
      "acc": 0.3142857142857143,
      "acc_stderr": 0.02971932942241746,
      "acc_norm": 0.19183673469387755,
      "acc_norm_stderr": 0.025206963154225378
    },
    "hendrycksTest-human_sexuality": {
      "acc": 0.2595419847328244,
      "acc_stderr": 0.03844876139785271,
      "acc_norm": 0.2595419847328244,
      "acc_norm_stderr": 0.03844876139785271
    },
    "crows_pairs_english_nationality": {
      "likelihood_difference": 6.443142361111111,
      "likelihood_difference_stderr": 0.539056794039704,
      "pct_stereotype": 0.44907407407407407,
      "pct_stereotype_stderr": 0.03392238405321617
    },
    "hendrycksTest-elementary_mathematics": {
      "acc": 0.2275132275132275,
      "acc_stderr": 0.021591269407823785,
      "acc_norm": 0.23544973544973544,
      "acc_norm_stderr": 0.021851509822031715
    },
    "hendrycksTest-high_school_psychology": {
      "acc": 0.25504587155963304,
      "acc_stderr": 0.018688500856535853,
      "acc_norm": 0.24587155963302754,
      "acc_norm_stderr": 0.018461940968708457
    },
    "hendrycksTest-anatomy": {
      "acc": 0.21481481481481482,
      "acc_stderr": 0.03547854198560823,
      "acc_norm": 0.2962962962962963,
      "acc_norm_stderr": 0.03944624162501117
    },
    "hendrycksTest-college_physics": {
      "acc": 0.23529411764705882,
      "acc_stderr": 0.04220773659171452,
      "acc_norm": 0.22549019607843138,
      "acc_norm_stderr": 0.04158307533083286
    },
    "hendrycksTest-high_school_geography": {
      "acc": 0.23737373737373738,
      "acc_stderr": 0.03031371053819888,
      "acc_norm": 0.2727272727272727,
      "acc_norm_stderr": 0.03173071239071724
    },
    "crows_pairs_english_gender": {
      "likelihood_difference": 5.01279296875,
      "likelihood_difference_stderr": 0.5134355620258765,
      "pct_stereotype": 0.5,
      "pct_stereotype_stderr": 0.027994625547792713
    },
    "hendrycksTest-us_foreign_policy": {
      "acc": 0.22,
      "acc_stderr": 0.04163331998932269,
      "acc_norm": 0.24,
      "acc_norm_stderr": 0.04292346959909283
    },
    "hendrycksTest-logical_fallacies": {
      "acc": 0.1656441717791411,
      "acc_stderr": 0.029208296231259104,
      "acc_norm": 0.2392638036809816,
      "acc_norm_stderr": 0.033519538795212696
    },
    "hendrycksTest-high_school_european_history": {
      "acc": 0.17575757575757575,
      "acc_stderr": 0.02972094300622445,
      "acc_norm": 0.18181818181818182,
      "acc_norm_stderr": 0.030117688929503585
    },
    "hendrycksTest-high_school_statistics": {
      "acc": 0.25462962962962965,
      "acc_stderr": 0.029711275860005354,
      "acc_norm": 0.2916666666666667,
      "acc_norm_stderr": 0.030998666304560534
    },
    "crows_pairs_french_disability": {
      "likelihood_difference": 14.813446969696969,
      "likelihood_difference_stderr": 1.4757253537933575,
      "pct_stereotype": 0.3484848484848485,
      "pct_stereotype_stderr": 0.0591013677911929
    },
    "wsc": {
      "acc": 0.5961538461538461,
      "acc_stderr": 0.048346889526540184
    },
    "hendrycksTest-moral_disputes": {
      "acc": 0.23410404624277456,
      "acc_stderr": 0.022797110278071138,
      "acc_norm": 0.21098265895953758,
      "acc_norm_stderr": 0.021966309947043117
    },
    "piqa": {
      "acc": 0.5228509249183896,
      "acc_stderr": 0.01165363483240117,
      "acc_norm": 0.5212187159956474,
      "acc_norm_stderr": 0.01165531473228886
    },
    "hendrycksTest-high_school_government_and_politics": {
      "acc": 0.21761658031088082,
      "acc_stderr": 0.029778663037752964,
      "acc_norm": 0.24352331606217617,
      "acc_norm_stderr": 0.030975436386845436
    },
    "crows_pairs_english_religion": {
      "likelihood_difference": 5.919763513513513,
      "likelihood_difference_stderr": 0.7154224085059995,
      "pct_stereotype": 0.5225225225225225,
      "pct_stereotype_stderr": 0.04762473917649626
    },
    "hendrycksTest-sociology": {
      "acc": 0.25870646766169153,
      "acc_stderr": 0.030965903123573044,
      "acc_norm": 0.2736318407960199,
      "acc_norm_stderr": 0.03152439186555402
    },
    "hendrycksTest-high_school_biology": {
      "acc": 0.20967741935483872,
      "acc_stderr": 0.02315787934908352,
      "acc_norm": 0.22258064516129034,
      "acc_norm_stderr": 0.02366421667164251
    },
    "hendrycksTest-high_school_computer_science": {
      "acc": 0.24,
      "acc_stderr": 0.042923469599092816,
      "acc_norm": 0.34,
      "acc_norm_stderr": 0.047609522856952344
    },
    "hendrycksTest-high_school_chemistry": {
      "acc": 0.22660098522167488,
      "acc_stderr": 0.029454863835293,
      "acc_norm": 0.21674876847290642,
      "acc_norm_stderr": 0.028990331252516235
    },
    "hendrycksTest-moral_scenarios": {
      "acc": 0.23798882681564246,
      "acc_stderr": 0.014242630070574915,
      "acc_norm": 0.27262569832402234,
      "acc_norm_stderr": 0.014893391735249588
    },
    "hendrycksTest-professional_psychology": {
      "acc": 0.2222222222222222,
      "acc_stderr": 0.01681902837573638,
      "acc_norm": 0.23366013071895425,
      "acc_norm_stderr": 0.017119158496044503
    },
    "crows_pairs_english_socioeconomic": {
      "likelihood_difference": 6.058059210526316,
      "likelihood_difference_stderr": 0.5512612681732508,
      "pct_stereotype": 0.47368421052631576,
      "pct_stereotype_stderr": 0.03631923996538703
    },
    "hendrycksTest-astronomy": {
      "acc": 0.18421052631578946,
      "acc_stderr": 0.031546980450822305,
      "acc_norm": 0.23684210526315788,
      "acc_norm_stderr": 0.03459777606810534
    },
    "hendrycksTest-econometrics": {
      "acc": 0.2543859649122807,
      "acc_stderr": 0.04096985139843669,
      "acc_norm": 0.30701754385964913,
      "acc_norm_stderr": 0.0433913832257986
    },
    "crows_pairs_french_physical_appearance": {
      "likelihood_difference": 10.164930555555555,
      "likelihood_difference_stderr": 1.3628801155784467,
      "pct_stereotype": 0.5138888888888888,
      "pct_stereotype_stderr": 0.05931618532716555
    },
    "crows_pairs_french_age": {
      "likelihood_difference": 5.870138888888889,
      "likelihood_difference_stderr": 0.7718652798506598,
      "pct_stereotype": 0.6444444444444445,
      "pct_stereotype_stderr": 0.05074011803597719
    },
    "crows_pairs_french_race_color": {
      "likelihood_difference": 8.873641304347826,
      "likelihood_difference_stderr": 0.3747500277540656,
      "pct_stereotype": 0.6413043478260869,
      "pct_stereotype_stderr": 0.02238663434141095
    },
    "crows_pairs_french_autre": {
      "likelihood_difference": 6.980769230769231,
      "likelihood_difference_stderr": 1.9247068413265722,
      "pct_stereotype": 0.38461538461538464,
      "pct_stereotype_stderr": 0.1404416814115811
    },
    "hendrycksTest-professional_law": {
      "acc": 0.23272490221642764,
      "acc_stderr": 0.010792595553888486,
      "acc_norm": 0.242503259452412,
      "acc_norm_stderr": 0.010946570966348788
    },
    "hendrycksTest-public_relations": {
      "acc": 0.2545454545454545,
      "acc_stderr": 0.041723430387053825,
      "acc_norm": 0.18181818181818182,
      "acc_norm_stderr": 0.03694284335337798
    },
    "hendrycksTest-professional_medicine": {
      "acc": 0.24632352941176472,
      "acc_stderr": 0.02617343857052,
      "acc_norm": 0.27941176470588236,
      "acc_norm_stderr": 0.02725720260611495
    },
    "hendrycksTest-clinical_knowledge": {
      "acc": 0.19245283018867926,
      "acc_stderr": 0.024262979839372277,
      "acc_norm": 0.2830188679245283,
      "acc_norm_stderr": 0.027724236492700904
    },
    "crows_pairs_english_physical_appearance": {
      "likelihood_difference": 6.251302083333333,
      "likelihood_difference_stderr": 0.8500284067467659,
      "pct_stereotype": 0.5416666666666666,
      "pct_stereotype_stderr": 0.05913268547421811
    },
    "crows_pairs_english_age": {
      "likelihood_difference": 3.84271978021978,
      "likelihood_difference_stderr": 0.6726241671396568,
      "pct_stereotype": 0.6593406593406593,
      "pct_stereotype_stderr": 0.049956709512768704
    },
    "hendrycksTest-high_school_mathematics": {
      "acc": 0.24074074074074073,
      "acc_stderr": 0.026067159222275794,
      "acc_norm": 0.2814814814814815,
      "acc_norm_stderr": 0.027420019350945263
    },
    "hendrycksTest-high_school_us_history": {
      "acc": 0.18627450980392157,
      "acc_stderr": 0.027325470966716326,
      "acc_norm": 0.2696078431372549,
      "acc_norm_stderr": 0.031145570659486782
    },
    "hendrycksTest-professional_accounting": {
      "acc": 0.2765957446808511,
      "acc_stderr": 0.026684564340460997,
      "acc_norm": 0.2624113475177305,
      "acc_norm_stderr": 0.026244920349843007
    },
    "hendrycksTest-computer_security": {
      "acc": 0.25,
      "acc_stderr": 0.04351941398892446,
      "acc_norm": 0.3,
      "acc_norm_stderr": 0.046056618647183814
    },
    "hendrycksTest-high_school_microeconomics": {
      "acc": 0.19327731092436976,
      "acc_stderr": 0.02564947026588919,
      "acc_norm": 0.2815126050420168,
      "acc_norm_stderr": 0.029213549414372163
    },
    "hendrycksTest-human_aging": {
      "acc": 0.2825112107623318,
      "acc_stderr": 0.03021683101150878,
      "acc_norm": 0.273542600896861,
      "acc_norm_stderr": 0.029918586707798817
    },
    "crows_pairs_french": {
      "likelihood_difference": 10.009703898330352,
      "likelihood_difference_stderr": 0.2345615645708032,
      "pct_stereotype": 0.5700655933214073,
      "pct_stereotype_stderr": 0.01209278993435711
    },
    "hendrycksTest-prehistory": {
      "acc": 0.23148148148148148,
      "acc_stderr": 0.02346842983245115,
      "acc_norm": 0.20679012345679013,
      "acc_norm_stderr": 0.022535006705942818
    },
    "crows_pairs_french_gender": {
      "likelihood_difference": 7.706775700934579,
      "likelihood_difference_stderr": 0.4545837073525256,
      "pct_stereotype": 0.5327102803738317,
      "pct_stereotype_stderr": 0.027890972865217984
    },
    "hendrycksTest-formal_logic": {
      "acc": 0.24603174603174602,
      "acc_stderr": 0.03852273364924318,
      "acc_norm": 0.23809523809523808,
      "acc_norm_stderr": 0.03809523809523809
    },
    "hendrycksTest-high_school_macroeconomics": {
      "acc": 0.21794871794871795,
      "acc_stderr": 0.020932445774463196,
      "acc_norm": 0.24358974358974358,
      "acc_norm_stderr": 0.02176373368417392
    },
    "hendrycksTest-college_biology": {
      "acc": 0.2638888888888889,
      "acc_stderr": 0.03685651095897532,
      "acc_norm": 0.2638888888888889,
      "acc_norm_stderr": 0.03685651095897532
    },
    "hendrycksTest-college_chemistry": {
      "acc": 0.17,
      "acc_stderr": 0.0377525168068637,
      "acc_norm": 0.19,
      "acc_norm_stderr": 0.03942772444036623
    },
    "crows_pairs_french_nationality": {
      "likelihood_difference": 10.080780632411066,
      "likelihood_difference_stderr": 0.5570562133400927,
      "pct_stereotype": 0.5652173913043478,
      "pct_stereotype_stderr": 0.03122795678881643
    },
    "hendrycksTest-virology": {
      "acc": 0.1927710843373494,
      "acc_stderr": 0.030709824050565274,
      "acc_norm": 0.22289156626506024,
      "acc_norm_stderr": 0.03240004825594688
    },
    "hendrycksTest-medical_genetics": {
      "acc": 0.23,
      "acc_stderr": 0.04229525846816505,
      "acc_norm": 0.24,
      "acc_norm_stderr": 0.04292346959909283
    }
  },
  "versions": {
    "crows_pairs_english_disability": 0,
    "crows_pairs_french_sexual_orientation": 0,
    "logiqa": 0,
    "sciq": 0,
    "arc_easy": 0,
    "hendrycksTest-business_ethics": 0,
    "hendrycksTest-world_religions": 0,
    "hendrycksTest-philosophy": 0,
    "crows_pairs_english_sexual_orientation": 0,
    "crows_pairs_english": 0,
    "hendrycksTest-high_school_world_history": 0,
    "hendrycksTest-international_law": 0,
    "hendrycksTest-machine_learning": 0,
    "hendrycksTest-conceptual_physics": 0,
    "arc_challenge": 0,
    "crows_pairs_french_religion": 0,
    "crows_pairs_english_race_color": 0,
    "hendrycksTest-abstract_algebra": 0,
    "hendrycksTest-miscellaneous": 0,
    "hendrycksTest-college_medicine": 0,
    "hendrycksTest-global_facts": 0,
    "lambada_openai": 0,
    "winogrande": 0,
    "hendrycksTest-marketing": 0,
    "hendrycksTest-high_school_physics": 0,
    "hendrycksTest-nutrition": 0,
    "hendrycksTest-jurisprudence": 0,
    "crows_pairs_french_socioeconomic": 0,
    "hendrycksTest-college_computer_science": 0,
    "hendrycksTest-electrical_engineering": 0,
    "hendrycksTest-college_mathematics": 0,
    "crows_pairs_english_autre": 0,
    "hendrycksTest-management": 0,
    "hendrycksTest-security_studies": 0,
    "hendrycksTest-human_sexuality": 0,
    "crows_pairs_english_nationality": 0,
    "hendrycksTest-elementary_mathematics": 0,
    "hendrycksTest-high_school_psychology": 0,
    "hendrycksTest-anatomy": 0,
    "hendrycksTest-college_physics": 0,
    "hendrycksTest-high_school_geography": 0,
    "crows_pairs_english_gender": 0,
    "hendrycksTest-us_foreign_policy": 0,
    "hendrycksTest-logical_fallacies": 0,
    "hendrycksTest-high_school_european_history": 0,
    "hendrycksTest-high_school_statistics": 0,
    "crows_pairs_french_disability": 0,
    "wsc": 0,
    "hendrycksTest-moral_disputes": 0,
    "piqa": 0,
    "hendrycksTest-high_school_government_and_politics": 0,
    "crows_pairs_english_religion": 0,
    "hendrycksTest-sociology": 0,
    "hendrycksTest-high_school_biology": 0,
    "hendrycksTest-high_school_computer_science": 0,
    "hendrycksTest-high_school_chemistry": 0,
    "hendrycksTest-moral_scenarios": 0,
    "hendrycksTest-professional_psychology": 0,
    "crows_pairs_english_socioeconomic": 0,
    "hendrycksTest-astronomy": 0,
    "hendrycksTest-econometrics": 0,
    "crows_pairs_french_physical_appearance": 0,
    "crows_pairs_french_age": 0,
    "crows_pairs_french_race_color": 0,
    "crows_pairs_french_autre": 0,
    "hendrycksTest-professional_law": 0,
    "hendrycksTest-public_relations": 0,
    "hendrycksTest-professional_medicine": 0,
    "hendrycksTest-clinical_knowledge": 0,
    "crows_pairs_english_physical_appearance": 0,
    "crows_pairs_english_age": 0,
    "hendrycksTest-high_school_mathematics": 0,
    "hendrycksTest-high_school_us_history": 0,
    "hendrycksTest-professional_accounting": 0,
    "hendrycksTest-computer_security": 0,
    "hendrycksTest-high_school_microeconomics": 0,
    "hendrycksTest-human_aging": 0,
    "crows_pairs_french": 0,
    "hendrycksTest-prehistory": 0,
    "crows_pairs_french_gender": 0,
    "hendrycksTest-formal_logic": 0,
    "hendrycksTest-high_school_macroeconomics": 0,
    "hendrycksTest-college_biology": 0,
    "hendrycksTest-college_chemistry": 0,
    "crows_pairs_french_nationality": 0,
    "hendrycksTest-virology": 0,
    "hendrycksTest-medical_genetics": 0
  },
  "config": {
    "model": "hf-causal",
    "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step2",
    "num_fewshot": 0,
    "batch_size": 32,
    "device": null,
    "no_cache": true,
    "limit": null,
    "bootstrap_iters": 100000,
    "description_dict": {}
  }
}